diff --git a/config/config.exs b/config/config.exs index 8f10a744..716dfdce 100644 --- a/config/config.exs +++ b/config/config.exs @@ -7,6 +7,8 @@ # General application configuration import Config +alias Microwaveprop.Workers.CanadianSoundingFetchWorker + # Configure esbuild (the version is required) config :esbuild, version: "0.27.4", @@ -75,7 +77,10 @@ config :microwaveprop, Oban, {"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker}, {"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker}, {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, - {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker} + {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, + # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch + {"30 1 * * *", CanadianSoundingFetchWorker}, + {"30 13 * * *", CanadianSoundingFetchWorker} ]} ] diff --git a/config/runtime.exs b/config/runtime.exs index 8dcd9aa4..a73d4d76 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -16,6 +16,8 @@ import Config # # Alternatively, you can use `mix phx.gen.release` to generate a `bin/server` # script that automatically sets the env var above. +alias Microwaveprop.Workers.CanadianSoundingFetchWorker + if System.get_env("PHX_SERVER") do config :microwaveprop, MicrowavepropWeb.Endpoint, server: true end @@ -188,7 +190,10 @@ if config_env() == :prod do {"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker}, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker, - args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}} + args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}}, + # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch + {"30 1 * * *", CanadianSoundingFetchWorker}, + {"30 13 * * *", CanadianSoundingFetchWorker} ]} ] diff --git a/config/test.exs b/config/test.exs index 80c27bc3..c31606bf 100644 --- a/config/test.exs +++ b/config/test.exs @@ -58,6 +58,7 @@ config :microwaveprop, nexrad_req_options: [plug: {Req.Test, Microwaveprop.Weath config :microwaveprop, solar_req_options: [plug: {Req.Test, Microwaveprop.Weather.SolarClient}] config :microwaveprop, srtm_req_options: [plug: {Req.Test, Microwaveprop.Terrain.Srtm}, retry: false] config :microwaveprop, start_freshness_monitor: false +config :microwaveprop, uwyo_req_options: [plug: {Req.Test, Microwaveprop.Weather.UwyoSoundingClient}, retry: false] # Initialize plugs at runtime for faster test compilation config :phoenix, :plug_init_mode, :runtime diff --git a/docs/plans/2026-04-13-hrdps-canadian-prop-grid.md b/docs/plans/2026-04-13-hrdps-canadian-prop-grid.md new file mode 100644 index 00000000..1a6d31ce --- /dev/null +++ b/docs/plans/2026-04-13-hrdps-canadian-prop-grid.md @@ -0,0 +1,311 @@ +# HRDPS Canadian Propagation Grid Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Extend the propagation scoring grid to cover all of Canada at HRDPS's 2.5 km resolution — the Canadian analog to HRRR. This lights up the `/map` for VE stations, gives terrain-diffraction analysis for VE QSOs, and unlocks cross-border path analysis for KP/W7-to-VE7 tropo paths. + +**Architecture:** Build a parallel NWP ingestion pipeline (`HrdpsClient` + `HrdpsGridWorker` + `hrdps_profiles` table) that matches the HRRR structure one-for-one, with three major differences: rotated lat/lon projection, CMC file naming (`CMC_hrdps_continental_*_ps2.5km_*_{FFF}.grib2`), and a different native-level grid spec for duct analysis. A new `propagation_scores` region tag distinguishes HRRR-sourced vs HRDPS-sourced cells so the front-end can render them uniformly. + +**Tech Stack:** Elixir 1.15, Req, Nx (optional for duct analysis), Postgres, Oban, wgrib2, proj (for projection conversions if needed) + +**Prerequisite:** Task 6 of the RDPS plan (`2026-04-13-rdps-vertical-profiles.md`) — or at minimum a working pattern for "score grid_data from a non-HRRR source". If you're starting HRDPS fresh, Task 6 below stands alone but will duplicate some work from the RDPS plan. + +--- + +## Pre-flight research (1–2 hours) + +**Step R.1: Confirm HRDPS datamart URL pattern** + +```bash +curl -s https://dd.weather.gc.ca/model_hrdps/continental/2.5km/ 2>&1 | head -30 +curl -s https://dd.weather.gc.ca/model_hrdps/continental/2.5km/00/000/ 2>&1 | head -30 +``` + +Expected: file list with entries like `CMC_hrdps_continental_TMP_TGL_2_ps2.5km_{YYYYMMDDHH}_P{FFF}-00.grib2`. + +**Step R.2: Download one sample + inspect with wgrib2** + +```bash +wgrib2 CMC_hrdps_continental_TMP_TGL_2_ps2.5km_2026041300_P000-00.grib2 -grid -v +``` + +Expected: confirms grid is rotated lat/lon, gives dimensions (typically ~2500×1800), and lists all variables. + +**Step R.3: Confirm pressure-level files exist** + +HRDPS publishes surface + pressure-level files separately (like HRRR wrfsfc vs wrfprs). Find the pressure-level variant: + +```bash +curl -s https://dd.weather.gc.ca/model_hrdps/continental/2.5km/00/000/ | grep -i 'ISBL' +``` + +If separate files per pressure level, the ingestion must fan out like `HrrrClient.pressure_messages/0`. If a single combined file, one download gets everything. + +**Step R.4: Decide projection strategy** + +wgrib2 can extract points from rotated lat/lon directly with `-lon LON LAT`. For extracting the full grid as a flat array, you need `proj` or a wgrib2 `-new_grid` reprojection. **Recommended**: never reproject — always extract point-by-point for the grid points we care about. Slower than bulk extraction but avoids projection bugs. + +**Step R.5: Confirm run frequency and forecast range** + +HRDPS: 4× daily (00/06/12/18Z), forecasts to 48h. Verify via datamart. This is half HRRR's run frequency (hourly) and 2.7× longer (48h vs 18h). + +**Decision gate:** After R.5, confirm with your human partner which forecast range to ingest. Recommendation: start with f00–f24 to match the "current + 1 day" use case. Extending to f48 doubles storage and compute and probably isn't worth it for amateur radio planning. + +--- + +## Task 1: `HrdpsClient` — surface + pressure fetch + +**Files:** +- Create: `lib/microwaveprop/weather/hrdps_client.ex` +- Test: `test/microwaveprop/weather/hrdps_client_test.exs` +- Fixture: `test/support/fixtures/hrdps_sample_sfc.grib2` + `test/support/fixtures/hrdps_sample_prs.grib2` + +**Step 1.1: Write failing URL builder test** + +```elixir +test "builds correct CMC datamart URL for surface temp" do + url = HrdpsClient.grib_url(~D[2026-04-13], 12, 6, :surface, :tmp) + assert url =~ "/model_hrdps/continental/2.5km/12/006/CMC_hrdps_continental_TMP_TGL_2_ps2.5km_2026041312_P006-00.grib2" +end +``` + +Adjust the format string after pre-flight research confirms the exact naming. + +**Step 1.2–1.4: Write → fail → implement → pass** + +Reuse `HrrrClient.parse_idx/1`, `HrrrClient.byte_ranges_for_messages/2`, and `HrrrClient.download_grib_ranges/2` — they're format-agnostic. Only the URL pattern and the message set differ. + +**Step 1.5: Implement `fetch_grid/3`** + +```elixir +@spec fetch_grid([{float(), float()}], DateTime.t(), keyword()) :: + {:ok, %{{float(), float()} => map()}} | {:error, term()} +def fetch_grid(points, run_time, opts \\ []) +``` + +Same signature as `HrrrClient.fetch_grid/3` so `PropagationGridWorker.process_forecast_hour/4` can call either one polymorphically in Task 6. + +**Step 1.6: Nail the projection** + +Write a focused test: pick a known Canadian airport (CYYR = Goose Bay, 53.32°N, 60.42°W) and verify the HRDPS profile at that point matches the UWYO radiosonde for the same `valid_time` within a reasonable tolerance (surface temp ±3°C, dewpoint ±4°C). If they don't match, the projection is wrong and you need to debug wgrib2's `-lon` extraction vs the CMC rotated grid. + +**Step 1.7: Commit** + +--- + +## Task 2: Native-level duct metrics via `HrdpsNativeClient` + +**Files:** +- Create: `lib/microwaveprop/weather/hrdps_native_client.ex` +- Test: `test/microwaveprop/weather/hrdps_native_client_test.exs` + +**Step 2.1: Confirm HRDPS publishes native hybrid levels** + +HRDPS uses CMC's GEM dynamical core with hybrid sigma-pressure levels (62 of them). The datamart file naming for native levels is `*_HY_*` or within `model_hrdps/continental/2.5km/native/` — confirm in pre-flight. + +**If native levels are NOT published**: this task is impossible; skip it and fall back to pressure-level duct analysis (coarser — 31 levels vs 62 — but still works). The map's ducting detection quality will be slightly worse over VE airspace than over CONUS, which is acceptable for a first pass. + +**If native levels ARE published**: mirror `HrrrNativeClient.fetch_native_duct_grid/4`. Use `Microwaveprop.Propagation.Duct.analyze/1` which is projection-agnostic — it just needs a list of levels with temperature, pressure, and specific humidity. + +**Step 2.2–2.5: TDD cycle as above** + +**Step 2.6: Commit** + +--- + +## Task 3: `hrdps_profiles` table + schema + +**Files:** +- Create: `priv/repo/migrations/{timestamp}_create_hrdps_profiles.exs` +- Create: `lib/microwaveprop/weather/hrdps_profile.ex` +- Test: `test/microwaveprop/weather/hrdps_profile_test.exs` + +**Step 3.1–3.4**: Mirror `hrrr_profiles` exactly. Same columns, same indexes. The only reason to have a separate table is operational isolation — don't contaminate the 10M-row HRRR partition with 8M-row HRDPS rows, and let you drop/reprocess HRDPS without touching HRRR. + +Use daily partitioning on `valid_time` just like `hrrr_profiles`. + +**Step 3.5: Partition-pruning migration** + +Add an `Oban.Plugins.Cron` entry for `HrdpsPartitionCleanup` similar to `HrrrPartitionCleanup` if one exists. + +**Step 3.6: Commit** + +--- + +## Task 4: `HrdpsGridWorker` — hourly compute over Canadian grid + +**Files:** +- Create: `lib/microwaveprop/workers/hrdps_grid_worker.ex` +- Test: `test/microwaveprop/workers/hrdps_grid_worker_test.exs` +- Modify: `lib/microwaveprop/propagation/grid.ex` (add `canadian_points/0`) + +**Step 4.1: Define the Canadian grid** + +At HRDPS 2.5 km resolution, Canada (42N-83N, -141 to -52W) has ~1.5M cells. Too many to score every hour. Instead: +- Sample at 0.125° (14 km) to match HRRR's output grid → ~100K points +- For the subset of points that are INSIDE HRRR's CONUS footprint, prefer HRRR; OUTSIDE, use HRDPS +- Store the "outside HRRR" mask as `Grid.hrdps_only_points/0` + +**Step 4.2: Worker perform/1** + +Mirror `PropagationGridWorker.process_forecast_hour/4` but with `HrdpsClient.fetch_grid/3` and `HrdpsNativeClient.fetch_native_duct_grid/4`. + +**Step 4.3: Scoring** + +Call `Propagation.score_grid_point/4` — it's already projection-agnostic. It takes a profile map and valid_time and produces per-band scores. No changes needed to the scorer. + +**Step 4.4: Upsert into `propagation_scores`** + +`grid_scores` is already indexed by `(lat, lon, valid_time, band_mhz)`. HRDPS and HRRR both write into the same table — the unique key prevents collisions because HRDPS cells have different (lat, lon) than HRRR cells at the shared 0.125° resolution IF you don't re-score HRRR's cells from HRDPS. Use the `hrdps_only_points/0` mask to guarantee no overlap. + +**Step 4.5: TDD cycle, commit** + +--- + +## Task 5: Oban queue + cron + +**Files:** +- Modify: `config/config.exs`, `config/runtime.exs` + +**Step 5.1: Add `hrdps` queue** + +```elixir +queues: [ + # ... existing + hrdps: 1 # HRDPS fetch is sequential like HRRR (memory pressure) +] +``` + +**Step 5.2: Add cron** + +```elixir +# HRDPS runs 00/06/12/18Z. Aligned with RDPS cadence. +{"35 */6 * * *", Microwaveprop.Workers.HrdpsGridWorker} +``` + +Note: 35 past the hour, not 5, because PropagationGridWorker (HRRR) fires at 5 past. Stagger by 30 minutes so they don't compete for the DB and the wgrib2 subprocess slot. + +**Step 5.3: `mix compile` sanity-check** + +**Step 5.4: Commit** + +--- + +## Task 6: Map overlay — extend prop map to cover Canada + +**Files:** +- Modify: `lib/microwaveprop_web/live/map_live.ex` — `@initial_bounds`, timeline +- Modify: `assets/js/propagation_map_hook.ts` — heatmap rendering bounds +- Modify: `lib/microwaveprop/propagation/grid.ex` — export `north_america_points/0` that returns HRRR ∪ HRDPS + +**Step 6.1: Write failing LiveView test** + +```elixir +test "/map renders scores for a Canadian grid cell (53N, -113W)" do + {:ok, view, html} = live(conn, ~p"/map") + # after PropagationGridWorker (HRRR) and HrdpsGridWorker (HRDPS) have both written scores + assert html =~ "53.0" # or whatever assertion proves the HRDPS cell made it to the client +end +``` + +**Step 6.2: Widen `@initial_bounds` in map_live.ex** + +From the CONUS-only `(23.5, 49.5, -124.5, -66.9)` to `(23.5, 83.0, -141.0, -52.0)`. Zoom out default to show all of North America. + +**Step 6.3: Verify front-end renders Canadian points** + +```bash +mix phx.server +# Open http://localhost:4000/map +# Confirm colored cells north of 49N after the first HRDPS run completes +``` + +**Step 6.4: Commit** + +--- + +## Task 7: Terrain + QSO path analysis north of the border + +**Files:** +- Modify: `lib/microwaveprop/terrain/terrain_analysis.ex` (if needed) +- Modify: `lib/microwaveprop/workers/terrain_profile_worker.ex` +- Possibly: mount SRTM data covering Canada onto the production pods + +**Step 7.1: Check SRTM coverage** + +SRTM1 covers 60°S–60°N. Canadian radiosondes and stations above 60°N (Yellowknife, Whitehorse, Inuvik, Resolute) have NO SRTM coverage. The terrain analysis pipeline needs a fallback — likely CDEM (Canadian Digital Elevation Model) or ASTER GDEM. + +**Decision gate with your human partner**: Does the feature need to work above 60°N? If yes, this is a multi-day sub-project (Task 7 becomes a whole new plan). If no — scope to 42–60°N and explicitly document that Inuvik paths won't get terrain analysis. + +**Step 7.2 onward**: Out of scope for this plan. Write a follow-up plan for Arctic terrain if your human partner wants coverage. + +**Step 7.3: Commit the scope decision to `docs/plans/` or a memory file** + +--- + +## Task 8: Freshness monitoring + observability + +**Files:** +- Modify: `lib/microwaveprop/propagation/freshness_monitor.ex` +- Modify: `lib/microwaveprop_web/live/status_live.ex` + +**Step 8.1: Extend FreshnessMonitor to track HRDPS run time** + +Add a `hrdps_last_run_at` field to the monitor state, surface it on `/status`. + +**Step 8.2: Alert when HRDPS run_time is > 4 hours stale** + +HRDPS runs every 6 hours, publishes ~90 min after, so a healthy pipeline never exceeds ~2.5 hours stale. 4 hours = something broken. + +**Step 8.3: Commit** + +--- + +## Task 9: Precommit, docs, PR + +**Step 9.1: `mix precommit`** + +Must show 0 failures. + +**Step 9.2: Update `CLAUDE.md`** + +- Add HRDPS to `### Key Data Sources` +- Add `HrdpsGridWorker` + `HrdpsFetchWorker` to the background queue table +- Add a note on the CONUS → North America scope change + +**Step 9.3: Update `memory/reference_hrdps_canada.md`** + +Mark the plan as executed — include the final ingestion rate, queue config, and any projection gotchas that surfaced. + +**Step 9.4: Commit, push to `github` and `origin`, let Flux deploy** + +Do NOT push to `dokku`. + +--- + +## Known risks + gotchas + +- **Projection**: HRDPS rotated lat/lon is the #1 source of bugs. Burn a day in Task 1.6 validating a dozen points against UWYO or ASOS surface obs before moving on. A 10 km projection offset is invisible on the propagation map but destroys terrain-path analysis. +- **Memory pressure**: The existing HRRR worker hits `:system_memory_high_watermark` alarms at ~4 GB. HRDPS grid is 3× larger in cell count. Plan for 8 GB per pod for the prop namespace, or offload HRDPS fetch to a dedicated node via Kubernetes node affinity. +- **Publish time**: CMC's publish delay can be erratic — 90 min on good days, 3+ hours under load. Build retry-with-backoff into `HrdpsGridWorker.perform/1` so the cron doesn't fire, find no data, and dead-letter immediately. +- **Arctic terrain**: SRTM stops at 60°N. Accept or fix — don't silently serve wrong diffraction losses to stations in Yellowknife. +- **HRRR/HRDPS overlap**: The two models disagree on the US/Canadian border. Pick a rule (prefer HRRR south of 49°N, prefer HRDPS north) and encode it in `Grid.point_source/2`. +- **Scope creep**: This plan is already ~5 days of work. Resist adding "while we're at it, let's also fix the native-level ducting for rotated grids" side-quests. Ship the minimum viable Canadian prop grid first. + +## Rough effort estimate + +| Task | Estimate | +|---|---| +| Pre-flight research | 2 hours | +| Task 1: HrdpsClient | 1 day (projection debug dominates) | +| Task 2: HrdpsNativeClient | 0.5 day (or skipped if no native) | +| Task 3: hrdps_profiles schema | 2 hours | +| Task 4: HrdpsGridWorker | 1 day | +| Task 5: Cron wiring | 1 hour | +| Task 6: Map overlay | 0.5 day | +| Task 7: Terrain coverage decision | 2 hours (punt if Arctic gets out-of-scope) | +| Task 8: Freshness monitoring | 2 hours | +| Task 9: Precommit + docs | 2 hours | +| **Total** | **~5 days** | + +Compare to RDPS plan (~2 days) and the UWYO Canadian soundings (~4 hours, already done). Do HRDPS only when the RDPS 10 km coverage feels limiting, not before. diff --git a/docs/plans/2026-04-13-rdps-vertical-profiles.md b/docs/plans/2026-04-13-rdps-vertical-profiles.md new file mode 100644 index 00000000..2003b71d --- /dev/null +++ b/docs/plans/2026-04-13-rdps-vertical-profiles.md @@ -0,0 +1,307 @@ +# RDPS Vertical Profiles (Canadian Ducting Outside HRRR Footprint) Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Extend the refractivity/ducting analysis beyond HRRR's Lambert CONUS footprint by ingesting ECCC's Regional Deterministic Prediction System (RDPS) vertical profiles. This gives us forecast ducting across all of Canada + northern-tier US gaps without the full HRDPS ingestion complexity. + +**Architecture:** RDPS publishes a dedicated "vertical profiles" subset at 10 km resolution, 84h forecast, 4×/day. The files contain pressure-level T/Td/height — exactly the shape `SoundingParams.derive/1` + `Propagation.Duct.analyze/1` already consume. Fetch one forecast cycle per run, decode, store as time-indexed profile rows in a new `rdps_profiles` table (parallel to `hrrr_profiles` but on a 10 km grid), and surface it through a new `/weather-ca` map or extend the existing weather map timeline to "fall back to RDPS outside the HRRR footprint". + +**Tech Stack:** Elixir 1.15, Req (HTTP), Nx optional, Postgres, Oban, wgrib2 (already installed for HRRR native duct grid) + +--- + +## Pre-flight: URL and format research + +Before writing any code, confirm the RDPS vertical-profiles format. MSC docs are thin; the cheapest way is to hit the datamart directly. + +**Files to touch:** None — this is read-only research. + +**Step R.1: Find today's run directory** + +Run: + +```bash +curl -s https://dd.weather.gc.ca/model_gem_regional/10km/grib2/00/000/ | grep -oE 'href="[^"]+\.grib2"' | head -20 +``` + +Expected: file list including a `*vertical*profile*.grib2` or similar naming. + +**Step R.2: Confirm vertical-profiles subdirectory** + +Actual RDPS directory on dd.weather.gc.ca is `/model_gem_regional/10km/grib2/{HH}/{FFF}/` — where HH is 00/06/12/18 and FFF is forecast hour. The vertical-profile subset may live under a different path; check: + +```bash +curl -s https://dd.weather.gc.ca/ensemble/rdps/grib2/ 2>/dev/null +curl -s https://dd.weather.gc.ca/20260413/WXO-DD/model_gem_regional/ 2>/dev/null | grep -oE 'href="[^"]+/?' +``` + +If no dedicated vertical-profiles subset exists, fall back to pulling TMP/DEPR/HGT at all 31 pressure levels of the regular RDPS pressure-level files (same approach `HrrrClient.fetch_grid/3` uses for HRRR). + +**Step R.3: Download one sample file and inspect with wgrib2** + +```bash +wgrib2 /tmp/sample.grib2 -var -v | head -30 +``` + +Confirm it contains TMP, DEPR (or SPFH), HGT at 1000, 925, 850, 700, 500, 400, 300, 250, 200 mb (minimum). + +**Decision after R.3:** If RDPS ships a ready-made vertical-profiles product with structured per-cell levels, continue with Task 1. If it's just the full pressure-level grid (like HRRR), this plan is still viable but the first task becomes "adapt HrrrClient pressure-level extraction to RDPS rotated lat/lon projection" — still simpler than full HRDPS. + +--- + +## Task 1: New `RdpsClient` module for GRIB2 fetch + decode + +**Files:** +- Create: `lib/microwaveprop/weather/rdps_client.ex` +- Test: `test/microwaveprop/weather/rdps_client_test.exs` +- Fixture: `test/support/fixtures/rdps_sample.grib2` (capture one real ~50 MB file via curl, commit it) + +**Step 1.1: Write failing test for URL builder** + +```elixir +test "builds the correct datamart URL for a run/forecast hour/pressure level" do + url = RdpsClient.grib_url(~D[2026-04-13], 12, 6, :tmp, 850) + assert url =~ "/20260413/WXO-DD/model_gem_regional/10km/grib2/12/006/" + assert url =~ "TMP_ISBL_850" +end +``` + +**Step 1.2: Verify test fails** + +``` +mix test test/microwaveprop/weather/rdps_client_test.exs -v +``` + +Expected: FAIL with UndefinedFunctionError. + +**Step 1.3: Implement `grib_url/5`** + +Match the actual dd.weather.gc.ca pattern discovered in pre-flight research. Copy the index-file + byte-range download pattern from `HrrrClient.fetch_product/4` — RDPS GRIB2 files also ship `.idx` sidecars. + +**Step 1.4: Run, confirm PASS** + +**Step 1.5: Write failing test for `fetch_cell_profile/3`** + +Given a lat/lon and run time, return `%{"pres", "hght", "tmpc", "dwpc"}` maps for all standard pressure levels, matching the shape `SoundingParams.derive/1` expects. Use the committed fixture GRIB2 via `Req.Test` stub or directly from disk. + +**Step 1.6: Implement fetch_cell_profile/3 using wgrib2 subprocess** + +Reuse `Microwaveprop.Weather.Grib2.Wgrib2.extract_points/3` — the module already wraps wgrib2 for HRRR extraction and supports arbitrary GRIB2 files. + +**Key nuance — rotated lat/lon projection:** RDPS uses rotated_latitude_longitude, not Lambert conformal. wgrib2 handles this internally for `-lon`, `-match ":(TMP|DEPR|HGT):(1000 mb|925 mb|850 mb|700 mb|500 mb|400 mb|300 mb|250 mb|200 mb)"` extraction — verify by comparing a known point against UWYO sounding for Winnipeg/YWG. + +**Step 1.7: Commit** + +```bash +git add lib/microwaveprop/weather/rdps_client.ex test/microwaveprop/weather/rdps_client_test.exs test/support/fixtures/rdps_sample.grib2 +git commit -m "Add RdpsClient for Canadian pressure-level weather fetch" +``` + +--- + +## Task 2: `rdps_profiles` schema + migration + +**Files:** +- Create: `priv/repo/migrations/{timestamp}_create_rdps_profiles.exs` (use `mix ecto.gen.migration create_rdps_profiles`) +- Create: `lib/microwaveprop/weather/rdps_profile.ex` +- Test: `test/microwaveprop/weather/rdps_profile_test.exs` + +**Step 2.1: Write failing changeset test** + +```elixir +test "requires valid_time, lat, lon, and profile" do + changeset = RdpsProfile.changeset(%RdpsProfile{}, %{}) + refute changeset.valid? + assert %{valid_time: [_], lat: [_], lon: [_], profile: [_]} = errors_on(changeset) +end +``` + +**Step 2.2: Generate migration** + +```bash +mix ecto.gen.migration create_rdps_profiles +``` + +Columns (mirror `hrrr_profiles` minus the native-duct fields RDPS doesn't provide): +- `id` binary_id +- `valid_time` utc_datetime +- `run_time` utc_datetime +- `lat` float +- `lon` float +- `profile` jsonb (array of pressure-level maps) +- `surface_pressure_mb`, `surface_temp_c`, `surface_dewpoint_c` floats +- `surface_refractivity`, `min_refractivity_gradient` floats +- `ducting_detected` boolean, `duct_characteristics` jsonb +- `pwat_mm`, `hpbl_m` floats (if RDPS publishes them; otherwise null) +- unique index on `(lat, lon, valid_time)` +- timestamps + +**Step 2.3: Implement schema module** + +Copy shape from `lib/microwaveprop/weather/hrrr_profile.ex`, strip anything RDPS doesn't supply. + +**Step 2.4: Run migration against dev DB and verify** + +```bash +mix ecto.migrate +psql -h localhost -d microwaveprop_dev -c "\d rdps_profiles" +``` + +**Step 2.5: Run schema test, verify PASS** + +**Step 2.6: Commit** + +--- + +## Task 3: `RdpsFetchWorker` — one cell per job, queue-limited + +**Files:** +- Create: `lib/microwaveprop/workers/rdps_fetch_worker.ex` +- Test: `test/microwaveprop/workers/rdps_fetch_worker_test.exs` + +**Step 3.1: Write failing test** + +Worker takes `%{"lat" => lat, "lon" => lon, "valid_time" => iso}`, stubs `RdpsClient.fetch_cell_profile/3`, asserts an `RdpsProfile` row is upserted with derived SoundingParams fields filled in. + +**Step 3.2: Implement worker** + +- `use Oban.Worker, queue: :rdps, max_attempts: 3, unique: ...` +- `perform/1` calls `RdpsClient.fetch_cell_profile`, runs `SoundingParams.derive/1` on the returned profile, upserts via `Weather.upsert_rdps_profile/1` (add this function). + +**Step 3.3: Verify test passes** + +**Step 3.4: Commit** + +--- + +## Task 4: `RdpsGridWorker` — hourly-ish batch driver + +**Files:** +- Create: `lib/microwaveprop/workers/rdps_grid_worker.ex` +- Test: `test/microwaveprop/workers/rdps_grid_worker_test.exs` + +**Step 4.1: Write failing test** + +Test that `perform/1` enqueues one `RdpsFetchWorker` per grid point in a Canadian bounding box `(south, north, west, east) = (42, 83, -141, -52)` sampled at 0.5° (≈1500 points). + +**Step 4.2: Implement** + +Mirror `PropagationGridWorker.perform/1` — compute run_time (2 hours ago nearest RDPS synoptic hour 00/06/12/18), loop over Canadian grid, enqueue. Don't couple to HRRR; RDPS runs on a different cadence (every 6 hours vs HRRR every hour) and this keeps the two pipelines independent. + +**Step 4.3: Verify** + +**Step 4.4: Commit** + +--- + +## Task 5: Oban cron wiring + queue limits + +**Files:** +- Modify: `config/config.exs` (dev + shared) +- Modify: `config/runtime.exs` (prod Oban config) + +**Step 5.1: Add `rdps` queue** + +```elixir +queues: [ + # ... existing + rdps: 2 +] +``` + +Dev: 2. Prod: 2 per pod × 3 pods = 6 workers, plus a global_limit if MSC starts rate-limiting. + +**Step 5.2: Add cron** + +```elixir +# RDPS runs 00/06/12/18Z. Data typically publishes ~90 minutes after synoptic. +{"30 */6 * * *", Microwaveprop.Workers.RdpsGridWorker} +``` + +**Step 5.3: `mix compile` sanity-check** + +**Step 5.4: Commit** + +--- + +## Task 6: Score integration — fall back to RDPS outside HRRR footprint + +**Files:** +- Modify: `lib/microwaveprop/propagation.ex:~244` (`scores_at/3`) +- Modify: `lib/microwaveprop/workers/propagation_grid_worker.ex:~74` (`process_forecast_hour/4`) +- Test: `test/microwaveprop/propagation/rdps_fallback_test.exs` + +**Step 6.1: Write failing test** + +For a lat/lon inside HRRR's CONUS footprint, `Propagation.score_grid_point/4` uses the HRRR profile. For a lat/lon in northern Canada (65N, -90W), it falls back to the `RdpsProfile` for the same valid_time. + +**Step 6.2: Add `rdps_profile_at/3` helper in `Weather`** + +```elixir +def rdps_profile_at(lat, lon, valid_time) do + # nearest-neighbor lookup in rdps_profiles +end +``` + +**Step 6.3: Modify `process_forecast_hour/4` to also score RDPS points** + +Two-pass: first the HRRR CONUS grid as today, then a second pass over RDPS cells that fall outside HRRR's Lambert domain. Extend `grid_data` to include both sources, mark each point with `source: :hrrr` or `source: :rdps` in factors. + +**Step 6.4: Verify tests + precommit** + +**Step 6.5: Commit** + +--- + +## Task 7: Map overlay — extend CONUS prop map north + +**Files:** +- Modify: `lib/microwaveprop/propagation/grid.ex` — `conus_points/0` → `north_america_points/0` or similar +- Modify: `assets/js/propagation_map_hook.ts` — extend bounds, label RDPS-sourced cells + +**Step 7.1: Decide UX** + +Pick one: +- **Option A**: Silent extension — the existing /map just shows scores north of 49N without visual distinction. +- **Option B**: Dashed line on the map at the HRRR/RDPS boundary, tooltip notes data source. + +Default to Option A unless your human partner prefers B. + +**Step 7.2: Extend grid bounds** + +Change the `conus_points/0` generator to include cells in `(42, 83, -141, -52)` for RDPS. Keep CONUS cells on the current 0.125° grid, add RDPS cells at 0.5° (coarser). + +**Step 7.3: Verify `/map` renders Canadian cells** + +Manual: `mix phx.server`, open http://localhost:4000/map, zoom to Ontario/Quebec, confirm colored cells appear. + +**Step 7.4: Commit** + +--- + +## Task 8: Precommit + PR + +**Step 8.1: `mix precommit`** + +Must show 0 failures. + +**Step 8.2: Update `CLAUDE.md` with the new pipeline** + +Add an `RdpsFetchWorker`/`RdpsGridWorker` row to the Background Job Queues table and a bullet under `## Key Data Sources` for RDPS. + +**Step 8.3: Update `memory/reference_hrdps_canada.md`** + +Add a note: "RDPS vertical profiles (10 km) ingestion landed YYYY-MM-DD — gives Canadian ducting without the full HRDPS rewrite." + +**Step 8.4: Commit and push to both `github` and `origin` remotes** + +Do NOT push to `dokku`. Let Flux roll out the new image from the CI build. + +--- + +## Notes / risks + +- **Rotated lat/lon projection**: wgrib2 handles it correctly for point extraction but caution needed if the team ever decides to store full-grid binary rasters. +- **File size**: RDPS pressure-level files are ~50 MB each. At 4 runs/day × ~10 forecast hours × ~5 variables × ~10 pressure levels, total download ≈ 2 GB/day. Not a problem on a home lab with skippy.w5isp.com cache, but worth caching aggressively. +- **Data latency**: RDPS publishes ~90 minutes after synoptic; the cron is already aligned. +- **Fallback when RDPS is unavailable**: `scores_at/3` should gracefully degrade to HRRR-only if `rdps_profiles` is empty for the requested `valid_time`. diff --git a/lib/microwaveprop/weather/uwyo_sounding_client.ex b/lib/microwaveprop/weather/uwyo_sounding_client.ex new file mode 100644 index 00000000..0072d3f6 --- /dev/null +++ b/lib/microwaveprop/weather/uwyo_sounding_client.ex @@ -0,0 +1,158 @@ +defmodule Microwaveprop.Weather.UwyoSoundingClient do + @moduledoc """ + Client for University of Wyoming atmospheric sounding archive + (`weather.uwyo.edu`), which serves near-real-time radiosonde data for + the global WMO network — including Canadian stations that the IEM RAOB + endpoint no longer keeps current. + + Produces the same shape as `Microwaveprop.Weather.IemClient.parse_raob_json/1`: + `{:ok, [%{observed_at: DateTime.t(), profile: [map()]}]}`, where each + profile level is `%{"pres", "hght", "tmpc", "dwpc", "drct", "sknt"}` with + string keys so `Microwaveprop.Weather.SoundingParams.derive/1` can consume + it directly. + + Station codes are UWYO's 3-letter form — for Canadian ICAO codes (CWSE, + CYYR, ...) the caller must strip the leading `C`. + """ + + @base "https://weather.uwyo.edu/cgi-bin/sounding" + + @type profile_level :: %{String.t() => float() | nil} + @type sounding :: %{observed_at: DateTime.t(), profile: [profile_level()]} + + @spec sounding_url(String.t(), DateTime.t()) :: String.t() + def sounding_url(station_code, %DateTime{} = dt) do + month = dt.month |> Integer.to_string() |> String.pad_leading(2, "0") + day = dt.day |> Integer.to_string() |> String.pad_leading(2, "0") + hour = dt.hour |> Integer.to_string() |> String.pad_leading(2, "0") + dayhour = day <> hour + + "#{@base}?region=naconf&TYPE=TEXT%3ALIST" <> + "&YEAR=#{dt.year}&MONTH=#{month}&FROM=#{dayhour}&TO=#{dayhour}&STNM=#{station_code}" + end + + @spec fetch_sounding(String.t(), DateTime.t()) :: {:ok, [sounding()]} | {:error, term()} + def fetch_sounding(station_code, %DateTime{} = dt) do + url = sounding_url(station_code, dt) + + case Req.get(url, req_options()) do + {:ok, %{status: 200, body: body}} -> {:ok, parse_sounding_html(body)} + {:ok, %{status: status}} -> {:error, "UWYO sounding HTTP #{status}"} + {:error, reason} -> {:error, reason} + end + end + + @spec parse_sounding_html(String.t()) :: [sounding()] + def parse_sounding_html(html) when is_binary(html) do + with {:ok, observed_at} <- parse_h2_header(html), + {:ok, pre_text} <- extract_pre_block(html) do + profile = + pre_text + |> String.split("\n") + |> Enum.flat_map(&parse_profile_row/1) + + [%{observed_at: observed_at, profile: profile}] + else + _ -> [] + end + end + + # ---------- Private ---------- + + defp req_options do + defaults = [retry: &retry?/2, max_retries: 3, retry_delay: &retry_delay/1] + overrides = Application.get_env(:microwaveprop, :uwyo_req_options, []) + Keyword.merge(defaults, overrides) + end + + defp retry?(_request, response) do + case response do + %Req.Response{status: status} when status in [429, 500, 502, 503, 504] -> true + %{__exception__: true} -> true + _ -> false + end + end + + defp retry_delay(n) do + base = Integer.pow(2, n) * 1_000 + jitter = :rand.uniform(1_000) + base + jitter + end + + # Header line example: + #

71816 YYR Goose Bay Observations at 00Z 13 Apr 2026

+ defp parse_h2_header(html) do + case Regex.run(~r/

\s*\d+\s+\w+.*?Observations at (\d{2})Z (\d{1,2}) (\w+) (\d{4})/i, html) do + [_, hh, dd, mon, yyyy] -> + with {:ok, month} <- month_num(mon), + {day, ""} <- Integer.parse(dd), + {hour, ""} <- Integer.parse(hh), + {year, ""} <- Integer.parse(yyyy), + {:ok, dt} <- DateTime.new(Date.new!(year, month, day), Time.new!(hour, 0, 0)) do + {:ok, dt} + else + _ -> :error + end + + _ -> + :error + end + end + + defp extract_pre_block(html) do + case Regex.run(~r/
(.*?)<\/PRE>/s, html) do
+      [_, body] -> {:ok, body}
+      _ -> :error
+    end
+  end
+
+  # UWYO rows are fixed-width, 7 chars per column. We care about the first 8
+  # columns: PRES, HGHT, TEMP, DWPT, RELH, MIXR, DRCT, SKNT. Any line without
+  # both a numeric pressure AND a numeric temperature is a header/separator
+  # line or an incomplete observation and is skipped.
+  defp parse_profile_row(line) when is_binary(line) do
+    if String.length(line) < 28 do
+      []
+    else
+      pres = parse_column(line, 0)
+      hght = parse_column(line, 7)
+      tmpc = parse_column(line, 14)
+      dwpc = parse_column(line, 21)
+      drct = parse_column(line, 42)
+      sknt = parse_column(line, 49)
+
+      if is_number(pres) and is_number(tmpc) do
+        [%{"pres" => pres, "hght" => hght, "tmpc" => tmpc, "dwpc" => dwpc, "drct" => drct, "sknt" => sknt}]
+      else
+        []
+      end
+    end
+  end
+
+  defp parse_column(line, offset) do
+    field = line |> String.slice(offset, 7) |> String.trim()
+
+    if field == "" do
+      nil
+    else
+      case Float.parse(field) do
+        {val, ""} -> val
+        _ -> nil
+      end
+    end
+  end
+
+  defp month_num("Jan"), do: {:ok, 1}
+  defp month_num("Feb"), do: {:ok, 2}
+  defp month_num("Mar"), do: {:ok, 3}
+  defp month_num("Apr"), do: {:ok, 4}
+  defp month_num("May"), do: {:ok, 5}
+  defp month_num("Jun"), do: {:ok, 6}
+  defp month_num("Jul"), do: {:ok, 7}
+  defp month_num("Aug"), do: {:ok, 8}
+  defp month_num("Sep"), do: {:ok, 9}
+  defp month_num("Oct"), do: {:ok, 10}
+  defp month_num("Nov"), do: {:ok, 11}
+  defp month_num("Dec"), do: {:ok, 12}
+  defp month_num(_), do: :error
+end
diff --git a/lib/microwaveprop/workers/canadian_sounding_fetch_worker.ex b/lib/microwaveprop/workers/canadian_sounding_fetch_worker.ex
new file mode 100644
index 00000000..a4740c3d
--- /dev/null
+++ b/lib/microwaveprop/workers/canadian_sounding_fetch_worker.ex
@@ -0,0 +1,129 @@
+defmodule Microwaveprop.Workers.CanadianSoundingFetchWorker do
+  @moduledoc """
+  Twice-daily batch fetcher for Canadian radiosonde soundings via the
+  University of Wyoming archive. IEM's RAOB endpoint has not been keeping
+  Canadian stations current (most stalled Sep 2024), so this worker fills
+  the gap — crucial because soundings drive the refractivity / ducting
+  calibration in `SoundingParams.derive/1`.
+
+  Finds every `weather_stations` row of type `sounding` whose `station_code`
+  starts with `C` (Canadian ICAO prefix), drops the leading `C` to get the
+  UWYO 3-letter code, fetches the latest available 00Z or 12Z sounding,
+  and upserts it into `soundings`.
+  """
+
+  use Oban.Worker,
+    queue: :weather,
+    max_attempts: 3,
+    unique: [period: 3600, states: [:available, :scheduled, :executing, :retryable]]
+
+  import Ecto.Query
+
+  alias Microwaveprop.Repo
+  alias Microwaveprop.Weather
+  alias Microwaveprop.Weather.SoundingParams
+  alias Microwaveprop.Weather.Station
+  alias Microwaveprop.Weather.UwyoSoundingClient
+
+  require Logger
+
+  # UWYO publishes ~90 minutes after the synoptic hour
+  @publish_delay_seconds 5_400
+
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: args}) do
+    sounding_time =
+      case args do
+        %{"sounding_time" => iso} when is_binary(iso) ->
+          {:ok, dt, _} = DateTime.from_iso8601(iso)
+          DateTime.truncate(dt, :second)
+
+        _ ->
+          most_recent_sounding_time(DateTime.utc_now())
+      end
+
+    stations = canadian_sounding_stations()
+
+    Logger.info("CanadianSoundings: fetching #{length(stations)} stations for #{DateTime.to_iso8601(sounding_time)}")
+
+    Enum.each(stations, &fetch_station(&1, sounding_time))
+
+    :ok
+  end
+
+  @doc """
+  Return the most recently-publishable 00Z or 12Z sounding time, given
+  the current UTC instant. UWYO needs ~90 minutes after launch before the
+  decoded profile lands on the site, so we step back from synoptic hour
+  until we find one that's at least 90 minutes old.
+  """
+  @spec most_recent_sounding_time(DateTime.t()) :: DateTime.t()
+  def most_recent_sounding_time(%DateTime{} = now) do
+    cutoff = DateTime.add(now, -@publish_delay_seconds, :second)
+
+    cond do
+      cutoff.hour >= 12 ->
+        %{cutoff | hour: 12, minute: 0, second: 0, microsecond: {0, 0}}
+
+      cutoff.hour >= 0 and DateTime.compare(cutoff, %{cutoff | hour: 0, minute: 0, second: 0, microsecond: {0, 0}}) != :lt ->
+        %{cutoff | hour: 0, minute: 0, second: 0, microsecond: {0, 0}}
+    end
+  end
+
+  # ---------- Internal ----------
+
+  defp canadian_sounding_stations do
+    Repo.all(from(s in Station, where: s.station_type == "sounding" and ilike(s.station_code, "c%")))
+  end
+
+  defp fetch_station(%Station{station_code: code} = station, sounding_time) do
+    uwyo_code = String.slice(code, 1..-1//1)
+
+    if Weather.has_sounding?(station.id, sounding_time) do
+      Logger.debug("CanadianSoundings: #{code} already has sounding for #{sounding_time}")
+      :ok
+    else
+      do_fetch(station, code, uwyo_code, sounding_time)
+    end
+  end
+
+  defp do_fetch(station, code, uwyo_code, sounding_time) do
+    case UwyoSoundingClient.fetch_sounding(uwyo_code, sounding_time) do
+      {:ok, [%{profile: profile} | _]} when profile != [] ->
+        attrs = build_attrs(sounding_time, profile)
+        Weather.upsert_sounding(station, attrs)
+
+        Logger.info("CanadianSoundings: #{code} ingested sounding with #{length(profile)} levels")
+
+      {:ok, _} ->
+        Logger.info("CanadianSoundings: #{code} no data for #{sounding_time}")
+
+      {:error, reason} ->
+        Logger.warning("CanadianSoundings: #{code} failed: #{inspect(reason)}")
+    end
+  end
+
+  defp build_attrs(sounding_time, profile) do
+    base = %{observed_at: sounding_time, profile: profile, level_count: length(profile)}
+
+    case SoundingParams.derive(profile) do
+      nil ->
+        base
+
+      params ->
+        Map.merge(base, %{
+          surface_pressure_mb: params.surface_pressure_mb,
+          surface_temp_c: params.surface_temp_c,
+          surface_dewpoint_c: params.surface_dewpoint_c,
+          surface_refractivity: params.surface_refractivity,
+          min_refractivity_gradient: params.min_refractivity_gradient,
+          boundary_layer_depth_m: params.boundary_layer_depth_m,
+          precipitable_water_mm: params.precipitable_water_mm,
+          k_index: params.k_index,
+          lifted_index: params.lifted_index,
+          ducting_detected: params.ducting_detected,
+          duct_characteristics: params.duct_characteristics
+        })
+    end
+  end
+end
diff --git a/test/microwaveprop/weather/uwyo_sounding_client_test.exs b/test/microwaveprop/weather/uwyo_sounding_client_test.exs
new file mode 100644
index 00000000..a8e6626d
--- /dev/null
+++ b/test/microwaveprop/weather/uwyo_sounding_client_test.exs
@@ -0,0 +1,122 @@
+defmodule Microwaveprop.Weather.UwyoSoundingClientTest do
+  use ExUnit.Case, async: true
+
+  alias Microwaveprop.Weather.UwyoSoundingClient
+
+  @fixture File.read!("test/support/fixtures/uwyo_sounding_yyr.html")
+
+  describe "sounding_url/2" do
+    test "builds the correct URL for a 3-letter station code + datetime" do
+      url = UwyoSoundingClient.sounding_url("YYR", ~U[2026-04-13 00:00:00Z])
+
+      assert url ==
+               "https://weather.uwyo.edu/cgi-bin/sounding" <>
+                 "?region=naconf&TYPE=TEXT%3ALIST" <>
+                 "&YEAR=2026&MONTH=04&FROM=1300&TO=1300&STNM=YYR"
+    end
+
+    test "pads month and day-hour to two digits" do
+      url = UwyoSoundingClient.sounding_url("WSE", ~U[2026-01-05 12:00:00Z])
+
+      assert url =~ "MONTH=01"
+      assert url =~ "FROM=0512"
+      assert url =~ "TO=0512"
+      assert url =~ "STNM=WSE"
+    end
+  end
+
+  describe "parse_sounding_html/1" do
+    test "returns one sounding entry for a valid response" do
+      assert [sounding] = UwyoSoundingClient.parse_sounding_html(@fixture)
+      assert sounding.observed_at == ~U[2026-04-13 00:00:00Z]
+      assert is_list(sounding.profile)
+      assert length(sounding.profile) > 20
+    end
+
+    test "parses the surface level with all fields populated" do
+      [sounding] = UwyoSoundingClient.parse_sounding_html(@fixture)
+      surface = hd(sounding.profile)
+
+      assert surface["pres"] == 1016.0
+      assert surface["hght"] == 36.0
+      assert surface["tmpc"] == -3.5
+      assert surface["dwpc"] == -9.5
+      assert surface["drct"] == 145.0
+      assert surface["sknt"] == 1.0
+    end
+
+    test "each level has the same key set (pres, hght, tmpc, dwpc, drct, sknt)" do
+      [sounding] = UwyoSoundingClient.parse_sounding_html(@fixture)
+
+      Enum.each(sounding.profile, fn level ->
+        assert Map.has_key?(level, "pres")
+        assert Map.has_key?(level, "hght")
+        assert Map.has_key?(level, "tmpc")
+        assert Map.has_key?(level, "dwpc")
+        assert Map.has_key?(level, "drct")
+        assert Map.has_key?(level, "sknt")
+      end)
+    end
+
+    test "levels with missing temp/dewpoint are omitted" do
+      # Lines like ' 1000.0     92' (only PRES + HGHT, rest blank) must not
+      # appear in the parsed profile because SoundingParams.derive/1 needs temp.
+      html = """
+      

71119 WSE Edmonton Stony Plain Observations at 00Z 13 Apr 2026

+
+      -----------------------------------------------------------------------------
+         PRES   HGHT   TEMP   DWPT   RELH   MIXR   DRCT   SKNT   THTA   THTE   THTV
+          hPa     m      C      C      %    g/kg    deg   knot     K      K      K
+      -----------------------------------------------------------------------------
+       1000.0     92
+        920.0    766    2.2   -3.8     65   3.15     85      6  282.0  291.1  282.5
+      
+ """ + + [sounding] = UwyoSoundingClient.parse_sounding_html(html) + assert length(sounding.profile) == 1 + assert hd(sounding.profile)["pres"] == 920.0 + end + + test "returns an empty list when the response contains no PRE block" do + html = "Sorry, the server is too busy to process your request.\n" + assert UwyoSoundingClient.parse_sounding_html(html) == [] + end + + test "returns an empty list when the header is missing" do + html = "
 1000.0 30 25.0 20.0 50 0.0 180 10 0 0 0
" + assert UwyoSoundingClient.parse_sounding_html(html) == [] + end + end + + describe "fetch_sounding/2" do + test "returns parsed sounding on a 200 response" do + Req.Test.stub(UwyoSoundingClient, fn conn -> + Req.Test.text(conn, @fixture) + end) + + assert {:ok, [sounding]} = + UwyoSoundingClient.fetch_sounding("YYR", ~U[2026-04-13 00:00:00Z]) + + assert sounding.observed_at == ~U[2026-04-13 00:00:00Z] + refute sounding.profile == [] + end + + test "returns an empty list when the server is busy" do + Req.Test.stub(UwyoSoundingClient, fn conn -> + Req.Test.text(conn, "Sorry, the server is too busy to process your request.\n") + end) + + assert {:ok, []} = UwyoSoundingClient.fetch_sounding("YYR", ~U[2026-04-13 00:00:00Z]) + end + + test "returns an error on a non-200 response" do + Req.Test.stub(UwyoSoundingClient, fn conn -> + Plug.Conn.send_resp(conn, 503, "Service Unavailable") + end) + + assert {:error, "UWYO sounding HTTP 503"} = + UwyoSoundingClient.fetch_sounding("YYR", ~U[2026-04-13 00:00:00Z]) + end + end +end diff --git a/test/microwaveprop/workers/canadian_sounding_fetch_worker_test.exs b/test/microwaveprop/workers/canadian_sounding_fetch_worker_test.exs new file mode 100644 index 00000000..2a3630a7 --- /dev/null +++ b/test/microwaveprop/workers/canadian_sounding_fetch_worker_test.exs @@ -0,0 +1,130 @@ +defmodule Microwaveprop.Workers.CanadianSoundingFetchWorkerTest do + use Microwaveprop.DataCase, async: false + + alias Microwaveprop.Repo + alias Microwaveprop.Weather + alias Microwaveprop.Weather.Sounding + alias Microwaveprop.Weather.Station + alias Microwaveprop.Weather.UwyoSoundingClient + alias Microwaveprop.Workers.CanadianSoundingFetchWorker + + @fixture File.read!("test/support/fixtures/uwyo_sounding_yyr.html") + + describe "most_recent_sounding_time/1" do + test "returns 00Z of the same day when called shortly after 02:00Z" do + # UWYO publishes 00Z soundings ~1h after launch; use 90 min buffer + assert CanadianSoundingFetchWorker.most_recent_sounding_time(~U[2026-04-13 02:00:00Z]) == + ~U[2026-04-13 00:00:00Z] + end + + test "returns 12Z of the same day after 14:00Z" do + assert CanadianSoundingFetchWorker.most_recent_sounding_time(~U[2026-04-13 14:00:00Z]) == + ~U[2026-04-13 12:00:00Z] + end + + test "returns prior day 12Z when called between midnight and 01:30Z" do + # Too early for today's 00Z run to be published + assert CanadianSoundingFetchWorker.most_recent_sounding_time(~U[2026-04-13 01:00:00Z]) == + ~U[2026-04-12 12:00:00Z] + end + + test "returns 00Z when called between 13:00Z and 13:30Z (before 12Z is ready)" do + assert CanadianSoundingFetchWorker.most_recent_sounding_time(~U[2026-04-13 13:00:00Z]) == + ~U[2026-04-13 00:00:00Z] + end + end + + describe "perform/1" do + setup do + cwse = insert_sounding_station("CWSE", "Edmonton Stony Plain") + cyyr = insert_sounding_station("CYYR", "Goose Bay") + us_sta = insert_sounding_station("KFWD", "Fort Worth") + %{cwse: cwse, cyyr: cyyr, us_sta: us_sta} + end + + test "fetches only Canadian sounding stations (station_code starts with 'C')", %{cwse: cwse, us_sta: us_sta} do + stub_uwyo_with_fixture() + + :ok = + CanadianSoundingFetchWorker.perform(%Oban.Job{ + args: %{"sounding_time" => "2026-04-13T00:00:00Z"} + }) + + assert soundings_for_station(cwse.id) == 1 + assert soundings_for_station(us_sta.id) == 0 + end + + test "stores a sounding with derived atmospheric parameters", %{cyyr: cyyr} do + stub_uwyo_with_fixture() + + :ok = + CanadianSoundingFetchWorker.perform(%Oban.Job{ + args: %{"sounding_time" => "2026-04-13T00:00:00Z"} + }) + + sounding = Repo.get_by!(Sounding, station_id: cyyr.id, observed_at: ~U[2026-04-13 00:00:00Z]) + + assert sounding.level_count > 0 + assert is_float(sounding.surface_refractivity) + assert is_float(sounding.min_refractivity_gradient) + end + + test "skips stations that already have a sounding for this observed_at", %{cwse: cwse, cyyr: cyyr} do + # Pre-existing stub sounding on BOTH Canadian stations so UWYO should + # never be called. + for sta <- [cwse, cyyr] do + {:ok, _} = + Weather.upsert_sounding(sta, %{ + observed_at: ~U[2026-04-13 00:00:00Z], + profile: [], + level_count: 0 + }) + end + + Req.Test.stub(UwyoSoundingClient, fn _conn -> + raise "UwyoSoundingClient was called for a station that already has a sounding" + end) + + :ok = + CanadianSoundingFetchWorker.perform(%Oban.Job{ + args: %{"sounding_time" => "2026-04-13T00:00:00Z"} + }) + + assert soundings_for_station(cwse.id) == 1 + assert soundings_for_station(cyyr.id) == 1 + end + + test "uses most_recent_sounding_time when no sounding_time arg is given", %{cwse: cwse} do + stub_uwyo_with_fixture() + + :ok = CanadianSoundingFetchWorker.perform(%Oban.Job{args: %{}}) + + # A sounding was created — use the function's own logic to figure out the observed_at + expected = CanadianSoundingFetchWorker.most_recent_sounding_time(DateTime.utc_now()) + + assert Repo.get_by(Sounding, station_id: cwse.id, observed_at: expected) + end + end + + defp insert_sounding_station(code, name) do + %Station{} + |> Station.changeset(%{ + station_code: code, + station_type: "sounding", + name: name, + lat: 50.0, + lon: -100.0 + }) + |> Repo.insert!() + end + + defp soundings_for_station(station_id) do + Repo.aggregate(from(s in Sounding, where: s.station_id == ^station_id), :count, :id) + end + + defp stub_uwyo_with_fixture do + Req.Test.stub(UwyoSoundingClient, fn conn -> + Req.Test.text(conn, @fixture) + end) + end +end diff --git a/test/support/fixtures/uwyo_sounding_yyr.html b/test/support/fixtures/uwyo_sounding_yyr.html new file mode 100644 index 00000000..215c5927 --- /dev/null +++ b/test/support/fixtures/uwyo_sounding_yyr.html @@ -0,0 +1,141 @@ + +University of Wyoming - Radiosonde Data + +

71816 YYR Goose Bay Observations at 00Z 13 Apr 2026

+
+-----------------------------------------------------------------------------
+   PRES   HGHT   TEMP   DWPT   RELH   MIXR   DRCT   SKNT   THTA   THTE   THTV
+    hPa     m      C      C      %    g/kg    deg   knot     K      K      K 
+-----------------------------------------------------------------------------
+ 1016.0     36   -3.5   -9.5     63   1.84    145      1  268.4  273.6  268.7
+ 1014.0     52   -0.1  -10.1     47   1.76    128      1  272.0  277.0  272.3
+ 1012.0     68    0.0  -11.0     43   1.64    111      1  272.2  276.9  272.5
+ 1007.0    109    0.2  -12.8     37   1.43     69      1  272.8  276.9  273.0
+ 1002.0    150   -0.3  -13.3     37   1.38     27      1  272.7  276.7  272.9
+ 1000.0    166   -0.3  -13.3     37   1.38     10      1  272.9  276.9  273.1
+  982.7    305   -1.3  -15.8     32   1.14     20      5  273.2  276.6  273.4
+  955.0    532   -2.9  -19.9     26   0.83    358      9  273.8  276.3  274.0
+  945.6    610   -3.6  -20.3     26   0.81    350     11  273.9  276.4  274.1
+  925.0    784   -5.1  -21.1     27   0.77    345     13  274.1  276.4  274.2
+  909.6    914   -6.2  -21.0     30   0.79    335     14  274.3  276.7  274.4
+  874.6   1219   -8.8  -20.8     37   0.84    310     17  274.6  277.2  274.8
+  850.0   1441  -10.7  -20.7     44   0.87    300     18  274.9  277.6  275.1
+  807.7   1829  -14.2  -20.5     59   0.93    310     20  275.3  278.1  275.4
+  778.0   2114  -16.7  -20.4     73   0.98    319     22  275.5  278.4  275.7
+  775.9   2134  -16.8  -20.5     73   0.97    320     22  275.6  278.5  275.8
+  745.0   2438  -18.8  -21.8     77   0.90    330     23  276.7  279.4  276.9
+  740.0   2488  -19.1  -22.0     78   0.89    328     23  276.9  279.6  277.0
+  714.9   2743  -21.0  -23.7     79   0.80    320     23  277.6  280.0  277.7
+  700.0   2899  -22.1  -24.7     79   0.74    310     22  278.0  280.3  278.1
+  672.0   3199  -23.7  -26.4     78   0.66    303     19  279.4  281.5  279.6
+  669.0   3231  -23.3  -30.3     53   0.46    303     18  280.3  281.7  280.3
+  661.0   3320  -23.5  -33.5     39   0.34    301     17  281.0  282.1  281.1
+  658.0   3353  -23.7  -35.2     34   0.29    300     17  281.1  282.1  281.2
+  653.0   3409  -24.1  -38.1     26   0.22    301     18  281.3  282.0  281.3
+  645.0   3499  -24.7  -33.7     43   0.35    302     19  281.6  282.8  281.7
+  631.0   3658  -25.6  -40.6     23   0.18    305     22  282.3  282.9  282.4
+  624.0   3739  -26.1  -44.1     17   0.12    306     23  282.7  283.1  282.7
+  611.0   3891  -27.1  -36.1     42   0.29    307     24  283.2  284.2  283.3
+  608.0   3927  -27.1  -42.1     23   0.16    307     24  283.6  284.2  283.7
+  604.0   3974  -27.3  -40.3     28   0.19    308     25  283.9  284.6  284.0
+  600.0   4022  -27.5  -45.5     16   0.11    308     25  284.2  284.6  284.3
+  592.0   4119  -27.9  -44.9     18   0.12    309     26  284.9  285.3  284.9
+  579.8   4267  -29.1  -44.4     21   0.13    310     27  285.2  285.7  285.2
+  555.6   4572  -31.4  -43.4     30   0.15    310     26  285.9  286.4  285.9
+  549.0   4657  -32.1  -43.1     33   0.16    309     25  286.1  286.6  286.1
+  536.0   4826  -33.7  -46.7     26   0.11    306     23  286.1  286.5  286.2
+  532.1   4877  -34.1  -45.1     32   0.13    305     22  286.3  286.7  286.3
+  520.0   5038  -35.3  -40.3     60   0.22    299     23  286.7  287.5  286.8
+  515.0   5105  -35.7  -46.7     31   0.11    297     24  287.0  287.4  287.0
+  500.0   5310  -37.7  -42.7     59   0.18    290     25  287.0  287.6  287.1
+  499.0   5324  -37.7  -43.7     53   0.16    290     25  287.2  287.7  287.2
+  497.0   5351  -37.7  -51.7     22   0.07    289     25  287.5  287.8  287.5
+  487.3   5486  -38.3  -55.7     14   0.04    285     25  288.4  288.5  288.4
+  469.0   5748  -39.5  -63.5      6   0.02    287     25  290.1  290.1  290.1
+  445.5   6096  -41.3  -61.6      9   0.02    290     24  292.1  292.2  292.1
+  425.0   6415  -42.9  -59.9     14   0.03    294     29  294.0  294.1  294.0
+  400.0   6820  -45.3  -60.3     17   0.03    300     36  296.0  296.1  296.0
+  380.0   7161  -46.9  -60.9     19   0.03    306     41  298.3  298.4  298.3
+  360.0   7518  -48.5  -65.5     12   0.02    313     46  300.8  300.9  300.8
+  354.4   7620  -48.7  -66.0     12   0.01    315     48  301.8  301.9  301.8
+  338.3   7925  -49.5  -67.7     10   0.01    320     53  304.9  304.9  304.9
+  309.0   8518  -50.9  -70.9      8   0.01    312     55  310.9  310.9  310.9
+  300.0   8710  -51.3  -70.3      9   0.01    310     56  312.9  313.0  312.9
+  280.6   9144  -52.5  -71.9      8   0.01    305     59  317.3  317.4  317.3
+  263.0   9563  -53.6  -73.5      7   0.01    310     65  321.6  321.6  321.6
+  261.0   9613  -53.7  -73.7      7   0.01    315     65  322.1  322.1  322.1
+  255.3   9754  -53.8  -75.3      5   0.01    310     65  324.0  324.0  324.0
+  250.0   9890  -53.9  -76.9      4   0.00    305     50  325.8  325.8  325.8
+  243.6  10058  -52.8  -77.9      3   0.00    310     50  329.9  329.9  329.9
+  235.0  10290  -51.3  -79.3      2   0.00    310     51  335.6  335.6  335.6
+  221.6  10668  -52.6  -80.6      2   0.00    310     53  339.2  339.2  339.2
+  211.4  10973  -53.7  -81.7      2   0.00    310     46  342.1  342.1  342.1
+  206.0  11140  -54.3  -82.3      2   0.00    308     48  343.7  343.7  343.7
+  200.0  11330  -54.1  -82.1      2   0.00    305     50  346.9  346.9  346.9
+  166.7  12497  -55.1  -83.1      2   0.00    300     53  363.8  363.8  363.8
+  155.0  12960  -55.5  -83.5      2   0.00    297     46  370.8  370.8  370.8
+  150.0  13170  -54.5  -83.5      2   0.00    295     43  376.0  376.0  376.0
+  131.3  14021  -54.0  -83.8      1   0.00    290     46  391.4  391.4  391.4
+  128.0  14185  -53.9  -83.9      1   0.00    285     42  394.5  394.5  394.5
+  125.2  14326  -54.4  -84.1      1   0.00    280     39  396.1  396.1  396.1
+  119.0  14650  -55.5  -84.5      2   0.00    282     40  399.8  399.9  399.8
+  108.5  15240  -54.8  -84.3      1   0.00    285     42  411.9  412.0  411.9
+  100.0  15760  -54.1  -84.1      1   0.00    285     36  422.9  422.9  422.9
+

Station information and sounding indices

+                         Station identifier: YYR
+                             Station number: 71816
+                           Observation time: 260413/0000
+                           Station latitude: 53.30
+                          Station longitude: -60.36
+                          Station elevation: 36.0
+                            Showalter index: 8.12
+                               Lifted index: 8.94
+    LIFT computed using virtual temperature: 8.95
+                                SWEAT index: 61.01
+                                    K index: 3.70
+                         Cross totals index: 17.00
+                      Vertical totals index: 27.00
+                        Totals totals index: 44.00
+      Convective Available Potential Energy: 0.00
+             CAPE using virtual temperature: 0.00
+                      Convective Inhibition: 0.00
+             CINS using virtual temperature: 0.00
+                     Bulk Richardson Number: 0.00
+          Bulk Richardson Number using CAPV: 0.00
+  Temp [K] of the Lifted Condensation Level: 255.05
+Pres [hPa] of the Lifted Condensation Level: 787.67
+   Equivalent potential temp [K] of the LCL: 276.50
+     Mean mixed layer potential temperature: 273.07
+              Mean mixed layer mixing ratio: 1.20
+              1000 hPa to 500 hPa thickness: 5144.00
+Precipitable water [mm] for entire sounding: 3.59
+
+

Description of the +data columns +or sounding indices. + +

+

+ + +
+
+Interested in graduate studies in atmospheric science? +Check out our program at the +University of Wyoming + +
+Questions about the weather data provided by this site can be +addressed to +Larry Oolman (ldoolman@uwyo.edu) +
+ + +