diff --git a/config/dev.exs b/config/dev.exs index 7d587a49..e9dfa067 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -117,10 +117,7 @@ config :microwaveprop, :propagation_scores_dir, Path.expand("priv/dev_scores", F # Enable dev routes for dashboard and mailbox config :microwaveprop, dev_routes: true -# HRRR caching handled by nginx proxy on skippy — no local filesystem cache - -# Proxy HRRR downloads through local caching nginx (set to nil to use S3 directly) -config :microwaveprop, hrrr_base_url: "http://skippy.w5isp.com:8080" +# HRRR fetched directly from NOAA S3 — no local filesystem cache # Load ML model at startup (Nx/Axon/EXLA only available in dev/test) config :microwaveprop, load_ml_model: true diff --git a/config/runtime.exs b/config/runtime.exs index 8cf58641..6b8b9585 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -492,7 +492,7 @@ if config_env() == :prod do # replica. Queue depth is better observed from the oban_web # dashboard anyway. config :microwaveprop, :prom_ex_include_oban_plugin, false - config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080") + config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "https://noaa-hrrr-bdp-pds.s3.amazonaws.com") config :microwaveprop, srtm_tiles_dir: "/data/srtm" config :microwaveprop, start_freshness_monitor: false end diff --git a/docs/plans/2026-04-13-rdps-vertical-profiles.md b/docs/plans/2026-04-13-rdps-vertical-profiles.md index 0fdb25eb..4d850a0a 100644 --- a/docs/plans/2026-04-13-rdps-vertical-profiles.md +++ b/docs/plans/2026-04-13-rdps-vertical-profiles.md @@ -302,6 +302,6 @@ Do NOT push to `dokku`. Let Flux roll out the new image from the CI build. ## Notes / risks - **Rotated lat/lon projection**: wgrib2 handles it correctly for point extraction but caution needed if the team ever decides to store full-grid binary rasters. -- **File size**: RDPS pressure-level files are ~50 MB each. At 4 runs/day × ~10 forecast hours × ~5 variables × ~10 pressure levels, total download ≈ 2 GB/day. Not a problem on a home lab with skippy.w5isp.com cache, but worth caching aggressively. +- **File size**: RDPS pressure-level files are ~50 MB each. At 4 runs/day × ~10 forecast hours × ~5 variables × ~10 pressure levels, total download ≈ 2 GB/day. Worth caching aggressively in a home lab. - **Data latency**: RDPS publishes ~90 minutes after synoptic; the cron is already aligned. - **Fallback when RDPS is unavailable**: `scores_at/3` should gracefully degrade to HRRR-only if `rdps_profiles` is empty for the requested `valid_time`. diff --git a/k8s/deployment-backfill.yaml b/k8s/deployment-backfill.yaml index 0b7c8bec..ca1be8c4 100644 --- a/k8s/deployment-backfill.yaml +++ b/k8s/deployment-backfill.yaml @@ -67,8 +67,6 @@ spec: value: "backfill" - name: PHX_HOST value: "prop.w5isp.com" - - name: HRRR_BASE_URL - value: "http://skippy.w5isp.com:8080" envFrom: - secretRef: name: prop-secrets diff --git a/k8s/deployment-grid-rs.yaml b/k8s/deployment-grid-rs.yaml index 7d5a2e32..f957d25e 100644 --- a/k8s/deployment-grid-rs.yaml +++ b/k8s/deployment-grid-rs.yaml @@ -74,12 +74,6 @@ spec: env: # Elixir secrets bundle carries DATABASE_URL already. Rust # reads it directly (sqlx connects with the same URL). - - name: HRRR_BASE_URL - value: "http://skippy.w5isp.com:8080" - # When the LAN caching proxy at skippy is unreachable, fetches - # retry once against the NOAA S3 public bucket so a proxy - # outage degrades to direct S3 reads instead of stalling the - # hourly pipeline. - name: HRRR_FALLBACK_BASE_URL value: "https://noaa-hrrr-bdp-pds.s3.amazonaws.com" - name: PROP_SCORES_DIR diff --git a/k8s/deployment-hrrr-point-rs.yaml b/k8s/deployment-hrrr-point-rs.yaml index c088788b..9da31612 100644 --- a/k8s/deployment-hrrr-point-rs.yaml +++ b/k8s/deployment-hrrr-point-rs.yaml @@ -67,8 +67,6 @@ spec: imagePullPolicy: IfNotPresent command: ["/usr/bin/tini", "--", "/usr/local/bin/hrrr-point-worker"] env: - - name: HRRR_BASE_URL - value: "http://skippy.w5isp.com:8080" - name: HRRR_FALLBACK_BASE_URL value: "https://noaa-hrrr-bdp-pds.s3.amazonaws.com" - name: RUST_LOG diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index 84652a98..4abf9183 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -89,8 +89,6 @@ spec: value: "prop.w5isp.com" - name: PORT value: "5000" - - name: HRRR_BASE_URL - value: "http://skippy.w5isp.com:8080" envFrom: - secretRef: name: prop-secrets diff --git a/k8s/network-policy.yaml b/k8s/network-policy.yaml index b55f8306..287d4dd4 100644 --- a/k8s/network-policy.yaml +++ b/k8s/network-policy.yaml @@ -5,7 +5,7 @@ # - Erlang distribution (EPMD + ephemeral ports) between prop + backfill pods # # Egress is unrestricted — pods need to reach Postgres, NFS (10.0.19.103), -# HRRR (skippy + NOAA S3), and IEM. +# HRRR (NOAA S3), and IEM. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: diff --git a/lib/microwaveprop/weather/hrrr_client.ex b/lib/microwaveprop/weather/hrrr_client.ex index 7b133a75..b03d66f3 100644 --- a/lib/microwaveprop/weather/hrrr_client.ex +++ b/lib/microwaveprop/weather/hrrr_client.ex @@ -490,8 +490,7 @@ defmodule Microwaveprop.Weather.HrrrClient do @doc """ Downloads the given byte ranges from a GRIB2 URL and concatenates them in offset order. Merges adjacent/overlapping ranges and runs - requests in parallel. Backed by a disk cache keyed on - `(url, ranges)` in dev. + requests in parallel. Public so the native-level client can reuse it rather than duplicating the HTTP/parallelism/cache logic. @@ -509,27 +508,7 @@ defmodule Microwaveprop.Weather.HrrrClient do end defp do_download_grib_ranges(url, ranges) do - cache_key = url_to_cache_key(url, ranges) - - case read_cache(cache_key) do - {:ok, binary} -> - Logger.info("HRRR cache hit: #{cache_key}") - {:ok, binary} - - :miss -> - download_and_cache_grib_ranges(url, ranges, cache_key) - end - end - - defp download_and_cache_grib_ranges(url, ranges, cache_key) do - case download_grib_ranges_remote(url, ranges) do - {:ok, binary} = ok -> - write_cache(cache_key, binary) - ok - - error -> - error - end + download_grib_ranges_remote(url, ranges) end @doc """ @@ -613,44 +592,6 @@ defmodule Microwaveprop.Weather.HrrrClient do end end - defp hrrr_cache_dir, do: Application.get_env(:microwaveprop, :hrrr_cache_dir) - - defp url_to_cache_key(url, ranges) do - # e.g. hrrr.20240922_conus_hrrr.t18z.wrfprsf00_r42.grib2 - uri = URI.parse(url) - path_part = uri.path |> String.trim_leading("/") |> String.replace("/", "_") - range_hash = ranges |> :erlang.phash2() |> Integer.to_string(16) - "#{path_part}_r#{range_hash}.grib2" - end - - defp read_cache(key) do - case hrrr_cache_dir() do - nil -> - :miss - - dir -> - path = Path.join(dir, key) - - case File.read(path) do - {:ok, binary} -> {:ok, binary} - {:error, _} -> :miss - end - end - end - - defp write_cache(key, binary) do - case hrrr_cache_dir() do - nil -> - :ok - - dir -> - File.mkdir_p!(dir) - path = Path.join(dir, key) - File.write!(path, binary) - Logger.info("HRRR cached: #{path} (#{byte_size(binary)} bytes)") - end - end - defp merge_ranges(ranges) do ranges |> Enum.sort_by(&elem(&1, 0)) diff --git a/test/mix/tasks/simple_tasks_test.exs b/test/mix/tasks/simple_tasks_test.exs index f21c7574..2b456287 100644 --- a/test/mix/tasks/simple_tasks_test.exs +++ b/test/mix/tasks/simple_tasks_test.exs @@ -309,7 +309,7 @@ defmodule Mix.Tasks.SimpleTasksTest do describe "Mix.Tasks.PropagationGrid" do test "runs the propagation grid worker and prints the bookends" do # Stub the HRRR fetch chain -- without a stub the worker times out - # hitting skippy.w5isp.com. Elevation client too for safety. + # hitting NOAA S3. Elevation client too for safety. Req.Test.stub(HrrrClient, fn conn -> Plug.Conn.send_resp(conn, 404, "not found") end)