feat: remove skippy HRRR proxy + disk cache, fetch directly from NOAA S3
Some checks failed
Build and Push / Build and Push Docker Image (push) Failing after 4m45s

- Drop skippy.w5isp.com:8080 as HRRR_BASE_URL in dev/prod config and
  all K8s manifests. Default to direct NOAA S3 reads.
- Strip hrrr_cache_dir disk-caching infrastructure from HrrrClient
  (read_cache, write_cache, download_and_cache_grib_ranges). Files are
  now fetched fresh each time — no local filesystem cache.
- Update stale skippy references in network-policy, test comments, docs.
This commit is contained in:
Graham McIntire 2026-07-28 14:36:54 -05:00
parent db6d231887
commit c4e71e41b3
10 changed files with 7 additions and 81 deletions

View file

@ -117,10 +117,7 @@ config :microwaveprop, :propagation_scores_dir, Path.expand("priv/dev_scores", F
# Enable dev routes for dashboard and mailbox
config :microwaveprop, dev_routes: true
# HRRR caching handled by nginx proxy on skippy — no local filesystem cache
# Proxy HRRR downloads through local caching nginx (set to nil to use S3 directly)
config :microwaveprop, hrrr_base_url: "http://skippy.w5isp.com:8080"
# HRRR fetched directly from NOAA S3 — no local filesystem cache
# Load ML model at startup (Nx/Axon/EXLA only available in dev/test)
config :microwaveprop, load_ml_model: true

View file

@ -492,7 +492,7 @@ if config_env() == :prod do
# replica. Queue depth is better observed from the oban_web
# dashboard anyway.
config :microwaveprop, :prom_ex_include_oban_plugin, false
config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080")
config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "https://noaa-hrrr-bdp-pds.s3.amazonaws.com")
config :microwaveprop, srtm_tiles_dir: "/data/srtm"
config :microwaveprop, start_freshness_monitor: false
end

View file

@ -302,6 +302,6 @@ Do NOT push to `dokku`. Let Flux roll out the new image from the CI build.
## Notes / risks
- **Rotated lat/lon projection**: wgrib2 handles it correctly for point extraction but caution needed if the team ever decides to store full-grid binary rasters.
- **File size**: RDPS pressure-level files are ~50 MB each. At 4 runs/day × ~10 forecast hours × ~5 variables × ~10 pressure levels, total download ≈ 2 GB/day. Not a problem on a home lab with skippy.w5isp.com cache, but worth caching aggressively.
- **File size**: RDPS pressure-level files are ~50 MB each. At 4 runs/day × ~10 forecast hours × ~5 variables × ~10 pressure levels, total download ≈ 2 GB/day. Worth caching aggressively in a home lab.
- **Data latency**: RDPS publishes ~90 minutes after synoptic; the cron is already aligned.
- **Fallback when RDPS is unavailable**: `scores_at/3` should gracefully degrade to HRRR-only if `rdps_profiles` is empty for the requested `valid_time`.

View file

@ -67,8 +67,6 @@ spec:
value: "backfill"
- name: PHX_HOST
value: "prop.w5isp.com"
- name: HRRR_BASE_URL
value: "http://skippy.w5isp.com:8080"
envFrom:
- secretRef:
name: prop-secrets

View file

@ -74,12 +74,6 @@ spec:
env:
# Elixir secrets bundle carries DATABASE_URL already. Rust
# reads it directly (sqlx connects with the same URL).
- name: HRRR_BASE_URL
value: "http://skippy.w5isp.com:8080"
# When the LAN caching proxy at skippy is unreachable, fetches
# retry once against the NOAA S3 public bucket so a proxy
# outage degrades to direct S3 reads instead of stalling the
# hourly pipeline.
- name: HRRR_FALLBACK_BASE_URL
value: "https://noaa-hrrr-bdp-pds.s3.amazonaws.com"
- name: PROP_SCORES_DIR

View file

@ -67,8 +67,6 @@ spec:
imagePullPolicy: IfNotPresent
command: ["/usr/bin/tini", "--", "/usr/local/bin/hrrr-point-worker"]
env:
- name: HRRR_BASE_URL
value: "http://skippy.w5isp.com:8080"
- name: HRRR_FALLBACK_BASE_URL
value: "https://noaa-hrrr-bdp-pds.s3.amazonaws.com"
- name: RUST_LOG

View file

@ -89,8 +89,6 @@ spec:
value: "prop.w5isp.com"
- name: PORT
value: "5000"
- name: HRRR_BASE_URL
value: "http://skippy.w5isp.com:8080"
envFrom:
- secretRef:
name: prop-secrets

View file

@ -5,7 +5,7 @@
# - Erlang distribution (EPMD + ephemeral ports) between prop + backfill pods
#
# Egress is unrestricted — pods need to reach Postgres, NFS (10.0.19.103),
# HRRR (skippy + NOAA S3), and IEM.
# HRRR (NOAA S3), and IEM.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:

View file

@ -490,8 +490,7 @@ defmodule Microwaveprop.Weather.HrrrClient do
@doc """
Downloads the given byte ranges from a GRIB2 URL and concatenates
them in offset order. Merges adjacent/overlapping ranges and runs
requests in parallel. Backed by a disk cache keyed on
`(url, ranges)` in dev.
requests in parallel.
Public so the native-level client can reuse it rather than
duplicating the HTTP/parallelism/cache logic.
@ -509,27 +508,7 @@ defmodule Microwaveprop.Weather.HrrrClient do
end
defp do_download_grib_ranges(url, ranges) do
cache_key = url_to_cache_key(url, ranges)
case read_cache(cache_key) do
{:ok, binary} ->
Logger.info("HRRR cache hit: #{cache_key}")
{:ok, binary}
:miss ->
download_and_cache_grib_ranges(url, ranges, cache_key)
end
end
defp download_and_cache_grib_ranges(url, ranges, cache_key) do
case download_grib_ranges_remote(url, ranges) do
{:ok, binary} = ok ->
write_cache(cache_key, binary)
ok
error ->
error
end
download_grib_ranges_remote(url, ranges)
end
@doc """
@ -613,44 +592,6 @@ defmodule Microwaveprop.Weather.HrrrClient do
end
end
defp hrrr_cache_dir, do: Application.get_env(:microwaveprop, :hrrr_cache_dir)
defp url_to_cache_key(url, ranges) do
# e.g. hrrr.20240922_conus_hrrr.t18z.wrfprsf00_r42.grib2
uri = URI.parse(url)
path_part = uri.path |> String.trim_leading("/") |> String.replace("/", "_")
range_hash = ranges |> :erlang.phash2() |> Integer.to_string(16)
"#{path_part}_r#{range_hash}.grib2"
end
defp read_cache(key) do
case hrrr_cache_dir() do
nil ->
:miss
dir ->
path = Path.join(dir, key)
case File.read(path) do
{:ok, binary} -> {:ok, binary}
{:error, _} -> :miss
end
end
end
defp write_cache(key, binary) do
case hrrr_cache_dir() do
nil ->
:ok
dir ->
File.mkdir_p!(dir)
path = Path.join(dir, key)
File.write!(path, binary)
Logger.info("HRRR cached: #{path} (#{byte_size(binary)} bytes)")
end
end
defp merge_ranges(ranges) do
ranges
|> Enum.sort_by(&elem(&1, 0))

View file

@ -309,7 +309,7 @@ defmodule Mix.Tasks.SimpleTasksTest do
describe "Mix.Tasks.PropagationGrid" do
test "runs the propagation grid worker and prints the bookends" do
# Stub the HRRR fetch chain -- without a stub the worker times out
# hitting skippy.w5isp.com. Elevation client too for safety.
# hitting NOAA S3. Elevation client too for safety.
Req.Test.stub(HrrrClient, fn conn ->
Plug.Conn.send_resp(conn, 404, "not found")
end)