feat: remove skippy HRRR proxy + disk cache, fetch directly from NOAA S3
Some checks failed
Build and Push / Build and Push Docker Image (push) Failing after 4m45s
Some checks failed
Build and Push / Build and Push Docker Image (push) Failing after 4m45s
- Drop skippy.w5isp.com:8080 as HRRR_BASE_URL in dev/prod config and all K8s manifests. Default to direct NOAA S3 reads. - Strip hrrr_cache_dir disk-caching infrastructure from HrrrClient (read_cache, write_cache, download_and_cache_grib_ranges). Files are now fetched fresh each time — no local filesystem cache. - Update stale skippy references in network-policy, test comments, docs.
This commit is contained in:
parent
db6d231887
commit
c4e71e41b3
10 changed files with 7 additions and 81 deletions
|
|
@ -117,10 +117,7 @@ config :microwaveprop, :propagation_scores_dir, Path.expand("priv/dev_scores", F
|
|||
# Enable dev routes for dashboard and mailbox
|
||||
config :microwaveprop, dev_routes: true
|
||||
|
||||
# HRRR caching handled by nginx proxy on skippy — no local filesystem cache
|
||||
|
||||
# Proxy HRRR downloads through local caching nginx (set to nil to use S3 directly)
|
||||
config :microwaveprop, hrrr_base_url: "http://skippy.w5isp.com:8080"
|
||||
# HRRR fetched directly from NOAA S3 — no local filesystem cache
|
||||
|
||||
# Load ML model at startup (Nx/Axon/EXLA only available in dev/test)
|
||||
config :microwaveprop, load_ml_model: true
|
||||
|
|
|
|||
|
|
@ -492,7 +492,7 @@ if config_env() == :prod do
|
|||
# replica. Queue depth is better observed from the oban_web
|
||||
# dashboard anyway.
|
||||
config :microwaveprop, :prom_ex_include_oban_plugin, false
|
||||
config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080")
|
||||
config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "https://noaa-hrrr-bdp-pds.s3.amazonaws.com")
|
||||
config :microwaveprop, srtm_tiles_dir: "/data/srtm"
|
||||
config :microwaveprop, start_freshness_monitor: false
|
||||
end
|
||||
|
|
|
|||
|
|
@ -302,6 +302,6 @@ Do NOT push to `dokku`. Let Flux roll out the new image from the CI build.
|
|||
## Notes / risks
|
||||
|
||||
- **Rotated lat/lon projection**: wgrib2 handles it correctly for point extraction but caution needed if the team ever decides to store full-grid binary rasters.
|
||||
- **File size**: RDPS pressure-level files are ~50 MB each. At 4 runs/day × ~10 forecast hours × ~5 variables × ~10 pressure levels, total download ≈ 2 GB/day. Not a problem on a home lab with skippy.w5isp.com cache, but worth caching aggressively.
|
||||
- **File size**: RDPS pressure-level files are ~50 MB each. At 4 runs/day × ~10 forecast hours × ~5 variables × ~10 pressure levels, total download ≈ 2 GB/day. Worth caching aggressively in a home lab.
|
||||
- **Data latency**: RDPS publishes ~90 minutes after synoptic; the cron is already aligned.
|
||||
- **Fallback when RDPS is unavailable**: `scores_at/3` should gracefully degrade to HRRR-only if `rdps_profiles` is empty for the requested `valid_time`.
|
||||
|
|
|
|||
|
|
@ -67,8 +67,6 @@ spec:
|
|||
value: "backfill"
|
||||
- name: PHX_HOST
|
||||
value: "prop.w5isp.com"
|
||||
- name: HRRR_BASE_URL
|
||||
value: "http://skippy.w5isp.com:8080"
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: prop-secrets
|
||||
|
|
|
|||
|
|
@ -74,12 +74,6 @@ spec:
|
|||
env:
|
||||
# Elixir secrets bundle carries DATABASE_URL already. Rust
|
||||
# reads it directly (sqlx connects with the same URL).
|
||||
- name: HRRR_BASE_URL
|
||||
value: "http://skippy.w5isp.com:8080"
|
||||
# When the LAN caching proxy at skippy is unreachable, fetches
|
||||
# retry once against the NOAA S3 public bucket so a proxy
|
||||
# outage degrades to direct S3 reads instead of stalling the
|
||||
# hourly pipeline.
|
||||
- name: HRRR_FALLBACK_BASE_URL
|
||||
value: "https://noaa-hrrr-bdp-pds.s3.amazonaws.com"
|
||||
- name: PROP_SCORES_DIR
|
||||
|
|
|
|||
|
|
@ -67,8 +67,6 @@ spec:
|
|||
imagePullPolicy: IfNotPresent
|
||||
command: ["/usr/bin/tini", "--", "/usr/local/bin/hrrr-point-worker"]
|
||||
env:
|
||||
- name: HRRR_BASE_URL
|
||||
value: "http://skippy.w5isp.com:8080"
|
||||
- name: HRRR_FALLBACK_BASE_URL
|
||||
value: "https://noaa-hrrr-bdp-pds.s3.amazonaws.com"
|
||||
- name: RUST_LOG
|
||||
|
|
|
|||
|
|
@ -89,8 +89,6 @@ spec:
|
|||
value: "prop.w5isp.com"
|
||||
- name: PORT
|
||||
value: "5000"
|
||||
- name: HRRR_BASE_URL
|
||||
value: "http://skippy.w5isp.com:8080"
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: prop-secrets
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
# - Erlang distribution (EPMD + ephemeral ports) between prop + backfill pods
|
||||
#
|
||||
# Egress is unrestricted — pods need to reach Postgres, NFS (10.0.19.103),
|
||||
# HRRR (skippy + NOAA S3), and IEM.
|
||||
# HRRR (NOAA S3), and IEM.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -490,8 +490,7 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
|||
@doc """
|
||||
Downloads the given byte ranges from a GRIB2 URL and concatenates
|
||||
them in offset order. Merges adjacent/overlapping ranges and runs
|
||||
requests in parallel. Backed by a disk cache keyed on
|
||||
`(url, ranges)` in dev.
|
||||
requests in parallel.
|
||||
|
||||
Public so the native-level client can reuse it rather than
|
||||
duplicating the HTTP/parallelism/cache logic.
|
||||
|
|
@ -509,27 +508,7 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
|||
end
|
||||
|
||||
defp do_download_grib_ranges(url, ranges) do
|
||||
cache_key = url_to_cache_key(url, ranges)
|
||||
|
||||
case read_cache(cache_key) do
|
||||
{:ok, binary} ->
|
||||
Logger.info("HRRR cache hit: #{cache_key}")
|
||||
{:ok, binary}
|
||||
|
||||
:miss ->
|
||||
download_and_cache_grib_ranges(url, ranges, cache_key)
|
||||
end
|
||||
end
|
||||
|
||||
defp download_and_cache_grib_ranges(url, ranges, cache_key) do
|
||||
case download_grib_ranges_remote(url, ranges) do
|
||||
{:ok, binary} = ok ->
|
||||
write_cache(cache_key, binary)
|
||||
ok
|
||||
|
||||
error ->
|
||||
error
|
||||
end
|
||||
download_grib_ranges_remote(url, ranges)
|
||||
end
|
||||
|
||||
@doc """
|
||||
|
|
@ -613,44 +592,6 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
|||
end
|
||||
end
|
||||
|
||||
defp hrrr_cache_dir, do: Application.get_env(:microwaveprop, :hrrr_cache_dir)
|
||||
|
||||
defp url_to_cache_key(url, ranges) do
|
||||
# e.g. hrrr.20240922_conus_hrrr.t18z.wrfprsf00_r42.grib2
|
||||
uri = URI.parse(url)
|
||||
path_part = uri.path |> String.trim_leading("/") |> String.replace("/", "_")
|
||||
range_hash = ranges |> :erlang.phash2() |> Integer.to_string(16)
|
||||
"#{path_part}_r#{range_hash}.grib2"
|
||||
end
|
||||
|
||||
defp read_cache(key) do
|
||||
case hrrr_cache_dir() do
|
||||
nil ->
|
||||
:miss
|
||||
|
||||
dir ->
|
||||
path = Path.join(dir, key)
|
||||
|
||||
case File.read(path) do
|
||||
{:ok, binary} -> {:ok, binary}
|
||||
{:error, _} -> :miss
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp write_cache(key, binary) do
|
||||
case hrrr_cache_dir() do
|
||||
nil ->
|
||||
:ok
|
||||
|
||||
dir ->
|
||||
File.mkdir_p!(dir)
|
||||
path = Path.join(dir, key)
|
||||
File.write!(path, binary)
|
||||
Logger.info("HRRR cached: #{path} (#{byte_size(binary)} bytes)")
|
||||
end
|
||||
end
|
||||
|
||||
defp merge_ranges(ranges) do
|
||||
ranges
|
||||
|> Enum.sort_by(&elem(&1, 0))
|
||||
|
|
|
|||
|
|
@ -309,7 +309,7 @@ defmodule Mix.Tasks.SimpleTasksTest do
|
|||
describe "Mix.Tasks.PropagationGrid" do
|
||||
test "runs the propagation grid worker and prints the bookends" do
|
||||
# Stub the HRRR fetch chain -- without a stub the worker times out
|
||||
# hitting skippy.w5isp.com. Elevation client too for safety.
|
||||
# hitting NOAA S3. Elevation client too for safety.
|
||||
Req.Test.stub(HrrrClient, fn conn ->
|
||||
Plug.Conn.send_resp(conn, 404, "not found")
|
||||
end)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue