From 024db2a5b4f4d8b86234c022c70c6d4d103d7a14 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Wed, 22 Apr 2026 16:50:45 -0500 Subject: [PATCH] perf(fleet): reduce IEM 429 storm and raise probe timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-pod IEM rate limiter was 700ms (~1.4 req/sec); four pods put the fleet at ~5.7 req/sec, above the observed 429 threshold. Retry storm (Req exponential backoff up to 32s) piled concurrent sleeps on top of live workers and tripped Bandit's acceptor + Oban's leader heartbeat. - Raise default interval to 1500ms (~2.7 req/sec fleet). - Hot pod /live timeout 3s → 5s (acceptor stalls under retry sleeps). - prop-backfill exec probe period 30s → 120s, timeout 15s → 30s; the release CLI round-trips through distributed Erlang and 15s wasn't enough under load, restart-looping every ~10 min. --- k8s/deployment-backfill.yaml | 24 ++++++++++--------- k8s/deployment.yaml | 6 ++++- lib/microwaveprop/weather/iem_rate_limiter.ex | 13 +++++++--- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/k8s/deployment-backfill.yaml b/k8s/deployment-backfill.yaml index 8ba1e02c..b995de25 100644 --- a/k8s/deployment-backfill.yaml +++ b/k8s/deployment-backfill.yaml @@ -73,25 +73,27 @@ spec: - secretRef: name: prop-secrets # No HTTP probes (no web server). Use exec probe against the - # running BEAM to confirm the VM is responsive. timeoutSeconds - # is generous because `rpc ':ok'` has to spawn a release CLI - # and round-trip through distributed Erlang; during an IEM - # 429 retry storm (WeatherFetchWorker sleeping 1-16 s between - # attempts) the scheduler is responsive but slow, and a 5 s - # probe timeout was tripping failureThreshold every ~10 min. + # running BEAM to confirm the VM is responsive. `rpc ':ok'` + # has to spawn a release CLI and round-trip through distributed + # Erlang; during an IEM 429 retry storm the scheduler is + # responsive but slow. 15 s still tripped failureThreshold + # every ~10 min, so the liveness probe now runs every 2 min + # with a 30 s ceiling — the BEAM itself would have crashed + # long before that if it were genuinely wedged, and rescue + # relies on the BEAM's own crash-loop handling anyway. startupProbe: exec: command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"] initialDelaySeconds: 10 - periodSeconds: 5 - failureThreshold: 24 - timeoutSeconds: 15 + periodSeconds: 10 + failureThreshold: 18 + timeoutSeconds: 30 livenessProbe: exec: command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"] - periodSeconds: 30 + periodSeconds: 120 failureThreshold: 3 - timeoutSeconds: 15 + timeoutSeconds: 30 resources: # 8GB T620 box — leave ~3GB for kubelet/containerd/OS. requests: diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index 171afe57..38342160 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -104,13 +104,17 @@ spec: # /health's `SELECT 1` to time out and the kubelet to SIGKILL # every replica at once. Readiness still points at /health so # DB outages drain the pod from Service endpoints gracefully. + # timeoutSeconds raised to 5s: under an IEM 429 retry storm + # Bandit's acceptor loop can stall briefly as the scheduler + # chews through retrying HTTP sleeps, and 3s was tripping + # failureThreshold across all 3 replicas at once. livenessProbe: httpGet: path: /live port: 5000 periodSeconds: 10 failureThreshold: 3 - timeoutSeconds: 3 + timeoutSeconds: 5 resources: requests: cpu: 100m diff --git a/lib/microwaveprop/weather/iem_rate_limiter.ex b/lib/microwaveprop/weather/iem_rate_limiter.ex index 07dc2ce7..8c4cf09c 100644 --- a/lib/microwaveprop/weather/iem_rate_limiter.ex +++ b/lib/microwaveprop/weather/iem_rate_limiter.ex @@ -12,14 +12,21 @@ defmodule Microwaveprop.Weather.IemRateLimiter do arrives. Configured via `config :microwaveprop, :iem_rate_limiter_interval_ms` - (defaults to 700ms ≈ 1.4 req/sec per pod; three pods give ~4 req/sec - cluster-wide, well under the empirically observed 429 threshold). + (defaults to 1500ms ≈ 0.67 req/sec per pod; four pods give ~2.7 + req/sec cluster-wide). Previously set at 700ms, which put four pods + at ~5.7 req/sec fleet-wide — above IEM's observed 429 threshold — + and the resulting retry storm (Req exponential backoff up to 32 s) + piled concurrent HTTP sleeps on top of live workers until Bandit's + acceptor and Oban's leader heartbeat both started tripping. The + limiter is per-pod, so cluster-synchronous bursts are still possible + when every pod reserves `now + interval` simultaneously; moving to + a global limiter is out of scope for this tuning pass. In tests, pass `interval_ms: 0` to disable. """ use GenServer - @default_interval_ms 700 + @default_interval_ms 1500 @spec start_link(keyword()) :: GenServer.on_start() def start_link(opts \\ []) do