Graham McIntire
23e4e3fef2
Two independent wins the latest prod telemetry pointed at: 1. Iowa Environmental Mesonet rate limiter. IEM throttles per source IP across all in-flight requests, so dropping the `:weather` Oban queue to 1/pod wasn't enough — workers were still issuing back-to- back requests inside each job and drawing a steady stream of HTTP 429s (1,396 retryable on the queue). A GenServer-based token bucket serialises acquire() with a 700ms min gap per pod; three pods give ~4 req/sec cluster-wide, well under the observed 429 threshold. Wrapped around every IemClient fetch. 2. Parallel HRRR surface + pressure grid fetch. The two products live in separate wrfsfcf / wrfprsf GRIB files, so their fetch → wgrib2 → decode pipelines are independent; running them sequentially was adding ~30s per forecast hour for no reason. Task.async the pair, merge at the end. Halves per-fh wall time and should unblock some of the missing-hour cases we saw on the 14:00Z chain.
81 lines
2.2 KiB
Elixir
81 lines
2.2 KiB
Elixir
defmodule Microwaveprop.Weather.IemRateLimiterTest do
|
|
use ExUnit.Case, async: false
|
|
|
|
alias Microwaveprop.Weather.IemRateLimiter
|
|
|
|
# Each test starts an isolated limiter under a unique name so the
|
|
# app-level IemRateLimiter (started by the application supervisor
|
|
# with interval_ms: 0) doesn't conflict.
|
|
defp start_limiter(interval_ms) do
|
|
name = :"iem_rate_limiter_test_#{System.unique_integer([:positive])}"
|
|
start_supervised!({IemRateLimiter, interval_ms: interval_ms, name: name})
|
|
name
|
|
end
|
|
|
|
describe "acquire/1" do
|
|
test "first call returns immediately when the limiter is idle" do
|
|
name = start_limiter(50)
|
|
|
|
{elapsed_ms, :ok} = :timer.tc(fn -> IemRateLimiter.acquire(name) end, :millisecond)
|
|
|
|
# Allow a generous upper bound for scheduling jitter; the point is that
|
|
# no rate-limit sleep was applied on the first call.
|
|
assert elapsed_ms < 20
|
|
end
|
|
|
|
test "serial calls are spaced at least interval_ms apart" do
|
|
name = start_limiter(50)
|
|
|
|
{elapsed_ms, _} =
|
|
:timer.tc(
|
|
fn ->
|
|
IemRateLimiter.acquire(name)
|
|
IemRateLimiter.acquire(name)
|
|
IemRateLimiter.acquire(name)
|
|
end,
|
|
:millisecond
|
|
)
|
|
|
|
# Three acquires at 50ms gap → floor at 100ms (gap after call 1 and 2).
|
|
assert elapsed_ms >= 100
|
|
end
|
|
|
|
test "concurrent callers each get a non-overlapping slot" do
|
|
name = start_limiter(50)
|
|
parent = self()
|
|
|
|
for _ <- 1..4 do
|
|
spawn(fn ->
|
|
IemRateLimiter.acquire(name)
|
|
send(parent, {:acquired, System.monotonic_time(:millisecond)})
|
|
end)
|
|
end
|
|
|
|
timestamps =
|
|
for _ <- 1..4 do
|
|
assert_receive {:acquired, ts}, 500
|
|
ts
|
|
end
|
|
|
|
sorted = Enum.sort(timestamps)
|
|
gaps = sorted |> Enum.zip(tl(sorted)) |> Enum.map(fn {a, b} -> b - a end)
|
|
min_gap = Enum.min(gaps)
|
|
# Allow 5ms scheduler jitter on the lower bound.
|
|
assert min_gap >= 45
|
|
end
|
|
|
|
test "interval_ms: 0 is a no-op" do
|
|
name = start_limiter(0)
|
|
|
|
{elapsed_ms, _} =
|
|
:timer.tc(
|
|
fn ->
|
|
for _ <- 1..10, do: IemRateLimiter.acquire(name)
|
|
end,
|
|
:millisecond
|
|
)
|
|
|
|
assert elapsed_ms < 20
|
|
end
|
|
end
|
|
end
|