Graham McIntire
23e4e3fef2
Two independent wins the latest prod telemetry pointed at: 1. Iowa Environmental Mesonet rate limiter. IEM throttles per source IP across all in-flight requests, so dropping the `:weather` Oban queue to 1/pod wasn't enough — workers were still issuing back-to- back requests inside each job and drawing a steady stream of HTTP 429s (1,396 retryable on the queue). A GenServer-based token bucket serialises acquire() with a 700ms min gap per pod; three pods give ~4 req/sec cluster-wide, well under the observed 429 threshold. Wrapped around every IemClient fetch. 2. Parallel HRRR surface + pressure grid fetch. The two products live in separate wrfsfcf / wrfprsf GRIB files, so their fetch → wgrib2 → decode pipelines are independent; running them sequentially was adding ~30s per forecast hour for no reason. Task.async the pair, merge at the end. Halves per-fh wall time and should unblock some of the missing-hour cases we saw on the 14:00Z chain.
90 lines
3.9 KiB
Elixir
90 lines
3.9 KiB
Elixir
import Config
|
|
|
|
alias Microwaveprop.Qrz.Client
|
|
|
|
# Only in tests, remove the complexity from the password hashing algorithm
|
|
config :bcrypt_elixir, :log_rounds, 1
|
|
|
|
# Print only warnings and errors during test
|
|
config :logger, level: :warning
|
|
|
|
config :microwaveprop, Client,
|
|
username: "testuser",
|
|
password: "testpass",
|
|
agent: "microwaveprop-test/1.0",
|
|
plug: {Req.Test, Client},
|
|
retry: false
|
|
|
|
config :microwaveprop, Microwaveprop.Geocoder,
|
|
plug: {Req.Test, Microwaveprop.Geocoder},
|
|
retry: false
|
|
|
|
# In test we don't send emails
|
|
config :microwaveprop, Microwaveprop.Mailer, adapter: Swoosh.Adapters.Test
|
|
config :microwaveprop, Microwaveprop.Qrz, cache_ttl_hours: 0
|
|
|
|
# Configure your database
|
|
#
|
|
# The MIX_TEST_PARTITION environment variable can be used
|
|
# to provide built-in test partitioning in CI environment.
|
|
# Run `mix help test` for more information.
|
|
config :microwaveprop, Microwaveprop.Repo,
|
|
username: "postgres",
|
|
password: "postgres",
|
|
hostname: "localhost",
|
|
database: "microwaveprop_test#{System.get_env("MIX_TEST_PARTITION")}",
|
|
pool: Ecto.Adapters.SQL.Sandbox,
|
|
pool_size: System.schedulers_online() * 2,
|
|
ownership_timeout: 60_000,
|
|
timeout: 60_000
|
|
|
|
# We don't run a server during test. If one is required,
|
|
# you can enable the server option below.
|
|
config :microwaveprop, MicrowavepropWeb.Endpoint,
|
|
http: [ip: {127, 0, 0, 1}, port: 4002],
|
|
secret_key_base: "f7KbnwUezGrPtrv/67l6Wi+W9/+wOqwFG5QJ/ZLs24cVM1gg2lBswZwrKuyWsBAZ",
|
|
server: false
|
|
|
|
# Run Oban jobs inline during tests
|
|
config :microwaveprop, Oban, testing: :inline
|
|
|
|
# Score files land under a per-run tmp directory so replace_scores tests
|
|
# don't write into a shared location (and we can assert on the file
|
|
# contents from inside tests).
|
|
config :microwaveprop,
|
|
:propagation_scores_dir,
|
|
Path.join(System.tmp_dir!(), "microwaveprop_test_scores_#{System.unique_integer([:positive])}")
|
|
|
|
config :microwaveprop, cache_contact_count: false
|
|
config :microwaveprop, cache_contact_map: false
|
|
config :microwaveprop, elevation_req_options: [plug: {Req.Test, Microwaveprop.Terrain.ElevationClient}, retry: false]
|
|
config :microwaveprop, gefs_req_options: [plug: {Req.Test, Microwaveprop.Weather.GefsClient}, retry: false]
|
|
config :microwaveprop, giro_req_options: [plug: {Req.Test, Microwaveprop.Ionosphere.GiroClient}, retry: false]
|
|
config :microwaveprop, hrrr_req_options: [plug: {Req.Test, Microwaveprop.Weather.HrrrClient}, retry: false]
|
|
config :microwaveprop, iem_rate_limiter_interval_ms: 0
|
|
config :microwaveprop, iem_req_options: [plug: {Req.Test, Microwaveprop.Weather.IemClient}, retry: false]
|
|
config :microwaveprop, mrms_req_options: [plug: {Req.Test, Microwaveprop.Weather.MrmsClient}, retry: false]
|
|
config :microwaveprop, narr_req_options: [plug: {Req.Test, Microwaveprop.Weather.NarrClient}, retry: false]
|
|
|
|
# Route HTTP requests through Req.Test stubs
|
|
config :microwaveprop, nexrad_req_options: [plug: {Req.Test, Microwaveprop.Weather.NexradClient}, retry: false]
|
|
config :microwaveprop, rtma_req_options: [plug: {Req.Test, Microwaveprop.Weather.RtmaClient}, retry: false]
|
|
config :microwaveprop, solar_req_options: [plug: {Req.Test, Microwaveprop.Weather.SolarClient}]
|
|
config :microwaveprop, srtm_req_options: [plug: {Req.Test, Microwaveprop.Terrain.Srtm}, retry: false]
|
|
config :microwaveprop, start_freshness_monitor: false
|
|
config :microwaveprop, swpc_req_options: [plug: {Req.Test, Microwaveprop.SpaceWeather.SwpcClient}, retry: false]
|
|
config :microwaveprop, uwyo_req_options: [plug: {Req.Test, Microwaveprop.Weather.UwyoSoundingClient}, retry: false]
|
|
|
|
# Initialize plugs at runtime for faster test compilation
|
|
config :phoenix, :plug_init_mode, :runtime
|
|
|
|
# Sort query params output of verified routes for robust url comparisons
|
|
config :phoenix,
|
|
sort_verified_routes_query_params: true
|
|
|
|
# Enable helpful, but potentially expensive runtime checks
|
|
config :phoenix_live_view,
|
|
enable_expensive_runtime_checks: true
|
|
|
|
# Disable swoosh api client as it is only required for production adapters
|
|
config :swoosh, :api_client, false
|