Graham McIntire
f7eb888123
Future-proof the queue config so a "let's bump concurrency for throughput" change has the per-pod RSS math in front of it. Includes the actual memory limit (6 GiB), observed steady state (3-5 GiB), the ~500 MiB peak per path-compute, and the explicit recommendation to add hot replicas via HPA or offload HRRR fetch to hrrr-point-rs before touching this slot count.
176 lines
6.6 KiB
Elixir
176 lines
6.6 KiB
Elixir
# This file is responsible for configuring your application
|
||
# and its dependencies with the aid of the Config module.
|
||
#
|
||
# This configuration file is loaded before any dependency and
|
||
# is restricted to this project.
|
||
|
||
# General application configuration
|
||
import Config
|
||
|
||
alias Microwaveprop.Workers.CanadianSoundingFetchWorker
|
||
|
||
# Configure esbuild (the version is required)
|
||
config :esbuild,
|
||
version: "0.27.4",
|
||
microwaveprop: [
|
||
args:
|
||
~w(js/app.ts --bundle --target=es2022 --format=esm --splitting --outdir=../priv/static/assets/js --external:/fonts/* --external:/images/* --alias:@=.),
|
||
cd: Path.expand("../assets", __DIR__),
|
||
env: %{"NODE_PATH" => [Path.expand("../deps", __DIR__), Mix.Project.build_path()]}
|
||
]
|
||
|
||
config :live_table,
|
||
app: :microwaveprop,
|
||
repo: Microwaveprop.Repo,
|
||
pubsub: Microwaveprop.PubSub,
|
||
defaults: %{
|
||
custom_footer: {MicrowavepropWeb.LiveTableFooter, :render}
|
||
}
|
||
|
||
# Configure Elixir's Logger
|
||
config :logger, :default_formatter,
|
||
format: "$time $metadata[$level] $message\n",
|
||
metadata: [:request_id, :remote_ip, :method, :request_path]
|
||
|
||
# Configure the mailer
|
||
#
|
||
# By default it uses the "Local" adapter which stores the emails
|
||
# locally. You can see the emails in your browser, at "/dev/mailbox".
|
||
#
|
||
# For production it's recommended to configure a different adapter
|
||
# at the `config/runtime.exs`.
|
||
config :microwaveprop, Microwaveprop.Mailer, adapter: Swoosh.Adapters.Local
|
||
|
||
# PromEx metrics exporter. We surface the `/metrics` Prometheus scrape
|
||
# endpoint via MicrowavepropWeb.Router rather than PromEx's own HTTP
|
||
# server, so the router (and optional bearer-token auth in
|
||
# MicrowavepropWeb.MetricsPlug) stay the single ingress path. Grafana
|
||
# auto-provisioning is off — dashboards live in the dedicated Grafana
|
||
# instance and are managed separately.
|
||
config :microwaveprop, Microwaveprop.PromEx,
|
||
disabled: false,
|
||
manual_metrics_start_delay: :no_delay,
|
||
drop_metrics_groups: [],
|
||
grafana: :disabled,
|
||
metrics_server: :disabled
|
||
|
||
# Configure the endpoint
|
||
config :microwaveprop, MicrowavepropWeb.Endpoint,
|
||
url: [host: "localhost"],
|
||
adapter: Bandit.PhoenixAdapter,
|
||
render_errors: [
|
||
formats: [html: MicrowavepropWeb.ErrorHTML, json: MicrowavepropWeb.ErrorJSON],
|
||
layout: false
|
||
],
|
||
pubsub_server: Microwaveprop.PubSub,
|
||
live_view: [signing_salt: "Gdq36xze"]
|
||
|
||
config :microwaveprop, Oban,
|
||
engine: Oban.Pro.Engines.Smart,
|
||
repo: Microwaveprop.Repo,
|
||
queues: [
|
||
solar: 1,
|
||
weather: 20,
|
||
enqueue: 1,
|
||
gefs: 2,
|
||
terrain: 4,
|
||
# DO NOT raise above 1 without measuring per-pod RSS first.
|
||
# RoverPathProfileWorker runs the full PathCompute pipeline
|
||
# (terrain sample analysis + 9 HRRR profile fetches/decodes +
|
||
# sounding readout + ionosphere readout + per-band scoring +
|
||
# loss + power + 18 h forecast). Per-job heap is roughly the
|
||
# same shape as the :propagation grid chain — ~500 MiB peak.
|
||
# With the deployment.yaml memory limit at 6 GiB and steady-state
|
||
# hot pods sitting at 3-5 GiB (ScoreCache + GridCache + LiveView
|
||
# state), even 2 concurrent path-computes can push a hot pod past
|
||
# the limit and trigger OOMKill. Verified on 2026-05-03: a
|
||
# backfill flood ran 3 slots/pod × 4 hot pods on the :terrain
|
||
# queue and OOMKilled two pods in a CrashLoopBackOff cycle.
|
||
# If you need more throughput, prefer adding hot replicas (HPA
|
||
# max is 4 in k8s/hpa.yaml) over raising this slot count, OR
|
||
# offload the HRRR profile fetch to hrrr-point-rs first.
|
||
rover_path: 1,
|
||
commercial: 2,
|
||
iemre: 10,
|
||
# 2 slots so PropagationPruneWorker can run alongside the
|
||
# forecast-hour chain job.
|
||
propagation: 2,
|
||
admin: 1,
|
||
nexrad: 2,
|
||
radar: 2,
|
||
ionosphere: 1,
|
||
space_weather: 1,
|
||
mechanism: 4,
|
||
contact_import: 4,
|
||
# HRDPS Canadian propagation chain seeder. Single slot — the worker
|
||
# only inserts grid_tasks rows, so concurrent execution buys nothing
|
||
# and uniqueness is enforced by the (run_time, kind, source) index.
|
||
# Dormant until the Rust prop-grid-rs HRDPS branch ships and the
|
||
# cron entry in runtime.exs is uncommented.
|
||
hrdps: 1
|
||
],
|
||
plugins: [
|
||
{Oban.Plugins.Pruner, max_age: 3600 * 24},
|
||
# See runtime.exs — producer-record-based orphan rescue, not
|
||
# timer-based. Fast recovery from rolling deploys.
|
||
{Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)},
|
||
{Oban.Plugins.Cron,
|
||
crontab: [
|
||
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
|
||
{"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
|
||
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
|
||
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
||
{"15 * * * *", Microwaveprop.Workers.GridCachePruneWorker},
|
||
# Heals rover-planning missions whose paths are stuck in :pending
|
||
# or :failed (transient elevation-API errors, interrupted enqueues).
|
||
{"30 * * * *", Microwaveprop.Workers.RoverMissionBackfillWorker},
|
||
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
|
||
{"30 1 * * *", CanadianSoundingFetchWorker},
|
||
{"30 13 * * *", CanadianSoundingFetchWorker},
|
||
# GIRO ionosonde stations publish at ~7.5 min cadence; poll every
|
||
# 10 min to pick up fresh foF2/foEs/hmF2 within one cycle.
|
||
{"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker},
|
||
# NOAA SWPC publishes Kp and GOES X-ray at 1-min cadence and F10.7
|
||
# hourly; poll every 5 min for reasonable freshness vs request volume.
|
||
{"*/5 * * * *", Microwaveprop.Workers.SpaceWeatherFetchWorker}
|
||
]}
|
||
]
|
||
|
||
config :microwaveprop, :scopes,
|
||
user: [
|
||
default: true,
|
||
module: Microwaveprop.Accounts.Scope,
|
||
assign_key: :current_scope,
|
||
access_path: [:user, :id],
|
||
schema_key: :user_id,
|
||
schema_type: :binary_id,
|
||
schema_table: :users,
|
||
test_data_fixture: Microwaveprop.AccountsFixtures,
|
||
test_setup_helper: :register_and_log_in_user
|
||
]
|
||
|
||
config :microwaveprop,
|
||
ecto_repos: [Microwaveprop.Repo],
|
||
generators: [timestamp_type: :utc_datetime, binary_id: true]
|
||
|
||
# Use EXLA as default Nx backend for accelerated tensor operations
|
||
config :nx, :default_backend, EXLA.Backend
|
||
|
||
# Use Jason for JSON parsing in Phoenix
|
||
config :phoenix, :json_library, Jason
|
||
|
||
# Configure tailwind (the version is required)
|
||
config :tailwind,
|
||
version: "4.2.2",
|
||
microwaveprop: [
|
||
args: ~w(
|
||
--input=assets/css/app.css
|
||
--output=priv/static/assets/css/app.css
|
||
),
|
||
|
||
# Import environment specific config. This must remain at the bottom
|
||
# of this file so it overrides the configuration defined above.
|
||
cd: Path.expand("..", __DIR__)
|
||
]
|
||
|
||
import_config "#{config_env()}.exs"
|