import Config # config/runtime.exs is executed for all environments, including # during releases. It is executed after compilation and before the # system starts, so it is typically used to load production configuration # and secrets from environment variables or elsewhere. Do not define # any compile-time configuration in here, as it won't be applied. # The block below contains prod specific runtime configuration. # ## Using releases # # If you use `mix release`, you need to explicitly enable the server # by passing the PHX_SERVER=true when you start it: # # PHX_SERVER=true bin/microwaveprop start # # Alternatively, you can use `mix phx.gen.release` to generate a `bin/server` # script that automatically sets the env var above. alias Microwaveprop.Workers.CanadianSoundingFetchWorker alias Oban.Plugins.Cron if System.get_env("PHX_SERVER") do config :microwaveprop, MicrowavepropWeb.Endpoint, server: true end # Log format: "text" (default) keeps Elixir's built-in line formatter for # local/dev use; "json" swaps the :default :logger handler to emit a single # JSON object per line via `LoggerJSON.Formatters.Basic`. The ingress + # Grafana pipeline is already JSON-aware, so prod defaults to JSON. log_format = System.get_env("LOG_FORMAT") || if(config_env() == :prod, do: "json", else: "text") if log_format == "json" do formatter = LoggerJSON.Formatters.Basic.new(metadata: [:request_id, :trace_id, :worker, :queue, :job_id]) :logger.update_handler_config(:default, :formatter, formatter) end config :microwaveprop, MicrowavepropWeb.Endpoint, http: [port: String.to_integer(System.get_env("PORT", "4000"))] # MicrowavepropWeb.Plugs.RemoteIp will only honour `Cf-Connecting-Ip` / # `X-Forwarded-For` when the immediate TCP peer (conn.remote_ip) sits inside # one of these CIDRs. Set TRUSTED_PROXY_CIDRS to a comma-separated list to # override; leave unset to keep the module default (loopback + RFC1918 + # ULA + link-local), which matches the k8s cluster pod/service network. if trusted_proxy_cidrs = System.get_env("TRUSTED_PROXY_CIDRS") do cidrs = trusted_proxy_cidrs |> String.split(",", trim: true) |> Enum.map(&String.trim/1) |> Enum.reject(&(&1 == "")) config :microwaveprop, :trusted_proxies, cidrs end if prometheus_token = System.get_env("PROMETHEUS_AUTH_TOKEN") do config :microwaveprop, :prometheus_auth_token, prometheus_token end if qrz_username = System.get_env("QRZ_USERNAME") do config :microwaveprop, Microwaveprop.Qrz.Client, username: qrz_username, password: System.get_env("QRZ_PASSWORD") || raise("QRZ_PASSWORD missing"), agent: System.get_env("QRZ_AGENT", "microwaveprop/1.0") end if google_api_key = System.get_env("GOOGLE_API_KEY") do config :microwaveprop, Microwaveprop.Geocoder, api_key: google_api_key end if config_env() == :prod do database_url = System.get_env("DATABASE_URL") || raise """ environment variable DATABASE_URL is missing. For example: ecto://USER:PASS@HOST/DATABASE """ maybe_ipv6 = if System.get_env("ECTO_IPV6") in ~w(true 1), do: [:inet6], else: [] # The secret key base is used to sign/encrypt cookies and other secrets. # A default value is used in config/dev.exs and config/test.exs but you # want to use a different value for prod and you most likely don't want # to check this value into version control, so we use an environment # variable instead. secret_key_base = System.get_env("SECRET_KEY_BASE") || raise """ environment variable SECRET_KEY_BASE is missing. You can generate one by calling: mix phx.gen.secret """ host = System.get_env("PHX_HOST") || "example.com" # ## SSL Support # # To get SSL working, you will need to add the `https` key # to your endpoint configuration: # # config :microwaveprop, MicrowavepropWeb.Endpoint, # https: [ # ..., # port: 443, # cipher_suite: :strong, # keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"), # certfile: System.get_env("SOME_APP_SSL_CERT_PATH") # ] # # The `cipher_suite` is set to `:strong` to support only the # latest and more secure SSL ciphers. This means old browsers # and clients may not be supported. You can set it to # `:compatible` for wider support. # # `:keyfile` and `:certfile` expect an absolute path to the key # and cert in disk or a relative path inside priv, for example # "priv/ssl/server.key". For all supported SSL configuration # options, see https://hexdocs.pm/plug/Plug.SSL.html#configure/1 # # We also recommend setting `force_ssl` in your config/prod.exs, # ensuring no data is ever sent via http, always redirecting to https: # # config :microwaveprop, MicrowavepropWeb.Endpoint, # force_ssl: [hsts: true] # # Check `Plug.SSL` for all available options in `force_ssl`. # ## Configuring the mailer # # In production you need to configure the mailer to use a different adapter. # Here is an example configuration for Mailgun: # # config :microwaveprop, Microwaveprop.Mailer, # adapter: Swoosh.Adapters.Mailgun, # tls: :always + explicit tls_options is required — SMTP2GO's endpoint sits # behind a wildcard cert (*.smtp2go.com) so SNI is mandatory, and gen_smtp # won't set it without explicit server_name_indication. Without this, the # handshake fails with an "unexpected_message" TLS alert. email_server = System.get_env("EMAIL_SERVER") # Production Oban: live scoring, polling, and on-demand QSO enrichment. # Runs on the Oban Pro Smart engine so we can use global_limit / rate_limit # to protect rate-limited upstreams (Copernicus CDS in particular). # # PROP_ROLE controls which workloads this pod picks up: # * "hot" (default) — full hot-path: web, crons, all queues. # * "backfill" — no web server (unset PHX_SERVER), no crons, and only # backfill-style queues. Intended for smaller nodes dedicated to # catching up historical enrichment without competing with the # hourly propagation grid chain. prop_role = System.get_env("PROP_ROLE", "hot") hot_only_queues = [ # 1 slot per pod. Two concurrent forecast-hour steps in a single # pod stack HRRR grid + native duct grid + scored band map into # ~5-6 GiB RSS, hitting the 6 GiB limit and OOM-killing the pod # mid-chain. With 3 pods we still get 3-way parallel fan-out # cluster-wide, which is enough to finish the f00–f18 chain # comfortably inside the hourly interval. PropagationPruneWorker # shares this queue at a lower priority; if a prune job blocks # the grid chain briefly, the chain's unique window protects # the next hourly fire. propagation: 1, commercial: 1, solar: 1 ] shared_queues = [ # 1 slot per pod (3 cluster-wide). Any higher than this and the # ASOS backfill hammers IEM hard enough to get HTTP 429s, which # fill the retryable queue and thrash pod CPU without making # forward progress. weather: 1, gefs: 1, # :hrrr queue removed — per-QSO HRRR fetches flow through # hrrr_fetch_tasks and the Rust hrrr-point-worker (Phase 3 Stream C). terrain: 3, iemre: 3, # Historical backfill for pre-2014 contacts (pre-HRRR archive). # NARR is fetched anonymously from NCEI with no quota, no job # queue. Replaced the ERA5/CDS pipeline which never produced a # single row in prod. era5_cds_jobs table is kept for inspection # but its workers/client are gone. See # docs/plans/2026-04-15-merra2-historical-backfill.md. narr: 6, rtma: 2, backfill_enqueue: 1, admin: 1, nexrad: 2, radar: 2, ionosphere: 1, space_weather: 1, mechanism: 4, exports: 1, contact_import: 4 ] # config/config.exs sets a base Oban config (queues + cron) that Config # deep-merges with runtime.exs. To actually drop the hot-path queues on # backfill pods we can't use `limit: 0` — Oban Pro's Smart engine # rejects it with "local_limit must be greater than 0". Instead override # each queue with `paused: true` (queue exists, processes nothing). paused_queue = [local_limit: 1, paused: true] disabled_hot_queues = [ propagation: paused_queue, commercial: paused_queue, solar: paused_queue, enqueue: paused_queue ] oban_queues = case prop_role do "backfill" -> disabled_hot_queues ++ shared_queues _ -> hot_only_queues ++ shared_queues end cron_plugin = {Cron, crontab: [ {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, {"*/5 * * * *", Microwaveprop.Commercial.PollWorker}, # Hourly at :05. With scores written as binary files (not # Postgres) and HRRR grid profile persistence removed, a # full f00-f18 chain runs in ~45-60 min, so hourly fits. # If one chain slips past 60 min the :propagation queue's # concurrency-of-2 lets the next chain interleave; files # are keyed by (band, valid_time) so last-writer-wins gives # newer analysis data naturally. {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker}, # GEFS runs publish ~3-4h after the 00/06/12/18Z cycle; seed the # Day 2-7 outlook 5h after each run to stay safely past NOMADS' # publication lag. Seeder enqueues f024-f168 at 6-hour cadence # (25 jobs per run, 100/day) into the :gefs queue. {"30 5,11,17,23 * * *", Microwaveprop.Workers.GefsFetchWorker}, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, {"15 * * * *", Microwaveprop.Workers.GridCachePruneWorker}, # The :narr type is virtual: it targets contacts with hrrr_status = # :unavailable (pre-2014, missing from the HRRR archive) and dispatches # NarrFetchWorker against NCEI. See narr_jobs_for_contact/1. # limit=2000 keeps each 30-min tick under the Oban job-execution # timeout that discards unlimited runs after ~17 min. At 4000 # contacts/hour the backlog of missing-data contacts drains # steadily without any single job getting killed mid-way. {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker, args: %{ "limit" => 2000, "types" => ["hrrr", "weather", "terrain", "iemre", "narr", "radar", "mechanism"] }}, # Hourly safety net for pos1/pos2/distance_km. Normally every contact # gets positions at insert time via Radio.resolve_grids_and_insert/1, # but direct DB writes (manual fixes, bulk imports) can bypass that. {"0 * * * *", Microwaveprop.Workers.ContactPositionBackfillWorker, args: %{"limit" => 500}}, # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch {"30 1 * * *", CanadianSoundingFetchWorker}, {"30 13 * * *", CanadianSoundingFetchWorker}, # GIRO DIDBase publishes foF2/foEs/hmF2 at ~7.5 min cadence; poll # every 10 min per station (Millstone Hill, Alpena) for the 144 # MHz sporadic-E and HF MUF scoring inputs. {"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker}, # NOAA SWPC: Kp / GOES X-ray at 1-min cadence, F10.7 hourly. # 5-min poll balances freshness vs request volume. {"*/5 * * * *", Microwaveprop.Workers.SpaceWeatherFetchWorker} ]} base_plugins = [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, # DynamicLifeline uses producer records (heartbeats from each # live node) to rescue orphans, not a timer. When a pod dies # mid-deploy its Producer row disappears and any in-flight job # with that producer in `attempted_by` gets moved back to # `available` on the next rescue cycle — within ~30s instead of # waiting for a timer-based `rescue_after` to expire. Critical # for PropagationGridWorker chain steps to recover from rolling # deploys before the next hourly cron fire. {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)} ] # Backfill pods must not fire the hourly cron. Config deep-merge by # module key means we can't just omit the Cron plugin — config.exs # registers it and that registration survives. Override it with an # empty crontab on backfill pods so the plugin exists but schedules # nothing. empty_cron_plugin = {Cron, crontab: []} oban_plugins = case prop_role do "backfill" -> base_plugins ++ [empty_cron_plugin] _ -> base_plugins ++ [cron_plugin] end config :libcluster, topologies: [ k8s: [ strategy: Cluster.Strategy.Kubernetes, config: [ mode: :ip, kubernetes_ip_lookup_mode: :pods, kubernetes_node_basename: "microwaveprop", kubernetes_selector: "app=prop", kubernetes_namespace: "prop" ] ] ] config :microwaveprop, Microwaveprop.Mailer, adapter: Swoosh.Adapters.SMTP, relay: email_server, port: 587, username: System.get_env("EMAIL_USERNAME"), password: System.get_env("EMAIL_PASSWORD"), tls: :always, tls_options: [ server_name_indication: String.to_charlist(email_server || ""), versions: [:"tlsv1.2", :"tlsv1.3"], verify: :verify_none ], auth: :always config :microwaveprop, Microwaveprop.Repo, # ssl: true, url: database_url, pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"), # For machines with several cores, consider starting multiple pools of `pool_size` # pool_count: 4, socket_options: maybe_ipv6 config :microwaveprop, MicrowavepropWeb.Endpoint, # SSL terminated by Cloudflare tunnel; generated URLs still use https url: [host: host, port: 443, scheme: "https"], http: [ # Enable IPv6 and bind on all interfaces. # Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access. # See the documentation on https://hexdocs.pm/bandit/Bandit.html#t:options/0 # for details about using IPv6 vs IPv4 and loopback vs public addresses. ip: {0, 0, 0, 0, 0, 0, 0, 0} ], secret_key_base: secret_key_base config :microwaveprop, Oban, engine: Oban.Pro.Engines.Smart, queues: oban_queues config :microwaveprop, Oban, plugins: oban_plugins config :microwaveprop, :email_from, {"NTMS Propagation", "prop@w5isp.com"} # Disable the PromEx Oban queue-depth poller in prod. Its 5 s # telemetry_poller runs an aggregate scan against the oban_jobs # table; with our current row count that query regularly holds a # Postgrex connection for > 15 s and starves the Ecto pool enough # that the /health probe times out and the kubelet SIGKILLs every # replica. Queue depth is better observed from the oban_web # dashboard anyway. config :microwaveprop, :prom_ex_include_oban_plugin, false config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080") config :microwaveprop, srtm_tiles_dir: "/data/srtm" # --- OpenTelemetry --- # config :microwaveprop, start_freshness_monitor: true # Ship traces via OTLP/gRPC to the cluster OpenTelemetry Collector # when OTEL_EXPORTER_OTLP_ENDPOINT is set (k8s deployment env). When # unset the exporter is replaced with the `:none` sampler so trace # collection is a no-op — keeps dev/test / unconfigured hosts quiet. config :microwaveprop, start_freshness_monitor: false end if otlp_endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") do config :opentelemetry, :resource, service: [ name: "microwaveprop", namespace: "prop" ] config :opentelemetry, span_processor: :batch, traces_exporter: :otlp config :opentelemetry_exporter, otlp_protocol: :grpc, otlp_endpoint: otlp_endpoint else config :opentelemetry, traces_exporter: :none end