prop/config/runtime.exs
Graham McIntire 2405c5f169
Stop PropagationGridWorker crash-looping on forecast hours
warm_grid_cache_and_broadcast was re-SELECTing 92k hrrr_profiles rows
with JSONB profile/duct_characteristics columns on every forecast hour.
Row decoding exceeded the 15s default DB connection timeout, killing
the worker before f01-f18 could run. Oban retried from f00 and got
stuck in a loop: for the last several hours, no forecast hours were
being written and even f00 was stale.

Build the weather cache rows directly from the in-memory grid_data
the worker already has (Weather.build_grid_cache_rows/2) and
GridCache.broadcast_put directly. No DB round trip, no JSONB decode.

Also widen the cron from hourly to every 3 hours so a full f00-f18
sweep (~95 min) can actually complete before the next run starts,
and drop the redundant prune_old_scores() at worker start
(PropagationPruneWorker already runs every 15 min). Add a 60s query
timeout to load_weather_grid_from_db as defense-in-depth for the
cold-cache fill path that still uses it.
2026-04-13 08:20:28 -05:00

200 lines
7.4 KiB
Elixir
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import Config
# config/runtime.exs is executed for all environments, including
# during releases. It is executed after compilation and before the
# system starts, so it is typically used to load production configuration
# and secrets from environment variables or elsewhere. Do not define
# any compile-time configuration in here, as it won't be applied.
# The block below contains prod specific runtime configuration.
# ## Using releases
#
# If you use `mix release`, you need to explicitly enable the server
# by passing the PHX_SERVER=true when you start it:
#
# PHX_SERVER=true bin/microwaveprop start
#
# Alternatively, you can use `mix phx.gen.release` to generate a `bin/server`
# script that automatically sets the env var above.
if System.get_env("PHX_SERVER") do
config :microwaveprop, MicrowavepropWeb.Endpoint, server: true
end
config :microwaveprop, MicrowavepropWeb.Endpoint, http: [port: String.to_integer(System.get_env("PORT", "4000"))]
if qrz_username = System.get_env("QRZ_USERNAME") do
config :microwaveprop, Microwaveprop.Qrz.Client,
username: qrz_username,
password: System.get_env("QRZ_PASSWORD") || raise("QRZ_PASSWORD missing"),
agent: System.get_env("QRZ_AGENT", "microwaveprop/1.0")
end
if google_api_key = System.get_env("GOOGLE_API_KEY") do
config :microwaveprop, Microwaveprop.Geocoder, api_key: google_api_key
end
if config_env() == :prod do
database_url =
System.get_env("DATABASE_URL") ||
raise """
environment variable DATABASE_URL is missing.
For example: ecto://USER:PASS@HOST/DATABASE
"""
maybe_ipv6 = if System.get_env("ECTO_IPV6") in ~w(true 1), do: [:inet6], else: []
# The secret key base is used to sign/encrypt cookies and other secrets.
# A default value is used in config/dev.exs and config/test.exs but you
# want to use a different value for prod and you most likely don't want
# to check this value into version control, so we use an environment
# variable instead.
secret_key_base =
System.get_env("SECRET_KEY_BASE") ||
raise """
environment variable SECRET_KEY_BASE is missing.
You can generate one by calling: mix phx.gen.secret
"""
host = System.get_env("PHX_HOST") || "example.com"
# ## SSL Support
#
# To get SSL working, you will need to add the `https` key
# to your endpoint configuration:
#
# config :microwaveprop, MicrowavepropWeb.Endpoint,
# https: [
# ...,
# port: 443,
# cipher_suite: :strong,
# keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"),
# certfile: System.get_env("SOME_APP_SSL_CERT_PATH")
# ]
#
# The `cipher_suite` is set to `:strong` to support only the
# latest and more secure SSL ciphers. This means old browsers
# and clients may not be supported. You can set it to
# `:compatible` for wider support.
#
# `:keyfile` and `:certfile` expect an absolute path to the key
# and cert in disk or a relative path inside priv, for example
# "priv/ssl/server.key". For all supported SSL configuration
# options, see https://hexdocs.pm/plug/Plug.SSL.html#configure/1
#
# We also recommend setting `force_ssl` in your config/prod.exs,
# ensuring no data is ever sent via http, always redirecting to https:
#
# config :microwaveprop, MicrowavepropWeb.Endpoint,
# force_ssl: [hsts: true]
#
# Check `Plug.SSL` for all available options in `force_ssl`.
# ## Configuring the mailer
#
# In production you need to configure the mailer to use a different adapter.
# Here is an example configuration for Mailgun:
#
# config :microwaveprop, Microwaveprop.Mailer,
# adapter: Swoosh.Adapters.Mailgun,
# tls: :always + explicit tls_options is required — SMTP2GO's endpoint sits
# behind a wildcard cert (*.smtp2go.com) so SNI is mandatory, and gen_smtp
# won't set it without explicit server_name_indication. Without this, the
# handshake fails with an "unexpected_message" TLS alert.
email_server = System.get_env("EMAIL_SERVER")
config :libcluster,
topologies: [
k8s: [
strategy: Cluster.Strategy.Kubernetes,
config: [
mode: :ip,
kubernetes_ip_lookup_mode: :pods,
kubernetes_node_basename: "microwaveprop",
kubernetes_selector: "app=prop",
kubernetes_namespace: "prop"
]
]
]
config :microwaveprop, Microwaveprop.Mailer,
adapter: Swoosh.Adapters.SMTP,
relay: email_server,
port: 587,
username: System.get_env("EMAIL_USERNAME"),
password: System.get_env("EMAIL_PASSWORD"),
tls: :always,
tls_options: [
server_name_indication: String.to_charlist(email_server || ""),
versions: [:"tlsv1.2", :"tlsv1.3"],
verify: :verify_none
],
auth: :always
config :microwaveprop, Microwaveprop.Repo,
# ssl: true,
url: database_url,
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"),
# For machines with several cores, consider starting multiple pools of `pool_size`
# pool_count: 4,
socket_options: maybe_ipv6
config :microwaveprop, MicrowavepropWeb.Endpoint,
# SSL terminated by Cloudflare tunnel; generated URLs still use https
url: [host: host, port: 443, scheme: "https"],
http: [
# Enable IPv6 and bind on all interfaces.
# Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access.
# See the documentation on https://hexdocs.pm/bandit/Bandit.html#t:options/0
# for details about using IPv6 vs IPv4 and loopback vs public addresses.
ip: {0, 0, 0, 0, 0, 0, 0, 0}
],
secret_key_base: secret_key_base
# Production Oban: live scoring, polling, and on-demand QSO enrichment.
# Runs on the Oban Pro Smart engine so we can use global_limit / rate_limit
# to protect rate-limited upstreams (Copernicus CDS in particular).
config :microwaveprop, Oban,
engine: Oban.Pro.Engines.Smart,
# Per-pod concurrency (×3 pods = effective cluster total)
queues: [
propagation: 1,
commercial: 1,
solar: 1,
weather: 3,
hrrr: 1,
terrain: 3,
iemre: 3,
era5: 2,
era5_batch: [
local_limit: 2,
global_limit: 4,
rate_limit: [allowed: 10, period: {1, :hour}]
],
rtma: 2,
backfill_enqueue: 1,
admin: 1,
nexrad: 2
],
plugins: [
{Oban.Plugins.Pruner, max_age: 3600 * 24},
{Oban.Plugins.Lifeline, rescue_after: 10 * 60 * 1000},
{Oban.Plugins.Cron,
crontab: [
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
{"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
# Every 3 hours: one full run does f00f18 (~5 min/hour × 19 ≈ 95 min).
# Hourly cron caused overlapping/failed runs because one full sweep
# exceeds a 60-minute window.
{"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker},
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}}
]}
]
config :microwaveprop, :email_from, {"NTMS Propagation", "graham@w5isp.com"}
config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080")
config :microwaveprop, srtm_tiles_dir: "/srtm"
# config :microwaveprop, start_freshness_monitor: true
config :microwaveprop, start_freshness_monitor: false
end