Graham McIntire
c6a4e88a4a
Filter port_died and write_failed/epipe errors from Honeybadger reports, email notifications, and logs. Reorder supervision tree so Oban drains before its dependencies shut down, and add 40s shutdown grace period.
310 lines
11 KiB
Elixir
310 lines
11 KiB
Elixir
import Config
|
|
|
|
# config/runtime.exs is executed for all environments, including
|
|
# during releases. It is executed after compilation and before the
|
|
# system starts, so it is typically used to load production configuration
|
|
# and secrets from environment variables or elsewhere. Do not define
|
|
# any compile-time configuration in here, as it won't be applied.
|
|
# The block below contains prod specific runtime configuration.
|
|
|
|
# ## Using releases
|
|
#
|
|
# If you use `mix release`, you need to explicitly enable the server
|
|
# by passing the PHX_SERVER=true when you start it:
|
|
#
|
|
# PHX_SERVER=true bin/towerops start
|
|
#
|
|
# Alternatively, you can use `mix phx.gen.release` to generate a `bin/server`
|
|
# script that automatically sets the env var above.
|
|
if System.get_env("PHX_SERVER") do
|
|
config :towerops, ToweropsWeb.Endpoint, server: true
|
|
end
|
|
|
|
config :towerops, ToweropsWeb.Endpoint, http: [port: String.to_integer(System.get_env("PORT", "4000"))]
|
|
|
|
if config_env() == :prod do
|
|
database_url =
|
|
System.get_env("DATABASE_URL") ||
|
|
raise """
|
|
environment variable DATABASE_URL is missing.
|
|
For example: ecto://USER:PASS@HOST/DATABASE
|
|
"""
|
|
|
|
maybe_ipv6 = if System.get_env("ECTO_IPV6") in ~w(true 1), do: [:inet6], else: []
|
|
|
|
# The secret key base is used to sign/encrypt cookies and other secrets.
|
|
# A default value is used in config/dev.exs and config/test.exs but you
|
|
# want to use a different value for prod and you most likely don't want
|
|
# to check this value into version control, so we use an environment
|
|
# variable instead.
|
|
secret_key_base =
|
|
System.get_env("SECRET_KEY_BASE") ||
|
|
raise """
|
|
environment variable SECRET_KEY_BASE is missing.
|
|
You can generate one by calling: mix phx.gen.secret
|
|
"""
|
|
|
|
# Cloak encryption key for MikroTik API passwords and other sensitive data
|
|
cloak_key =
|
|
System.get_env("CLOAK_KEY") ||
|
|
raise """
|
|
environment variable CLOAK_KEY is missing.
|
|
You can generate one by calling: openssl rand -base64 32
|
|
Store the generated key in 1Password and set as environment variable.
|
|
"""
|
|
|
|
host = System.get_env("PHX_HOST") || "example.com"
|
|
|
|
# Configure Redis/Valkey connection
|
|
# Dokku sets REDIS_URL, K8s uses individual env vars
|
|
redis_config =
|
|
case System.get_env("REDIS_URL") do
|
|
nil ->
|
|
# Use individual env vars (K8s pattern)
|
|
redis_host = System.get_env("REDIS_HOST") || "localhost"
|
|
redis_port = String.to_integer(System.get_env("REDIS_PORT") || "6379")
|
|
redis_password = System.get_env("REDIS_PASSWORD")
|
|
|
|
base_config = [
|
|
host: redis_host,
|
|
port: redis_port
|
|
]
|
|
|
|
if redis_password do
|
|
Keyword.put(base_config, :password, redis_password)
|
|
else
|
|
base_config
|
|
end
|
|
|
|
redis_url ->
|
|
# Parse REDIS_URL (Dokku pattern)
|
|
uri = URI.parse(redis_url)
|
|
|
|
base_config = [
|
|
host: uri.host || "localhost",
|
|
port: uri.port || 6379
|
|
]
|
|
|
|
# Extract password from userinfo if present
|
|
case uri.userinfo do
|
|
nil ->
|
|
base_config
|
|
|
|
userinfo ->
|
|
# userinfo can be "password" or "username:password"
|
|
password =
|
|
case String.split(userinfo, ":", parts: 2) do
|
|
[pass] -> pass
|
|
[_user, pass] -> pass
|
|
end
|
|
|
|
Keyword.put(base_config, :password, password)
|
|
end
|
|
end
|
|
|
|
# Configure SSL options based on environment variables
|
|
ssl_config =
|
|
case System.get_env("DATABASE_SSL") do
|
|
"true" ->
|
|
case System.get_env("DATABASE_SSL_VERIFY") do
|
|
"false" ->
|
|
[
|
|
verify: :verify_none,
|
|
server_name_indication: :disable,
|
|
verify_fun: {fn _, _, _ -> {:valid, :ok} end, :ok}
|
|
]
|
|
|
|
_ ->
|
|
[verify: :verify_peer]
|
|
end
|
|
|
|
_ ->
|
|
false
|
|
end
|
|
|
|
# Only enable libcluster in Kubernetes deployments
|
|
# Kubernetes automatically sets KUBERNETES_SERVICE_HOST env var
|
|
libcluster_topologies =
|
|
if System.get_env("KUBERNETES_SERVICE_HOST") do
|
|
[
|
|
k8s: [
|
|
strategy: Cluster.Strategy.Kubernetes.DNS,
|
|
config: [
|
|
service: "towerops-headless",
|
|
application_name: "towerops",
|
|
namespace: "towerops",
|
|
polling_interval: 10_000
|
|
]
|
|
]
|
|
]
|
|
else
|
|
# No clustering for non-K8s deployments (Dokku, single-node, etc.)
|
|
[]
|
|
end
|
|
|
|
config :libcluster, topologies: libcluster_topologies
|
|
|
|
# Configure Swoosh to use Req for API requests
|
|
config :swoosh, :api_client, Swoosh.ApiClient.Req
|
|
|
|
# Configure Oban for distributed job processing
|
|
config :towerops, Oban,
|
|
engine: Oban.Pro.Engines.Smart,
|
|
repo: Towerops.Repo,
|
|
shutdown_grace_period: to_timeout(second: 40),
|
|
queues: [
|
|
default: 10,
|
|
discovery: 10,
|
|
# SNMP polling jobs - one per device
|
|
pollers: 50,
|
|
# Device monitoring jobs - health checks
|
|
monitors: 50,
|
|
maintenance: 5
|
|
],
|
|
plugins: [
|
|
# Cron jobs for periodic maintenance tasks
|
|
{Oban.Plugins.Cron,
|
|
crontab: [
|
|
# Run neighbor cleanup every hour
|
|
{"0 * * * *", Towerops.Snmp.NeighborCleanupWorker},
|
|
# Check for stale agents every minute
|
|
{"* * * * *", Towerops.Workers.StaleAgentWorker},
|
|
# Evaluate latency-based agent reassignment every 5 minutes
|
|
{"*/5 * * * *", Towerops.Workers.AgentLatencyEvaluator},
|
|
# Health check for missing jobs every 10 minutes
|
|
{"*/10 * * * *", Towerops.Workers.JobHealthCheckWorker},
|
|
# Mark stale backup requests as timeout every 10 minutes
|
|
{"*/10 * * * *", Towerops.Workers.BackupTimeoutWorker},
|
|
# Clean up old login history daily at 2 AM
|
|
{"0 2 * * *", Towerops.Workers.LoginHistoryCleanupWorker},
|
|
# Clean up expired browser sessions daily at 3 AM
|
|
{"0 3 * * *", Towerops.Workers.SessionCleanupWorker},
|
|
# Fetch latest firmware versions daily at 4 AM
|
|
{"0 4 * * *", Towerops.Workers.FirmwareVersionFetcherWorker},
|
|
# MikroTik configuration backups daily at 7 AM UTC
|
|
{"0 7 * * *", Towerops.Workers.MikrotikBackupWorker},
|
|
# Send backup summary email daily at 8 AM
|
|
{"0 8 * * *", Towerops.Workers.BackupSummaryWorker},
|
|
# Delete expired IP bans every hour
|
|
{"0 * * * *", Towerops.Workers.ExpiredBanCleanupWorker},
|
|
# Delete stale violation records daily at 2 AM
|
|
{"0 2 * * *", Towerops.Workers.StaleViolationCleanupWorker}
|
|
]},
|
|
# Automatically delete completed jobs after 60 seconds
|
|
{Oban.Plugins.Pruner, max_age: 60},
|
|
# Rescue orphaned jobs (when node crashes)
|
|
{Oban.Plugins.Reindexer, schedule: "0 2 * * *"},
|
|
# Rescue jobs stuck in executing state (mark as cancelled after 10 minutes)
|
|
{Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 10)}
|
|
]
|
|
|
|
# ## SSL Support
|
|
#
|
|
# To get SSL working, you will need to add the `https` key
|
|
# to your endpoint configuration:
|
|
#
|
|
# config :towerops, ToweropsWeb.Endpoint,
|
|
# https: [
|
|
# ...,
|
|
# port: 443,
|
|
# cipher_suite: :strong,
|
|
# keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"),
|
|
# certfile: System.get_env("SOME_APP_SSL_CERT_PATH")
|
|
# ]
|
|
#
|
|
# The `cipher_suite` is set to `:strong` to support only the
|
|
# latest and more secure SSL ciphers. This means old browsers
|
|
# and clients may not be supported. You can set it to
|
|
# `:compatible` for wider support.
|
|
#
|
|
# `:keyfile` and `:certfile` expect an absolute path to the key
|
|
# and cert in disk or a relative path inside priv, for example
|
|
# "priv/ssl/server.key". For all supported SSL configuration
|
|
# options, see https://hexdocs.pm/plug/Plug.SSL.html#configure/1
|
|
#
|
|
# We also recommend setting `force_ssl` in your config/prod.exs,
|
|
# ensuring no data is ever sent via http, always redirecting to https:
|
|
#
|
|
# config :towerops, ToweropsWeb.Endpoint,
|
|
# force_ssl: [hsts: true]
|
|
#
|
|
# Check `Plug.SSL` for all available options in `force_ssl`.
|
|
|
|
# ## Configuring the mailer
|
|
#
|
|
# In production you need to configure the mailer to use a different adapter.
|
|
# Here is an example configuration for Mailgun:
|
|
#
|
|
# config :towerops, Towerops.Mailer,
|
|
# adapter: Swoosh.Adapters.Mailgun,
|
|
# api_key: System.get_env("MAILGUN_API_KEY"),
|
|
# domain: System.get_env("MAILGUN_DOMAIN")
|
|
#
|
|
# Most non-SMTP adapters require an API client. Swoosh supports Req, Hackney,
|
|
# and Finch out-of-the-box. This configuration is typically done at
|
|
# compile-time in your config/prod.exs:
|
|
#
|
|
# config :swoosh, :api_client, Swoosh.ApiClient.Req
|
|
#
|
|
# See https://hexdocs.pm/swoosh/Swoosh.html#module-installation for details.
|
|
|
|
# Configure Amazon SES for production email
|
|
config :towerops, Towerops.Mailer,
|
|
adapter: Swoosh.Adapters.AmazonSES,
|
|
region: System.get_env("AWS_REGION") || "us-east-1",
|
|
access_key: System.get_env("AWS_ACCESS_KEY_ID"),
|
|
secret: System.get_env("AWS_SECRET_ACCESS_KEY")
|
|
|
|
config :towerops, Towerops.Repo,
|
|
ssl: ssl_config,
|
|
url: database_url,
|
|
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10"),
|
|
# For machines with several cores, consider starting multiple pools of `pool_size`
|
|
# pool_count: 4,
|
|
socket_options: maybe_ipv6
|
|
|
|
config :towerops, Towerops.Vault,
|
|
ciphers: [
|
|
default: {
|
|
Cloak.Ciphers.AES.GCM,
|
|
tag: "AES.GCM.V1", key: Base.decode64!(cloak_key)
|
|
}
|
|
]
|
|
|
|
config :towerops, ToweropsWeb.Endpoint,
|
|
url: [host: host, port: 443, scheme: "https"],
|
|
check_origin: [
|
|
"//towerops.net",
|
|
"//*.towerops.net"
|
|
],
|
|
http: [
|
|
# Enable IPv6 and bind on all interfaces.
|
|
# Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access.
|
|
# See the documentation on https://hexdocs.pm/bandit/Bandit.html#t:options/0
|
|
# for details about using IPv6 vs IPv4 and loopback vs public addresses.
|
|
ip: {0, 0, 0, 0, 0, 0, 0, 0},
|
|
# Bandit uses ThousandIsland for connection management
|
|
# Timeout for reading client data before closing the connection.
|
|
# Set high enough to accommodate long-lived WebSocket connections
|
|
# (agents send Phoenix transport heartbeats every 25s).
|
|
thousand_island_options: [
|
|
read_timeout: 120_000
|
|
]
|
|
],
|
|
secret_key_base: secret_key_base
|
|
|
|
# Webhook secret for CI-triggered mass agent updates
|
|
config :towerops, :agent_webhook_secret, System.get_env("AGENT_WEBHOOK_SECRET")
|
|
config :towerops, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY")
|
|
# Set environment identifier for runtime checks
|
|
config :towerops, :env, :prod
|
|
|
|
# Set default sender for all emails
|
|
config :towerops, :mailer_from, {"Towerops", "hi@towerops.net"}
|
|
config :towerops, :redis, redis_config
|
|
|
|
# Cloudflare API credentials for brute force protection
|
|
config :towerops,
|
|
cloudflare_zone_id: System.get_env("CLOUDFLARE_ZONE_ID"),
|
|
cloudflare_api_token: System.get_env("CLOUDFLARE_API_TOKEN")
|
|
end
|