Graham McIntie
5b786aac08
Maintenance Windows: - maintenance_windows table with org/site/device scoping - Full CRUD context (Towerops.Maintenance) - device_in_maintenance?/1 checks device → site → org hierarchy - Alert suppression in DeviceMonitorWorker (skips alerts during maintenance) - maintenance_badge/1 helper for UI display Weather Overlay: - weather_observations + weather_alerts tables - OpenWeatherMap free API client (lat/lon based) - WeatherSyncWorker (Oban, 15min interval, respects rate limits) - Observation parsing from OWM response format - Correlation helpers: severe_weather_near?/2, weather_at/2 - Severity classification tuned for WISP ops (wind, rain, ice thresholds) - Per-org latest observations query for dashboard - 30-day automatic cleanup Both features are backend-only — no UI yet.
348 lines
13 KiB
Elixir
348 lines
13 KiB
Elixir
import Config
|
|
|
|
# config/runtime.exs is executed for all environments, including
|
|
# during releases. It is executed after compilation and before the
|
|
# system starts, so it is typically used to load production configuration
|
|
# and secrets from environment variables or elsewhere. Do not define
|
|
# any compile-time configuration in here, as it won't be applied.
|
|
# The block below contains prod specific runtime configuration.
|
|
|
|
# ## Using releases
|
|
#
|
|
# If you use `mix release`, you need to explicitly enable the server
|
|
# by passing the PHX_SERVER=true when you start it:
|
|
#
|
|
# PHX_SERVER=true bin/towerops start
|
|
#
|
|
# Alternatively, you can use `mix phx.gen.release` to generate a `bin/server`
|
|
# script that automatically sets the env var above.
|
|
if System.get_env("PHX_SERVER") do
|
|
config :towerops, ToweropsWeb.Endpoint, server: true
|
|
end
|
|
|
|
config :towerops, ToweropsWeb.Endpoint, http: [port: String.to_integer(System.get_env("PORT", "4000"))]
|
|
|
|
# Configure Google Maps API key for geocoding (available in all environments)
|
|
config :towerops, google_maps_api_key: System.get_env("GOOGLE_MAPS_API_KEY")
|
|
|
|
if config_env() == :prod do
|
|
database_url =
|
|
System.get_env("DATABASE_URL") ||
|
|
raise """
|
|
environment variable DATABASE_URL is missing.
|
|
For example: ecto://USER:PASS@HOST/DATABASE
|
|
"""
|
|
|
|
maybe_ipv6 = if System.get_env("ECTO_IPV6") in ~w(true 1), do: [:inet6], else: []
|
|
|
|
# The secret key base is used to sign/encrypt cookies and other secrets.
|
|
# A default value is used in config/dev.exs and config/test.exs but you
|
|
# want to use a different value for prod and you most likely don't want
|
|
# to check this value into version control, so we use an environment
|
|
# variable instead.
|
|
secret_key_base =
|
|
System.get_env("SECRET_KEY_BASE") ||
|
|
raise """
|
|
environment variable SECRET_KEY_BASE is missing.
|
|
You can generate one by calling: mix phx.gen.secret
|
|
"""
|
|
|
|
# Cloak encryption key for MikroTik API passwords and other sensitive data
|
|
cloak_key =
|
|
System.get_env("CLOAK_KEY") ||
|
|
raise """
|
|
environment variable CLOAK_KEY is missing.
|
|
You can generate one by calling: openssl rand -base64 32
|
|
Store the generated key in 1Password and set as environment variable.
|
|
"""
|
|
|
|
host = System.get_env("PHX_HOST") || "example.com"
|
|
|
|
# Configure Redis/Valkey connection
|
|
# Dokku sets REDIS_URL, K8s uses individual env vars
|
|
redis_config =
|
|
case System.get_env("REDIS_URL") do
|
|
nil ->
|
|
# Use individual env vars (K8s pattern)
|
|
redis_host = System.get_env("REDIS_HOST") || "localhost"
|
|
redis_port = String.to_integer(System.get_env("REDIS_PORT") || "6379")
|
|
redis_password = System.get_env("REDIS_PASSWORD")
|
|
|
|
base_config = [
|
|
host: redis_host,
|
|
port: redis_port
|
|
]
|
|
|
|
if redis_password do
|
|
Keyword.put(base_config, :password, redis_password)
|
|
else
|
|
base_config
|
|
end
|
|
|
|
redis_url ->
|
|
# Parse REDIS_URL (Dokku pattern)
|
|
uri = URI.parse(redis_url)
|
|
|
|
base_config = [
|
|
host: uri.host || "localhost",
|
|
port: uri.port || 6379
|
|
]
|
|
|
|
# Extract password from userinfo if present
|
|
case uri.userinfo do
|
|
nil ->
|
|
base_config
|
|
|
|
userinfo ->
|
|
# userinfo can be "password" or "username:password"
|
|
password =
|
|
case String.split(userinfo, ":", parts: 2) do
|
|
[pass] -> pass
|
|
[_user, pass] -> pass
|
|
end
|
|
|
|
Keyword.put(base_config, :password, password)
|
|
end
|
|
end
|
|
|
|
# Configure SSL options based on environment variables
|
|
ssl_config =
|
|
case System.get_env("DATABASE_SSL") do
|
|
"true" ->
|
|
case System.get_env("DATABASE_SSL_VERIFY") do
|
|
"false" ->
|
|
[
|
|
verify: :verify_none,
|
|
server_name_indication: :disable,
|
|
verify_fun: {fn _, _, _ -> {:valid, :ok} end, :ok}
|
|
]
|
|
|
|
_ ->
|
|
[verify: :verify_peer]
|
|
end
|
|
|
|
_ ->
|
|
false
|
|
end
|
|
|
|
# Only enable libcluster in Kubernetes deployments
|
|
# Kubernetes automatically sets KUBERNETES_SERVICE_HOST env var
|
|
libcluster_topologies =
|
|
if System.get_env("KUBERNETES_SERVICE_HOST") do
|
|
[
|
|
k8s: [
|
|
strategy: Cluster.Strategy.Kubernetes.DNS,
|
|
config: [
|
|
service: "towerops-headless",
|
|
application_name: "towerops",
|
|
namespace: "towerops",
|
|
polling_interval: 10_000
|
|
]
|
|
]
|
|
]
|
|
else
|
|
# No clustering for non-K8s deployments (Dokku, single-node, etc.)
|
|
[]
|
|
end
|
|
|
|
config :libcluster, topologies: libcluster_topologies
|
|
|
|
# Configure Swoosh to use Req for API requests
|
|
config :swoosh, :api_client, Swoosh.ApiClient.Req
|
|
|
|
# Configure Oban for distributed job processing
|
|
config :towerops, Oban,
|
|
engine: Oban.Pro.Engines.Smart,
|
|
repo: Towerops.Repo,
|
|
shutdown_grace_period: to_timeout(second: 40),
|
|
queues: [
|
|
default: 10,
|
|
discovery: 10,
|
|
# SNMP polling jobs - one per device
|
|
pollers: 50,
|
|
# Device monitoring jobs - health checks
|
|
monitors: 50,
|
|
# Service checks - HTTP/TCP/DNS
|
|
checks: 50,
|
|
maintenance: 5,
|
|
weather: 2
|
|
],
|
|
plugins: [
|
|
# Cron jobs for periodic maintenance tasks
|
|
{Oban.Plugins.Cron,
|
|
crontab: [
|
|
# Run neighbor cleanup every hour
|
|
{"0 * * * *", Towerops.Snmp.NeighborCleanupWorker},
|
|
# Check for stale agents every minute
|
|
{"* * * * *", Towerops.Workers.StaleAgentWorker},
|
|
# Cloud latency probes every 8 hours (00:00, 08:00, 16:00)
|
|
{"0 */8 * * *", Towerops.Workers.CloudLatencyProbeWorker},
|
|
# Evaluate latency-based agent reassignment 30 min after each probe batch
|
|
{"30 0,8,16 * * *", Towerops.Workers.AgentLatencyEvaluator},
|
|
# Health check for missing jobs every 10 minutes
|
|
{"*/10 * * * *", Towerops.Workers.JobHealthCheckWorker},
|
|
# Mark stale backup requests as timeout every 10 minutes
|
|
{"*/10 * * * *", Towerops.Workers.BackupTimeoutWorker},
|
|
# Clean up old login history daily at 2 AM
|
|
{"0 2 * * *", Towerops.Workers.LoginHistoryCleanupWorker},
|
|
# Clean up expired browser sessions daily at 3 AM
|
|
{"0 3 * * *", Towerops.Workers.SessionCleanupWorker},
|
|
# Fetch latest firmware versions daily at 4 AM
|
|
{"0 4 * * *", Towerops.Workers.FirmwareVersionFetcherWorker},
|
|
# MikroTik configuration backups daily at 7 AM UTC
|
|
{"0 7 * * *", Towerops.Workers.MikrotikBackupWorker},
|
|
# Send backup summary email daily at 8 AM
|
|
{"0 8 * * *", Towerops.Workers.BackupSummaryWorker},
|
|
# Delete expired IP bans every hour
|
|
{"0 * * * *", Towerops.Workers.ExpiredBanCleanupWorker},
|
|
# Delete stale violation records daily at 2 AM
|
|
{"0 2 * * *", Towerops.Workers.StaleViolationCleanupWorker},
|
|
# Sync Preseem data every 10 minutes
|
|
{"*/5 * * * *", Towerops.Workers.PreseemSyncWorker},
|
|
# Compute Preseem baselines and fleet profiles nightly at 2:30 AM
|
|
{"30 2 * * *", Towerops.Workers.PreseemBaselineWorker},
|
|
# Sync Gaiia data every 15 minutes
|
|
{"*/5 * * * *", Towerops.Workers.GaiiaSyncWorker},
|
|
# Sync NetBox data every 30 minutes
|
|
{"*/5 * * * *", Towerops.Workers.NetBoxSyncWorker},
|
|
# Sync Sonar billing data every 5 minutes
|
|
{"*/5 * * * *", Towerops.Workers.SonarSyncWorker},
|
|
# Sync Splynx billing data every 5 minutes
|
|
{"*/5 * * * *", Towerops.Workers.SplynxSyncWorker},
|
|
# Sync VISP billing data every 5 minutes
|
|
{"*/5 * * * *", Towerops.Workers.VispSyncWorker},
|
|
# Device health insights nightly at 3:30 AM
|
|
{"30 3 * * *", Towerops.Workers.DeviceHealthInsightWorker},
|
|
# System insights (agent offline detection) every 5 minutes
|
|
{"*/5 * * * *", Towerops.Workers.SystemInsightWorker},
|
|
# Gaiia reconciliation insights nightly at 4:30 AM
|
|
{"30 4 * * *", Towerops.Workers.GaiiaInsightWorker}
|
|
]},
|
|
# Automatically delete completed jobs after 60 seconds
|
|
{Oban.Plugins.Pruner, max_age: 60},
|
|
# Rescue orphaned jobs (when node crashes)
|
|
{Oban.Plugins.Reindexer, schedule: "0 2 * * *"},
|
|
# Rescue jobs stuck in executing state (mark as cancelled after 10 minutes)
|
|
{Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 10)}
|
|
]
|
|
|
|
# ## SSL Support
|
|
#
|
|
# To get SSL working, you will need to add the `https` key
|
|
# to your endpoint configuration:
|
|
#
|
|
# config :towerops, ToweropsWeb.Endpoint,
|
|
# https: [
|
|
# ...,
|
|
# port: 443,
|
|
# cipher_suite: :strong,
|
|
# keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"),
|
|
# certfile: System.get_env("SOME_APP_SSL_CERT_PATH")
|
|
# ]
|
|
#
|
|
# The `cipher_suite` is set to `:strong` to support only the
|
|
# latest and more secure SSL ciphers. This means old browsers
|
|
# and clients may not be supported. You can set it to
|
|
# `:compatible` for wider support.
|
|
#
|
|
# `:keyfile` and `:certfile` expect an absolute path to the key
|
|
# and cert in disk or a relative path inside priv, for example
|
|
# "priv/ssl/server.key". For all supported SSL configuration
|
|
# options, see https://hexdocs.pm/plug/Plug.SSL.html#configure/1
|
|
#
|
|
# We also recommend setting `force_ssl` in your config/prod.exs,
|
|
# ensuring no data is ever sent via http, always redirecting to https:
|
|
#
|
|
# config :towerops, ToweropsWeb.Endpoint,
|
|
# force_ssl: [hsts: true]
|
|
#
|
|
# Check `Plug.SSL` for all available options in `force_ssl`.
|
|
|
|
# ## Configuring the mailer
|
|
#
|
|
# In production you need to configure the mailer to use a different adapter.
|
|
# Here is an example configuration for Mailgun:
|
|
#
|
|
# config :towerops, Towerops.Mailer,
|
|
# adapter: Swoosh.Adapters.Mailgun,
|
|
# api_key: System.get_env("MAILGUN_API_KEY"),
|
|
# domain: System.get_env("MAILGUN_DOMAIN")
|
|
#
|
|
# Most non-SMTP adapters require an API client. Swoosh supports Req, Hackney,
|
|
# and Finch out-of-the-box. This configuration is typically done at
|
|
# compile-time in your config/prod.exs:
|
|
#
|
|
# config :swoosh, :api_client, Swoosh.ApiClient.Req
|
|
#
|
|
# See https://hexdocs.pm/swoosh/Swoosh.html#module-installation for details.
|
|
|
|
# Configure Amazon SES for production email
|
|
config :towerops, Towerops.Mailer,
|
|
adapter: Swoosh.Adapters.AmazonSES,
|
|
region: System.get_env("AWS_REGION") || "us-east-1",
|
|
access_key: System.get_env("AWS_ACCESS_KEY_ID"),
|
|
secret: System.get_env("AWS_SECRET_ACCESS_KEY")
|
|
|
|
config :towerops, Towerops.Repo,
|
|
ssl: ssl_config,
|
|
url: database_url,
|
|
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10"),
|
|
# For machines with several cores, consider starting multiple pools of `pool_size`
|
|
# pool_count: 4,
|
|
socket_options: maybe_ipv6
|
|
|
|
config :towerops, Towerops.Vault,
|
|
ciphers: [
|
|
default: {
|
|
Cloak.Ciphers.AES.GCM,
|
|
tag: "AES.GCM.V1", key: Base.decode64!(cloak_key)
|
|
}
|
|
]
|
|
|
|
config :towerops, ToweropsWeb.Endpoint,
|
|
url: [host: host, port: 443, scheme: "https"],
|
|
check_origin: [
|
|
"//towerops.net",
|
|
"//*.towerops.net"
|
|
],
|
|
http: [
|
|
# Enable IPv6 and bind on all interfaces.
|
|
# Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access.
|
|
# See the documentation on https://hexdocs.pm/bandit/Bandit.html#t:options/0
|
|
# for details about using IPv6 vs IPv4 and loopback vs public addresses.
|
|
ip: {0, 0, 0, 0, 0, 0, 0, 0},
|
|
# Bandit uses ThousandIsland for connection management
|
|
# Timeout for reading client data before closing the connection.
|
|
# Set high enough to accommodate long-lived WebSocket connections
|
|
# (agents send Phoenix transport heartbeats every 25s).
|
|
thousand_island_options: [
|
|
read_timeout: 120_000
|
|
]
|
|
],
|
|
secret_key_base: secret_key_base
|
|
|
|
# Webhook secret for CI-triggered mass agent updates
|
|
config :towerops, :agent_webhook_secret, System.get_env("AGENT_WEBHOOK_SECRET")
|
|
config :towerops, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY")
|
|
# Set environment identifier for runtime checks
|
|
config :towerops, :env, :prod
|
|
|
|
# Set default sender for all emails
|
|
config :towerops, :mailer_from, {"Towerops", "hi@towerops.net"}
|
|
config :towerops, :redis, redis_config
|
|
|
|
# Cloudflare API credentials for brute force protection
|
|
config :towerops,
|
|
cloudflare_zone_id: System.get_env("CLOUDFLARE_ZONE_ID"),
|
|
cloudflare_api_token: System.get_env("CLOUDFLARE_API_TOKEN")
|
|
|
|
# Disable automatic timezone database updates in production
|
|
# Timezone data is updated through dependency updates instead
|
|
# This prevents crashes when network timeouts occur during automatic updates
|
|
config :tzdata, :autoupdate, :disabled
|
|
|
|
# OpenWeatherMap API key for site weather data
|
|
if owm_key = System.get_env("OPENWEATHERMAP_API_KEY") do
|
|
config :towerops, :openweathermap_api_key, owm_key
|
|
end
|
|
end
|