towerops/lib/towerops/application.ex
Graham McIntire ee8a3220c4
refactor: convert periodic workers to Oban Cron for better resilience
Replaced GenServer-based periodic workers with Oban Cron jobs to improve
pod rollover resilience and simplify architecture.

Worker Changes:
- NeighborCleanupWorker: GenServer → Oban Cron (hourly)
  - Cleans stale neighbors, ARP entries, and MAC addresses
  - Runs every hour via Oban.Plugins.Cron

- StaleAgentWorker: GenServer → Oban Cron (every minute)
  - Detects agents that haven't checked in for 10+ minutes
  - Refactored to reduce nesting (extracted helper functions)
  - Removed stateful tracking (now stateless, re-evaluates each run)

- AgentLatencyEvaluator: GenServer → Oban Cron (every 5 minutes)
  - Latency-based agent reassignment with 20% threshold
  - Removed trigger_evaluation/0 (no longer needed)

- JobHealthCheckWorker: NEW Oban Cron worker (every 10 minutes)
  - Safety net to recover missing monitor/poller jobs
  - Auto-creates jobs for devices with monitoring/SNMP enabled

Infrastructure Changes:
- Removed Monitoring.Supervisor (no longer needed)
- Updated application.ex to remove GenServer workers from supervision tree
- Added Oban.Plugins.Cron to dev.exs and runtime.exs
- All workers now run cluster-wide via PostgreSQL-backed coordination

Test Updates:
- Updated all worker tests to call perform(%Oban.Job{args: %{}})
- Removed GenServer lifecycle tests (start_link, send messages, etc.)
- Removed async sleep calls (no longer needed)

Benefits:
- Better pod rollover resilience (Cron jobs run cluster-wide)
- Simpler architecture (no GenServers for periodic tasks)
- Better observability (all jobs visible in Oban dashboard)
- Safety net for missing jobs (JobHealthCheckWorker)
- Stateless workers (easier to reason about and test)

Documentation:
- Updated CLAUDE.md with Background Job Architecture section
- Documented job types, queues, and resilience features

All tests passing (3,686 tests, 0 failures).
2026-01-24 16:56:28 -06:00

163 lines
5.1 KiB
Elixir

defmodule Towerops.Application do
@moduledoc """
OTP Application module for Towerops.
Supervises the Repo, PubSub, Telemetry, Endpoint, and monitoring workers.
"""
use Application
# Capture the build timestamp at compile time (fallback for development)
@build_timestamp DateTime.utc_now()
@impl true
def start(_type, _args) do
# Run migrations on startup (Ecto handles locking for concurrent runs)
_ =
if Application.get_env(:towerops, Towerops.Repo)[:database] != "towerops_test" do
Towerops.Release.migrate()
end
topologies = Application.get_env(:libcluster, :topologies, [])
children =
[
{Cluster.Supervisor, [topologies, [name: Towerops.ClusterSupervisor]]},
ToweropsWeb.Telemetry,
Towerops.Repo,
{Oban, Application.fetch_env!(:towerops, Oban)},
{DNSCluster, query: Application.get_env(:towerops, :dns_cluster_query) || :ignore},
pubsub_spec(),
# Load YAML profiles into ETS cache
Towerops.Profiles.YamlProfiles,
# SnmpKit services for SNMP operations
SnmpKit.SnmpMgr.Config,
SnmpKit.SnmpMgr.MIB
] ++
background_workers() ++
[
# Start to serve requests, typically the last entry
ToweropsWeb.Endpoint
]
# See https://hexdocs.pm/elixir/Supervisor.html
# for other strategies and supported options
opts = [strategy: :one_for_one, name: Towerops.Supervisor]
Supervisor.start_link(children, opts)
end
# Tell Phoenix to update the endpoint configuration
# whenever the application is updated.
@impl true
def config_change(changed, _new, removed) do
ToweropsWeb.Endpoint.config_change(changed, removed)
:ok
end
# Configure background workers - don't start in test environment
defp background_workers do
if Application.get_env(:towerops, :env) == :test do
[]
else
[
# Start event logger (subscribes to PubSub)
Towerops.Devices.EventLogger
# Note: NeighborCleanupWorker, StaleAgentWorker, AgentLatencyEvaluator,
# and JobHealthCheckWorker are now Oban Cron jobs (see config/runtime.exs)
]
end
end
# Returns PubSub spec - uses Redis in production/clustered environments,
# falls back to default PG2 adapter for Mix tasks and development
defp pubsub_spec do
redis_config = Application.get_env(:towerops, :redis, [])
node_name = node()
# Use Redis adapter only when:
# 1. Redis config exists with host
# 2. Node has a name (not nonode@nohost)
use_redis? =
Keyword.has_key?(redis_config, :host) &&
node_name != :nonode@nohost
if use_redis? do
base_config = [name: Towerops.PubSub, adapter: Phoenix.PubSub.Redis]
redis_opts = redis_pubsub_config()
{Phoenix.PubSub, Keyword.merge(base_config, redis_opts)}
else
# Default PG2 adapter for Mix tasks and development
{Phoenix.PubSub, name: Towerops.PubSub}
end
end
defp redis_pubsub_config do
redis_config = Application.get_env(:towerops, :redis, [])
# Base configuration for Phoenix.PubSub.Redis
base_opts = [
host: Keyword.get(redis_config, :host, "localhost"),
port: Keyword.get(redis_config, :port, 6379),
node_name: node()
]
# Add password if configured (read from application config set by runtime.exs)
base_opts =
case Keyword.get(redis_config, :password) do
nil -> base_opts
"" -> base_opts
password -> Keyword.put(base_opts, :password, password)
end
# Add resilience options for better reconnection handling
resilience_opts = [
# Redix connection options for better resilience
socket_opts: [
# TCP keepalive to detect dead connections
keepalive: true
],
# Connection timeout (5 seconds)
timeout: 5_000,
# Backoff settings for reconnection
backoff_initial: 500,
backoff_max: 30_000,
# Don't exit process on disconnection - let PubSub handle it
exit_on_disconnection: false,
# Sync connect to fail fast if Redis unavailable at startup
sync_connect: false
]
Keyword.merge(base_opts, resilience_opts)
end
@doc """
Returns the deployment timestamp.
In production (Kubernetes), this reads from the DEPLOY_TIMESTAMP environment
variable which is automatically set by GitLab CI during deployment. Falls back
to compile time in development.
The timestamp is set by GitLab CI using:
```bash
kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -n towerops
```
This ensures all pods in the deployment show the same deployment time, regardless
of when individual pods were created or restarted.
"""
@spec build_timestamp() :: DateTime.t()
def build_timestamp do
case System.get_env("DEPLOY_TIMESTAMP") do
nil ->
# Development/fallback: use compile time
@build_timestamp
timestamp_string ->
# Production: parse from environment variable
case DateTime.from_iso8601(timestamp_string) do
{:ok, datetime, _offset} -> datetime
{:error, _} -> @build_timestamp
end
end
end
end