towerops/lib/towerops/application.ex

defmodule Towerops.Application do
  @moduledoc """
  OTP Application module for Towerops.

  Supervises the Repo, PubSub, Telemetry, Endpoint, and monitoring workers.
  """

  use Application

  alias Towerops.Coverages.Antenna
  alias Towerops.Workers.JobCleanupTask

  require Logger

  # Capture the build timestamp at compile time (fallback for development)
  @build_timestamp DateTime.utc_now()

  @impl true
  def start(_type, _args) do
    # Ensure parsetools is started for MIB parsing (yecc dependency)
    _ = Application.ensure_all_started(:parsetools)

    # Register file logger backend for development (using new LoggerBackends API)
    if Application.get_env(:towerops, :env) == :dev do
      register_file_logger()
    end

    # Add snmpkit ebin directory to code path for mib_grammar_elixir.beam
    # This is needed because snmpkit has vendored Erlang modules (compiled from .yrl)
    # Note: With pre-compiled MIBs, this may no longer be needed
    snmpkit_ebin = Application.app_dir(:towerops, "../snmpkit/ebin")

    _ =
      if File.dir?(snmpkit_ebin) do
        :code.add_patha(to_charlist(snmpkit_ebin))
      end

    # Run migrations on startup (Ecto handles locking for concurrent runs)
    _ =
      if Application.get_env(:towerops, Towerops.Repo)[:database] != "towerops_test" do
        Towerops.Release.migrate()
      end

    # Configure MIB directories for NIF resolution (before supervisor starts)
    Logger.info("Configuring MIB directories for NIF resolution...")
    mib_dir = Application.app_dir(:towerops, "priv/mibs")

    # Build list of all MIB directories (including subdirectories for vendor MIBs)
    mib_dirs =
      if File.dir?(mib_dir) do
        # Get main directory plus all subdirectories
        subdirs =
          mib_dir
          |> File.ls!()
          |> Enum.map(&Path.join(mib_dir, &1))
          |> Enum.filter(&File.dir?/1)

        [mib_dir | subdirs]
      else
        [mib_dir]
      end

    # Set MIBDIRS environment variable for net-snmp (must be set BEFORE init)
    # Join with colon for Unix-style path list
    mibdirs_env = Enum.join(mib_dirs, ":")
    System.put_env("MIBDIRS", mibdirs_env)
    System.put_env("MIBS", "ALL")

    Logger.info("MIB directories configured: #{length(mib_dirs)} directories")

    # Add each directory to net-snmp's search path
    Enum.each(mib_dirs, fn dir ->
      :ok = ToweropsNative.load_mib_directory(dir)
    end)

    # Initialize the MIB library (loads all MIBs into memory)
    case ToweropsNative.init_mib_library() do
      result when result in ["initialized", "already_initialized"] ->
        Logger.info("MIB library initialized: #{result}")

      other ->
        Logger.warning("Unexpected MIB library init result: #{inspect(other)}")
    end

    # Load antenna pattern registry (priv/antennas/*.ant) into :persistent_term
    Antenna.load_registry()

    topologies = Application.get_env(:libcluster, :topologies, [])

    # Supervision tree order matters for shutdown: children shut down in reverse start order.
    # Oban is placed AFTER its dependencies (SnmpKit, TaskSupervisor) so it shuts down
    # BEFORE them, allowing in-flight jobs to complete cleanly during the grace period.
    children =
      [
        {Cluster.Supervisor, [topologies, [name: Towerops.ClusterSupervisor]]},
        ToweropsWeb.Telemetry,
        Towerops.PromEx,
        Towerops.Repo,
        # HTTP client pool with CA certificate configuration
        # Ensures HTTPS works in containers where OS CA store is unavailable
        {Finch,
         name: Towerops.Finch,
         pools: %{
           default: [
             conn_opts: [
               transport_opts: [
                 cacertfile: CAStore.file_path()
               ]
             ]
           ]
         }},
        # Encryption vault for sensitive data (passwords, API tokens)
        Towerops.Vault,
        # Error notifications via email (production only, gracefully shuts down in dev/test)
        {ErrorTrackerNotifier, []},
        # Redis connection for brute force protection and rate limiting
        redis_spec(),
        # Cache for device agent assignment lookups (reduces DB load from workers)
        Towerops.Agents.AgentCache,
        # Rate limiting backend (ETS-based, per-node)
        {Towerops.RateLimit, [clean_period: to_timeout(minute: 1)]},
        {DNSCluster, query: Application.get_env(:towerops, :dns_cluster_query) || :ignore},
        pubsub_spec(),
        # MIB name cache for fast profile matching (starts asynchronous pre-resolution)
        Towerops.Profiles.MibCache,
        # Load YAML profiles into ETS cache (depends on MibCache)
        Towerops.Profiles.YamlProfiles,
        # SnmpKit services for SNMP operations (used for SNMP protocol, not MIB resolution)
        SnmpKit.SnmpMgr.Config,
        SnmpKit.SnmpMgr.MIB,
        # Task supervisor for fire-and-forget background tasks (e.g. polling data processing)
        {Task.Supervisor, name: Towerops.TaskSupervisor},
        # Oban after its dependencies so it drains jobs before they shut down
        {Oban, Application.fetch_env!(:towerops, Oban)}
      ] ++
        dev_only_workers() ++
        background_workers() ++
        [
          # Start to serve requests, typically the last entry
          ToweropsWeb.Endpoint,
          # GraphQL subscription PubSub (must start after endpoint)
          {Absinthe.Subscription, ToweropsWeb.Endpoint}
        ]

    # See https://hexdocs.pm/elixir/Supervisor.html
    # for other strategies and supported options
    opts = [strategy: :one_for_one, name: Towerops.Supervisor]
    result = Supervisor.start_link(children, opts)

    if match?({:ok, _}, result), do: post_startup()

    result
  end

  # Post-startup work runs only when the supervisor came up cleanly. The Task
  # supervisor call is wrapped in try/catch so a transient noproc (the named
  # supervisor may not be ready or may already be dying) cannot exit the boot
  # process and mask the real error from Supervisor.start_link.
  defp post_startup do
    ToweropsWeb.TelemetryFilter.attach()

    _ =
      try do
        Task.Supervisor.start_child(Towerops.TaskSupervisor, fn ->
          Process.sleep(2000)
          JobCleanupTask.run()
        end)
      catch
        :exit, reason ->
          Logger.warning("Skipping JobCleanupTask: #{inspect(reason)}")
          :error
      end

    :ok
  end

  # Tell Phoenix to update the endpoint configuration
  # whenever the application is updated.
  @impl true
  def config_change(changed, _new, removed) do
    ToweropsWeb.Endpoint.config_change(changed, removed)
    :ok
  end

  # Configure background workers - don't start in test environment
  defp background_workers do
    if Application.get_env(:towerops, :env) == :test do
      []
    else
      [
        # Start event logger (subscribes to PubSub)
        Towerops.Devices.EventLogger,
        # Alert storm detection (sliding window per org)
        {Towerops.Alerts.StormDetector, []}
        # Note: NeighborCleanupWorker, StaleAgentWorker, AgentLatencyEvaluator,
        # and JobHealthCheckWorker are now Oban Cron jobs (see config/runtime.exs)
      ]
    end
  end

  # Configure dev-only workers - profile watcher for auto-reload
  defp dev_only_workers do
    if Application.get_env(:towerops, :env) == :dev do
      [
        # Watch YAML profiles for changes and auto-reload
        Towerops.Profiles.ProfileWatcher
      ]
    else
      []
    end
  end

  # Returns Redis connection spec for brute force protection
  defp redis_spec do
    redis_config = Application.get_env(:towerops, :redis, [])

    if Keyword.has_key?(redis_config, :host) do
      opts = [
        host: Keyword.get(redis_config, :host, "localhost"),
        port: Keyword.get(redis_config, :port, 6379),
        name: Towerops.Redix
      ]

      # Add password if configured
      opts =
        case Keyword.get(redis_config, :password) do
          nil -> opts
          "" -> opts
          password -> Keyword.put(opts, :password, password)
        end

      {Redix, opts}
    else
      # No Redis configured - start a stub process that will fail gracefully
      # This allows the app to start in environments without Redis
      Supervisor.child_spec({Agent, fn -> nil end}, id: Towerops.Redix)
    end
  end

  # Returns PubSub spec - uses Redis in production/clustered environments,
  # falls back to default PG2 adapter for Mix tasks and development
  defp pubsub_spec do
    redis_config = Application.get_env(:towerops, :redis, [])
    node_name = node()

    # Use Redis adapter only when:
    # 1. Redis config exists with host
    # 2. Node has a name (not nonode@nohost)
    use_redis? =
      Keyword.has_key?(redis_config, :host) &&
        node_name != :nonode@nohost

    if use_redis? do
      base_config = [name: Towerops.PubSub, adapter: Phoenix.PubSub.Redis]
      redis_opts = redis_pubsub_config()
      {Phoenix.PubSub, Keyword.merge(base_config, redis_opts)}
    else
      # Default PG2 adapter for Mix tasks and development
      {Phoenix.PubSub, name: Towerops.PubSub}
    end
  end

  defp redis_pubsub_config do
    redis_config = Application.get_env(:towerops, :redis, [])

    # Build Redix connection options (phoenix_pubsub_redis 3.1.0+ requires :redis_opts key)
    redis_opts = [
      host: Keyword.get(redis_config, :host, "localhost"),
      port: Keyword.get(redis_config, :port, 6379),
      # TCP keepalive to detect dead connections
      socket_opts: [keepalive: true],
      # Connection timeout (5 seconds)
      timeout: 5_000,
      # Backoff settings for reconnection
      backoff_initial: 500,
      backoff_max: 30_000,
      # Don't exit process on disconnection - let PubSub handle it
      exit_on_disconnection: false
      # Note: sync_connect is automatically set to true by phoenix_pubsub_redis 3.1.0
    ]

    # Add password if configured (read from application config set by runtime.exs)
    redis_opts =
      case Keyword.get(redis_config, :password) do
        nil -> redis_opts
        "" -> redis_opts
        password -> Keyword.put(redis_opts, :password, password)
      end

    # Phoenix.PubSub.Redis 3.1.0+ configuration structure
    [
      node_name: node(),
      redis_opts: redis_opts
    ]
  end

  # Register file logger backend for development (only available in :dev environment)
  defp register_file_logger do
    if Code.ensure_loaded?(LoggerBackends) and Code.ensure_loaded?(LoggerFileBackend) do
      # Add the backend with a unique identifier (using apply to avoid compile-time warnings)
      apply(LoggerBackends, :add, [{LoggerFileBackend, :file_log}])

      # Configure the backend with path and level
      Logger.configure_backend({LoggerFileBackend, :file_log},
        path: "_build/dev.log",
        level: :debug
      )
    else
      Logger.debug("LoggerBackends not available - file logging disabled (dev-only feature)")
    end
  end

  @doc """
  Returns the deployment timestamp.

  In production (Kubernetes), this reads from the DEPLOY_TIMESTAMP environment
  variable which is automatically set by GitLab CI during deployment. Falls back
  to compile time in development.

  The timestamp is set by GitLab CI using:
  ```bash
  kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -n towerops
  ```

  This ensures all pods in the deployment show the same deployment time, regardless
  of when individual pods were created or restarted.
  """
  @spec build_timestamp() :: DateTime.t()
  def build_timestamp do
    case System.get_env("DEPLOY_TIMESTAMP") do
      nil ->
        # Development/fallback: use compile time
        @build_timestamp

      timestamp_string ->
        # Production: parse from environment variable
        case DateTime.from_iso8601(timestamp_string) do
          {:ok, datetime, _offset} -> datetime
          {:error, _} -> @build_timestamp
        end
    end
  end
end