towerops/lib/towerops/workers/device_monitor_worker.ex

defmodule Towerops.Workers.DeviceMonitorWorker do
  @moduledoc """
  Oban worker that monitors device health by pinging it.

  Uses Oban's unique job feature to ensure only one monitor per device
  runs cluster-wide. Monitors check device connectivity and create alerts
  on status changes.
  """
  use Oban.Worker,
    queue: :monitors,
    max_attempts: 1,
    unique: [
      period: :infinity,
      keys: [:device_id],
      states: [:available, :scheduled, :retryable]
    ],
    replace: [
      scheduled: [:scheduled_at],
      available: [:scheduled_at]
    ]

  alias Towerops.Agents
  alias Towerops.Alerts
  alias Towerops.Devices
  alias Towerops.Maintenance
  alias Towerops.Monitoring
  alias Towerops.Snmp.Client
  alias Towerops.Workers.AlertNotificationWorker
  alias Towerops.Workers.PollingOffset

  require Logger

  @ping_module Application.compile_env(:towerops, :ping_module, Towerops.Monitoring.Ping)
  @compile {:no_warn_undefined, Towerops.Monitoring.PingMock}

  @monitor_interval 30

  @impl Oban.Worker
  @spec perform(Oban.Job.t()) :: :ok
  def perform(%Oban.Job{args: %{"device_id" => device_id}}) do
    case Devices.get_device(device_id) do
      nil ->
        Logger.debug("Device #{device_id} no longer exists, skipping monitor and not rescheduling")
        :ok

      device ->
        # Only reschedule if device still has monitoring enabled
        if should_continue_monitoring?(device) do
          maybe_perform_check(device)
          schedule_next_check_with_error_handling(device_id)
        else
          Logger.debug("Device #{device_id} monitoring disabled, not rescheduling",
            device_id: device_id,
            monitoring_enabled: device.monitoring_enabled
          )
        end

        :ok
    end
  end

  # Check if device should continue being monitored
  # Returns false if Phoenix SNMP is disabled, monitoring is disabled, or device is assigned to an agent
  defp should_continue_monitoring?(device) do
    !Client.phoenix_snmp_disabled() && device.monitoring_enabled &&
      Agents.should_phoenix_poll_device?(device)
  end

  defp maybe_perform_check(device) do
    cond do
      Client.phoenix_snmp_disabled() ->
        :ok

      not device.monitoring_enabled ->
        :ok

      not Agents.should_phoenix_poll_device?(device) ->
        Logger.debug("Skipping Phoenix monitoring for device #{device.name} - assigned to non-cloud-poller agent")
        :ok

      true ->
        perform_check(device)
    end
  end

  defp schedule_next_check_with_error_handling(device_id) do
    # Verify device still exists before scheduling next check
    case Devices.get_device(device_id) do
      nil ->
        Logger.debug("Device deleted, not rescheduling monitoring", device_id: device_id)
        :ok

      _device ->
        case schedule_next_check(device_id) do
          {:ok, _job} ->
            :ok

          {:error, changeset} ->
            Logger.error("Failed to schedule next monitoring check for device #{device_id}: #{inspect(changeset.errors)}")
            :ok
        end
    end
  end

  @doc """
  Starts monitoring for a device.
  """
  def start_monitoring(device_id) do
    offset = PollingOffset.calculate_offset(device_id, @monitor_interval)

    %{device_id: device_id}
    |> new(schedule_in: offset)
    |> Oban.insert()
  end

  @doc """
  Stops monitoring for a device by cancelling its jobs.
  """
  def stop_monitoring(device_id) do
    import Ecto.Query

    Oban.cancel_all_jobs(
      from(j in Oban.Job,
        where: j.worker == "Towerops.Workers.DeviceMonitorWorker",
        where: fragment("args->>'device_id' = ?", ^device_id),
        where: j.state in ["available", "scheduled", "executing", "retryable"]
      )
    )
  end

  @doc """
  Triggers an immediate check for a device.
  """
  def trigger_check(device_id) do
    %{device_id: device_id}
    |> new()
    |> Oban.insert()
  end

  # Private Functions

  defp perform_check(device) do
    check_result = check_device_health(device)

    now = DateTime.truncate(DateTime.utc_now(), :second)

    {status, response_time_ms} =
      case check_result do
        {:ok, time} -> {:success, time}
        {:error, _reason} -> {:failure, nil}
      end

    case Monitoring.create_monitoring_check(%{
           device_id: device.id,
           status: to_string(status),
           response_time_ms: response_time_ms,
           checked_at: now
         }) do
      {:ok, _check} ->
        :ok

      {:error, changeset} ->
        Logger.error("Failed to create monitoring check for device #{device.id}: #{inspect(changeset.errors)}")
    end

    new_status = if status == :success, do: :up, else: :down
    old_status = device.status

    _ = Devices.update_device_status(device, new_status)

    _ =
      if old_status != new_status do
        handle_status_change(device, old_status, new_status)
      end

    # Broadcast status change via PubSub
    _ =
      Phoenix.PubSub.broadcast(
        Towerops.PubSub,
        "device:#{device.id}",
        {:device_status_changed, device.id, new_status, nil}
      )
  end

  defp check_device_health(device) do
    @ping_module.ping(device.ip_address)
  end

  defp handle_status_change(device, old_status, new_status) do
    now = DateTime.truncate(DateTime.utc_now(), :second)

    case {old_status, new_status} do
      {_, :down} ->
        handle_equipment_down(device, now)

      {_, :up} ->
        handle_equipment_up(device, now)
    end
  end

  defp handle_equipment_down(device, now) do
    if Alerts.has_active_alert?(device.id, "device_down") do
      :ok
    else
      # Single maintenance check (removed duplicate from create_device_down_alert)
      if Maintenance.device_in_maintenance?(device.id) do
        Logger.info("Device #{device.id} (#{device.name}) is down but in maintenance window — suppressing alert")
        :ok
      else
        create_device_down_alert(device, now)
      end
    end
  end

  defp create_device_down_alert(device, now) do
    alert_message = get_down_alert_message(device)

    # Use case instead of pattern match to handle errors gracefully
    case Alerts.create_alert(%{
           device_id: device.id,
           alert_type: "device_down",
           triggered_at: now,
           message: alert_message
         }) do
      {:ok, alert} ->
        # Enqueue notification job - Oban handles retries and persistence
        AlertNotificationWorker.enqueue_trigger(alert.id)

        _ =
          Phoenix.PubSub.broadcast(
            Towerops.PubSub,
            "alerts:org:#{device.organization_id}:new",
            {:new_alert, device.id, :device_down}
          )

        :ok

      {:error, reason} ->
        require Logger

        Logger.error("Failed to create device down alert for #{device.name}: #{inspect(reason)}",
          device_id: device.id
        )

        {:error, reason}
    end
  end

  defp get_down_alert_message(device) do
    base_message =
      if device.snmp_enabled do
        "Device is not responding to SNMP"
      else
        "Device is not responding to ping"
      end

    case Towerops.Gaiia.get_device_impact(device.id) do
      %{subscriber_count: count, mrr: mrr} when count > 0 ->
        "#{device.name} is down — #{count} subscribers affected ($#{Decimal.round(mrr, 2)}/mo at risk)"

      _ ->
        base_message
    end
  end

  defp handle_equipment_up(device, now) do
    recovery_message = get_recovery_message(device)

    # Use case instead of pattern match to handle errors gracefully
    case Alerts.create_alert(%{
           device_id: device.id,
           alert_type: "device_up",
           triggered_at: now,
           resolved_at: now,
           message: recovery_message
         }) do
      {:ok, alert} ->
        # Enqueue notification job - Oban handles retries and persistence
        AlertNotificationWorker.enqueue_trigger(alert.id)

        resolve_down_alert(device)

        _ =
          Phoenix.PubSub.broadcast(
            Towerops.PubSub,
            "alerts:org:#{device.organization_id}:resolved",
            {:alert_resolved, device.id, :device_down}
          )

        :ok

      {:error, reason} ->
        require Logger

        Logger.error("Failed to create device up alert for #{device.name}: #{inspect(reason)}",
          device_id: device.id
        )

        {:error, reason}
    end
  end

  defp get_recovery_message(device) do
    if device.snmp_enabled do
      "Device is now responding to SNMP"
    else
      "Device is now responding to ping"
    end
  end

  defp resolve_down_alert(device) do
    case Alerts.get_active_alert(device.id, "device_down") do
      nil -> :ok
      alert -> Alerts.resolve_alert(alert)
    end
  end

  defp schedule_next_check(device_id) do
    offset = PollingOffset.calculate_offset(device_id, @monitor_interval)

    %{device_id: device_id}
    |> new(schedule_in: @monitor_interval + offset)
    |> Oban.insert()
  end
end