towerops/lib/towerops/workers/check_worker.ex

defmodule Towerops.Workers.CheckWorker do
  @moduledoc """
  Oban worker that executes service checks (HTTP, TCP, DNS).

  Follows same self-scheduling pattern as DeviceMonitorWorker.
  Phoenix executes checks unless assigned to an agent (agent_token_id set).
  """
  use Oban.Worker,
    queue: :checks,
    unique: [
      period: 60,
      keys: [:check_id],
      states: [:available, :scheduled, :executing, :retryable, :suspended]
    ]

  alias Towerops.Agents
  alias Towerops.Alerts
  alias Towerops.Monitoring
  alias Towerops.Monitoring.Executor
  alias Towerops.Snmp.Client
  alias Towerops.Workers.PollingOffset

  require Logger

  @impl Oban.Worker
  def perform(%Oban.Job{args: %{"check_id" => check_id}}) do
    case Monitoring.get_check(check_id) do
      nil ->
        Logger.debug("Check #{check_id} no longer exists, skipping and not rescheduling")
        :ok

      check ->
        if should_continue_checking?(check) do
          maybe_perform_check(check)
          schedule_next_check_with_error_handling(check_id, check.interval_seconds)
        else
          Logger.debug("Check #{check_id} disabled or assigned to agent, not rescheduling",
            check_id: check_id,
            enabled: check.enabled,
            agent_assigned: !is_nil(check.agent_token_id)
          )
        end

        :ok
    end
  end

  # Check should continue if:
  # - Enabled
  # - Not a passive check (passive checks don't execute actively)
  # - Either no agent assigned OR agent is cloud poller (Phoenix executes)
  defp should_continue_checking?(check) do
    check.enabled &&
      check.check_type != "passive" &&
      !Client.phoenix_snmp_disabled() &&
      should_phoenix_execute?(check)
  end

  defp should_phoenix_execute?(check) do
    # Never execute if the device has any effective agent at any cascade level
    if check.device_id != nil and Agents.device_has_effective_agent?(check.device_id) do
      false
    else
      case check.agent_token_id do
        nil ->
          true

        agent_token_id ->
          try do
            agent_token = Agents.get_agent_token!(agent_token_id)
            agent_token.is_cloud_poller
          rescue
            Ecto.NoResultsError -> true
          end
      end
    end
  end

  # Agent check already done by should_continue_checking?, no need to re-check
  defp maybe_perform_check(check) do
    perform_check(check)
  end

  defp schedule_next_check_with_error_handling(check_id, interval_seconds) do
    case Monitoring.get_check(check_id) do
      nil ->
        Logger.debug("Check deleted, not rescheduling", check_id: check_id)
        :ok

      _check ->
        case schedule_next_check(check_id, interval_seconds) do
          {:ok, _job} ->
            :ok

          {:error, changeset} ->
            Logger.error("Failed to schedule next check for #{check_id}: #{inspect(changeset.errors)}")
            :ok
        end
    end
  end

  @doc """
  Starts executing a check (schedules first execution).
  """
  def start_check(check_id, interval_seconds \\ 60) do
    offset = PollingOffset.calculate_offset(check_id, interval_seconds)

    %{check_id: check_id}
    |> new(schedule_in: offset)
    |> Oban.insert()
  end

  @doc """
  Stops executing a check by cancelling its jobs.
  """
  def stop_check(check_id) do
    import Ecto.Query

    Oban.cancel_all_jobs(
      from(j in Oban.Job,
        where: j.worker == "Towerops.Workers.CheckWorker",
        where: fragment("args->>'check_id' = ?", ^check_id),
        where: j.state in ["available", "scheduled", "executing", "retryable"]
      )
    )
  end

  @doc """
  Triggers an immediate check execution.
  """
  def trigger_check(check_id) do
    %{check_id: check_id}
    |> new()
    |> Oban.insert()
  end

  # Private Functions

  defp perform_check(check) do
    Logger.debug("Executing check: #{check.name} (#{check.check_type})", check_id: check.id)

    # Execute the check
    result = Executor.execute(check)

    # Convert result to status and extract details
    {status, response_time, output} =
      case result do
        {:ok, time, msg} -> {0, time, msg}
        {:error, reason} -> {2, nil, reason}
      end

    # Record the result
    now = DateTime.utc_now()

    case Monitoring.create_check_result(%{
           check_id: check.id,
           organization_id: check.organization_id,
           status: status,
           output: output,
           response_time_ms: response_time,
           checked_at: now
         }) do
      {:ok, _result} ->
        Logger.debug("Recorded check result: #{output}", check_id: check.id, status: status)

      {:error, changeset} ->
        Logger.error("Failed to create check result: #{inspect(changeset.errors)}")
    end

    # Update check state (handles soft/hard state transitions)
    old_state = check.current_state
    {:ok, updated_check} = Monitoring.update_check_state(check, status, output)

    # Handle state changes (create/resolve alerts)
    if state_changed_to_problem?(old_state, updated_check) do
      handle_check_problem(updated_check, output)
    end

    if state_changed_to_ok?(old_state, updated_check) do
      handle_check_recovery(updated_check)
    end

    :ok
  end

  @doc """
  True when a check transitioned from OK (0) to a hard non-OK state (1 or 2).

  Requires `current_state_type == "hard"` to avoid alerting on soft-state flaps.
  """
  @spec state_changed_to_problem?(integer(), map()) :: boolean()
  def state_changed_to_problem?(0, %{current_state: s, current_state_type: "hard"}) when s in [1, 2], do: true

  def state_changed_to_problem?(_old, _check), do: false

  @doc """
  True when a check transitioned from a non-OK state (1 or 2) back to OK (0).
  """
  @spec state_changed_to_ok?(integer(), map()) :: boolean()
  def state_changed_to_ok?(old, %{current_state: 0}) when old in [1, 2], do: true
  def state_changed_to_ok?(_old, _check), do: false

  defp handle_check_problem(check, output) do
    Logger.warning("Check #{check.name} entered problem state",
      check_id: check.id,
      state: check.current_state,
      output: output
    )

    # Create alert if one doesn't exist
    if !Alerts.has_active_check_alert?(check.id) do
      create_check_alert(check, output)
    end
  end

  defp handle_check_recovery(check) do
    Logger.info("Check #{check.name} recovered",
      check_id: check.id
    )

    # Resolve any active alerts
    Alerts.resolve_check_alerts(check.id)
  end

  defp create_check_alert(check, output) do
    severity = if check.current_state == 1, do: 1, else: 2

    attrs = %{
      check_id: check.id,
      device_id: check.device_id,
      organization_id: check.organization_id,
      alert_type: "check_#{check.check_type}",
      severity: severity,
      message: "Check '#{check.name}' is #{Monitoring.Check.state_label(check.current_state)}",
      output: output,
      triggered_at: DateTime.utc_now()
    }

    case Alerts.create_alert(attrs) do
      {:ok, alert} ->
        Logger.info("Created alert for check #{check.name}", alert_id: alert.id)

      {:error, changeset} ->
        Logger.error("Failed to create alert: #{inspect(changeset.errors)}")
    end
  end

  defp schedule_next_check(check_id, interval_seconds) do
    offset = PollingOffset.calculate_offset(check_id, interval_seconds)

    %{check_id: check_id}
    |> new(schedule_in: offset)
    |> Oban.insert()
  end
end