towerops/test/towerops/workers/agent_latency_evaluator_test.exs

defmodule Towerops.Workers.AgentLatencyEvaluatorTest do
  use Towerops.DataCase, async: false
  use Oban.Testing, repo: Towerops.Repo

  import Towerops.AccountsFixtures
  import Towerops.OrganizationsFixtures

  alias Towerops.Agents
  alias Towerops.Monitoring
  alias Towerops.Sites
  alias Towerops.Workers.AgentLatencyEvaluator

  describe "perform/1" do
    test "returns :ok when no reassignment candidates exist" do
      assert :ok = perform_job(AgentLatencyEvaluator, %{})
    end

    test "reassigns device to faster agent when 20%+ improvement available" do
      user = user_fixture()
      org = organization_fixture(user.id)
      {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
      {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")

      {:ok, site} =
        Sites.create_site(%{
          name: "Test Site",
          organization_id: org.id,
          agent_token_id: slow_agent.id
        })

      {:ok, device} =
        Towerops.Devices.create_device(%{
          name: "Router",
          ip_address: "192.168.1.1",
          site_id: site.id,
          organization_id: site.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      # Create checks: slow = 100ms, fast = 50ms (50% improvement)
      for _ <- 1..15 do
        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: slow_agent.id,
          status: :success,
          response_time_ms: 100,
          checked_at: DateTime.utc_now()
        })

        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: fast_agent.id,
          status: :success,
          response_time_ms: 50,
          checked_at: DateTime.utc_now()
        })
      end

      # Trigger evaluation
      assert :ok = AgentLatencyEvaluator.perform(%Oban.Job{args: %{}})

      # Verify site was reassigned
      updated_site = Sites.get_site!(site.id)
      assert updated_site.agent_token_id == fast_agent.id
    end

    test "does not reassign when improvement is below 20% threshold" do
      user = user_fixture()
      org = organization_fixture(user.id)
      {:ok, agent1, _} = Agents.create_agent_token(org.id, "Agent 1")
      {:ok, agent2, _} = Agents.create_agent_token(org.id, "Agent 2")

      {:ok, site} =
        Sites.create_site(%{
          name: "Test Site",
          organization_id: org.id,
          agent_token_id: agent1.id
        })

      {:ok, device} =
        Towerops.Devices.create_device(%{
          name: "Router",
          ip_address: "192.168.1.1",
          site_id: site.id,
          organization_id: site.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      # Create checks: agent1 = 100ms, agent2 = 85ms (15% improvement, below 20%)
      for _ <- 1..15 do
        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: agent1.id,
          status: :success,
          response_time_ms: 100,
          checked_at: DateTime.utc_now()
        })

        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: agent2.id,
          status: :success,
          response_time_ms: 85,
          checked_at: DateTime.utc_now()
        })
      end

      # Trigger evaluation
      assert :ok = AgentLatencyEvaluator.perform(%Oban.Job{args: %{}})

      # Verify site was NOT reassigned
      updated_site = Sites.get_site!(site.id)
      assert updated_site.agent_token_id == agent1.id
    end

    test "reassigns organization-level assignment" do
      user = user_fixture()

      {:ok, slow_agent, _} = Agents.create_cloud_poller("Slow Cloud")
      {:ok, fast_agent, _} = Agents.create_cloud_poller("Fast Cloud")

      org =
        organization_fixture(user.id, %{
          default_agent_token_id: slow_agent.id
        })

      {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})

      {:ok, device} =
        Towerops.Devices.create_device(%{
          name: "Router",
          ip_address: "192.168.1.1",
          site_id: site.id,
          organization_id: site.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      # Create checks showing fast agent is better
      for _ <- 1..15 do
        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: slow_agent.id,
          status: :success,
          response_time_ms: 150,
          checked_at: DateTime.utc_now()
        })

        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: fast_agent.id,
          status: :success,
          response_time_ms: 50,
          checked_at: DateTime.utc_now()
        })
      end

      # Trigger evaluation
      assert :ok = AgentLatencyEvaluator.perform(%Oban.Job{args: %{}})

      # Verify organization was reassigned
      updated_org = Towerops.Organizations.get_organization!(org.id)
      assert updated_org.default_agent_token_id == fast_agent.id
    end

    test "creates device assignment for global default assignment" do
      user = user_fixture()
      org = organization_fixture(user.id)

      {:ok, slow_cloud, _} = Agents.create_cloud_poller("Slow Global")
      {:ok, fast_cloud, _} = Agents.create_cloud_poller("Fast Global")

      # Set slow cloud as global default
      {:ok, _} = Towerops.Settings.set_global_default_cloud_poller(slow_cloud.id)

      {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})

      {:ok, device} =
        Towerops.Devices.create_device(%{
          name: "Router",
          ip_address: "192.168.1.1",
          site_id: site.id,
          organization_id: site.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      # Create checks showing fast cloud is better
      for _ <- 1..15 do
        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: slow_cloud.id,
          status: :success,
          response_time_ms: 200,
          checked_at: DateTime.utc_now()
        })

        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: fast_cloud.id,
          status: :success,
          response_time_ms: 60,
          checked_at: DateTime.utc_now()
        })
      end

      # Trigger evaluation
      assert :ok = AgentLatencyEvaluator.perform(%Oban.Job{args: %{}})

      # Verify device assignment was created
      assignment = Agents.get_device_assignment(device.id)
      assert assignment.agent_token_id == fast_cloud.id

      # Cleanup global default
      {:ok, _} = Towerops.Settings.set_global_default_cloud_poller(nil)
    end

    test "creates device assignment for cloud polling (no assignment)" do
      user = user_fixture()
      org = organization_fixture(user.id)

      {:ok, slow_cloud, _} = Agents.create_cloud_poller("Slow Cloud")
      {:ok, fast_cloud, _} = Agents.create_cloud_poller("Fast Cloud")

      {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})

      {:ok, device} =
        Towerops.Devices.create_device(%{
          name: "Router",
          ip_address: "192.168.1.1",
          site_id: site.id,
          organization_id: site.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      # Create checks from both cloud agents (bulk insert for speed)
      now = DateTime.truncate(DateTime.utc_now(), :second)

      for_result =
        for _ <- 1..10 do
          [
            %{
              id: Ecto.UUID.generate(),
              device_id: device.id,
              agent_token_id: slow_cloud.id,
              status: :success,
              response_time_ms: 100.0,
              checked_at: now,
              inserted_at: now
            },
            %{
              id: Ecto.UUID.generate(),
              device_id: device.id,
              agent_token_id: fast_cloud.id,
              status: :success,
              response_time_ms: 50.0,
              checked_at: now,
              inserted_at: now
            }
          ]
        end

      checks = List.flatten(for_result)

      Repo.insert_all(Towerops.Monitoring.Check, checks)

      # Trigger evaluation
      assert :ok = AgentLatencyEvaluator.perform(%Oban.Job{args: %{}})

      # Verify device assignment was created to the fast agent
      assignment = Agents.get_device_assignment(device.id)
      assert assignment != nil, "Expected device assignment to be created"
      assert assignment.agent_token_id == fast_cloud.id
    end

    test "does not reassign direct device assignments" do
      user = user_fixture()
      org = organization_fixture(user.id)
      {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
      {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")

      {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})

      {:ok, device} =
        Towerops.Devices.create_device(%{
          name: "Router",
          ip_address: "192.168.1.1",
          site_id: site.id,
          organization_id: site.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      # Create direct device assignment
      {:ok, _} = Agents.assign_device_to_agent(slow_agent.id, device.id)

      # Create checks showing fast agent is better
      for _ <- 1..15 do
        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: slow_agent.id,
          status: :success,
          response_time_ms: 100,
          checked_at: DateTime.utc_now()
        })

        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: fast_agent.id,
          status: :success,
          response_time_ms: 50,
          checked_at: DateTime.utc_now()
        })
      end

      # Trigger evaluation
      assert :ok = AgentLatencyEvaluator.perform(%Oban.Job{args: %{}})

      # Verify device assignment was NOT changed
      assignment = Agents.get_device_assignment(device.id)
      assert assignment.agent_token_id == slow_agent.id
    end

    test "requires minimum check count" do
      user = user_fixture()
      org = organization_fixture(user.id)
      {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
      {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")

      {:ok, site} =
        Sites.create_site(%{
          name: "Test Site",
          organization_id: org.id,
          agent_token_id: slow_agent.id
        })

      {:ok, device} =
        Towerops.Devices.create_device(%{
          name: "Router",
          ip_address: "192.168.1.1",
          site_id: site.id,
          organization_id: site.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      # Create only 5 checks (below default minimum of 10)
      for _ <- 1..5 do
        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: slow_agent.id,
          status: :success,
          response_time_ms: 100,
          checked_at: DateTime.utc_now()
        })

        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: fast_agent.id,
          status: :success,
          response_time_ms: 50,
          checked_at: DateTime.utc_now()
        })
      end

      # Trigger evaluation
      assert :ok = AgentLatencyEvaluator.perform(%Oban.Job{args: %{}})

      # Verify site was NOT reassigned (insufficient data)
      updated_site = Sites.get_site!(site.id)
      assert updated_site.agent_token_id == slow_agent.id
    end

    test "handles multiple devices in single evaluation" do
      user = user_fixture()
      org = organization_fixture(user.id)
      {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
      {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")

      {:ok, site1} =
        Sites.create_site(%{
          name: "Site 1",
          organization_id: org.id,
          agent_token_id: slow_agent.id
        })

      {:ok, site2} =
        Sites.create_site(%{
          name: "Site 2",
          organization_id: org.id,
          agent_token_id: slow_agent.id
        })

      {:ok, device1} =
        Towerops.Devices.create_device(%{
          name: "Router 1",
          ip_address: "192.168.1.1",
          site_id: site1.id,
          organization_id: site1.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      {:ok, device2} =
        Towerops.Devices.create_device(%{
          name: "Router 2",
          ip_address: "192.168.2.1",
          site_id: site2.id,
          organization_id: site2.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      # Create checks for both devices
      for device_id <- [device1.id, device2.id] do
        for _ <- 1..15 do
          Monitoring.create_check(%{
            device_id: device_id,
            agent_token_id: slow_agent.id,
            status: :success,
            response_time_ms: 100,
            checked_at: DateTime.utc_now()
          })

          Monitoring.create_check(%{
            device_id: device_id,
            agent_token_id: fast_agent.id,
            status: :success,
            response_time_ms: 50,
            checked_at: DateTime.utc_now()
          })
        end
      end

      # Trigger evaluation
      assert :ok = AgentLatencyEvaluator.perform(%Oban.Job{args: %{}})

      # Verify both sites were reassigned
      updated_site1 = Sites.get_site!(site1.id)
      updated_site2 = Sites.get_site!(site2.id)

      assert updated_site1.agent_token_id == fast_agent.id
      assert updated_site2.agent_token_id == fast_agent.id
    end

    test "broadcasts PubSub event when device is reassigned" do
      user = user_fixture()
      org = organization_fixture(user.id)
      {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
      {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")

      {:ok, site} =
        Sites.create_site(%{
          name: "Test Site",
          organization_id: org.id,
          agent_token_id: slow_agent.id
        })

      {:ok, device} =
        Towerops.Devices.create_device(%{
          name: "Router",
          ip_address: "192.168.1.1",
          site_id: site.id,
          organization_id: site.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      # Create checks showing significant improvement
      for _ <- 1..15 do
        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: slow_agent.id,
          status: :success,
          response_time_ms: 100,
          checked_at: DateTime.utc_now()
        })

        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: fast_agent.id,
          status: :success,
          response_time_ms: 50,
          checked_at: DateTime.utc_now()
        })
      end

      # Subscribe to PubSub
      Phoenix.PubSub.subscribe(Towerops.PubSub, "devices:latency")

      # Trigger evaluation
      assert :ok = perform_job(AgentLatencyEvaluator, %{})

      # Verify PubSub broadcast was sent
      assert_received {:device_reassigned, candidate}
      assert candidate.device_id == device.id
      assert candidate.best_agent_token_id == fast_agent.id
      assert candidate.current_agent_token_id == slow_agent.id
      assert candidate.improvement_percent >= 20
    end

    test "does not broadcast when no reassignment occurs" do
      user = user_fixture()
      org = organization_fixture(user.id)
      {:ok, agent1, _} = Agents.create_agent_token(org.id, "Agent 1")

      {:ok, site} =
        Sites.create_site(%{
          name: "Test Site",
          organization_id: org.id,
          agent_token_id: agent1.id
        })

      {:ok, device} =
        Towerops.Devices.create_device(%{
          name: "Router",
          ip_address: "192.168.1.1",
          site_id: site.id,
          organization_id: site.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      # Create checks with only one agent (no alternative)
      for _ <- 1..15 do
        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: agent1.id,
          status: :success,
          response_time_ms: 100,
          checked_at: DateTime.utc_now()
        })
      end

      # Subscribe to PubSub
      Phoenix.PubSub.subscribe(Towerops.PubSub, "devices:latency")

      # Trigger evaluation
      assert :ok = perform_job(AgentLatencyEvaluator, %{})

      # Verify no broadcast was sent
      refute_received {:device_reassigned, _}
    end

    test "handles reassignment failures gracefully" do
      user = user_fixture()
      org = organization_fixture(user.id)
      {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
      {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")

      {:ok, site} =
        Sites.create_site(%{
          name: "Test Site",
          organization_id: org.id,
          agent_token_id: slow_agent.id
        })

      {:ok, device} =
        Towerops.Devices.create_device(%{
          name: "Router",
          ip_address: "192.168.1.1",
          site_id: site.id,
          organization_id: site.organization_id,
          snmp_enabled: true,
          snmp_community: "public",
          snmp_version: "2c"
        })

      # Create checks showing improvement
      for _ <- 1..15 do
        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: slow_agent.id,
          status: :success,
          response_time_ms: 100,
          checked_at: DateTime.utc_now()
        })

        Monitoring.create_check(%{
          device_id: device.id,
          agent_token_id: fast_agent.id,
          status: :success,
          response_time_ms: 50,
          checked_at: DateTime.utc_now()
        })
      end

      # Delete the site to cause a failure during reassignment
      Sites.delete_site(site)

      # Worker should complete successfully despite the error
      assert :ok = perform_job(AgentLatencyEvaluator, %{})
    end
  end
end