- create MonitoringCheck schema for the monitoring_checks table, fixing silent data loss where ping results were dropped by the wrong schema - update agent channel and device monitor worker to use new schema - fix Stats queries to use MonitoringCheck instead of Check schema - add list_online_cloud_pollers and list_cloud_polled_devices queries - add CloudLatencyProbeWorker to dispatch cross-agent latency probes every 8 hours via PubSub to all online cloud pollers - scope AgentLatencyEvaluator to cloud pollers only with cloud_only filter, lower min_checks (5), wider time window (48h), and device-level assignments to avoid changing site/org defaults - update cron schedule: probes at 0/8/16h, evaluation at 0:30/8:30/16:30 - refactor monitoring.ex and http_executor.ex to fix credo complexity
1100 lines
34 KiB
Elixir
1100 lines
34 KiB
Elixir
defmodule Towerops.Agents.StatsTest do
|
|
use Towerops.DataCase
|
|
|
|
import Towerops.AccountsFixtures
|
|
import Towerops.OrganizationsFixtures
|
|
|
|
alias Towerops.Agents
|
|
alias Towerops.Agents.Stats
|
|
alias Towerops.Devices.Device
|
|
alias Towerops.Monitoring
|
|
alias Towerops.Monitoring.MonitoringCheck
|
|
alias Towerops.Repo
|
|
alias Towerops.Sites
|
|
alias Towerops.Snmp.Device
|
|
alias Towerops.Snmp.Interface
|
|
alias Towerops.Snmp.InterfaceStat
|
|
alias Towerops.Snmp.Sensor
|
|
alias Towerops.Snmp.SensorReading
|
|
|
|
describe "get_organization_agent_health/1" do
|
|
test "returns empty list for organization with no agents" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
|
|
assert Stats.get_organization_agent_health(org.id) == []
|
|
end
|
|
|
|
test "returns online agents with device counts" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Test Agent")
|
|
|
|
# Touch last_seen_at to make it online
|
|
agent_token
|
|
|> Ecto.Changeset.change(%{
|
|
last_seen_at: DateTime.truncate(DateTime.utc_now(), :second),
|
|
metadata: %{"version" => "0.1.0", "hostname" => "test-host"}
|
|
})
|
|
|> Repo.update!()
|
|
|
|
health = Stats.get_organization_agent_health(org.id)
|
|
|
|
assert length(health) == 1
|
|
agent_health = hd(health)
|
|
assert agent_health.id == agent_token.id
|
|
assert agent_health.name == "Test Agent"
|
|
assert agent_health.online == true
|
|
assert agent_health.device_count == 0
|
|
assert agent_health.version == "0.1.0"
|
|
assert agent_health.hostname == "test-host"
|
|
end
|
|
|
|
test "marks agents as offline after 5 minutes" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Offline Agent")
|
|
|
|
# Set last_seen_at to 6 minutes ago
|
|
six_minutes_ago = DateTime.utc_now() |> DateTime.add(-6, :minute) |> DateTime.truncate(:second)
|
|
|
|
agent_token
|
|
|> Ecto.Changeset.change(%{last_seen_at: six_minutes_ago})
|
|
|> Repo.update!()
|
|
|
|
health = Stats.get_organization_agent_health(org.id)
|
|
|
|
assert length(health) == 1
|
|
agent_health = hd(health)
|
|
assert agent_health.online == false
|
|
end
|
|
|
|
test "includes device count from hierarchical assignments" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Test Agent")
|
|
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Test Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
{:ok, _assignment} = Agents.assign_device_to_agent(agent_token.id, device.id)
|
|
|
|
agent_token
|
|
|> Ecto.Changeset.change(%{last_seen_at: DateTime.truncate(DateTime.utc_now(), :second)})
|
|
|> Repo.update!()
|
|
|
|
health = Stats.get_organization_agent_health(org.id)
|
|
|
|
assert length(health) == 1
|
|
agent_health = hd(health)
|
|
assert agent_health.device_count == 1
|
|
end
|
|
end
|
|
|
|
describe "get_device_assignment_breakdown/1" do
|
|
test "returns zero counts for organization with no devices" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
|
|
breakdown = Stats.get_device_assignment_breakdown(org.id)
|
|
|
|
assert breakdown == %{direct: 0, site: 0, organization: 0, cloud: 0}
|
|
end
|
|
|
|
test "counts device by assignment type" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Test Agent")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
# Direct assignment
|
|
{:ok, device1} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router 1",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
{:ok, _assignment} = Agents.assign_device_to_agent(agent_token.id, device1.id)
|
|
|
|
# Cloud polling (no agent)
|
|
{:ok, _device2} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router 2",
|
|
ip_address: "192.168.1.2",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
breakdown = Stats.get_device_assignment_breakdown(org.id)
|
|
|
|
assert breakdown.direct == 1
|
|
assert breakdown.cloud == 1
|
|
end
|
|
end
|
|
|
|
describe "get_offline_agents/1" do
|
|
test "returns empty list when all agents are online" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Online Agent")
|
|
|
|
agent_token
|
|
|> Ecto.Changeset.change(%{last_seen_at: DateTime.truncate(DateTime.utc_now(), :second)})
|
|
|> Repo.update!()
|
|
|
|
offline = Stats.get_offline_agents(org.id)
|
|
|
|
assert offline == []
|
|
end
|
|
|
|
test "returns agents not seen in last 5 minutes" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Offline Agent")
|
|
|
|
six_minutes_ago = DateTime.utc_now() |> DateTime.add(-6, :minute) |> DateTime.truncate(:second)
|
|
|
|
agent_token
|
|
|> Ecto.Changeset.change(%{last_seen_at: six_minutes_ago})
|
|
|> Repo.update!()
|
|
|
|
offline = Stats.get_offline_agents(org.id)
|
|
|
|
assert length(offline) == 1
|
|
agent = hd(offline)
|
|
assert agent.id == agent_token.id
|
|
assert agent.name == "Offline Agent"
|
|
end
|
|
|
|
test "returns agents that have never connected" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Never Connected")
|
|
|
|
offline = Stats.get_offline_agents(org.id)
|
|
|
|
assert length(offline) == 1
|
|
agent = hd(offline)
|
|
assert agent.id == agent_token.id
|
|
assert is_nil(agent.last_seen_at)
|
|
end
|
|
end
|
|
|
|
describe "get_high_load_agents/2" do
|
|
test "returns empty list when no agents exceed threshold" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, _agent_token, _token} = Agents.create_agent_token(org.id, "Low Load Agent")
|
|
|
|
high_load = Stats.get_high_load_agents(org.id, 50)
|
|
|
|
assert high_load == []
|
|
end
|
|
|
|
test "returns agents with device count above threshold" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "High Load Agent")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
# Create 3 devices and assign all to agent
|
|
for i <- 1..3 do
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router #{i}",
|
|
ip_address: "192.168.1.#{i}",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
{:ok, _assignment} = Agents.assign_device_to_agent(agent_token.id, device.id)
|
|
end
|
|
|
|
high_load = Stats.get_high_load_agents(org.id, 2)
|
|
|
|
assert length(high_load) == 1
|
|
agent = hd(high_load)
|
|
assert agent.id == agent_token.id
|
|
assert agent.name == "High Load Agent"
|
|
assert agent.device_count == 3
|
|
end
|
|
|
|
test "respects custom threshold parameter" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Agent")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
# Create 5 equipment
|
|
for i <- 1..5 do
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router #{i}",
|
|
ip_address: "192.168.1.#{i}",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
{:ok, _assignment} = Agents.assign_device_to_agent(agent_token.id, device.id)
|
|
end
|
|
|
|
# Should appear with threshold 4
|
|
high_load_4 = Stats.get_high_load_agents(org.id, 4)
|
|
assert length(high_load_4) == 1
|
|
|
|
# Should not appear with threshold 10
|
|
high_load_10 = Stats.get_high_load_agents(org.id, 10)
|
|
assert high_load_10 == []
|
|
end
|
|
end
|
|
|
|
describe "get_unmonitored_devices/1" do
|
|
test "returns empty list when all devices has agents" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Agent")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c",
|
|
monitoring_enabled: true
|
|
})
|
|
|
|
{:ok, _assignment} = Agents.assign_device_to_agent(agent_token.id, device.id)
|
|
|
|
unmonitored = Stats.get_unmonitored_devices(org.id)
|
|
|
|
assert unmonitored == []
|
|
end
|
|
|
|
test "returns device with no agent assigned" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Unmonitored Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c",
|
|
monitoring_enabled: true
|
|
})
|
|
|
|
unmonitored = Stats.get_unmonitored_devices(org.id)
|
|
|
|
assert length(unmonitored) == 1
|
|
eq = hd(unmonitored)
|
|
assert eq.id == device.id
|
|
assert eq.name == "Unmonitored Router"
|
|
assert eq.ip_address == "192.168.1.1"
|
|
assert eq.site_name == "Test Site"
|
|
end
|
|
|
|
test "excludes device with SNMP disabled" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, _device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "SNMP Disabled",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: false,
|
|
monitoring_enabled: true
|
|
})
|
|
|
|
unmonitored = Stats.get_unmonitored_devices(org.id)
|
|
|
|
assert unmonitored == []
|
|
end
|
|
|
|
test "excludes device with monitoring disabled" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, _device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Monitoring Disabled",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c",
|
|
monitoring_enabled: false
|
|
})
|
|
|
|
unmonitored = Stats.get_unmonitored_devices(org.id)
|
|
|
|
assert unmonitored == []
|
|
end
|
|
end
|
|
|
|
describe "get_agent_metric_stats/1" do
|
|
test "returns zero stats for agent with no metrics" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Agent")
|
|
|
|
stats = Stats.get_agent_metric_stats(agent_token.id)
|
|
|
|
assert stats.total_metrics == 0
|
|
assert stats.sensor_readings == 0
|
|
assert stats.interface_stats == 0
|
|
assert stats.avg_per_hour == 0
|
|
assert is_nil(stats.last_submission)
|
|
end
|
|
|
|
test "counts sensor readings and interface stats from last 24 hours" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Agent")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
{:ok, _assignment} = Agents.assign_device_to_agent(agent_token.id, device.id)
|
|
|
|
# Create SNMP device, sensor, and interface
|
|
device =
|
|
%Device{}
|
|
|> Device.changeset(%{
|
|
device_id: device.id,
|
|
sys_name: "Test Device",
|
|
sys_descr: "Test"
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
sensor =
|
|
%Sensor{}
|
|
|> Sensor.changeset(%{
|
|
snmp_device_id: device.id,
|
|
sensor_type: "temperature",
|
|
sensor_index: "1",
|
|
sensor_oid: "1.2.3.4"
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
interface =
|
|
%Interface{}
|
|
|> Interface.changeset(%{
|
|
snmp_device_id: device.id,
|
|
if_index: 1,
|
|
if_name: "GigabitEthernet0/1"
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
# Insert sensor reading
|
|
%SensorReading{}
|
|
|> SensorReading.changeset(%{
|
|
sensor_id: sensor.id,
|
|
value: 45.5,
|
|
status: "ok",
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
# Insert interface stat
|
|
%InterfaceStat{}
|
|
|> InterfaceStat.changeset(%{
|
|
interface_id: interface.id,
|
|
if_in_octets: 1_234_567,
|
|
if_out_octets: 9_876_543,
|
|
if_in_errors: 0,
|
|
if_out_errors: 0,
|
|
if_in_discards: 0,
|
|
if_out_discards: 0,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
stats = Stats.get_agent_metric_stats(agent_token.id)
|
|
|
|
assert stats.total_metrics == 2
|
|
assert stats.sensor_readings == 1
|
|
assert stats.interface_stats == 1
|
|
assert stats.avg_per_hour > 0
|
|
assert stats.last_submission
|
|
end
|
|
end
|
|
|
|
describe "get_device_latency_by_agent/2" do
|
|
test "returns empty list when no monitoring checks exist" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
latency_by_agent = Stats.get_device_latency_by_agent(device.id)
|
|
|
|
assert latency_by_agent == []
|
|
end
|
|
|
|
test "calculates average latency per agent for a device" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent1, _} = Agents.create_agent_token(org.id, "Agent 1")
|
|
{:ok, agent2, _} = Agents.create_agent_token(org.id, "Agent 2")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
# Create checks from agent1 with avg 50ms (bulk insert for speed)
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
checks_agent1 =
|
|
for _ <- 1..10 do
|
|
%{
|
|
id: Ecto.UUID.generate(),
|
|
device_id: device.id,
|
|
agent_token_id: agent1.id,
|
|
status: "success",
|
|
response_time_ms: 50.0,
|
|
checked_at: now,
|
|
inserted_at: now
|
|
}
|
|
end
|
|
|
|
Repo.insert_all(MonitoringCheck, checks_agent1)
|
|
|
|
# Create checks from agent2 with avg 100ms (bulk insert for speed)
|
|
checks_agent2 =
|
|
for _ <- 1..10 do
|
|
%{
|
|
id: Ecto.UUID.generate(),
|
|
device_id: device.id,
|
|
agent_token_id: agent2.id,
|
|
status: "success",
|
|
response_time_ms: 100.0,
|
|
checked_at: now,
|
|
inserted_at: now
|
|
}
|
|
end
|
|
|
|
Repo.insert_all(MonitoringCheck, checks_agent2)
|
|
|
|
latency_by_agent = Stats.get_device_latency_by_agent(device.id)
|
|
|
|
assert length(latency_by_agent) == 2
|
|
|
|
agent1_latency = Enum.find(latency_by_agent, &(&1.agent_token_id == agent1.id))
|
|
agent2_latency = Enum.find(latency_by_agent, &(&1.agent_token_id == agent2.id))
|
|
|
|
assert agent1_latency.avg_latency_ms == 50.0
|
|
assert agent1_latency.check_count == 10
|
|
assert agent2_latency.avg_latency_ms == 100.0
|
|
assert agent2_latency.check_count == 10
|
|
end
|
|
|
|
test "filters by successful checks only" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent, _} = Agents.create_agent_token(org.id, "Agent")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
# Create successful checks with 50ms
|
|
for _ <- 1..10 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: agent.id,
|
|
status: "success",
|
|
response_time_ms: 50,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
# Create failed checks (should be ignored)
|
|
for _ <- 1..5 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: agent.id,
|
|
status: "error",
|
|
response_time_ms: nil,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
latency_by_agent = Stats.get_device_latency_by_agent(device.id)
|
|
|
|
assert length(latency_by_agent) == 1
|
|
agent_latency = hd(latency_by_agent)
|
|
assert agent_latency.avg_latency_ms == 50.0
|
|
assert agent_latency.check_count == 10
|
|
end
|
|
|
|
test "respects minimum check count filter" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent1, _} = Agents.create_agent_token(org.id, "Agent 1")
|
|
{:ok, agent2, _} = Agents.create_agent_token(org.id, "Agent 2")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
# Agent1 has 15 checks (above min)
|
|
for _ <- 1..15 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: agent1.id,
|
|
status: "success",
|
|
response_time_ms: 50,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
# Agent2 has only 5 checks (below min)
|
|
for _ <- 1..5 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: agent2.id,
|
|
status: "success",
|
|
response_time_ms: 30,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
latency_by_agent = Stats.get_device_latency_by_agent(device.id, min_checks: 10)
|
|
|
|
assert length(latency_by_agent) == 1
|
|
assert hd(latency_by_agent).agent_token_id == agent1.id
|
|
end
|
|
|
|
test "respects time window parameter" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent, _} = Agents.create_agent_token(org.id, "Agent")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
# Create recent checks
|
|
for _ <- 1..10 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: agent.id,
|
|
status: "success",
|
|
response_time_ms: 50,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
# Create old checks (25 hours ago, outside default 24h window)
|
|
old_time = DateTime.add(DateTime.utc_now(), -25, :hour)
|
|
|
|
for _ <- 1..5 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: agent.id,
|
|
status: "success",
|
|
response_time_ms: 100,
|
|
checked_at: old_time
|
|
})
|
|
end
|
|
|
|
# Default 24h window should only include recent checks
|
|
latency_by_agent = Stats.get_device_latency_by_agent(device.id)
|
|
|
|
assert length(latency_by_agent) == 1
|
|
agent_latency = hd(latency_by_agent)
|
|
assert agent_latency.avg_latency_ms == 50.0
|
|
assert agent_latency.check_count == 10
|
|
|
|
# 48h window should include both
|
|
latency_by_agent_48h = Stats.get_device_latency_by_agent(device.id, hours_ago: 48)
|
|
|
|
assert length(latency_by_agent_48h) == 1
|
|
agent_latency_48h = hd(latency_by_agent_48h)
|
|
assert agent_latency_48h.check_count == 15
|
|
end
|
|
end
|
|
|
|
describe "get_device_assignment_with_latency/1" do
|
|
test "returns assignment details with latency stats" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent1, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
|
{:ok, agent2, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
# Assign to agent2
|
|
{:ok, _} = Agents.assign_device_to_agent(agent2.id, device.id)
|
|
|
|
# Create checks from both agents
|
|
for _ <- 1..15 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: agent1.id,
|
|
status: "success",
|
|
response_time_ms: 30,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: agent2.id,
|
|
status: "success",
|
|
response_time_ms: 100,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
result = Stats.get_device_assignment_with_latency(device.id)
|
|
|
|
assert result.device_id == device.id
|
|
assert result.current_agent_token_id == agent2.id
|
|
assert result.assignment_source == :device
|
|
assert length(result.latency_stats) == 2
|
|
|
|
fast_agent = Enum.find(result.latency_stats, &(&1.agent_token_id == agent1.id))
|
|
slow_agent = Enum.find(result.latency_stats, &(&1.agent_token_id == agent2.id))
|
|
|
|
assert fast_agent.avg_latency_ms == 30.0
|
|
assert slow_agent.avg_latency_ms == 100.0
|
|
end
|
|
|
|
test "returns nil current_agent_id when using cloud polling" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
result = Stats.get_device_assignment_with_latency(device.id)
|
|
|
|
assert result.device_id == device.id
|
|
assert is_nil(result.current_agent_token_id)
|
|
assert result.assignment_source == :none
|
|
end
|
|
end
|
|
|
|
describe "find_reassignment_candidates/1" do
|
|
test "returns empty list when no devices have better options" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent, _} = Agents.create_agent_token(org.id, "Agent")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
{:ok, _} = Agents.assign_device_to_agent(agent.id, device.id)
|
|
|
|
# Create checks showing this is the only/best agent
|
|
for _ <- 1..15 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: agent.id,
|
|
status: "success",
|
|
response_time_ms: 50,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
candidates = Stats.find_reassignment_candidates()
|
|
|
|
assert candidates == []
|
|
end
|
|
|
|
test "identifies devices with 20%+ improvement available" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
|
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
|
|
|
{:ok, site} =
|
|
Sites.create_site(%{
|
|
name: "Test Site",
|
|
organization_id: org.id,
|
|
agent_token_id: slow_agent.id
|
|
})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
# Create checks: slow agent = 100ms, fast agent = 50ms (50% improvement)
|
|
for _ <- 1..15 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: slow_agent.id,
|
|
status: "success",
|
|
response_time_ms: 100,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: fast_agent.id,
|
|
status: "success",
|
|
response_time_ms: 50,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
candidates = Stats.find_reassignment_candidates(min_improvement_percent: 20)
|
|
|
|
assert length(candidates) == 1
|
|
candidate = hd(candidates)
|
|
|
|
assert candidate.device_id == device.id
|
|
assert candidate.current_agent_token_id == slow_agent.id
|
|
assert candidate.best_agent_token_id == fast_agent.id
|
|
assert candidate.current_latency_ms == 100.0
|
|
assert candidate.best_latency_ms == 50.0
|
|
assert candidate.improvement_percent == 50.0
|
|
end
|
|
|
|
test "only considers automatic assignments" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
|
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
# Direct device assignment (should be excluded from automatic reassignment)
|
|
{:ok, _} = Agents.assign_device_to_agent(slow_agent.id, device.id)
|
|
|
|
# Create checks showing fast agent is better
|
|
for _ <- 1..15 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: slow_agent.id,
|
|
status: "success",
|
|
response_time_ms: 100,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: fast_agent.id,
|
|
status: "success",
|
|
response_time_ms: 50,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
# Should not reassign direct assignments
|
|
candidates = Stats.find_reassignment_candidates()
|
|
|
|
assert candidates == []
|
|
end
|
|
|
|
test "includes site-level assignments for automatic reassignment" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
|
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
|
|
|
{:ok, site} =
|
|
Sites.create_site(%{
|
|
name: "Test Site",
|
|
organization_id: org.id,
|
|
agent_token_id: slow_agent.id
|
|
})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
# Create checks showing fast agent is better
|
|
for _ <- 1..15 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: slow_agent.id,
|
|
status: "success",
|
|
response_time_ms: 100,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: fast_agent.id,
|
|
status: "success",
|
|
response_time_ms: 50,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
candidates = Stats.find_reassignment_candidates()
|
|
|
|
assert length(candidates) == 1
|
|
candidate = hd(candidates)
|
|
assert candidate.assignment_source == :site
|
|
assert candidate.best_agent_token_id == fast_agent.id
|
|
end
|
|
|
|
test "requires minimum check count" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
|
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
|
|
|
{:ok, site} =
|
|
Sites.create_site(%{
|
|
name: "Test Site",
|
|
organization_id: org.id,
|
|
agent_token_id: slow_agent.id
|
|
})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
# Create only 5 checks (below min of 10)
|
|
for _ <- 1..5 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: slow_agent.id,
|
|
status: "success",
|
|
response_time_ms: 100,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: fast_agent.id,
|
|
status: "success",
|
|
response_time_ms: 50,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
# Should not reassign with insufficient data
|
|
candidates = Stats.find_reassignment_candidates(min_checks_per_agent: 10)
|
|
|
|
assert candidates == []
|
|
end
|
|
end
|
|
|
|
describe "edge cases" do
|
|
test "get_device_assignment_breakdown/1 handles organization with SNMP-disabled devices" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
# Create device with SNMP disabled
|
|
{:ok, _device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Non-SNMP Device",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: false
|
|
})
|
|
|
|
breakdown = Stats.get_device_assignment_breakdown(org.id)
|
|
|
|
# Should return all zeros since no SNMP-enabled devices exist
|
|
assert breakdown == %{direct: 0, site: 0, organization: 0, cloud: 0}
|
|
end
|
|
|
|
test "get_high_load_agents/2 returns empty list with zero device count" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, _agent_token, _token} = Agents.create_agent_token(org.id, "Zero Device Agent")
|
|
|
|
# No devices assigned at all
|
|
high_load = Stats.get_high_load_agents(org.id, 1)
|
|
|
|
assert high_load == []
|
|
end
|
|
|
|
test "find_reassignment_candidates/1 handles devices with no current agent assignment" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
{:ok, agent, _} = Agents.create_agent_token(org.id, "Agent")
|
|
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Unassigned Router",
|
|
ip_address: "192.168.1.1",
|
|
site_id: site.id,
|
|
organization_id: org.id,
|
|
snmp_enabled: true,
|
|
snmp_community: "public",
|
|
snmp_version: "2c"
|
|
})
|
|
|
|
# Create checks from agent even though device isn't assigned
|
|
for _ <- 1..15 do
|
|
Monitoring.create_monitoring_check(%{
|
|
device_id: device.id,
|
|
agent_token_id: agent.id,
|
|
status: "success",
|
|
response_time_ms: 50,
|
|
checked_at: DateTime.utc_now()
|
|
})
|
|
end
|
|
|
|
# Device with no agent shouldn't be in candidates
|
|
candidates = Stats.find_reassignment_candidates()
|
|
|
|
assert candidates == []
|
|
end
|
|
|
|
test "get_offline_agents/1 returns empty for organization with no agents" do
|
|
user = user_fixture()
|
|
org = organization_fixture(user.id)
|
|
|
|
offline = Stats.get_offline_agents(org.id)
|
|
|
|
assert offline == []
|
|
end
|
|
end
|
|
end
|