600 lines
19 KiB
Elixir
600 lines
19 KiB
Elixir
defmodule ToweropsWeb.AgentChannelExtraCoverageTest do
|
||
@moduledoc """
|
||
Targeted coverage for ToweropsWeb.AgentChannel paths not exercised in the
|
||
primary channel tests:
|
||
|
||
* Stuck device_down/device_up alert resolution after a successful poll
|
||
* Deprecated PING result delivered as SnmpResult (legacy agents)
|
||
* Live poll result with malformed job_id triggers warning branch
|
||
* MikroTik device job building when device.snmp_device matches RouterOS
|
||
"""
|
||
use Towerops.DataCase, async: false
|
||
|
||
import Ecto.Query, only: [from: 2]
|
||
import Phoenix.ChannelTest
|
||
|
||
alias Towerops.AccountsFixtures
|
||
alias Towerops.Agent.AgentHeartbeat
|
||
alias Towerops.Agent.AgentJobList
|
||
alias Towerops.Agent.CheckResult, as: CheckResultProto
|
||
alias Towerops.Agent.MonitoringCheck, as: MonitoringCheckProto
|
||
alias Towerops.Agent.SnmpResult
|
||
alias Towerops.Agents.AgentAssignment
|
||
alias Towerops.AgentsFixtures
|
||
alias Towerops.Alerts
|
||
alias Towerops.DevicesFixtures
|
||
alias Towerops.OrganizationsFixtures
|
||
alias Towerops.Repo
|
||
alias Towerops.Snmp.AgentDiscovery
|
||
alias Towerops.SnmpFixtures
|
||
alias ToweropsWeb.AgentSocket
|
||
|
||
@endpoint ToweropsWeb.Endpoint
|
||
|
||
setup do
|
||
user = AccountsFixtures.user_fixture()
|
||
organization = OrganizationsFixtures.organization_fixture(user.id)
|
||
|
||
device =
|
||
DevicesFixtures.device_fixture(%{
|
||
organization_id: organization.id,
|
||
snmp_version: "2c",
|
||
snmp_community: "public"
|
||
})
|
||
|
||
{:ok, agent_token, token_string} =
|
||
AgentsFixtures.agent_token_fixture(organization.id)
|
||
|
||
{:ok, _assignment} =
|
||
AgentsFixtures.agent_assignment_fixture(agent_token.id, device.id)
|
||
|
||
{:ok, socket} = connect(AgentSocket, %{})
|
||
|
||
{:ok, _, socket} =
|
||
subscribe_and_join(socket, "agent:#{agent_token.id}", %{"token" => token_string})
|
||
|
||
assert_push "jobs", _initial_jobs
|
||
|
||
%{
|
||
socket: socket,
|
||
device: device,
|
||
agent_token: agent_token,
|
||
token_string: token_string,
|
||
organization: organization
|
||
}
|
||
end
|
||
|
||
defp encode_payload(struct) do
|
||
binary = struct.__struct__.encode(struct)
|
||
%{"binary" => Base.encode64(binary)}
|
||
end
|
||
|
||
defp build_heartbeat do
|
||
%AgentHeartbeat{
|
||
version: "1.0.0",
|
||
hostname: "test-agent",
|
||
uptime_seconds: 3600,
|
||
ip_address: "",
|
||
arch: "x86_64"
|
||
}
|
||
end
|
||
|
||
defp poll_until(fun, opts \\ []) do
|
||
max_attempts = Keyword.get(opts, :max_attempts, 100)
|
||
delay_ms = Keyword.get(opts, :delay_ms, 20)
|
||
|
||
Enum.reduce_while(1..max_attempts, nil, fn _, _ ->
|
||
case fun.() do
|
||
nil ->
|
||
receive do
|
||
after
|
||
delay_ms -> :ok
|
||
end
|
||
|
||
{:cont, nil}
|
||
|
||
false ->
|
||
receive do
|
||
after
|
||
delay_ms -> :ok
|
||
end
|
||
|
||
{:cont, nil}
|
||
|
||
result ->
|
||
{:halt, result}
|
||
end
|
||
end) || fun.()
|
||
end
|
||
|
||
# Returns all pending "jobs" pushes already in the test mailbox (bounded to
|
||
# 5 messages so a flood from multiple sockets cannot stall the test).
|
||
# Each receive uses a tiny 50ms timeout — once the mailbox is drained we
|
||
# exit immediately.
|
||
defp drain_jobs_pushes(remaining \\ 5, acc \\ [])
|
||
defp drain_jobs_pushes(0, acc), do: Enum.reverse(acc)
|
||
|
||
defp drain_jobs_pushes(remaining, acc) do
|
||
receive do
|
||
%Phoenix.Socket.Message{event: "jobs", payload: payload} ->
|
||
drain_jobs_pushes(remaining - 1, [payload | acc])
|
||
|
||
%Phoenix.Socket.Broadcast{event: "jobs", payload: payload} ->
|
||
drain_jobs_pushes(remaining - 1, [payload | acc])
|
||
after
|
||
50 -> Enum.reverse(acc)
|
||
end
|
||
end
|
||
|
||
defp find_jobs_with(payloads, predicate) do
|
||
Enum.find(payloads, fn %{binary: binary_b64} ->
|
||
{:ok, decoded} = Base.decode64(binary_b64)
|
||
{:ok, job_list} = AgentJobList.decode(decoded)
|
||
predicate.(job_list.jobs)
|
||
end)
|
||
end
|
||
|
||
# Set up a device that has been discovered (so poll path runs without
|
||
# hitting the discovery branch first).
|
||
defp discover_device!(device) do
|
||
oid_values = %{
|
||
"1.3.6.1.2.1.1.1.0" => "Test Device",
|
||
"1.3.6.1.2.1.1.2.0" => "1.3.6.1.4.1.9.1.1",
|
||
"1.3.6.1.2.1.1.3.0" => "123456",
|
||
"1.3.6.1.2.1.1.4.0" => "admin@test.com",
|
||
"1.3.6.1.2.1.1.5.0" => "test-device",
|
||
"1.3.6.1.2.1.1.6.0" => "Test Location"
|
||
}
|
||
|
||
{:ok, _discovered} = AgentDiscovery.process_agent_discovery(device, oid_values)
|
||
Towerops.Devices.get_device_with_details(device.id)
|
||
end
|
||
|
||
# ── stuck device_down alert resolution ──────────────────────────
|
||
|
||
describe "resolve_any_stuck_down_alerts" do
|
||
setup %{device: device} do
|
||
device = discover_device!(device)
|
||
# Mark device as up — this exercises the "stuck alert" branch where
|
||
# device is up but a device_down alert is still active.
|
||
{:ok, device} = Towerops.Devices.update_device_status(device, :up)
|
||
%{device: device}
|
||
end
|
||
|
||
test "resolves stuck device_down alert after successful poll", %{
|
||
socket: socket,
|
||
device: device
|
||
} do
|
||
now = DateTime.utc_now()
|
||
|
||
{:ok, alert} =
|
||
Alerts.create_alert(%{
|
||
device_id: device.id,
|
||
alert_type: :device_down,
|
||
triggered_at: now,
|
||
message: "stuck"
|
||
})
|
||
|
||
# Sanity check — alert is active
|
||
assert Alerts.get_active_alert(device.id, :device_down)
|
||
|
||
result = %SnmpResult{
|
||
device_id: device.id,
|
||
job_type: :POLL,
|
||
job_id: "poll:#{device.id}",
|
||
timestamp: DateTime.to_unix(now),
|
||
oid_values: %{}
|
||
}
|
||
|
||
push(socket, "result", encode_payload(result))
|
||
|
||
# Wait for alert to be resolved
|
||
resolved =
|
||
poll_until(fn ->
|
||
a = Alerts.get_alert!(alert.id)
|
||
if a.resolved_at, do: a
|
||
end)
|
||
|
||
assert resolved.resolved_at
|
||
refute Alerts.get_active_alert(device.id, :device_down)
|
||
end
|
||
|
||
test "resolves stuck device_up alert after successful poll", %{
|
||
socket: socket,
|
||
device: device
|
||
} do
|
||
now = DateTime.utc_now()
|
||
|
||
{:ok, alert} =
|
||
Alerts.create_alert(%{
|
||
device_id: device.id,
|
||
alert_type: :device_up,
|
||
triggered_at: now,
|
||
message: "stuck up"
|
||
})
|
||
|
||
assert Alerts.get_active_alert(device.id, :device_up)
|
||
|
||
result = %SnmpResult{
|
||
device_id: device.id,
|
||
job_type: :POLL,
|
||
job_id: "poll:#{device.id}",
|
||
timestamp: DateTime.to_unix(now),
|
||
oid_values: %{}
|
||
}
|
||
|
||
push(socket, "result", encode_payload(result))
|
||
|
||
resolved =
|
||
poll_until(fn ->
|
||
a = Alerts.get_alert!(alert.id)
|
||
if a.resolved_at, do: a
|
||
end)
|
||
|
||
assert resolved.resolved_at
|
||
end
|
||
end
|
||
|
||
# ── deprecated PING via SnmpResult ──────────────────────────────
|
||
|
||
describe "deprecated PING SnmpResult path" do
|
||
test "PING job_type in SnmpResult creates a monitoring check and does not crash",
|
||
%{socket: socket, device: device} do
|
||
device = discover_device!(device)
|
||
|
||
result = %SnmpResult{
|
||
device_id: device.id,
|
||
job_type: :PING,
|
||
job_id: "ping:#{device.id}",
|
||
timestamp: DateTime.to_unix(DateTime.utc_now()),
|
||
oid_values: %{}
|
||
}
|
||
|
||
push(socket, "result", encode_payload(result))
|
||
|
||
# Channel should still respond to subsequent messages
|
||
ref = push(socket, "heartbeat", encode_payload(build_heartbeat()))
|
||
refute_reply ref, :error
|
||
end
|
||
end
|
||
|
||
# ── live poll with invalid job_id ───────────────────────────────
|
||
|
||
describe "handle_live_poll_result invalid job_id" do
|
||
test "result with malformed live_poll job_id is logged and ignored",
|
||
%{socket: socket, device: device} do
|
||
# job_id starts with "live_poll:" but doesn't have 3 parts
|
||
result = %SnmpResult{
|
||
device_id: device.id,
|
||
job_type: :POLL,
|
||
# Missing reply_topic segment - exercises the catch-all warning branch
|
||
job_id: "live_poll:invalid",
|
||
timestamp: DateTime.to_unix(DateTime.utc_now()),
|
||
oid_values: %{}
|
||
}
|
||
|
||
push(socket, "result", encode_payload(result))
|
||
|
||
ref = push(socket, "heartbeat", encode_payload(build_heartbeat()))
|
||
refute_reply ref, :error
|
||
end
|
||
end
|
||
|
||
# ── MikroTik job building when device matches RouterOS ─────────
|
||
|
||
describe "MikroTik job included when device sys_descr matches RouterOS" do
|
||
test "device with discovered MikroTik snmp_device gets a MIKROTIK job",
|
||
%{organization: organization} do
|
||
mt_device =
|
||
DevicesFixtures.device_fixture(%{
|
||
organization_id: organization.id,
|
||
snmp_version: "2c",
|
||
snmp_community: "public",
|
||
mikrotik_enabled: true,
|
||
mikrotik_username: "admin",
|
||
mikrotik_password: "password123",
|
||
mikrotik_credential_source: "device"
|
||
})
|
||
|
||
_snmp_device =
|
||
SnmpFixtures.snmp_device_fixture(%{
|
||
device_id: mt_device.id,
|
||
manufacturer: "MikroTik",
|
||
sys_descr: "RouterOS 7.10",
|
||
sys_object_id: "1.3.6.1.4.1.14988.1"
|
||
})
|
||
|
||
# Force needs_discovery? = true by leaving last_discovery_at nil
|
||
# (default), so build_jobs_for_device runs the MikroTik branch.
|
||
|
||
{:ok, agent_token, token_string} =
|
||
AgentsFixtures.agent_token_fixture(organization.id)
|
||
|
||
{:ok, _} = AgentsFixtures.agent_assignment_fixture(agent_token.id, mt_device.id)
|
||
|
||
{:ok, socket} = connect(AgentSocket, %{})
|
||
|
||
{:ok, _, _socket} =
|
||
subscribe_and_join(socket, "agent:#{agent_token.id}", %{"token" => token_string})
|
||
|
||
# Drain all pending "jobs" pushes. Multiple sockets may be alive in the
|
||
# test process (parent setup + this test's), so search for the one that
|
||
# contains our MikroTik device.
|
||
payloads = drain_jobs_pushes()
|
||
|
||
mt_payload =
|
||
find_jobs_with(payloads, fn jobs ->
|
||
Enum.any?(jobs, &(&1.device_id == mt_device.id))
|
||
end)
|
||
|
||
assert mt_payload, "no jobs payload contained mt_device #{mt_device.id}"
|
||
|
||
{:ok, decoded} = Base.decode64(mt_payload.binary)
|
||
{:ok, job_list} = AgentJobList.decode(decoded)
|
||
mt_jobs = Enum.filter(job_list.jobs, &(&1.device_id == mt_device.id))
|
||
|
||
mikrotik_job = Enum.find(mt_jobs, &(&1.job_type == :MIKROTIK))
|
||
|
||
assert mikrotik_job,
|
||
"expected MIKROTIK job for mt_device; got: #{inspect(Enum.map(mt_jobs, & &1.job_type))}"
|
||
|
||
assert mikrotik_job.device_id == mt_device.id
|
||
assert mikrotik_job.job_id == "mikrotik:#{mt_device.id}"
|
||
assert mikrotik_job.mikrotik_device.username == "admin"
|
||
assert mikrotik_job.mikrotik_device.password == "password123"
|
||
assert is_list(mikrotik_job.mikrotik_commands) and mikrotik_job.mikrotik_commands != []
|
||
end
|
||
|
||
test "device sys_descr containing 'RouterOS' (without MikroTik manufacturer) still triggers MIKROTIK job",
|
||
%{organization: organization} do
|
||
mt_device =
|
||
DevicesFixtures.device_fixture(%{
|
||
organization_id: organization.id,
|
||
snmp_version: "2c",
|
||
snmp_community: "public",
|
||
mikrotik_enabled: true,
|
||
mikrotik_username: "admin",
|
||
mikrotik_password: "password123",
|
||
mikrotik_credential_source: "device"
|
||
})
|
||
|
||
_snmp_device =
|
||
SnmpFixtures.snmp_device_fixture(%{
|
||
device_id: mt_device.id,
|
||
# Manufacturer is NOT MikroTik but sys_descr mentions RouterOS
|
||
manufacturer: "Generic Vendor",
|
||
sys_descr: "Some device running RouterOS 6.x",
|
||
sys_object_id: "1.3.6.1.4.1.99999.1"
|
||
})
|
||
|
||
{:ok, agent_token, token_string} =
|
||
AgentsFixtures.agent_token_fixture(organization.id)
|
||
|
||
{:ok, _} = AgentsFixtures.agent_assignment_fixture(agent_token.id, mt_device.id)
|
||
|
||
{:ok, socket} = connect(AgentSocket, %{})
|
||
|
||
{:ok, _, _socket} =
|
||
subscribe_and_join(socket, "agent:#{agent_token.id}", %{"token" => token_string})
|
||
|
||
payloads = drain_jobs_pushes()
|
||
|
||
mt_payload =
|
||
find_jobs_with(payloads, fn jobs ->
|
||
Enum.any?(jobs, &(&1.device_id == mt_device.id))
|
||
end)
|
||
|
||
{:ok, decoded} = Base.decode64(mt_payload.binary)
|
||
{:ok, job_list} = AgentJobList.decode(decoded)
|
||
mt_jobs = Enum.filter(job_list.jobs, &(&1.device_id == mt_device.id))
|
||
|
||
assert Enum.any?(mt_jobs, &(&1.job_type == :MIKROTIK))
|
||
end
|
||
|
||
test "MikroTik device without credentials does not produce a MIKROTIK job",
|
||
%{organization: organization} do
|
||
mt_device =
|
||
DevicesFixtures.device_fixture(%{
|
||
organization_id: organization.id,
|
||
snmp_version: "2c",
|
||
snmp_community: "public",
|
||
# API not enabled
|
||
mikrotik_enabled: false,
|
||
mikrotik_username: nil,
|
||
mikrotik_password: nil
|
||
})
|
||
|
||
_snmp_device =
|
||
SnmpFixtures.snmp_device_fixture(%{
|
||
device_id: mt_device.id,
|
||
manufacturer: "MikroTik",
|
||
sys_descr: "RouterOS 7"
|
||
})
|
||
|
||
{:ok, agent_token, token_string} =
|
||
AgentsFixtures.agent_token_fixture(organization.id)
|
||
|
||
{:ok, _} = AgentsFixtures.agent_assignment_fixture(agent_token.id, mt_device.id)
|
||
|
||
{:ok, socket} = connect(AgentSocket, %{})
|
||
|
||
{:ok, _, _socket} =
|
||
subscribe_and_join(socket, "agent:#{agent_token.id}", %{"token" => token_string})
|
||
|
||
payloads = drain_jobs_pushes()
|
||
|
||
mt_payload =
|
||
find_jobs_with(payloads, fn jobs ->
|
||
Enum.any?(jobs, &(&1.device_id == mt_device.id))
|
||
end)
|
||
|
||
{:ok, decoded} = Base.decode64(mt_payload.binary)
|
||
{:ok, job_list} = AgentJobList.decode(decoded)
|
||
mt_jobs = Enum.filter(job_list.jobs, &(&1.device_id == mt_device.id))
|
||
|
||
refute Enum.any?(mt_jobs, &(&1.job_type == :MIKROTIK))
|
||
end
|
||
end
|
||
|
||
# ── poll with extra OIDs to trigger process_additional_polling_data ──
|
||
|
||
describe "process_additional_polling_data" do
|
||
test "poll result with extra OIDs triggers additional processing without crash",
|
||
%{socket: socket, device: device} do
|
||
device = discover_device!(device)
|
||
{:ok, _device} = Towerops.Devices.update_device_status(device, :up)
|
||
|
||
# Generate enough OIDs to exceed sensors + interfaces*6 threshold
|
||
# (which on a freshly discovered device is small).
|
||
extra_oids =
|
||
for i <- 1..50, into: %{} do
|
||
{"1.3.6.1.2.1.999.#{i}.0", "value-#{i}"}
|
||
end
|
||
|
||
result = %SnmpResult{
|
||
device_id: device.id,
|
||
job_type: :POLL,
|
||
job_id: "poll:#{device.id}",
|
||
timestamp: DateTime.to_unix(DateTime.utc_now()),
|
||
oid_values: extra_oids
|
||
}
|
||
|
||
push(socket, "result", encode_payload(result))
|
||
|
||
# The Task.Supervisor child runs async, but channel must still be alive.
|
||
ref = push(socket, "heartbeat", encode_payload(build_heartbeat()))
|
||
refute_reply ref, :error
|
||
end
|
||
end
|
||
|
||
# ── verify_device_assignment: explicit disabled assignment ─────
|
||
|
||
describe "device with explicitly disabled assignment" do
|
||
test "result for device whose assignment is disabled is treated as reassigned",
|
||
%{socket: socket, agent_token: agent_token, device: device} do
|
||
device = discover_device!(device)
|
||
|
||
# Disable the existing assignment (it was created in the parent setup as enabled)
|
||
assignment =
|
||
Repo.one!(
|
||
from(a in AgentAssignment,
|
||
where: a.agent_token_id == ^agent_token.id and a.device_id == ^device.id
|
||
)
|
||
)
|
||
|
||
_ =
|
||
assignment
|
||
|> Ecto.Changeset.change(%{enabled: false})
|
||
|> Repo.update!()
|
||
|
||
# Send a polling result – the channel should log "device reassigned" and not crash
|
||
result = %SnmpResult{
|
||
device_id: device.id,
|
||
job_type: :POLL,
|
||
job_id: "poll:#{device.id}",
|
||
timestamp: DateTime.to_unix(DateTime.utc_now()),
|
||
oid_values: %{}
|
||
}
|
||
|
||
push(socket, "result", encode_payload(result))
|
||
|
||
ref = push(socket, "heartbeat", encode_payload(build_heartbeat()))
|
||
refute_reply ref, :error
|
||
end
|
||
end
|
||
|
||
# ── store_monitoring_check error branches ───────────────────────
|
||
|
||
describe "monitoring_check with bogus device_id" do
|
||
test "result for unknown device is logged and ignored", %{socket: socket} do
|
||
check = %MonitoringCheckProto{
|
||
device_id: Ecto.UUID.generate(),
|
||
status: "success",
|
||
response_time_ms: 5.0,
|
||
timestamp: DateTime.to_unix(DateTime.utc_now())
|
||
}
|
||
|
||
push(socket, "monitoring_check", encode_payload(check))
|
||
|
||
ref = push(socket, "heartbeat", encode_payload(build_heartbeat()))
|
||
refute_reply ref, :error
|
||
end
|
||
|
||
test "result for device in a different organization is rejected",
|
||
%{socket: socket} do
|
||
# Create an entirely separate org+device that this agent token cannot see.
|
||
other_user = AccountsFixtures.user_fixture()
|
||
other_org = OrganizationsFixtures.organization_fixture(other_user.id)
|
||
|
||
other_device =
|
||
DevicesFixtures.device_fixture(%{
|
||
organization_id: other_org.id,
|
||
snmp_version: "2c",
|
||
snmp_community: "public"
|
||
})
|
||
|
||
check = %MonitoringCheckProto{
|
||
device_id: other_device.id,
|
||
status: "success",
|
||
response_time_ms: 5.0,
|
||
timestamp: DateTime.to_unix(DateTime.utc_now())
|
||
}
|
||
|
||
push(socket, "monitoring_check", encode_payload(check))
|
||
|
||
ref = push(socket, "heartbeat", encode_payload(build_heartbeat()))
|
||
refute_reply ref, :error
|
||
end
|
||
end
|
||
|
||
# ── store_check_result error branches ──────────────────────────
|
||
|
||
describe "check_result with unknown / wrong-org check_id" do
|
||
test "unknown check_id is logged and ignored", %{socket: socket} do
|
||
result = %CheckResultProto{
|
||
check_id: Ecto.UUID.generate(),
|
||
status: 0,
|
||
output: "All good",
|
||
response_time_ms: 12.3,
|
||
timestamp: DateTime.to_unix(DateTime.utc_now())
|
||
}
|
||
|
||
push(socket, "check_result", encode_payload(result))
|
||
|
||
ref = push(socket, "heartbeat", encode_payload(build_heartbeat()))
|
||
refute_reply ref, :error
|
||
end
|
||
|
||
test "check belonging to different org is rejected", %{socket: socket} do
|
||
other_user = AccountsFixtures.user_fixture()
|
||
other_org = OrganizationsFixtures.organization_fixture(other_user.id)
|
||
|
||
{:ok, other_token, _} = AgentsFixtures.agent_token_fixture(other_org.id)
|
||
|
||
{:ok, foreign_check} =
|
||
Towerops.Monitoring.create_check(%{
|
||
organization_id: other_org.id,
|
||
agent_token_id: other_token.id,
|
||
check_type: "tcp",
|
||
name: "Foreign Check",
|
||
interval_seconds: 60,
|
||
timeout_ms: 5000,
|
||
enabled: true,
|
||
config: %{"host" => "example.com", "port" => 443}
|
||
})
|
||
|
||
result = %CheckResultProto{
|
||
check_id: foreign_check.id,
|
||
status: 0,
|
||
output: "All good",
|
||
response_time_ms: 12.3,
|
||
timestamp: DateTime.to_unix(DateTime.utc_now())
|
||
}
|
||
|
||
push(socket, "check_result", encode_payload(result))
|
||
|
||
ref = push(socket, "heartbeat", encode_payload(build_heartbeat()))
|
||
refute_reply ref, :error
|
||
end
|
||
end
|
||
end
|