test: DevicePollerWorker error branches and state-change broadcasting

This commit is contained in:
Graham McIntire 2026-05-09 12:39:26 -05:00
parent f87dbcf933
commit 08cf3a67ff

View file

@ -0,0 +1,660 @@
defmodule Towerops.Workers.DevicePollerWorkerBranchesTest do
@moduledoc """
Branch coverage for `Towerops.Workers.DevicePollerWorker` focusing on:
* sensor error log clauses (`:no_such_object`, `:no_such_instance`, `:end_of_mib_view`)
* unknown processor type
* UCD CPU error path
* state sensor change broadcast (value change between polls)
* interface change events (oper status, admin status, speed, MAC)
* storage / mempool error paths
"""
use Towerops.DataCase, async: false
import Mox
import Towerops.AccountsFixtures
alias Towerops.Devices
alias Towerops.Organizations
alias Towerops.Repo
alias Towerops.Sites
alias Towerops.Snmp.Device, as: SnmpDevice
alias Towerops.Snmp.Interface
alias Towerops.Snmp.Mempool
alias Towerops.Snmp.Processor
alias Towerops.Snmp.ProcessorReading
alias Towerops.Snmp.Sensor
alias Towerops.Snmp.SensorReading
alias Towerops.Snmp.SnmpMock
alias Towerops.Snmp.StateSensor
alias Towerops.Snmp.Storage
alias Towerops.Workers.DevicePollerWorker
setup :verify_on_exit!
setup do
old_adapter = Application.get_env(:towerops, :snmp_adapter)
Application.put_env(:towerops, :snmp_adapter, SnmpMock)
on_exit(fn ->
if old_adapter do
Application.put_env(:towerops, :snmp_adapter, old_adapter)
else
Application.delete_env(:towerops, :snmp_adapter)
end
end)
user = user_fixture()
{:ok, organization} = Organizations.create_organization(%{name: "Branches Org"}, user.id)
{:ok, site} = Sites.create_site(%{name: "Branches Site", organization_id: organization.id})
%{organization: organization, site: site}
end
defp create_polled_device(site, attrs \\ %{}) do
base = %{
name: "Dev #{System.unique_integer([:positive])}",
ip_address: "10.0.0.#{Enum.random(1..250)}",
site_id: site.id,
organization_id: site.organization_id,
snmp_enabled: true,
snmp_version: "2c",
snmp_community: "public",
check_interval_seconds: 600
}
{:ok, device} = Devices.create_device(Map.merge(base, attrs))
Repo.delete_all(Oban.Job)
device
end
defp create_snmp_device(device, attrs \\ %{}) do
%SnmpDevice{}
|> SnmpDevice.changeset(Map.merge(%{device_id: device.id, sys_name: "host", sys_descr: "Test"}, attrs))
|> Repo.insert!()
end
defp insert_dummy_sensor(snmp) do
%Sensor{}
|> Sensor.changeset(%{
snmp_device_id: snmp.id,
sensor_type: "temperature",
sensor_index: "999",
sensor_oid: "1.3.6.1.99.99.99.999",
sensor_descr: "Gate Sensor"
})
|> Repo.insert!()
end
defp stub_walks_empty do
stub(SnmpMock, :walk, fn _, _, _ -> {:ok, []} end)
end
defp run_perform(device) do
DevicePollerWorker.perform(%Oban.Job{args: %{"device_id" => device.id}})
end
describe "sensor error logging branches" do
test ":no_such_object error path", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
sensor =
%Sensor{}
|> Sensor.changeset(%{
snmp_device_id: snmp.id,
sensor_type: "temperature",
sensor_index: "1",
sensor_oid: "1.3.6.1.2.1.99.1.1.1.4.1",
sensor_descr: "NSO Sensor"
})
|> Repo.insert!()
stub(SnmpMock, :get, fn _, _, _ -> {:error, :no_such_object} end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
reading = Repo.one(Ecto.Query.where(SensorReading, sensor_id: ^sensor.id))
assert reading.status == "error"
end
test ":no_such_instance error path", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
sensor =
%Sensor{}
|> Sensor.changeset(%{
snmp_device_id: snmp.id,
sensor_type: "temperature",
sensor_index: "1",
sensor_oid: "1.3.6.1.2.1.99.1.1.1.4.1",
sensor_descr: "NSI Sensor"
})
|> Repo.insert!()
stub(SnmpMock, :get, fn _, _, _ -> {:error, :no_such_instance} end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
reading = Repo.one(Ecto.Query.where(SensorReading, sensor_id: ^sensor.id))
assert reading.status == "error"
end
test ":end_of_mib_view error path", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
sensor =
%Sensor{}
|> Sensor.changeset(%{
snmp_device_id: snmp.id,
sensor_type: "temperature",
sensor_index: "1",
sensor_oid: "1.3.6.1.2.1.99.1.1.1.4.1",
sensor_descr: "EoM Sensor"
})
|> Repo.insert!()
stub(SnmpMock, :get, fn _, _, _ -> {:error, :end_of_mib_view} end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
reading = Repo.one(Ecto.Query.where(SensorReading, sensor_id: ^sensor.id))
assert reading.status == "error"
end
end
describe "processor branches" do
test "ucd_cpu with one OID failing falls into error branch", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
insert_dummy_sensor(snmp)
proc =
%Processor{}
|> Processor.changeset(%{
snmp_device_id: snmp.id,
processor_index: "ucd_1",
processor_type: "ucd_cpu",
description: "UCD CPU partial"
})
|> Repo.insert!()
stub(SnmpMock, :get, fn _, oid, _ ->
case oid do
# user OK
"1.3.6.1.4.1.2021.11.9.0" -> {:ok, 10}
# system fails
"1.3.6.1.4.1.2021.11.10.0" -> {:error, :timeout}
_ -> {:error, :no_such_object}
end
end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
reading = Repo.one(Ecto.Query.where(ProcessorReading, processor_id: ^proc.id))
assert reading.status == "error"
end
test "hr_processor with non-integer SNMP value gives error", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
insert_dummy_sensor(snmp)
proc =
%Processor{}
|> Processor.changeset(%{
snmp_device_id: snmp.id,
processor_index: "hr_1",
processor_type: "hr_processor",
description: "HR partial"
})
|> Repo.insert!()
stub(SnmpMock, :get, fn _, oid, _ ->
case oid do
"1.3.6.1.2.1.25.3.3.1.2.1" -> {:ok, "not-an-int"}
_ -> {:error, :no_such_object}
end
end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
reading = Repo.one(Ecto.Query.where(ProcessorReading, processor_id: ^proc.id))
assert reading.status == "error"
end
end
describe "state sensor change broadcasting" do
test "broadcasts state change when state_value changes between polls", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
insert_dummy_sensor(snmp)
# State sensor with previous value=2 (ok / enabled), now changing to 3
# (warning / disabled). The broadcast_state_sensor_change branch fires
# because old_state_value != nil && old_state_value != new_state_value.
ss =
%StateSensor{}
|> StateSensor.changeset(%{
snmp_device_id: snmp.id,
sensor_descr: "Power Supply 1",
sensor_oid: "1.3.6.1.2.1.47.1.1.1.1.5.1",
sensor_index: "1",
state_value: 2,
state_descr: "enabled",
status: "ok"
})
|> Repo.insert!()
Phoenix.PubSub.subscribe(Towerops.PubSub, "device:#{device.id}")
Phoenix.PubSub.subscribe(Towerops.PubSub, "device:events")
Phoenix.PubSub.subscribe(Towerops.PubSub, "device_events:org:#{device.organization_id}")
stub(SnmpMock, :get, fn _, oid, _ ->
case oid do
"1.3.6.1.2.1.47.1.1.1.1.5.1" -> {:ok, 3}
_ -> {:error, :no_such_object}
end
end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
updated = Repo.get!(StateSensor, ss.id)
assert updated.state_value == 3
assert updated.status == "warning"
assert updated.state_descr == "disabled"
# We expect at least one device_event broadcast describing the change
assert_receive {:device_event, %{message: msg}}, 2_000
assert msg =~ "Power Supply 1"
assert msg =~ "enabled"
assert msg =~ "disabled"
end
test "broadcasts state change for ok→critical (severity = critical)", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
insert_dummy_sensor(snmp)
# Previous: state_value=2, status=ok. New: an unknown state value (5)
# that maps to status="unknown". We can't drive critical from
# entity_state_to_status which only outputs ok/warning/unknown, but
# broadcast_state_sensor_change does run for ok→unknown transitions.
ss =
%StateSensor{}
|> StateSensor.changeset(%{
snmp_device_id: snmp.id,
sensor_descr: "Fan 1",
sensor_oid: "1.3.6.1.2.1.47.1.1.1.1.5.2",
sensor_index: "2",
state_value: 2,
state_descr: "enabled",
status: "ok"
})
|> Repo.insert!()
Phoenix.PubSub.subscribe(Towerops.PubSub, "device:events")
stub(SnmpMock, :get, fn _, oid, _ ->
case oid do
"1.3.6.1.2.1.47.1.1.1.1.5.2" -> {:ok, 5}
_ -> {:error, :no_such_object}
end
end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
updated = Repo.get!(StateSensor, ss.id)
assert updated.state_value == 5
assert_receive {:device_event, _event}, 2_000
end
test "no broadcast when state_value is unchanged", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
insert_dummy_sensor(snmp)
ss =
%StateSensor{}
|> StateSensor.changeset(%{
snmp_device_id: snmp.id,
sensor_descr: "Steady",
sensor_oid: "1.3.6.1.2.1.47.1.1.1.1.5.3",
sensor_index: "3",
state_value: 2,
state_descr: "enabled",
status: "ok"
})
|> Repo.insert!()
stub(SnmpMock, :get, fn _, oid, _ ->
case oid do
"1.3.6.1.2.1.47.1.1.1.1.5.3" -> {:ok, 2}
_ -> {:error, :no_such_object}
end
end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
updated = Repo.get!(StateSensor, ss.id)
assert updated.state_value == 2
end
end
describe "interface change detection events" do
test "speed change generates a speed_change event", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
_interface =
%Interface{}
|> Interface.changeset(%{
snmp_device_id: snmp.id,
if_index: 7,
if_name: "eth7",
if_descr: "ethernet",
if_type: 6,
if_speed: 1_000_000_000,
if_phys_address: "00:11:22:33:44:55",
if_admin_status: "up",
if_oper_status: "up",
monitored: true
})
|> Repo.insert!()
Phoenix.PubSub.subscribe(Towerops.PubSub, "device:#{device.id}")
# Speed dropped to 100Mbps (warning), MAC and admin/oper unchanged.
stub(SnmpMock, :get, fn _, oid, _ ->
case oid do
"1.3.6.1.2.1.31.1.1.1.6.7" -> {:ok, 1_000}
"1.3.6.1.2.1.31.1.1.1.10.7" -> {:ok, 2_000}
"1.3.6.1.2.1.2.2.1.14.7" -> {:ok, 0}
"1.3.6.1.2.1.2.2.1.20.7" -> {:ok, 0}
"1.3.6.1.2.1.2.2.1.13.7" -> {:ok, 0}
"1.3.6.1.2.1.2.2.1.19.7" -> {:ok, 0}
_ -> {:error, :no_such_object}
end
end)
stub(SnmpMock, :get_multiple, fn _, _oids, _ ->
{:ok,
%{
# speed changed
"1.3.6.1.2.1.2.2.1.5.7" => {:integer, 100_000_000},
"1.3.6.1.2.1.2.2.1.6.7" => {:"OCTET STRING", <<0, 17, 34, 51, 68, 85>>},
"1.3.6.1.2.1.2.2.1.7.7" => {:integer, 1},
"1.3.6.1.2.1.2.2.1.8.7" => {:integer, 1}
}}
end)
stub_walks_empty()
assert :ok = run_perform(device)
received_events = collect_events([], 1_500)
assert Enum.any?(received_events, fn e ->
Map.get(e, :event_type) == "interface_speed_change"
end)
end
test "MAC change generates a mac_change event", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
_interface =
%Interface{}
|> Interface.changeset(%{
snmp_device_id: snmp.id,
if_index: 8,
if_name: "eth8",
if_descr: "ethernet",
if_type: 6,
if_speed: 100_000_000,
if_phys_address: "00:11:22:33:44:55",
if_admin_status: "up",
if_oper_status: "up",
monitored: true
})
|> Repo.insert!()
Phoenix.PubSub.subscribe(Towerops.PubSub, "device:#{device.id}")
stub(SnmpMock, :get, fn _, oid, _ ->
case oid do
"1.3.6.1.2.1.31.1.1.1.6.8" -> {:ok, 1_000}
"1.3.6.1.2.1.31.1.1.1.10.8" -> {:ok, 2_000}
"1.3.6.1.2.1.2.2.1.14.8" -> {:ok, 0}
"1.3.6.1.2.1.2.2.1.20.8" -> {:ok, 0}
"1.3.6.1.2.1.2.2.1.13.8" -> {:ok, 0}
"1.3.6.1.2.1.2.2.1.19.8" -> {:ok, 0}
_ -> {:error, :no_such_object}
end
end)
stub(SnmpMock, :get_multiple, fn _, _, _ ->
{:ok,
%{
"1.3.6.1.2.1.2.2.1.5.8" => {:integer, 100_000_000},
# changed MAC
"1.3.6.1.2.1.2.2.1.6.8" => {:"OCTET STRING", <<0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF>>},
"1.3.6.1.2.1.2.2.1.7.8" => {:integer, 1},
"1.3.6.1.2.1.2.2.1.8.8" => {:integer, 1}
}}
end)
stub_walks_empty()
assert :ok = run_perform(device)
received_events = collect_events([], 1_500)
assert Enum.any?(received_events, fn e ->
Map.get(e, :event_type) == "interface_mac_change"
end)
end
test "oper_status change down→up generates interface_up event", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
_interface =
%Interface{}
|> Interface.changeset(%{
snmp_device_id: snmp.id,
if_index: 9,
if_name: "eth9",
if_descr: "ethernet",
if_type: 6,
if_speed: 100_000_000,
if_phys_address: "00:11:22:33:44:55",
if_admin_status: "up",
if_oper_status: "down",
monitored: true
})
|> Repo.insert!()
Phoenix.PubSub.subscribe(Towerops.PubSub, "device:#{device.id}")
stub(SnmpMock, :get, fn _, oid, _ ->
case oid do
"1.3.6.1.2.1.31.1.1.1.6.9" -> {:ok, 1_000}
"1.3.6.1.2.1.31.1.1.1.10.9" -> {:ok, 2_000}
"1.3.6.1.2.1.2.2.1.14.9" -> {:ok, 0}
"1.3.6.1.2.1.2.2.1.20.9" -> {:ok, 0}
"1.3.6.1.2.1.2.2.1.13.9" -> {:ok, 0}
"1.3.6.1.2.1.2.2.1.19.9" -> {:ok, 0}
_ -> {:error, :no_such_object}
end
end)
stub(SnmpMock, :get_multiple, fn _, _, _ ->
{:ok,
%{
"1.3.6.1.2.1.2.2.1.5.9" => {:integer, 100_000_000},
"1.3.6.1.2.1.2.2.1.6.9" => {:"OCTET STRING", <<0, 17, 34, 51, 68, 85>>},
"1.3.6.1.2.1.2.2.1.7.9" => {:integer, 1},
# now up
"1.3.6.1.2.1.2.2.1.8.9" => {:integer, 1}
}}
end)
stub_walks_empty()
assert :ok = run_perform(device)
events = collect_events([], 1_500)
assert Enum.any?(events, fn e ->
Map.get(e, :event_type) == "interface_up"
end)
end
end
describe "storage and mempool error paths" do
test "storage SNMP error skips reading and does not raise", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
insert_dummy_sensor(snmp)
_storage =
%Storage{}
|> Storage.changeset(%{
snmp_device_id: snmp.id,
storage_index: 11,
storage_type: "ram",
description: "Mem"
})
|> Repo.insert!()
stub(SnmpMock, :get, fn _, _, _ -> {:error, :timeout} end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
end
test "mempool SNMP error skips reading and does not raise", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
insert_dummy_sensor(snmp)
_mempool =
%Mempool{}
|> Mempool.changeset(%{
snmp_device_id: snmp.id,
mempool_index: "12",
mempool_type: "hr_memory",
description: "Bad"
})
|> Repo.insert!()
stub(SnmpMock, :get, fn _, _, _ -> {:error, :timeout} end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
end
test "storage with non-integer values triggers invalid_values branch", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
insert_dummy_sensor(snmp)
_storage =
%Storage{}
|> Storage.changeset(%{
snmp_device_id: snmp.id,
storage_index: 13,
storage_type: "fixed_disk",
description: "Strange"
})
|> Repo.insert!()
stub(SnmpMock, :get, fn _, oid, _ ->
case oid do
"1.3.6.1.2.1.25.2.3.1.6.13" -> {:ok, "not-int"}
"1.3.6.1.2.1.25.2.3.1.5.13" -> {:ok, "not-int"}
"1.3.6.1.2.1.25.2.3.1.4.13" -> {:ok, "not-int"}
_ -> {:error, :no_such_object}
end
end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
end
test "mempool with non-integer values triggers invalid_values branch", %{site: site} do
device = create_polled_device(site)
snmp = create_snmp_device(device)
insert_dummy_sensor(snmp)
_mempool =
%Mempool{}
|> Mempool.changeset(%{
snmp_device_id: snmp.id,
mempool_index: "14",
mempool_type: "hr_memory",
description: "Strange"
})
|> Repo.insert!()
stub(SnmpMock, :get, fn _, oid, _ ->
case oid do
"1.3.6.1.2.1.25.2.3.1.6.14" -> {:ok, "not-int"}
"1.3.6.1.2.1.25.2.3.1.5.14" -> {:ok, "not-int"}
"1.3.6.1.2.1.25.2.3.1.4.14" -> {:ok, "not-int"}
_ -> {:error, :no_such_object}
end
end)
stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end)
stub_walks_empty()
assert :ok = run_perform(device)
end
end
describe "decode_snmp_value larger sizes" do
test "decodes binary larger than 16 bytes (s > 8 fallback offset path)" do
# 12-byte binary: prefix 4 zero bytes, then 8-byte counter
bin = <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7>>
assert 7 == DevicePollerWorker.decode_snmp_value(bin)
end
end
# Walks message queue collecting :device_event payloads up to a deadline.
defp collect_events(acc, ms_remaining) do
receive do
{:device_event, e} -> collect_events([e | acc], ms_remaining)
after
ms_remaining -> Enum.reverse(acc)
end
end
end