diff --git a/test/towerops/workers/device_poller_worker_branches_test.exs b/test/towerops/workers/device_poller_worker_branches_test.exs new file mode 100644 index 00000000..476dde42 --- /dev/null +++ b/test/towerops/workers/device_poller_worker_branches_test.exs @@ -0,0 +1,660 @@ +defmodule Towerops.Workers.DevicePollerWorkerBranchesTest do + @moduledoc """ + Branch coverage for `Towerops.Workers.DevicePollerWorker` focusing on: + * sensor error log clauses (`:no_such_object`, `:no_such_instance`, `:end_of_mib_view`) + * unknown processor type + * UCD CPU error path + * state sensor change broadcast (value change between polls) + * interface change events (oper status, admin status, speed, MAC) + * storage / mempool error paths + """ + use Towerops.DataCase, async: false + + import Mox + import Towerops.AccountsFixtures + + alias Towerops.Devices + alias Towerops.Organizations + alias Towerops.Repo + alias Towerops.Sites + alias Towerops.Snmp.Device, as: SnmpDevice + alias Towerops.Snmp.Interface + alias Towerops.Snmp.Mempool + alias Towerops.Snmp.Processor + alias Towerops.Snmp.ProcessorReading + alias Towerops.Snmp.Sensor + alias Towerops.Snmp.SensorReading + alias Towerops.Snmp.SnmpMock + alias Towerops.Snmp.StateSensor + alias Towerops.Snmp.Storage + alias Towerops.Workers.DevicePollerWorker + + setup :verify_on_exit! + + setup do + old_adapter = Application.get_env(:towerops, :snmp_adapter) + Application.put_env(:towerops, :snmp_adapter, SnmpMock) + + on_exit(fn -> + if old_adapter do + Application.put_env(:towerops, :snmp_adapter, old_adapter) + else + Application.delete_env(:towerops, :snmp_adapter) + end + end) + + user = user_fixture() + {:ok, organization} = Organizations.create_organization(%{name: "Branches Org"}, user.id) + {:ok, site} = Sites.create_site(%{name: "Branches Site", organization_id: organization.id}) + + %{organization: organization, site: site} + end + + defp create_polled_device(site, attrs \\ %{}) do + base = %{ + name: "Dev #{System.unique_integer([:positive])}", + ip_address: "10.0.0.#{Enum.random(1..250)}", + site_id: site.id, + organization_id: site.organization_id, + snmp_enabled: true, + snmp_version: "2c", + snmp_community: "public", + check_interval_seconds: 600 + } + + {:ok, device} = Devices.create_device(Map.merge(base, attrs)) + Repo.delete_all(Oban.Job) + device + end + + defp create_snmp_device(device, attrs \\ %{}) do + %SnmpDevice{} + |> SnmpDevice.changeset(Map.merge(%{device_id: device.id, sys_name: "host", sys_descr: "Test"}, attrs)) + |> Repo.insert!() + end + + defp insert_dummy_sensor(snmp) do + %Sensor{} + |> Sensor.changeset(%{ + snmp_device_id: snmp.id, + sensor_type: "temperature", + sensor_index: "999", + sensor_oid: "1.3.6.1.99.99.99.999", + sensor_descr: "Gate Sensor" + }) + |> Repo.insert!() + end + + defp stub_walks_empty do + stub(SnmpMock, :walk, fn _, _, _ -> {:ok, []} end) + end + + defp run_perform(device) do + DevicePollerWorker.perform(%Oban.Job{args: %{"device_id" => device.id}}) + end + + describe "sensor error logging branches" do + test ":no_such_object error path", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + + sensor = + %Sensor{} + |> Sensor.changeset(%{ + snmp_device_id: snmp.id, + sensor_type: "temperature", + sensor_index: "1", + sensor_oid: "1.3.6.1.2.1.99.1.1.1.4.1", + sensor_descr: "NSO Sensor" + }) + |> Repo.insert!() + + stub(SnmpMock, :get, fn _, _, _ -> {:error, :no_such_object} end) + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + + reading = Repo.one(Ecto.Query.where(SensorReading, sensor_id: ^sensor.id)) + assert reading.status == "error" + end + + test ":no_such_instance error path", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + + sensor = + %Sensor{} + |> Sensor.changeset(%{ + snmp_device_id: snmp.id, + sensor_type: "temperature", + sensor_index: "1", + sensor_oid: "1.3.6.1.2.1.99.1.1.1.4.1", + sensor_descr: "NSI Sensor" + }) + |> Repo.insert!() + + stub(SnmpMock, :get, fn _, _, _ -> {:error, :no_such_instance} end) + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + + reading = Repo.one(Ecto.Query.where(SensorReading, sensor_id: ^sensor.id)) + assert reading.status == "error" + end + + test ":end_of_mib_view error path", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + + sensor = + %Sensor{} + |> Sensor.changeset(%{ + snmp_device_id: snmp.id, + sensor_type: "temperature", + sensor_index: "1", + sensor_oid: "1.3.6.1.2.1.99.1.1.1.4.1", + sensor_descr: "EoM Sensor" + }) + |> Repo.insert!() + + stub(SnmpMock, :get, fn _, _, _ -> {:error, :end_of_mib_view} end) + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + + reading = Repo.one(Ecto.Query.where(SensorReading, sensor_id: ^sensor.id)) + assert reading.status == "error" + end + end + + describe "processor branches" do + test "ucd_cpu with one OID failing falls into error branch", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + insert_dummy_sensor(snmp) + + proc = + %Processor{} + |> Processor.changeset(%{ + snmp_device_id: snmp.id, + processor_index: "ucd_1", + processor_type: "ucd_cpu", + description: "UCD CPU partial" + }) + |> Repo.insert!() + + stub(SnmpMock, :get, fn _, oid, _ -> + case oid do + # user OK + "1.3.6.1.4.1.2021.11.9.0" -> {:ok, 10} + # system fails + "1.3.6.1.4.1.2021.11.10.0" -> {:error, :timeout} + _ -> {:error, :no_such_object} + end + end) + + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + + reading = Repo.one(Ecto.Query.where(ProcessorReading, processor_id: ^proc.id)) + assert reading.status == "error" + end + + test "hr_processor with non-integer SNMP value gives error", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + insert_dummy_sensor(snmp) + + proc = + %Processor{} + |> Processor.changeset(%{ + snmp_device_id: snmp.id, + processor_index: "hr_1", + processor_type: "hr_processor", + description: "HR partial" + }) + |> Repo.insert!() + + stub(SnmpMock, :get, fn _, oid, _ -> + case oid do + "1.3.6.1.2.1.25.3.3.1.2.1" -> {:ok, "not-an-int"} + _ -> {:error, :no_such_object} + end + end) + + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + + reading = Repo.one(Ecto.Query.where(ProcessorReading, processor_id: ^proc.id)) + assert reading.status == "error" + end + end + + describe "state sensor change broadcasting" do + test "broadcasts state change when state_value changes between polls", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + insert_dummy_sensor(snmp) + + # State sensor with previous value=2 (ok / enabled), now changing to 3 + # (warning / disabled). The broadcast_state_sensor_change branch fires + # because old_state_value != nil && old_state_value != new_state_value. + ss = + %StateSensor{} + |> StateSensor.changeset(%{ + snmp_device_id: snmp.id, + sensor_descr: "Power Supply 1", + sensor_oid: "1.3.6.1.2.1.47.1.1.1.1.5.1", + sensor_index: "1", + state_value: 2, + state_descr: "enabled", + status: "ok" + }) + |> Repo.insert!() + + Phoenix.PubSub.subscribe(Towerops.PubSub, "device:#{device.id}") + Phoenix.PubSub.subscribe(Towerops.PubSub, "device:events") + Phoenix.PubSub.subscribe(Towerops.PubSub, "device_events:org:#{device.organization_id}") + + stub(SnmpMock, :get, fn _, oid, _ -> + case oid do + "1.3.6.1.2.1.47.1.1.1.1.5.1" -> {:ok, 3} + _ -> {:error, :no_such_object} + end + end) + + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + + updated = Repo.get!(StateSensor, ss.id) + assert updated.state_value == 3 + assert updated.status == "warning" + assert updated.state_descr == "disabled" + + # We expect at least one device_event broadcast describing the change + assert_receive {:device_event, %{message: msg}}, 2_000 + assert msg =~ "Power Supply 1" + assert msg =~ "enabled" + assert msg =~ "disabled" + end + + test "broadcasts state change for ok→critical (severity = critical)", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + insert_dummy_sensor(snmp) + + # Previous: state_value=2, status=ok. New: an unknown state value (5) + # that maps to status="unknown". We can't drive critical from + # entity_state_to_status which only outputs ok/warning/unknown, but + # broadcast_state_sensor_change does run for ok→unknown transitions. + ss = + %StateSensor{} + |> StateSensor.changeset(%{ + snmp_device_id: snmp.id, + sensor_descr: "Fan 1", + sensor_oid: "1.3.6.1.2.1.47.1.1.1.1.5.2", + sensor_index: "2", + state_value: 2, + state_descr: "enabled", + status: "ok" + }) + |> Repo.insert!() + + Phoenix.PubSub.subscribe(Towerops.PubSub, "device:events") + + stub(SnmpMock, :get, fn _, oid, _ -> + case oid do + "1.3.6.1.2.1.47.1.1.1.1.5.2" -> {:ok, 5} + _ -> {:error, :no_such_object} + end + end) + + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + + updated = Repo.get!(StateSensor, ss.id) + assert updated.state_value == 5 + + assert_receive {:device_event, _event}, 2_000 + end + + test "no broadcast when state_value is unchanged", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + insert_dummy_sensor(snmp) + + ss = + %StateSensor{} + |> StateSensor.changeset(%{ + snmp_device_id: snmp.id, + sensor_descr: "Steady", + sensor_oid: "1.3.6.1.2.1.47.1.1.1.1.5.3", + sensor_index: "3", + state_value: 2, + state_descr: "enabled", + status: "ok" + }) + |> Repo.insert!() + + stub(SnmpMock, :get, fn _, oid, _ -> + case oid do + "1.3.6.1.2.1.47.1.1.1.1.5.3" -> {:ok, 2} + _ -> {:error, :no_such_object} + end + end) + + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + + updated = Repo.get!(StateSensor, ss.id) + assert updated.state_value == 2 + end + end + + describe "interface change detection events" do + test "speed change generates a speed_change event", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + + _interface = + %Interface{} + |> Interface.changeset(%{ + snmp_device_id: snmp.id, + if_index: 7, + if_name: "eth7", + if_descr: "ethernet", + if_type: 6, + if_speed: 1_000_000_000, + if_phys_address: "00:11:22:33:44:55", + if_admin_status: "up", + if_oper_status: "up", + monitored: true + }) + |> Repo.insert!() + + Phoenix.PubSub.subscribe(Towerops.PubSub, "device:#{device.id}") + + # Speed dropped to 100Mbps (warning), MAC and admin/oper unchanged. + stub(SnmpMock, :get, fn _, oid, _ -> + case oid do + "1.3.6.1.2.1.31.1.1.1.6.7" -> {:ok, 1_000} + "1.3.6.1.2.1.31.1.1.1.10.7" -> {:ok, 2_000} + "1.3.6.1.2.1.2.2.1.14.7" -> {:ok, 0} + "1.3.6.1.2.1.2.2.1.20.7" -> {:ok, 0} + "1.3.6.1.2.1.2.2.1.13.7" -> {:ok, 0} + "1.3.6.1.2.1.2.2.1.19.7" -> {:ok, 0} + _ -> {:error, :no_such_object} + end + end) + + stub(SnmpMock, :get_multiple, fn _, _oids, _ -> + {:ok, + %{ + # speed changed + "1.3.6.1.2.1.2.2.1.5.7" => {:integer, 100_000_000}, + "1.3.6.1.2.1.2.2.1.6.7" => {:"OCTET STRING", <<0, 17, 34, 51, 68, 85>>}, + "1.3.6.1.2.1.2.2.1.7.7" => {:integer, 1}, + "1.3.6.1.2.1.2.2.1.8.7" => {:integer, 1} + }} + end) + + stub_walks_empty() + + assert :ok = run_perform(device) + + received_events = collect_events([], 1_500) + + assert Enum.any?(received_events, fn e -> + Map.get(e, :event_type) == "interface_speed_change" + end) + end + + test "MAC change generates a mac_change event", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + + _interface = + %Interface{} + |> Interface.changeset(%{ + snmp_device_id: snmp.id, + if_index: 8, + if_name: "eth8", + if_descr: "ethernet", + if_type: 6, + if_speed: 100_000_000, + if_phys_address: "00:11:22:33:44:55", + if_admin_status: "up", + if_oper_status: "up", + monitored: true + }) + |> Repo.insert!() + + Phoenix.PubSub.subscribe(Towerops.PubSub, "device:#{device.id}") + + stub(SnmpMock, :get, fn _, oid, _ -> + case oid do + "1.3.6.1.2.1.31.1.1.1.6.8" -> {:ok, 1_000} + "1.3.6.1.2.1.31.1.1.1.10.8" -> {:ok, 2_000} + "1.3.6.1.2.1.2.2.1.14.8" -> {:ok, 0} + "1.3.6.1.2.1.2.2.1.20.8" -> {:ok, 0} + "1.3.6.1.2.1.2.2.1.13.8" -> {:ok, 0} + "1.3.6.1.2.1.2.2.1.19.8" -> {:ok, 0} + _ -> {:error, :no_such_object} + end + end) + + stub(SnmpMock, :get_multiple, fn _, _, _ -> + {:ok, + %{ + "1.3.6.1.2.1.2.2.1.5.8" => {:integer, 100_000_000}, + # changed MAC + "1.3.6.1.2.1.2.2.1.6.8" => {:"OCTET STRING", <<0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF>>}, + "1.3.6.1.2.1.2.2.1.7.8" => {:integer, 1}, + "1.3.6.1.2.1.2.2.1.8.8" => {:integer, 1} + }} + end) + + stub_walks_empty() + + assert :ok = run_perform(device) + + received_events = collect_events([], 1_500) + + assert Enum.any?(received_events, fn e -> + Map.get(e, :event_type) == "interface_mac_change" + end) + end + + test "oper_status change down→up generates interface_up event", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + + _interface = + %Interface{} + |> Interface.changeset(%{ + snmp_device_id: snmp.id, + if_index: 9, + if_name: "eth9", + if_descr: "ethernet", + if_type: 6, + if_speed: 100_000_000, + if_phys_address: "00:11:22:33:44:55", + if_admin_status: "up", + if_oper_status: "down", + monitored: true + }) + |> Repo.insert!() + + Phoenix.PubSub.subscribe(Towerops.PubSub, "device:#{device.id}") + + stub(SnmpMock, :get, fn _, oid, _ -> + case oid do + "1.3.6.1.2.1.31.1.1.1.6.9" -> {:ok, 1_000} + "1.3.6.1.2.1.31.1.1.1.10.9" -> {:ok, 2_000} + "1.3.6.1.2.1.2.2.1.14.9" -> {:ok, 0} + "1.3.6.1.2.1.2.2.1.20.9" -> {:ok, 0} + "1.3.6.1.2.1.2.2.1.13.9" -> {:ok, 0} + "1.3.6.1.2.1.2.2.1.19.9" -> {:ok, 0} + _ -> {:error, :no_such_object} + end + end) + + stub(SnmpMock, :get_multiple, fn _, _, _ -> + {:ok, + %{ + "1.3.6.1.2.1.2.2.1.5.9" => {:integer, 100_000_000}, + "1.3.6.1.2.1.2.2.1.6.9" => {:"OCTET STRING", <<0, 17, 34, 51, 68, 85>>}, + "1.3.6.1.2.1.2.2.1.7.9" => {:integer, 1}, + # now up + "1.3.6.1.2.1.2.2.1.8.9" => {:integer, 1} + }} + end) + + stub_walks_empty() + + assert :ok = run_perform(device) + + events = collect_events([], 1_500) + + assert Enum.any?(events, fn e -> + Map.get(e, :event_type) == "interface_up" + end) + end + end + + describe "storage and mempool error paths" do + test "storage SNMP error skips reading and does not raise", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + insert_dummy_sensor(snmp) + + _storage = + %Storage{} + |> Storage.changeset(%{ + snmp_device_id: snmp.id, + storage_index: 11, + storage_type: "ram", + description: "Mem" + }) + |> Repo.insert!() + + stub(SnmpMock, :get, fn _, _, _ -> {:error, :timeout} end) + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + end + + test "mempool SNMP error skips reading and does not raise", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + insert_dummy_sensor(snmp) + + _mempool = + %Mempool{} + |> Mempool.changeset(%{ + snmp_device_id: snmp.id, + mempool_index: "12", + mempool_type: "hr_memory", + description: "Bad" + }) + |> Repo.insert!() + + stub(SnmpMock, :get, fn _, _, _ -> {:error, :timeout} end) + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + end + + test "storage with non-integer values triggers invalid_values branch", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + insert_dummy_sensor(snmp) + + _storage = + %Storage{} + |> Storage.changeset(%{ + snmp_device_id: snmp.id, + storage_index: 13, + storage_type: "fixed_disk", + description: "Strange" + }) + |> Repo.insert!() + + stub(SnmpMock, :get, fn _, oid, _ -> + case oid do + "1.3.6.1.2.1.25.2.3.1.6.13" -> {:ok, "not-int"} + "1.3.6.1.2.1.25.2.3.1.5.13" -> {:ok, "not-int"} + "1.3.6.1.2.1.25.2.3.1.4.13" -> {:ok, "not-int"} + _ -> {:error, :no_such_object} + end + end) + + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + end + + test "mempool with non-integer values triggers invalid_values branch", %{site: site} do + device = create_polled_device(site) + snmp = create_snmp_device(device) + insert_dummy_sensor(snmp) + + _mempool = + %Mempool{} + |> Mempool.changeset(%{ + snmp_device_id: snmp.id, + mempool_index: "14", + mempool_type: "hr_memory", + description: "Strange" + }) + |> Repo.insert!() + + stub(SnmpMock, :get, fn _, oid, _ -> + case oid do + "1.3.6.1.2.1.25.2.3.1.6.14" -> {:ok, "not-int"} + "1.3.6.1.2.1.25.2.3.1.5.14" -> {:ok, "not-int"} + "1.3.6.1.2.1.25.2.3.1.4.14" -> {:ok, "not-int"} + _ -> {:error, :no_such_object} + end + end) + + stub(SnmpMock, :get_multiple, fn _, _, _ -> {:error, :timeout} end) + stub_walks_empty() + + assert :ok = run_perform(device) + end + end + + describe "decode_snmp_value larger sizes" do + test "decodes binary larger than 16 bytes (s > 8 fallback offset path)" do + # 12-byte binary: prefix 4 zero bytes, then 8-byte counter + bin = <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7>> + assert 7 == DevicePollerWorker.decode_snmp_value(bin) + end + end + + # Walks message queue collecting :device_event payloads up to a deadline. + defp collect_events(acc, ms_remaining) do + receive do + {:device_event, e} -> collect_events([e | acc], ms_remaining) + after + ms_remaining -> Enum.reverse(acc) + end + end +end