towerops/test/towerops/snmp/batch_insert_test.exs
Graham McIntire aa1bac0d46
perf: batch inserts, selective LiveView reloading, and database indexes for scale
Replace individual Repo.insert() with Repo.insert_all() for sensor readings,
interface stats, processor readings, and storage readings in agent channel and
device poller worker. Add partial/composite database indexes for common query
patterns. Dashboard now uses GROUP BY aggregation instead of loading all devices.
DeviceLive.Show only reloads data for the active tab on PubSub events.
DeviceLive.Index debounces status changes and skips quota queries on updates.
Agent polling preloads filtered to monitored sensors/interfaces only.
2026-02-12 10:16:10 -06:00

459 lines
13 KiB
Elixir

defmodule Towerops.Snmp.BatchInsertTest do
use Towerops.DataCase, async: true
import Towerops.AccountsFixtures
alias Towerops.Repo
alias Towerops.Snmp
alias Towerops.Snmp.Device
alias Towerops.Snmp.Interface
alias Towerops.Snmp.InterfaceStat
alias Towerops.Snmp.Processor
alias Towerops.Snmp.ProcessorReading
alias Towerops.Snmp.Sensor
alias Towerops.Snmp.SensorReading
alias Towerops.Snmp.Storage
alias Towerops.Snmp.StorageReading
setup do
user = user_fixture()
{:ok, organization} = Towerops.Organizations.create_organization(%{name: "Test Org"}, user.id)
{:ok, site} =
Towerops.Sites.create_site(%{
name: "Test Site",
organization_id: organization.id
})
{:ok, device} =
Towerops.Devices.create_device(%{
name: "Test Router",
ip_address: "192.168.1.1",
snmp_enabled: true,
snmp_version: "2c",
snmp_community: "public",
snmp_port: 161,
site_id: site.id,
organization_id: organization.id
})
snmp_device =
%Device{}
|> Device.changeset(%{
device_id: device.id,
sys_name: "test-router",
sys_descr: "Test Device"
})
|> Repo.insert!()
sensor =
%Sensor{}
|> Sensor.changeset(%{
snmp_device_id: snmp_device.id,
sensor_type: "temperature",
sensor_index: "temp_1",
sensor_oid: "1.3.6.1.4.1.9.9.91.1.1.1.1.4.1"
})
|> Repo.insert!()
sensor2 =
%Sensor{}
|> Sensor.changeset(%{
snmp_device_id: snmp_device.id,
sensor_type: "voltage",
sensor_index: "volt_1",
sensor_oid: "1.3.6.1.4.1.9.9.91.1.1.1.1.4.2"
})
|> Repo.insert!()
interface =
%Interface{}
|> Interface.changeset(%{
snmp_device_id: snmp_device.id,
if_index: 1,
if_name: "eth0"
})
|> Repo.insert!()
interface2 =
%Interface{}
|> Interface.changeset(%{
snmp_device_id: snmp_device.id,
if_index: 2,
if_name: "eth1"
})
|> Repo.insert!()
processor =
%Processor{}
|> Processor.changeset(%{
snmp_device_id: snmp_device.id,
processor_index: "1",
processor_type: "hr_processor",
description: "CPU 1"
})
|> Repo.insert!()
processor2 =
%Processor{}
|> Processor.changeset(%{
snmp_device_id: snmp_device.id,
processor_index: "2",
processor_type: "hr_processor",
description: "CPU 2"
})
|> Repo.insert!()
storage =
%Storage{}
|> Storage.changeset(%{
snmp_device_id: snmp_device.id,
storage_index: 1,
storage_type: "fixed_disk",
description: "/dev/sda1"
})
|> Repo.insert!()
storage2 =
%Storage{}
|> Storage.changeset(%{
snmp_device_id: snmp_device.id,
storage_index: 2,
storage_type: "ram",
description: "Physical Memory"
})
|> Repo.insert!()
%{
snmp_device: snmp_device,
sensor: sensor,
sensor2: sensor2,
interface: interface,
interface2: interface2,
processor: processor,
processor2: processor2,
storage: storage,
storage2: storage2
}
end
describe "create_sensor_readings_batch/1" do
test "inserts multiple sensor readings", %{sensor: sensor, sensor2: sensor2} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{sensor_id: sensor.id, value: 25.5, status: "ok", checked_at: now},
%{sensor_id: sensor2.id, value: 3.3, status: "warning", checked_at: now}
]
assert {2, nil} = Snmp.create_sensor_readings_batch(entries)
readings = Repo.all(SensorReading)
assert length(readings) == 2
reading1 = Enum.find(readings, &(&1.sensor_id == sensor.id))
assert reading1.value == 25.5
assert reading1.status == "ok"
assert reading1.checked_at == now
assert reading1.id
assert reading1.inserted_at
reading2 = Enum.find(readings, &(&1.sensor_id == sensor2.id))
assert reading2.value == 3.3
assert reading2.status == "warning"
end
test "returns {0, nil} for empty list" do
assert {0, nil} = Snmp.create_sensor_readings_batch([])
assert Repo.all(SensorReading) == []
end
test "generates unique UUIDs for each entry", %{sensor: sensor} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{sensor_id: sensor.id, value: 10.0, status: "ok", checked_at: now},
%{sensor_id: sensor.id, value: 11.0, status: "ok", checked_at: now}
]
{2, nil} = Snmp.create_sensor_readings_batch(entries)
readings = Repo.all(SensorReading)
ids = Enum.map(readings, & &1.id)
assert length(Enum.uniq(ids)) == 2
end
test "handles optional state_descr field", %{sensor: sensor} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{
sensor_id: sensor.id,
value: 25.5,
status: "ok",
state_descr: "Normal operating temperature",
checked_at: now
}
]
assert {1, nil} = Snmp.create_sensor_readings_batch(entries)
[reading] = Repo.all(SensorReading)
assert reading.state_descr == "Normal operating temperature"
end
test "handles entries with nil value", %{sensor: sensor} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{sensor_id: sensor.id, value: nil, status: "error", checked_at: now}
]
assert {1, nil} = Snmp.create_sensor_readings_batch(entries)
[reading] = Repo.all(SensorReading)
assert reading.value == nil
assert reading.status == "error"
end
end
describe "create_interface_stats_batch/1" do
test "inserts multiple interface stats", %{interface: interface, interface2: interface2} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{
interface_id: interface.id,
if_in_octets: 1000,
if_out_octets: 2000,
if_in_errors: 0,
if_out_errors: 0,
if_in_discards: 0,
if_out_discards: 0,
checked_at: now
},
%{
interface_id: interface2.id,
if_in_octets: 5000,
if_out_octets: 6000,
if_in_errors: 1,
if_out_errors: 2,
if_in_discards: 3,
if_out_discards: 4,
checked_at: now
}
]
assert {2, nil} = Snmp.create_interface_stats_batch(entries)
stats = Repo.all(InterfaceStat)
assert length(stats) == 2
stat1 = Enum.find(stats, &(&1.interface_id == interface.id))
assert stat1.if_in_octets == 1000
assert stat1.if_out_octets == 2000
assert stat1.if_in_errors == 0
assert stat1.if_out_errors == 0
assert stat1.if_in_discards == 0
assert stat1.if_out_discards == 0
assert stat1.checked_at == now
assert stat1.id
assert stat1.inserted_at
stat2 = Enum.find(stats, &(&1.interface_id == interface2.id))
assert stat2.if_in_octets == 5000
assert stat2.if_out_octets == 6000
assert stat2.if_in_errors == 1
assert stat2.if_out_errors == 2
end
test "returns {0, nil} for empty list" do
assert {0, nil} = Snmp.create_interface_stats_batch([])
assert Repo.all(InterfaceStat) == []
end
test "generates unique UUIDs for each entry", %{interface: interface} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{interface_id: interface.id, if_in_octets: 100, if_out_octets: 200, checked_at: now},
%{interface_id: interface.id, if_in_octets: 300, if_out_octets: 400, checked_at: now}
]
{2, nil} = Snmp.create_interface_stats_batch(entries)
stats = Repo.all(InterfaceStat)
ids = Enum.map(stats, & &1.id)
assert length(Enum.uniq(ids)) == 2
end
test "handles entries with nil counter values", %{interface: interface} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{interface_id: interface.id, if_in_octets: nil, if_out_octets: nil, checked_at: now}
]
assert {1, nil} = Snmp.create_interface_stats_batch(entries)
[stat] = Repo.all(InterfaceStat)
assert stat.if_in_octets == nil
assert stat.if_out_octets == nil
end
end
describe "create_processor_readings_batch/1" do
test "inserts multiple processor readings", %{processor: processor, processor2: processor2} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{processor_id: processor.id, load_percent: 45.0, status: "ok", checked_at: now},
%{processor_id: processor2.id, load_percent: 92.0, status: "critical", checked_at: now}
]
assert {2, nil} = Snmp.create_processor_readings_batch(entries)
readings = Repo.all(ProcessorReading)
assert length(readings) == 2
reading1 = Enum.find(readings, &(&1.processor_id == processor.id))
assert reading1.load_percent == 45.0
assert reading1.status == "ok"
assert reading1.checked_at == now
assert reading1.id
assert reading1.inserted_at
reading2 = Enum.find(readings, &(&1.processor_id == processor2.id))
assert reading2.load_percent == 92.0
assert reading2.status == "critical"
end
test "returns {0, nil} for empty list" do
assert {0, nil} = Snmp.create_processor_readings_batch([])
assert Repo.all(ProcessorReading) == []
end
test "generates unique UUIDs for each entry", %{processor: processor} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{processor_id: processor.id, load_percent: 10.0, status: "ok", checked_at: now},
%{processor_id: processor.id, load_percent: 20.0, status: "ok", checked_at: now}
]
{2, nil} = Snmp.create_processor_readings_batch(entries)
readings = Repo.all(ProcessorReading)
ids = Enum.map(readings, & &1.id)
assert length(Enum.uniq(ids)) == 2
end
test "handles entries with nil load_percent", %{processor: processor} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{processor_id: processor.id, load_percent: nil, status: "error", checked_at: now}
]
assert {1, nil} = Snmp.create_processor_readings_batch(entries)
[reading] = Repo.all(ProcessorReading)
assert reading.load_percent == nil
assert reading.status == "error"
end
end
describe "create_storage_readings_batch/1" do
test "inserts multiple storage readings", %{storage: storage, storage2: storage2} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{
storage_id: storage.id,
used_bytes: 5_000_000,
total_bytes: 10_000_000,
usage_percent: 50.0,
checked_at: now
},
%{
storage_id: storage2.id,
used_bytes: 2_000_000,
total_bytes: 8_000_000,
usage_percent: 25.0,
checked_at: now
}
]
assert {2, nil} = Snmp.create_storage_readings_batch(entries)
readings = Repo.all(StorageReading)
assert length(readings) == 2
reading1 = Enum.find(readings, &(&1.storage_id == storage.id))
assert reading1.used_bytes == 5_000_000
assert reading1.total_bytes == 10_000_000
assert reading1.usage_percent == 50.0
assert reading1.checked_at == now
assert reading1.id
assert reading1.inserted_at
reading2 = Enum.find(readings, &(&1.storage_id == storage2.id))
assert reading2.used_bytes == 2_000_000
assert reading2.total_bytes == 8_000_000
assert reading2.usage_percent == 25.0
end
test "returns {0, nil} for empty list" do
assert {0, nil} = Snmp.create_storage_readings_batch([])
assert Repo.all(StorageReading) == []
end
test "generates unique UUIDs for each entry", %{storage: storage} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{
storage_id: storage.id,
used_bytes: 1000,
total_bytes: 2000,
usage_percent: 50.0,
checked_at: now
},
%{
storage_id: storage.id,
used_bytes: 1500,
total_bytes: 2000,
usage_percent: 75.0,
checked_at: now
}
]
{2, nil} = Snmp.create_storage_readings_batch(entries)
readings = Repo.all(StorageReading)
ids = Enum.map(readings, & &1.id)
assert length(Enum.uniq(ids)) == 2
end
test "handles entries with nil optional fields", %{storage: storage} do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries = [
%{
storage_id: storage.id,
used_bytes: nil,
total_bytes: nil,
usage_percent: nil,
checked_at: now
}
]
assert {1, nil} = Snmp.create_storage_readings_batch(entries)
[reading] = Repo.all(StorageReading)
assert reading.used_bytes == nil
assert reading.total_bytes == nil
assert reading.usage_percent == nil
end
end
end