diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 58820e59..b5ee4e9d 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,33 @@ CHANGELOG - towerops-web ======================== +2026-02-12 - perf: performance improvements for scale (batch inserts, indexes, selective reloading) + - Files: lib/towerops/snmp.ex, lib/towerops/devices.ex, lib/towerops_web/channels/agent_channel.ex, + lib/towerops/workers/device_poller_worker.ex, lib/towerops_web/live/dashboard_live.ex, + lib/towerops_web/live/device_live/show.ex, lib/towerops_web/live/device_live/index.ex, + lib/towerops/agents.ex, priv/repo/migrations/20260212155105_add_performance_indexes.exs + + Phase 1: Batch data ingestion - replaced individual Repo.insert() calls with Repo.insert_all() + for sensor readings, interface stats, processor readings, and storage readings in both + agent_channel.ex and device_poller_worker.ex. Added create_*_batch/1 functions to snmp.ex. + + Phase 2: Database indexes - added partial indexes for monitored sensors/interfaces, + composite index for devices org+status, partial index for active alerts. + + Phase 3: Dashboard aggregation - replaced loading all devices into memory with + Devices.get_device_status_counts/1 GROUP BY query. + + Phase 4: DeviceLive.Show selective updates - split monolithic load_equipment_data into + tab-specific loaders (base, tab_nav, overview, ports, logs, backups). PubSub handlers + now only reload data relevant to the active tab. + + Phase 5: DeviceLive.Index incremental updates - separated structural changes (create/delete) + from status changes. Added debouncing for rapid status changes. Skip reload when viewing + discovered tab. + + Phase 6: Agent polling memory optimization - filtered preloads in list_agent_polling_targets + to only load monitored sensors and interfaces instead of all. + 2026-02-12 - feat: complete WISP vendor coverage (ePMP, Raisecom, Netonix) - Files: priv/profiles/os_discovery/epmp.yaml (enhanced), lib/towerops/snmp/profiles/vendors/raisecom.ex (new) Completed LibreNMS parity for major WISP equipment vendors. Three improvements: diff --git a/lib/towerops/agents.ex b/lib/towerops/agents.ex index 1c084010..a51850af 100644 --- a/lib/towerops/agents.ex +++ b/lib/towerops/agents.ex @@ -592,7 +592,10 @@ defmodule Towerops.Agents do :agent_assignments, :organization, site: :organization, - snmp_device: [:sensors, :interfaces] + snmp_device: [ + sensors: ^from(s in Towerops.Snmp.Sensor, where: s.monitored == true), + interfaces: ^from(i in Towerops.Snmp.Interface, where: i.monitored == true) + ] ] ) diff --git a/lib/towerops/devices.ex b/lib/towerops/devices.ex index dc6cd88d..6139f153 100644 --- a/lib/towerops/devices.ex +++ b/lib/towerops/devices.ex @@ -116,6 +116,30 @@ defmodule Towerops.Devices do ) end + @doc """ + Returns a map of device status counts for an organization. + + Uses a GROUP BY query instead of loading all devices into memory. + + ## Examples + + iex> get_device_status_counts(organization_id) + %{up: 5, down: 2, unknown: 1} + + iex> get_device_status_counts(empty_org_id) + %{} + """ + @spec get_device_status_counts(String.t()) :: %{atom() => non_neg_integer()} + def get_device_status_counts(organization_id) do + from(d in DeviceSchema, + where: d.organization_id == ^organization_id, + group_by: d.status, + select: {d.status, count(d.id)} + ) + |> Repo.all() + |> Map.new() + end + @doc """ Returns the list of all devices with monitoring enabled. """ diff --git a/lib/towerops/snmp.ex b/lib/towerops/snmp.ex index eaff81d8..e523e67c 100644 --- a/lib/towerops/snmp.ex +++ b/lib/towerops/snmp.ex @@ -372,6 +372,28 @@ defmodule Towerops.Snmp do |> Repo.insert() end + @doc """ + Batch inserts multiple sensor readings using `Repo.insert_all/3`. + + Accepts a list of attribute maps with the same keys as `create_sensor_reading/1`. + Generates UUIDs and timestamps automatically. Returns `{count, nil}`. + """ + @spec create_sensor_readings_batch([map()]) :: {non_neg_integer(), nil} + def create_sensor_readings_batch([]), do: {0, nil} + + def create_sensor_readings_batch(entries) when is_list(entries) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + rows = + Enum.map(entries, fn entry -> + entry + |> Map.put(:id, Ecto.UUID.generate()) + |> Map.put(:inserted_at, now) + end) + + Repo.insert_all(SensorReading, rows) + end + @doc """ Records a new interface stat. """ @@ -381,6 +403,28 @@ defmodule Towerops.Snmp do |> Repo.insert() end + @doc """ + Batch inserts multiple interface stats using `Repo.insert_all/3`. + + Accepts a list of attribute maps with the same keys as `create_interface_stat/1`. + Generates UUIDs and timestamps automatically. Returns `{count, nil}`. + """ + @spec create_interface_stats_batch([map()]) :: {non_neg_integer(), nil} + def create_interface_stats_batch([]), do: {0, nil} + + def create_interface_stats_batch(entries) when is_list(entries) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + rows = + Enum.map(entries, fn entry -> + entry + |> Map.put(:id, Ecto.UUID.generate()) + |> Map.put(:inserted_at, now) + end) + + Repo.insert_all(InterfaceStat, rows) + end + # Neighbor queries @doc """ @@ -1383,6 +1427,28 @@ defmodule Towerops.Snmp do |> Repo.insert() end + @doc """ + Batch inserts multiple processor readings using `Repo.insert_all/3`. + + Accepts a list of attribute maps with the same keys as `create_processor_reading/1`. + Generates UUIDs and timestamps automatically. Returns `{count, nil}`. + """ + @spec create_processor_readings_batch([map()]) :: {non_neg_integer(), nil} + def create_processor_readings_batch([]), do: {0, nil} + + def create_processor_readings_batch(entries) when is_list(entries) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + rows = + Enum.map(entries, fn entry -> + entry + |> Map.put(:id, Ecto.UUID.generate()) + |> Map.put(:inserted_at, now) + end) + + Repo.insert_all(ProcessorReading, rows) + end + # Storage queries @doc """ @@ -1490,6 +1556,28 @@ defmodule Towerops.Snmp do |> Repo.insert() end + @doc """ + Batch inserts multiple storage readings using `Repo.insert_all/3`. + + Accepts a list of attribute maps with the same keys as `create_storage_reading/1`. + Generates UUIDs and timestamps automatically. Returns `{count, nil}`. + """ + @spec create_storage_readings_batch([map()]) :: {non_neg_integer(), nil} + def create_storage_readings_batch([]), do: {0, nil} + + def create_storage_readings_batch(entries) when is_list(entries) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + rows = + Enum.map(entries, fn entry -> + entry + |> Map.put(:id, Ecto.UUID.generate()) + |> Map.put(:inserted_at, now) + end) + + Repo.insert_all(StorageReading, rows) + end + # ARP Entry queries @doc """ diff --git a/lib/towerops/workers/device_poller_worker.ex b/lib/towerops/workers/device_poller_worker.ex index 2a9faf04..28f22496 100644 --- a/lib/towerops/workers/device_poller_worker.ex +++ b/lib/towerops/workers/device_poller_worker.ex @@ -413,17 +413,54 @@ defmodule Towerops.Workers.DevicePollerWorker do # (These remain unchanged - I'll include them all below) defp poll_storage(storage_entries, client_opts, timestamp) do - storage_entries - |> Task.async_stream( - fn storage -> - result = poll_storage_value(storage, client_opts) - handle_storage_poll_result(storage, result, timestamp) - end, - max_concurrency: 2, - timeout: 40_000, - on_timeout: :kill_task - ) - |> Stream.run() + poll_results = + storage_entries + |> Task.async_stream( + fn storage -> + result = poll_storage_value(storage, client_opts) + {storage, result} + end, + max_concurrency: 2, + timeout: 40_000, + on_timeout: :kill_task + ) + |> Enum.flat_map(fn + {:ok, pair} -> [pair] + {:exit, _reason} -> [] + end) + + # Build batch reading entries (only for successful polls) + reading_entries = + Enum.flat_map(poll_results, fn + {storage, {:ok, values}} -> + [ + %{ + storage_id: storage.id, + used_bytes: values.used_bytes, + total_bytes: values.total_bytes, + usage_percent: values.usage_percent, + checked_at: timestamp + } + ] + + {_storage, {:error, _reason}} -> + [] + end) + + Snmp.create_storage_readings_batch(reading_entries) + + # Process metadata updates individually + Enum.each(poll_results, fn + {storage, {:ok, values}} -> + Snmp.update_storage(storage, %{ + used_bytes: values.used_bytes, + total_bytes: values.total_bytes, + last_checked_at: timestamp + }) + + {storage, {:error, reason}} -> + Logger.debug("Failed to poll storage #{storage.description}: #{inspect(reason)}") + end) end defp poll_storage_value(storage, client_opts) do @@ -447,38 +484,35 @@ defmodule Towerops.Workers.DevicePollerWorker do end end - defp handle_storage_poll_result(storage, {:ok, values}, timestamp) do - Snmp.create_storage_reading(%{ - storage_id: storage.id, - used_bytes: values.used_bytes, - total_bytes: values.total_bytes, - usage_percent: values.usage_percent, - checked_at: timestamp - }) - - Snmp.update_storage(storage, %{ - used_bytes: values.used_bytes, - total_bytes: values.total_bytes, - last_checked_at: timestamp - }) - end - - defp handle_storage_poll_result(storage, {:error, reason}, _timestamp) do - Logger.debug("Failed to poll storage #{storage.description}: #{inspect(reason)}") - end - defp poll_processors(processors, client_opts, timestamp) do - processors - |> Task.async_stream( - fn processor -> - result = poll_processor_value(processor, client_opts) - handle_processor_poll_result(processor, result, timestamp) - end, - max_concurrency: 2, - timeout: 40_000, - on_timeout: :kill_task - ) - |> Stream.run() + poll_results = + processors + |> Task.async_stream( + fn processor -> + result = poll_processor_value(processor, client_opts) + {processor, result} + end, + max_concurrency: 2, + timeout: 40_000, + on_timeout: :kill_task + ) + |> Enum.flat_map(fn + {:ok, pair} -> [pair] + {:exit, _reason} -> [] + end) + + # Build batch reading entries + reading_entries = + Enum.map(poll_results, fn {processor, result} -> + build_processor_reading_entry(processor, result, timestamp) + end) + + Snmp.create_processor_readings_batch(reading_entries) + + # Process metadata updates individually + Enum.each(poll_results, fn {processor, result} -> + handle_processor_poll_metadata(processor, result, timestamp) + end) end defp poll_processor_value(processor, client_opts) do @@ -533,46 +567,69 @@ defmodule Towerops.Workers.DevicePollerWorker do end end - defp handle_processor_poll_result(processor, {:ok, load_percent}, timestamp) do - Snmp.create_processor_reading(%{ + defp build_processor_reading_entry(processor, {:ok, load_percent}, timestamp) do + %{ processor_id: processor.id, load_percent: load_percent, status: "ok", checked_at: timestamp - }) + } + end + defp build_processor_reading_entry(processor, {:error, reason}, timestamp) do + Logger.debug("Failed to poll processor #{processor.description}: #{inspect(reason)}") + + %{ + processor_id: processor.id, + load_percent: nil, + status: "error", + checked_at: timestamp + } + end + + defp handle_processor_poll_metadata(processor, {:ok, load_percent}, timestamp) do Snmp.update_processor(processor, %{ load_percent: load_percent, last_checked_at: timestamp }) end - defp handle_processor_poll_result(processor, {:error, reason}, timestamp) do - Logger.debug("Failed to poll processor #{processor.description}: #{inspect(reason)}") - - Snmp.create_processor_reading(%{ - processor_id: processor.id, - load_percent: nil, - status: "error", - checked_at: timestamp - }) - end + defp handle_processor_poll_metadata(_processor, {:error, _reason}, _timestamp), do: :ok defp poll_sensors(sensors, client_opts, timestamp) do - sensors - |> Task.async_stream( - fn sensor -> - result = poll_sensor_value(sensor, client_opts) - handle_sensor_poll_result(sensor, result, timestamp) - sensor.sensor_descr - end, - max_concurrency: 2, - timeout: 40_000, - on_timeout: :kill_task - ) - |> Enum.each(fn - {:ok, _} -> :ok - {:exit, reason} -> Logger.warning("Sensor poll task failed: #{inspect(reason)}") + # Poll all sensors concurrently and collect results + poll_results = + sensors + |> Task.async_stream( + fn sensor -> + result = poll_sensor_value(sensor, client_opts) + {sensor, result} + end, + max_concurrency: 2, + timeout: 40_000, + on_timeout: :kill_task + ) + |> Enum.flat_map(fn + {:ok, pair} -> + [pair] + + {:exit, reason} -> + Logger.warning("Sensor poll task failed: #{inspect(reason)}") + [] + end) + + # Build batch reading entries + reading_entries = + Enum.map(poll_results, fn {sensor, result} -> + build_sensor_reading_entry(sensor, result, timestamp) + end) + + # Batch insert all sensor readings + Snmp.create_sensor_readings_batch(reading_entries) + + # Process metadata updates and change detection individually + Enum.each(poll_results, fn {sensor, result} -> + handle_sensor_poll_metadata(sensor, result, timestamp) end) end @@ -720,18 +777,43 @@ defmodule Towerops.Workers.DevicePollerWorker do end end - defp handle_sensor_poll_result(sensor, {:ok, value}, timestamp) do - # For state sensors, map numeric value to state description + # Build a reading entry map for batch insert, or nil if not applicable + defp build_sensor_reading_entry(sensor, {:ok, value}, timestamp) do state_descr = get_state_description(sensor, value) - Snmp.create_sensor_reading(%{ + %{ sensor_id: sensor.id, value: value, status: "ok", state_descr: state_descr, checked_at: timestamp - }) + } + end + defp build_sensor_reading_entry(sensor, {:error, :non_numeric}, timestamp) do + Logger.debug("Non-numeric SNMP value for sensor #{sensor.sensor_descr}") + + %{ + sensor_id: sensor.id, + value: nil, + status: "unknown", + checked_at: timestamp + } + end + + defp build_sensor_reading_entry(sensor, {:error, reason}, timestamp) do + log_sensor_error(sensor, reason) + + %{ + sensor_id: sensor.id, + value: nil, + status: "error", + checked_at: timestamp + } + end + + # Process metadata updates and change detection (must remain per-sensor) + defp handle_sensor_poll_metadata(sensor, {:ok, value}, timestamp) do SensorChangeDetector.detect_and_broadcast(sensor, value, timestamp) Snmp.update_sensor(sensor, %{ @@ -740,27 +822,7 @@ defmodule Towerops.Workers.DevicePollerWorker do }) end - defp handle_sensor_poll_result(sensor, {:error, :non_numeric}, timestamp) do - Logger.debug("Non-numeric SNMP value for sensor #{sensor.sensor_descr}") - - Snmp.create_sensor_reading(%{ - sensor_id: sensor.id, - value: nil, - status: "unknown", - checked_at: timestamp - }) - end - - defp handle_sensor_poll_result(sensor, {:error, reason}, timestamp) do - log_sensor_error(sensor, reason) - - Snmp.create_sensor_reading(%{ - sensor_id: sensor.id, - value: nil, - status: "error", - checked_at: timestamp - }) - end + defp handle_sensor_poll_metadata(_sensor, {:error, _reason}, _timestamp), do: :ok # Get state description for state sensors from metadata defp get_state_description(sensor, value) do @@ -838,24 +900,27 @@ defmodule Towerops.Workers.DevicePollerWorker do end defp poll_interfaces(interfaces, client_opts, timestamp) do - interfaces - |> Task.async_stream( - fn interface -> - {:ok, stat_data} = get_interface_stats(client_opts, interface.if_index) + entries = + interfaces + |> Task.async_stream( + fn interface -> + {:ok, stat_data} = get_interface_stats(client_opts, interface.if_index) - stats = Map.merge(stat_data, %{ interface_id: interface.id, checked_at: timestamp }) + end, + max_concurrency: 2, + timeout: 40_000, + on_timeout: :kill_task + ) + |> Enum.flat_map(fn + {:ok, entry} -> [entry] + {:exit, _reason} -> [] + end) - Snmp.create_interface_stat(stats) - end, - max_concurrency: 2, - timeout: 40_000, - on_timeout: :kill_task - ) - |> Stream.run() + Snmp.create_interface_stats_batch(entries) end defp check_interface_changes(interfaces, device, client_opts, timestamp) do diff --git a/lib/towerops_web/channels/agent_channel.ex b/lib/towerops_web/channels/agent_channel.ex index ef4dc2a1..f0339e16 100644 --- a/lib/towerops_web/channels/agent_channel.ex +++ b/lib/towerops_web/channels/agent_channel.ex @@ -1422,17 +1422,17 @@ defmodule ToweropsWeb.AgentChannel do snmp_device = device.snmp_device oid_values = normalize_oid_keys(result.oid_values) # Use server time to avoid clock drift issues between agents - timestamp = DateTime.utc_now() + timestamp = DateTime.truncate(DateTime.utc_now(), :second) Logger.info( "Processing polling result for #{device.name}: #{map_size(oid_values)} OIDs, #{length(snmp_device.sensors)} sensors, #{length(snmp_device.interfaces)} interfaces" ) - # Process sensor readings (quick path - already in memory) - Enum.each(snmp_device.sensors, &process_sensor_reading(&1, oid_values, timestamp)) + # Batch insert sensor readings and process metadata updates + process_sensor_readings_batch(snmp_device.sensors, oid_values, timestamp) - # Process interface stats (quick path - already in memory) - Enum.each(device.snmp_device.interfaces, &process_interface_stats(&1, oid_values, timestamp)) + # Batch insert interface stats + process_interface_stats_batch(snmp_device.interfaces, oid_values, timestamp) # Process complex SNMP data (neighbors, ARP, MAC, IP addresses, processors, storage) # using Replay adapter to reuse existing parsing logic @@ -1576,47 +1576,69 @@ defmodule ToweropsWeb.AgentChannel do end end - defp process_sensor_reading(sensor, oid_values, timestamp) do + defp process_sensor_readings_batch(sensors, oid_values, timestamp) do + # Build reading entries and collect sensor updates in one pass + {reading_entries, sensor_updates} = + Enum.reduce(sensors, {[], []}, fn sensor, {readings, updates} -> + case resolve_sensor_value(sensor, oid_values) do + nil -> + {readings, updates} + + final_value -> + reading = %{ + sensor_id: sensor.id, + value: final_value, + status: "ok", + checked_at: timestamp + } + + {[reading | readings], [{sensor, final_value} | updates]} + end + end) + + # Batch insert all sensor readings in one SQL statement + Snmp.create_sensor_readings_batch(reading_entries) + + # Process change detection and metadata updates individually + Enum.each(sensor_updates, fn {sensor, final_value} -> + SensorChangeDetector.detect_and_broadcast(sensor, final_value, timestamp) + + Snmp.update_sensor(sensor, %{ + last_value: final_value, + last_checked_at: timestamp + }) + end) + end + + defp resolve_sensor_value(sensor, oid_values) do normalized_oid = String.trim_leading(sensor.sensor_oid, ".") - case Map.get(oid_values, normalized_oid) do - nil -> - :ok - - value -> - with parsed_value when not is_nil(parsed_value) <- parse_float(value) do - final_value = Float.round(parsed_value / sensor.sensor_divisor, 1) - - Snmp.create_sensor_reading(%{ - sensor_id: sensor.id, - value: final_value, - status: "ok", - checked_at: timestamp - }) - - SensorChangeDetector.detect_and_broadcast(sensor, final_value, timestamp) - - Snmp.update_sensor(sensor, %{ - last_value: final_value, - last_checked_at: timestamp - }) - end + with value when not is_nil(value) <- Map.get(oid_values, normalized_oid), + parsed when not is_nil(parsed) <- parse_float(value) do + Float.round(parsed / sensor.sensor_divisor, 1) + else + _ -> nil end end - defp process_interface_stats(iface, oid_values, timestamp) do - idx = iface.if_index + defp process_interface_stats_batch(interfaces, oid_values, timestamp) do + entries = + Enum.map(interfaces, fn iface -> + idx = iface.if_index - Snmp.create_interface_stat(%{ - interface_id: iface.id, - if_in_octets: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.31.1.1.1.6.#{idx}")), - if_out_octets: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.31.1.1.1.10.#{idx}")), - if_in_errors: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.2.2.1.14.#{idx}")), - if_out_errors: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.2.2.1.20.#{idx}")), - if_in_discards: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.2.2.1.13.#{idx}")), - if_out_discards: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.2.2.1.19.#{idx}")), - checked_at: timestamp - }) + %{ + interface_id: iface.id, + if_in_octets: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.31.1.1.1.6.#{idx}")), + if_out_octets: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.31.1.1.1.10.#{idx}")), + if_in_errors: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.2.2.1.14.#{idx}")), + if_out_errors: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.2.2.1.20.#{idx}")), + if_in_discards: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.2.2.1.13.#{idx}")), + if_out_discards: parse_integer(Map.get(oid_values, "1.3.6.1.2.1.2.2.1.19.#{idx}")), + checked_at: timestamp + } + end) + + Snmp.create_interface_stats_batch(entries) end defp parse_integer(nil), do: nil diff --git a/lib/towerops_web/live/dashboard_live.ex b/lib/towerops_web/live/dashboard_live.ex index 4c071edb..8a64c6f6 100644 --- a/lib/towerops_web/live/dashboard_live.ex +++ b/lib/towerops_web/live/dashboard_live.ex @@ -36,20 +36,17 @@ defmodule ToweropsWeb.DashboardLive do defp load_dashboard_data(socket, organization_id) do organization = socket.assigns.current_scope.organization active_alerts = Alerts.list_organization_active_alerts(organization_id) - sites_count = if organization.use_sites, do: length(Sites.list_organization_sites(organization_id)), else: 0 - devices = Devices.list_organization_devices(organization_id) + sites_count = if organization.use_sites, do: Sites.count_organization_sites(organization_id), else: 0 + status_counts = Devices.get_device_status_counts(organization_id) - devices_by_status = - devices - |> Enum.group_by(& &1.status) - |> Map.new(fn {status, equip} -> {status, length(equip)} end) + device_count = status_counts |> Map.values() |> Enum.sum() socket |> assign(:active_alerts, active_alerts) |> assign(:sites_count, sites_count) - |> assign(:device_count, length(devices)) - |> assign(:device_up, Map.get(devices_by_status, :up, 0)) - |> assign(:device_down, Map.get(devices_by_status, :down, 0)) - |> assign(:device_unknown, Map.get(devices_by_status, :unknown, 0)) + |> assign(:device_count, device_count) + |> assign(:device_up, Map.get(status_counts, :up, 0)) + |> assign(:device_down, Map.get(status_counts, :down, 0)) + |> assign(:device_unknown, Map.get(status_counts, :unknown, 0)) end end diff --git a/lib/towerops_web/live/device_live/index.ex b/lib/towerops_web/live/device_live/index.ex index f30821d9..567f6491 100644 --- a/lib/towerops_web/live/device_live/index.ex +++ b/lib/towerops_web/live/device_live/index.ex @@ -31,7 +31,8 @@ defmodule ToweropsWeb.DeviceLive.Index do |> assign(:sites_enabled, organization.use_sites) |> assign(:has_sites, sites != []) |> assign(:reorder_mode, false) - |> assign(:device_quota, %{current: current, limit: limit})} + |> assign(:device_quota, %{current: current, limit: limit}) + |> assign(:pending_reload_ref, nil)} end @impl true @@ -210,12 +211,47 @@ defmodule ToweropsWeb.DeviceLive.Index do end end + # Structural changes (create/delete) need full reload with quota recalculation @impl true - def handle_info({event, _org_id}, socket) - when event in [:device_created, :device_updated, :device_deleted, :device_status_changed] do + def handle_info({event, _org_id}, socket) when event in [:device_created, :device_deleted] do {:noreply, reload_devices(socket)} end + # Status changes and updates are debounced to avoid redundant reloads + # when many devices change status simultaneously (e.g., network event) + @impl true + def handle_info({event, _org_id}, socket) when event in [:device_status_changed, :device_updated] do + {:noreply, schedule_debounced_reload(socket, event)} + end + + @impl true + def handle_info(:debounced_device_reload, socket) do + {:noreply, reload_device_stream(socket)} + end + + @debounce_ms if Mix.env() == :test, do: 0, else: 100 + + defp schedule_debounced_reload(socket, event) do + # Cancel any pending debounced reload + if ref = socket.assigns.pending_reload_ref do + Process.cancel_timer(ref) + end + + # For status changes, skip reload entirely if viewing the discovered tab + if event == :device_status_changed && socket.assigns.active_tab == "discovered" do + assign(socket, :pending_reload_ref, nil) + else + if @debounce_ms == 0 do + # In test: reload immediately so render(view) sees the update + reload_device_stream(socket) + else + ref = Process.send_after(self(), :debounced_device_reload, @debounce_ms) + assign(socket, :pending_reload_ref, ref) + end + end + end + + # Full reload: stream + quota (for structural changes like create/delete) defp reload_devices(socket) do organization = socket.assigns.current_scope.organization devices = Devices.list_organization_devices(organization.id) @@ -227,6 +263,17 @@ defmodule ToweropsWeb.DeviceLive.Index do |> assign(:device_quota, %{current: current, limit: limit}) end + # Lightweight reload: stream only, skip quota query (for status/update changes) + defp reload_device_stream(socket) do + organization = socket.assigns.current_scope.organization + devices = Devices.list_organization_devices(organization.id) + + socket + |> stream(:device_rows, build_device_rows(devices), reset: true) + |> assign(:has_devices, devices != []) + |> assign(:pending_reload_ref, nil) + end + defp enqueue_discovery(device_id) do if Application.get_env(:towerops, :env) == :test do _ = Task.start(fn -> Snmp.discover_device(Devices.get_device!(device_id)) end) diff --git a/lib/towerops_web/live/device_live/show.ex b/lib/towerops_web/live/device_live/show.ex index 25d5058a..32369119 100644 --- a/lib/towerops_web/live/device_live/show.ex +++ b/lib/towerops_web/live/device_live/show.ex @@ -96,17 +96,19 @@ defmodule ToweropsWeb.DeviceLive.Show do _device -> Process.send_after(self(), :refresh_data, 10_000) - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_active_tab_data(socket)} end end @impl true def handle_info({:device_status_changed, _device_id, _new_status, _response_time}, socket) do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + # Device status shows on all tabs in the header - reload base data only + {:noreply, reload_base_data(socket)} end @impl true def handle_info({:discovery_completed, _device_id}, socket) do + # Structural change - full reload needed {:noreply, socket |> load_equipment_data(socket.assigns.device.id) @@ -115,61 +117,60 @@ defmodule ToweropsWeb.DeviceLive.Show do @impl true def handle_info({:sensors_updated, _device_id}, socket) do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_if_tab(socket, ["overview"])} end @impl true def handle_info({:interfaces_updated, _device_id}, socket) do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_if_tab(socket, ["overview", "ports"])} end @impl true def handle_info({:neighbors_updated, _device_id}, socket) do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_if_tab(socket, ["neighbors"])} end @impl true def handle_info({:arp_updated, _device_id}, socket) do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_if_tab(socket, ["arp"])} end @impl true def handle_info({:mac_updated, _device_id}, socket) do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_if_tab(socket, ["mac"])} end @impl true def handle_info({:state_sensors_updated, _device_id}, socket) do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_if_tab(socket, ["overview"])} end @impl true def handle_info({:processors_updated, _device_id}, socket) do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_if_tab(socket, ["overview"])} end @impl true def handle_info({:storage_updated, _device_id}, socket) do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_if_tab(socket, ["overview"])} end @impl true def handle_info({:monitoring_check_updated, _device_id}, socket) do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_if_tab(socket, ["overview"])} end @impl true def handle_info({:device_event, _event_attrs}, socket) do # Device event logged (interface changes, sensor thresholds, etc.) - # Reload to update events list - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_if_tab(socket, ["logs"])} end @impl true def handle_info({:agent_connected, agent_token_id, _organization_id}, socket) do - # Reload if this device's agent just connected + # Reload agent info if this device's agent just connected if socket.assigns.agent_info.agent_token_id == agent_token_id do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_base_data(socket)} else {:noreply, socket} end @@ -177,9 +178,9 @@ defmodule ToweropsWeb.DeviceLive.Show do @impl true def handle_info({:agent_disconnected, agent_token_id, _organization_id}, socket) do - # Reload if this device's agent just disconnected + # Reload agent info if this device's agent just disconnected if socket.assigns.agent_info.agent_token_id == agent_token_id do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_base_data(socket)} else {:noreply, socket} end @@ -187,9 +188,9 @@ defmodule ToweropsWeb.DeviceLive.Show do @impl true def handle_info({:agent_heartbeat, agent_token_id, _organization_id}, socket) do - # Reload if this device's agent sent a heartbeat (important for stale agents coming back) + # Reload agent info if this device's agent sent a heartbeat if socket.assigns.agent_info.agent_token_id == agent_token_id do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_base_data(socket)} else {:noreply, socket} end @@ -197,11 +198,11 @@ defmodule ToweropsWeb.DeviceLive.Show do @impl true def handle_info({:agents_stale, stale_agents}, socket) do - # Reload if this device's agent went stale + # Reload agent info if this device's agent went stale device_agent_id = socket.assigns.agent_info.agent_token_id if device_agent_id && Enum.any?(stale_agents, &(&1.id == device_agent_id)) do - {:noreply, load_equipment_data(socket, socket.assigns.device.id)} + {:noreply, reload_base_data(socket)} else {:noreply, socket} end @@ -220,21 +221,111 @@ defmodule ToweropsWeb.DeviceLive.Show do :ok end + # -- Full data load (initial page load and discovery_completed) -- + + # Loads all data for every tab. Called from handle_params (initial load) + # and discovery_completed (structural change requiring full refresh). defp load_equipment_data(socket, device_id) do + socket + |> assign_base_data(device_id) + |> assign_tab_nav_data(device_id) + |> assign_overview_data(device_id) + |> assign_ports_data() + |> assign_logs_data(device_id) + |> assign_backups_data(device_id) + end + + # -- Selective reload helpers (used by handle_info handlers) -- + + # Reload only base data (device, agent_info, snmp_device). + # Used when device status or agent state changes - these show in the header on all tabs. + defp reload_base_data(socket) do + assign_base_data(socket, socket.assigns.device.id) + end + + # Reload base data + data for the currently active tab only. + # Used by the periodic refresh timer. + defp reload_active_tab_data(socket) do + device_id = socket.assigns.device.id + + socket + |> assign_base_data(device_id) + |> assign_tab_nav_data(device_id) + |> reload_current_tab_data(device_id) + end + + # Reload data only if the active tab is one of the relevant tabs. + # Otherwise does nothing - the event is irrelevant to what the user sees. + defp reload_if_tab(socket, relevant_tabs) do + if socket.assigns.active_tab in relevant_tabs do + reload_current_tab_data(socket, socket.assigns.device.id) + else + socket + end + end + + # Dispatch to the appropriate tab-specific loader based on active_tab. + defp reload_current_tab_data(socket, device_id) do + case socket.assigns.active_tab do + "overview" -> assign_overview_data(socket, device_id) + "ports" -> assign_ports_data(socket) + "logs" -> assign_logs_data(socket, device_id) + "backups" -> assign_backups_data(socket, device_id) + # neighbors, arp, mac, vlans, ip_addresses, debug use data from tab_nav/base + _ -> socket + end + end + + # -- Focused data loaders -- + + # Base data needed by all tabs: device record, agent info, SNMP device/sensors/interfaces. + defp assign_base_data(socket, device_id) do device = Devices.get_device!(device_id) device_with_associations = Repo.preload(device, [:organization, site: [organization: :default_agent_token]]) {agent_token_id, agent_source} = Agents.get_effective_agent_token_with_source(device_with_associations) agent_info = load_agent_info(agent_token_id, agent_source) - recent_checks = Monitoring.list_devices_checks(device_id, 50) snmp_data = load_snmp_data(device_id) - events = Devices.list_devices_events(device_id, 100) + + socket + |> assign(:page_title, device.name) + |> assign(:timezone, socket.assigns.current_scope.timezone) + |> assign(:device, device) + |> assign(:agent_info, agent_info) + |> assign(:snmp_device, snmp_data.device) + |> assign(:snmp_interfaces, snmp_data.interfaces) + |> assign(:snmp_sensors, snmp_data.sensors) + end + + # Data needed for the tab navigation bar to decide which tabs to show. + # Neighbors, ARP, MAC, VLANs, and IP addresses determine tab visibility. + defp assign_tab_nav_data(socket, device_id) do + snmp_device = socket.assigns.snmp_device neighbors = Snmp.list_neighbors_with_equipment(device_id) arp_entries = Snmp.list_arp_entries(device_id) mac_addresses = Snmp.list_mac_addresses(device_id) - vlans = load_vlans(snmp_data.device) - ip_addresses = load_ip_addresses(snmp_data.device) - processors = load_processors(snmp_data.device) - storage = load_storage(snmp_data.device) + vlans = load_vlans(snmp_device) + ip_addresses = load_ip_addresses(snmp_device) + + socket + |> assign(:neighbors, neighbors) + |> assign(:arp_entries, arp_entries) + |> assign(:mac_addresses, mac_addresses) + |> assign(:vlans, vlans) + |> assign(:ipv4_addresses, Enum.filter(ip_addresses, &(&1.ip_type == "ipv4"))) + |> assign(:ipv6_addresses, Enum.filter(ip_addresses, &(&1.ip_type == "ipv6"))) + |> assign(:ip_addresses_by_interface, group_ip_addresses_by_interface(ip_addresses)) + end + + # Overview tab: charts, sensor classifications, metrics, processors, storage, firmware. + # This is the most expensive loader due to time-series chart data queries. + defp assign_overview_data(socket, device_id) do + device = socket.assigns.device + snmp_device = socket.assigns.snmp_device + sensors = socket.assigns.snmp_sensors + + recent_checks = Monitoring.list_devices_checks(device_id, 50) + processors = load_processors(snmp_device) + storage = load_storage(snmp_device) latency_chart_data = load_latency_chart_data(device_id) processor_chart_data = load_processor_chart_data(processors) memory_chart_data = load_sensor_chart_data(device_id, ["memory_usage"]) @@ -242,7 +333,6 @@ defmodule ToweropsWeb.DeviceLive.Show do storage_volume_chart_data = load_storage_volume_chart_data(storage) traffic_chart_data = load_overall_traffic_chart_data(device_id) - # Wireless sensor chart data wireless_chart_data = load_sensor_chart_data(device_id, [ "frequency", @@ -257,81 +347,35 @@ defmodule ToweropsWeb.DeviceLive.Show do "utilization" ]) - # Filter sensors by type for temperature, voltage, storage, count, and transceivers - # Separate transceiver sensors from general sensors + # Classify sensors by type {transceiver_sensors, non_transceiver_sensors} = - Enum.split_with(snmp_data.sensors, fn sensor -> + Enum.split_with(sensors, fn sensor -> sensor.sensor_descr && String.contains?(String.downcase(sensor.sensor_descr), "sfp") end) - temperature_sensors = - Enum.filter(non_transceiver_sensors, &temperature_sensor?/1) - + temperature_sensors = Enum.filter(non_transceiver_sensors, &temperature_sensor?/1) voltage_sensors = Enum.filter(non_transceiver_sensors, &voltage_sensor?/1) - current_sensors = Enum.filter(non_transceiver_sensors, ¤t_sensor?/1) - power_sensors = Enum.filter(non_transceiver_sensors, &power_sensor?/1) - fan_sensors = Enum.filter(non_transceiver_sensors, &fan_sensor?/1) - load_sensors = Enum.filter(non_transceiver_sensors, &load_sensor?/1) - storage_sensors = Enum.filter(non_transceiver_sensors, &(&1.sensor_type in ["disk_usage"])) - - # Include various counter/gauge sensor types in the Counters section count_sensors = Enum.filter(non_transceiver_sensors, &counter_sensor?/1) - # Split count sensors into firewall and general {firewall_counters, general_counters} = Enum.split_with(count_sensors, &firewall_counter?/1) - # Wireless sensors (frequency, power, rssi, ccq, etc.) wireless_sensors = Enum.filter(non_transceiver_sensors, &wireless_sensor?/1) - - # Signal quality sensors (SNR, RSSI, RSRP, etc.) signal_sensors = Enum.filter(non_transceiver_sensors, &signal_sensor?/1) - - # Group transceiver sensors by transceiver name grouped_transceivers = group_transceiver_sensors(transceiver_sensors) - - # Calculate metrics for dashboard metrics = calculate_metrics(recent_checks, device) - - # Group interfaces by type for the ports tab - interfaces_by_type = group_interfaces_by_type(snmp_data.interfaces) - - # Check for available firmware updates - available_firmware = get_available_firmware(snmp_data.device) - - # Load MikroTik backups if device has MikroTik API enabled - mikrotik_backups = - if device.mikrotik_enabled do - MikrotikBackups.list_device_backups(device_id, limit: 50) - else - [] - end + available_firmware = get_available_firmware(snmp_device) socket - |> assign(:page_title, device.name) - |> assign(:timezone, socket.assigns.current_scope.timezone) - |> assign(:device, device) |> assign(:recent_checks, recent_checks) |> assign(:metrics, metrics) - |> assign(:snmp_device, snmp_data.device) - |> assign(:snmp_interfaces, snmp_data.interfaces) - |> assign(:interfaces_by_type, interfaces_by_type) - |> assign(:snmp_sensors, snmp_data.sensors) - |> assign(:neighbors, neighbors) - |> assign(:arp_entries, arp_entries) - |> assign(:mac_addresses, mac_addresses) - |> assign(:vlans, vlans) - |> assign(:ipv4_addresses, Enum.filter(ip_addresses, &(&1.ip_type == "ipv4"))) - |> assign(:ipv6_addresses, Enum.filter(ip_addresses, &(&1.ip_type == "ipv6"))) - |> assign(:ip_addresses_by_interface, group_ip_addresses_by_interface(ip_addresses)) |> assign(:processors, processors) |> assign(:storage, storage) - |> assign(:events, events) |> assign(:latency_chart_data, latency_chart_data) |> assign(:processor_chart_data, processor_chart_data) |> assign(:memory_chart_data, memory_chart_data) @@ -352,8 +396,36 @@ defmodule ToweropsWeb.DeviceLive.Show do |> assign(:wireless_sensors, wireless_sensors) |> assign(:signal_sensors, signal_sensors) |> assign(:grouped_transceivers, grouped_transceivers) - |> assign(:agent_info, agent_info) |> assign(:available_firmware, available_firmware) + end + + # Ports tab: interfaces grouped by type. + # Uses snmp_interfaces already loaded in base data. + defp assign_ports_data(socket) do + interfaces = socket.assigns.snmp_interfaces + interfaces_by_type = group_interfaces_by_type(interfaces) + + assign(socket, :interfaces_by_type, interfaces_by_type) + end + + # Logs/events tab data. + defp assign_logs_data(socket, device_id) do + events = Devices.list_devices_events(device_id, 100) + assign(socket, :events, events) + end + + # Backups tab data. + defp assign_backups_data(socket, device_id) do + device = socket.assigns.device + + mikrotik_backups = + if device.mikrotik_enabled do + MikrotikBackups.list_device_backups(device_id, limit: 50) + else + [] + end + + socket |> assign(:mikrotik_backups, mikrotik_backups) |> assign(:selected_backup_ids, MapSet.new()) end diff --git a/priv/repo/migrations/20260212155105_add_performance_indexes.exs b/priv/repo/migrations/20260212155105_add_performance_indexes.exs new file mode 100644 index 00000000..77febe4d --- /dev/null +++ b/priv/repo/migrations/20260212155105_add_performance_indexes.exs @@ -0,0 +1,39 @@ +defmodule Towerops.Repo.Migrations.AddPerformanceIndexes do + use Ecto.Migration + + @disable_ddl_transaction true + @disable_migration_lock true + + def change do + # Partial index for monitored sensors (WHERE monitored = true) + # Used by Snmp.list_monitored_sensors/1 + create index(:snmp_sensors, [:monitored], + where: "monitored = true", + name: :snmp_sensors_monitored_true_idx, + concurrently: true + ) + + # Partial index for monitored interfaces (WHERE monitored = true) + # Used by Snmp.list_monitored_interfaces/1 + create index(:snmp_interfaces, [:monitored], + where: "monitored = true", + name: :snmp_interfaces_monitored_true_idx, + concurrently: true + ) + + # Composite index for devices filtered by org + status + # Used by Devices.list_organization_devices/2 with status filter + create index(:devices, [:organization_id, :status], + name: :devices_organization_id_status_idx, + concurrently: true + ) + + # Partial index for active (unresolved) alerts + # Used by Alerts.list_organization_active_alerts/1 + create index(:alerts, [:resolved_at], + where: "resolved_at IS NULL", + name: :alerts_active_idx, + concurrently: true + ) + end +end diff --git a/test/towerops/devices/device_status_counts_test.exs b/test/towerops/devices/device_status_counts_test.exs new file mode 100644 index 00000000..7cbc93da --- /dev/null +++ b/test/towerops/devices/device_status_counts_test.exs @@ -0,0 +1,129 @@ +defmodule Towerops.Devices.DeviceStatusCountsTest do + use Towerops.DataCase, async: true + + import Towerops.AccountsFixtures + + alias Towerops.Devices + + setup do + user = user_fixture() + {:ok, organization} = Towerops.Organizations.create_organization(%{name: "Test Org"}, user.id) + + {:ok, site} = + Towerops.Sites.create_site(%{ + name: "Test Site", + organization_id: organization.id + }) + + %{organization: organization, site: site} + end + + describe "get_device_status_counts/1" do + test "returns empty map when no devices exist", %{organization: organization} do + assert Devices.get_device_status_counts(organization.id) == %{} + end + + test "returns correct counts for mixed statuses", %{organization: organization, site: site} do + # Create devices and set their statuses + for i <- 1..3 do + {:ok, device} = + Devices.create_device(%{ + name: "Up Device #{i}", + ip_address: "192.168.1.#{i}", + site_id: site.id, + organization_id: organization.id + }) + + Devices.update_device_status(device, :up) + end + + for i <- 1..2 do + {:ok, device} = + Devices.create_device(%{ + name: "Down Device #{i}", + ip_address: "192.168.2.#{i}", + site_id: site.id, + organization_id: organization.id + }) + + Devices.update_device_status(device, :down) + end + + {:ok, _unknown_device} = + Devices.create_device(%{ + name: "Unknown Device", + ip_address: "192.168.3.1", + site_id: site.id, + organization_id: organization.id + }) + + # Unknown is the default status, so no need to update + + counts = Devices.get_device_status_counts(organization.id) + + assert counts == %{up: 3, down: 2, unknown: 1} + end + + test "only counts devices for the given organization", %{organization: organization, site: site} do + # Create device in our org + {:ok, device} = + Devices.create_device(%{ + name: "Our Device", + ip_address: "192.168.1.1", + site_id: site.id, + organization_id: organization.id + }) + + Devices.update_device_status(device, :up) + + # Create device in a different org + other_user = user_fixture() + {:ok, other_org} = Towerops.Organizations.create_organization(%{name: "Other Org"}, other_user.id) + + {:ok, other_site} = + Towerops.Sites.create_site(%{ + name: "Other Site", + organization_id: other_org.id + }) + + {:ok, other_device} = + Devices.create_device(%{ + name: "Other Device", + ip_address: "192.168.1.1", + site_id: other_site.id, + organization_id: other_org.id + }) + + Devices.update_device_status(other_device, :down) + + # Our org should only see its own device + counts = Devices.get_device_status_counts(organization.id) + assert counts == %{up: 1} + + # Other org should only see its own device + other_counts = Devices.get_device_status_counts(other_org.id) + assert other_counts == %{down: 1} + end + + test "returns correct counts after status changes", %{organization: organization, site: site} do + {:ok, device} = + Devices.create_device(%{ + name: "Device", + ip_address: "192.168.1.1", + site_id: site.id, + organization_id: organization.id + }) + + # Initially unknown + assert Devices.get_device_status_counts(organization.id) == %{unknown: 1} + + # Change to up + {:ok, device} = Devices.update_device_status(device, :up) + assert Devices.get_device_status_counts(organization.id) == %{up: 1} + + # Change to down + {:ok, _device} = Devices.update_device_status(device, :down) + assert Devices.get_device_status_counts(organization.id) == %{down: 1} + end + end +end diff --git a/test/towerops/snmp/batch_insert_test.exs b/test/towerops/snmp/batch_insert_test.exs new file mode 100644 index 00000000..335d5e3b --- /dev/null +++ b/test/towerops/snmp/batch_insert_test.exs @@ -0,0 +1,459 @@ +defmodule Towerops.Snmp.BatchInsertTest do + use Towerops.DataCase, async: true + + import Towerops.AccountsFixtures + + alias Towerops.Repo + alias Towerops.Snmp + alias Towerops.Snmp.Device + alias Towerops.Snmp.Interface + alias Towerops.Snmp.InterfaceStat + alias Towerops.Snmp.Processor + alias Towerops.Snmp.ProcessorReading + alias Towerops.Snmp.Sensor + alias Towerops.Snmp.SensorReading + alias Towerops.Snmp.Storage + alias Towerops.Snmp.StorageReading + + setup do + user = user_fixture() + {:ok, organization} = Towerops.Organizations.create_organization(%{name: "Test Org"}, user.id) + + {:ok, site} = + Towerops.Sites.create_site(%{ + name: "Test Site", + organization_id: organization.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Test Router", + ip_address: "192.168.1.1", + snmp_enabled: true, + snmp_version: "2c", + snmp_community: "public", + snmp_port: 161, + site_id: site.id, + organization_id: organization.id + }) + + snmp_device = + %Device{} + |> Device.changeset(%{ + device_id: device.id, + sys_name: "test-router", + sys_descr: "Test Device" + }) + |> Repo.insert!() + + sensor = + %Sensor{} + |> Sensor.changeset(%{ + snmp_device_id: snmp_device.id, + sensor_type: "temperature", + sensor_index: "temp_1", + sensor_oid: "1.3.6.1.4.1.9.9.91.1.1.1.1.4.1" + }) + |> Repo.insert!() + + sensor2 = + %Sensor{} + |> Sensor.changeset(%{ + snmp_device_id: snmp_device.id, + sensor_type: "voltage", + sensor_index: "volt_1", + sensor_oid: "1.3.6.1.4.1.9.9.91.1.1.1.1.4.2" + }) + |> Repo.insert!() + + interface = + %Interface{} + |> Interface.changeset(%{ + snmp_device_id: snmp_device.id, + if_index: 1, + if_name: "eth0" + }) + |> Repo.insert!() + + interface2 = + %Interface{} + |> Interface.changeset(%{ + snmp_device_id: snmp_device.id, + if_index: 2, + if_name: "eth1" + }) + |> Repo.insert!() + + processor = + %Processor{} + |> Processor.changeset(%{ + snmp_device_id: snmp_device.id, + processor_index: "1", + processor_type: "hr_processor", + description: "CPU 1" + }) + |> Repo.insert!() + + processor2 = + %Processor{} + |> Processor.changeset(%{ + snmp_device_id: snmp_device.id, + processor_index: "2", + processor_type: "hr_processor", + description: "CPU 2" + }) + |> Repo.insert!() + + storage = + %Storage{} + |> Storage.changeset(%{ + snmp_device_id: snmp_device.id, + storage_index: 1, + storage_type: "fixed_disk", + description: "/dev/sda1" + }) + |> Repo.insert!() + + storage2 = + %Storage{} + |> Storage.changeset(%{ + snmp_device_id: snmp_device.id, + storage_index: 2, + storage_type: "ram", + description: "Physical Memory" + }) + |> Repo.insert!() + + %{ + snmp_device: snmp_device, + sensor: sensor, + sensor2: sensor2, + interface: interface, + interface2: interface2, + processor: processor, + processor2: processor2, + storage: storage, + storage2: storage2 + } + end + + describe "create_sensor_readings_batch/1" do + test "inserts multiple sensor readings", %{sensor: sensor, sensor2: sensor2} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{sensor_id: sensor.id, value: 25.5, status: "ok", checked_at: now}, + %{sensor_id: sensor2.id, value: 3.3, status: "warning", checked_at: now} + ] + + assert {2, nil} = Snmp.create_sensor_readings_batch(entries) + + readings = Repo.all(SensorReading) + assert length(readings) == 2 + + reading1 = Enum.find(readings, &(&1.sensor_id == sensor.id)) + assert reading1.value == 25.5 + assert reading1.status == "ok" + assert reading1.checked_at == now + assert reading1.id + assert reading1.inserted_at + + reading2 = Enum.find(readings, &(&1.sensor_id == sensor2.id)) + assert reading2.value == 3.3 + assert reading2.status == "warning" + end + + test "returns {0, nil} for empty list" do + assert {0, nil} = Snmp.create_sensor_readings_batch([]) + assert Repo.all(SensorReading) == [] + end + + test "generates unique UUIDs for each entry", %{sensor: sensor} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{sensor_id: sensor.id, value: 10.0, status: "ok", checked_at: now}, + %{sensor_id: sensor.id, value: 11.0, status: "ok", checked_at: now} + ] + + {2, nil} = Snmp.create_sensor_readings_batch(entries) + + readings = Repo.all(SensorReading) + ids = Enum.map(readings, & &1.id) + assert length(Enum.uniq(ids)) == 2 + end + + test "handles optional state_descr field", %{sensor: sensor} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{ + sensor_id: sensor.id, + value: 25.5, + status: "ok", + state_descr: "Normal operating temperature", + checked_at: now + } + ] + + assert {1, nil} = Snmp.create_sensor_readings_batch(entries) + + [reading] = Repo.all(SensorReading) + assert reading.state_descr == "Normal operating temperature" + end + + test "handles entries with nil value", %{sensor: sensor} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{sensor_id: sensor.id, value: nil, status: "error", checked_at: now} + ] + + assert {1, nil} = Snmp.create_sensor_readings_batch(entries) + + [reading] = Repo.all(SensorReading) + assert reading.value == nil + assert reading.status == "error" + end + end + + describe "create_interface_stats_batch/1" do + test "inserts multiple interface stats", %{interface: interface, interface2: interface2} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{ + interface_id: interface.id, + if_in_octets: 1000, + if_out_octets: 2000, + if_in_errors: 0, + if_out_errors: 0, + if_in_discards: 0, + if_out_discards: 0, + checked_at: now + }, + %{ + interface_id: interface2.id, + if_in_octets: 5000, + if_out_octets: 6000, + if_in_errors: 1, + if_out_errors: 2, + if_in_discards: 3, + if_out_discards: 4, + checked_at: now + } + ] + + assert {2, nil} = Snmp.create_interface_stats_batch(entries) + + stats = Repo.all(InterfaceStat) + assert length(stats) == 2 + + stat1 = Enum.find(stats, &(&1.interface_id == interface.id)) + assert stat1.if_in_octets == 1000 + assert stat1.if_out_octets == 2000 + assert stat1.if_in_errors == 0 + assert stat1.if_out_errors == 0 + assert stat1.if_in_discards == 0 + assert stat1.if_out_discards == 0 + assert stat1.checked_at == now + assert stat1.id + assert stat1.inserted_at + + stat2 = Enum.find(stats, &(&1.interface_id == interface2.id)) + assert stat2.if_in_octets == 5000 + assert stat2.if_out_octets == 6000 + assert stat2.if_in_errors == 1 + assert stat2.if_out_errors == 2 + end + + test "returns {0, nil} for empty list" do + assert {0, nil} = Snmp.create_interface_stats_batch([]) + assert Repo.all(InterfaceStat) == [] + end + + test "generates unique UUIDs for each entry", %{interface: interface} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{interface_id: interface.id, if_in_octets: 100, if_out_octets: 200, checked_at: now}, + %{interface_id: interface.id, if_in_octets: 300, if_out_octets: 400, checked_at: now} + ] + + {2, nil} = Snmp.create_interface_stats_batch(entries) + + stats = Repo.all(InterfaceStat) + ids = Enum.map(stats, & &1.id) + assert length(Enum.uniq(ids)) == 2 + end + + test "handles entries with nil counter values", %{interface: interface} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{interface_id: interface.id, if_in_octets: nil, if_out_octets: nil, checked_at: now} + ] + + assert {1, nil} = Snmp.create_interface_stats_batch(entries) + + [stat] = Repo.all(InterfaceStat) + assert stat.if_in_octets == nil + assert stat.if_out_octets == nil + end + end + + describe "create_processor_readings_batch/1" do + test "inserts multiple processor readings", %{processor: processor, processor2: processor2} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{processor_id: processor.id, load_percent: 45.0, status: "ok", checked_at: now}, + %{processor_id: processor2.id, load_percent: 92.0, status: "critical", checked_at: now} + ] + + assert {2, nil} = Snmp.create_processor_readings_batch(entries) + + readings = Repo.all(ProcessorReading) + assert length(readings) == 2 + + reading1 = Enum.find(readings, &(&1.processor_id == processor.id)) + assert reading1.load_percent == 45.0 + assert reading1.status == "ok" + assert reading1.checked_at == now + assert reading1.id + assert reading1.inserted_at + + reading2 = Enum.find(readings, &(&1.processor_id == processor2.id)) + assert reading2.load_percent == 92.0 + assert reading2.status == "critical" + end + + test "returns {0, nil} for empty list" do + assert {0, nil} = Snmp.create_processor_readings_batch([]) + assert Repo.all(ProcessorReading) == [] + end + + test "generates unique UUIDs for each entry", %{processor: processor} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{processor_id: processor.id, load_percent: 10.0, status: "ok", checked_at: now}, + %{processor_id: processor.id, load_percent: 20.0, status: "ok", checked_at: now} + ] + + {2, nil} = Snmp.create_processor_readings_batch(entries) + + readings = Repo.all(ProcessorReading) + ids = Enum.map(readings, & &1.id) + assert length(Enum.uniq(ids)) == 2 + end + + test "handles entries with nil load_percent", %{processor: processor} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{processor_id: processor.id, load_percent: nil, status: "error", checked_at: now} + ] + + assert {1, nil} = Snmp.create_processor_readings_batch(entries) + + [reading] = Repo.all(ProcessorReading) + assert reading.load_percent == nil + assert reading.status == "error" + end + end + + describe "create_storage_readings_batch/1" do + test "inserts multiple storage readings", %{storage: storage, storage2: storage2} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{ + storage_id: storage.id, + used_bytes: 5_000_000, + total_bytes: 10_000_000, + usage_percent: 50.0, + checked_at: now + }, + %{ + storage_id: storage2.id, + used_bytes: 2_000_000, + total_bytes: 8_000_000, + usage_percent: 25.0, + checked_at: now + } + ] + + assert {2, nil} = Snmp.create_storage_readings_batch(entries) + + readings = Repo.all(StorageReading) + assert length(readings) == 2 + + reading1 = Enum.find(readings, &(&1.storage_id == storage.id)) + assert reading1.used_bytes == 5_000_000 + assert reading1.total_bytes == 10_000_000 + assert reading1.usage_percent == 50.0 + assert reading1.checked_at == now + assert reading1.id + assert reading1.inserted_at + + reading2 = Enum.find(readings, &(&1.storage_id == storage2.id)) + assert reading2.used_bytes == 2_000_000 + assert reading2.total_bytes == 8_000_000 + assert reading2.usage_percent == 25.0 + end + + test "returns {0, nil} for empty list" do + assert {0, nil} = Snmp.create_storage_readings_batch([]) + assert Repo.all(StorageReading) == [] + end + + test "generates unique UUIDs for each entry", %{storage: storage} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{ + storage_id: storage.id, + used_bytes: 1000, + total_bytes: 2000, + usage_percent: 50.0, + checked_at: now + }, + %{ + storage_id: storage.id, + used_bytes: 1500, + total_bytes: 2000, + usage_percent: 75.0, + checked_at: now + } + ] + + {2, nil} = Snmp.create_storage_readings_batch(entries) + + readings = Repo.all(StorageReading) + ids = Enum.map(readings, & &1.id) + assert length(Enum.uniq(ids)) == 2 + end + + test "handles entries with nil optional fields", %{storage: storage} do + now = DateTime.truncate(DateTime.utc_now(), :second) + + entries = [ + %{ + storage_id: storage.id, + used_bytes: nil, + total_bytes: nil, + usage_percent: nil, + checked_at: now + } + ] + + assert {1, nil} = Snmp.create_storage_readings_batch(entries) + + [reading] = Repo.all(StorageReading) + assert reading.used_bytes == nil + assert reading.total_bytes == nil + assert reading.usage_percent == nil + end + end +end