graham/towerops
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: Add comprehensive agent polling for all discoverable data

Browse source 
Extended agent polling to collect all available SNMP data during regular
polls, not just sensors and interfaces. This ensures data stays current
and reduces reliance on periodic discovery runs.

Changes:
- Added polling queries for: neighbors (LLDP/CDP), ARP tables, MAC tables,
  IP addresses, HOST-RESOURCES processors and storage
- Process additional data using Replay adapter pattern to reuse existing
  parsing logic from discovery modules
- Made discovery helper functions public for reuse in polling context:
  save_neighbors/2, save_arp_entries/3, sync_ip_addresses/2,
  sync_processors/2, sync_storage/2
- Optimized by fetching interfaces once and reusing across operations
- Fixed compilation warnings by using proper guard syntax (list != [])

Technical details:
- Detection threshold: polls with more OIDs than sensors+interfaces
  are processed for additional data
- Background processing: spawns async process to avoid blocking metric
  recording
- Replay adapter: allows reusing discovery parsing logic without
  re-querying SNMP device
This commit is contained in:
Graham McIntire 2026-02-06 11:18:36 -06:00 • committed by Graham McIntire
parent 5f0549908e
commit bf4fee4846
No known key found for this signature in database
2 changed files with 139 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
lib/towerops/snmp/discovery.ex
View file

@ -825,7 +825,10 @@ defmodule Towerops.Snmp.Discovery do
end
@spec sync_ip_addresses(Device.t(), [map()]) :: :ok
defp sync_ip_addresses(device, discovered_ip_addresses) do
@doc """
Sync IP addresses from discovery/polling results to database.
"""
def sync_ip_addresses(device, discovered_ip_addresses) do
# Build a map of if_index to interface ID from the device's interfaces
if_index_to_interface =
Map.new(device.interfaces, fn iface -> {iface.if_index, iface.id} end)
@ -912,7 +915,10 @@ defmodule Towerops.Snmp.Discovery do
end
@spec sync_processors(Device.t(), [map()]) :: :ok
defp sync_processors(device, discovered_processors) do
@doc """
Sync processors from discovery/polling results to database.
"""
def sync_processors(device, discovered_processors) do
# Get existing processors for this device, indexed by processor_index
existing_processors =
from(p in Processor, where: p.snmp_device_id == ^device.id)
@ -959,7 +965,10 @@ defmodule Towerops.Snmp.Discovery do
end
@spec sync_storage(Device.t(), [map()]) :: :ok
defp sync_storage(device, discovered_storage) do
@doc """
Sync storage from discovery/polling results to database.
"""
def sync_storage(device, discovered_storage) do
# Get existing storage for this device, indexed by storage_index
existing_storage =
from(s in Storage, where: s.snmp_device_id == ^device.id)
@ -1024,7 +1033,10 @@ defmodule Towerops.Snmp.Discovery do
end
@spec save_neighbors(binary(), [map()]) :: :ok
defp save_neighbors(device_id, neighbors) do
@doc """
Save discovered neighbors to database.
"""
def save_neighbors(device_id, neighbors) do
# Delete stale neighbors (not seen in last 5 minutes)
cutoff = DateTime.add(DateTime.utc_now(), -5, :minute)
Towerops.Snmp.delete_stale_neighbors(device_id, cutoff)
@ -1039,7 +1051,10 @@ defmodule Towerops.Snmp.Discovery do
end
@spec save_arp_entries(binary(), [map()], [Interface.t()]) :: :ok
defp save_arp_entries(device_id, arp_entries, interfaces) do
@doc """
Save discovered ARP entries to database.
"""
def save_arp_entries(device_id, arp_entries, interfaces) do
# Delete stale ARP entries (not seen in last 30 minutes)
cutoff = DateTime.add(DateTime.utc_now(), -30, :minute)
Towerops.Snmp.delete_stale_arp_entries(device_id, cutoff)

125
lib/towerops_web/channels/agent_channel.ex
View file

@ -719,6 +719,36 @@ defmodule ToweropsWeb.AgentChannel do
"1.3.6.1.4.1.9.9.23"
]
# ARP table OIDs
arp_oids = [
# IP-MIB ipNetToMediaTable (IPv4 ARP)
"1.3.6.1.2.1.4.22",
# IP-MIB ipNetToPhysicalTable (RFC 4293 - unified IPv4/IPv6)
"1.3.6.1.2.1.4.35"
]
# MAC address table OIDs
mac_oids = [
# BRIDGE-MIB dot1dTpFdbTable
"1.3.6.1.2.1.17.4.3"
]
# IP address OIDs (for polling changes)
ip_address_oids = [
# IP-MIB ipAddrTable (IPv4)
"1.3.6.1.2.1.4.20",
# IP-MIB ipAddressTable (IPv6)
"1.3.6.1.2.1.4.34"
]
# HOST-RESOURCES-MIB for processors and storage
host_resources_oids = [
# hrProcessorTable
"1.3.6.1.2.1.25.3.3",
# hrStorageTable
"1.3.6.1.2.1.25.2.3"
]
[
%SnmpQuery{
query_type: :GET,
@ -726,7 +756,7 @@ defmodule ToweropsWeb.AgentChannel do
},
%SnmpQuery{
query_type: :WALK,
oids: neighbor_oids
oids: neighbor_oids ++ arp_oids ++ mac_oids ++ ip_address_oids ++ host_resources_oids
}
]
end
@ -887,17 +917,100 @@ defmodule ToweropsWeb.AgentChannel do
timestamp = DateTime.from_unix!(result.timestamp, :second)
Logger.info(
"Processing polling result for #{device.name}: #{map_size(oid_values)} OIDs, #{length(snmp_device.sensors)} sensors"
"Processing polling result for #{device.name}: #{map_size(oid_values)} OIDs, #{length(snmp_device.sensors)} sensors, #{length(snmp_device.interfaces)} interfaces"
)
# Process sensor readings
# Process sensor readings (quick path - already in memory)
Enum.each(snmp_device.sensors, &process_sensor_reading(&1, oid_values, timestamp))
# Process interface stats
# Process interface stats (quick path - already in memory)
Enum.each(device.snmp_device.interfaces, &process_interface_stats(&1, oid_values, timestamp))
# Note: Neighbor discovery is handled by separate polling worker
# since it requires complex LLDP/CDP parsing
# Process complex SNMP data (neighbors, ARP, MAC, IP addresses, processors, storage)
# using Replay adapter to reuse existing parsing logic
if map_size(oid_values) > length(snmp_device.sensors) + (length(snmp_device.interfaces) * 6) do
process_additional_polling_data(device, oid_values)
end
end
# Process neighbors, ARP, MAC, IP addresses, processors, storage from agent polling
defp process_additional_polling_data(device, oid_values) do
# Build Replay adapter opts to reuse existing parsing logic
client_opts = [
ip: device.ip_address,
adapter: Towerops.Snmp.Adapters.Replay,
oid_map: oid_values
]
# Reuse existing polling functions with Replay adapter
spawn(fn ->
alias Towerops.Snmp.Profiles.Base
alias Towerops.Snmp.NeighborDiscovery
# Fetch interfaces once for reuse across multiple operations
interfaces = Towerops.Snmp.list_interfaces(device.snmp_device.id)
# Process neighbors (LLDP/CDP)
case NeighborDiscovery.discover_neighbors(client_opts, interfaces) do
{:ok, neighbors} when neighbors != [] ->
Towerops.Snmp.Discovery.save_neighbors(device.id, neighbors)
_ ->
:ok
end
# Process ARP entries
case Towerops.Snmp.ArpDiscovery.discover_arp_table(client_opts) do
{:ok, arp_entries} when arp_entries != [] ->
Towerops.Snmp.Discovery.save_arp_entries(device.id, arp_entries, interfaces)
_ ->
:ok
end
# Process MAC addresses
case Towerops.Snmp.MacDiscovery.discover_mac_table(client_opts) do
{:ok, mac_addresses} when mac_addresses != [] ->
Towerops.Snmp.upsert_mac_addresses(device.id, mac_addresses, interfaces)
_ ->
:ok
end
# Process IP addresses
case Base.discover_all_ip_addresses(client_opts) do
{:ok, ip_addresses} when ip_addresses != [] ->
discovered_device = %{
device_id: device.id,
interfaces: interfaces
}
Towerops.Snmp.Discovery.sync_ip_addresses(discovered_device, ip_addresses)
_ ->
:ok
end
# Process processors
case Base.discover_processors(client_opts) do
{:ok, processors} when processors != [] ->
discovered_device = %{device_id: device.id}
Towerops.Snmp.Discovery.sync_processors(discovered_device, processors)
_ ->
:ok
end
# Process storage
case Base.discover_storage(client_opts) do
{:ok, storage} when storage != [] ->
discovered_device = %{device_id: device.id}
Towerops.Snmp.Discovery.sync_storage(discovered_device, storage)
_ ->
:ok
end
end)
end
defp process_sensor_reading(sensor, oid_values, timestamp) do