towerops/lib/towerops/topology.ex
Graham McIntire 7371ceb942
feat: add automatic network topology inference
Build a rich network topology from SNMP polling data using evidence-based
confidence scoring. LLDP/CDP neighbors, MAC address tables, and ARP data
are combined to infer device links with weighted confidence merging.

- Add DeviceLink and DeviceLinkEvidence schemas for persistent topology
- Implement evidence collectors: LLDP (0.95), CDP (0.95), MAC (0.7), ARP (0.6)
- Add device role inference from sysObjectID/sysDescr patterns
- Hook topology inference into DevicePollerWorker pipeline
- Add stale link cleanup (24h mark stale, 72h delete) via NeighborCleanupWorker
- Update NetworkMapLive with "Added" vs "All Devices" tabs
- Add connected devices section to device detail page
- Add device role selector to device edit form
- Update Cytoscape.js with role-based node shapes/colors and confidence edges
2026-02-12 13:28:01 -06:00

658 lines
20 KiB
Elixir

defmodule Towerops.Topology do
@moduledoc """
Topology inference engine. Builds and maintains a persistent model of
network device relationships by analyzing SNMP polling data.
"""
import Ecto.Query
alias Towerops.Devices.Device
alias Towerops.Repo
alias Towerops.Snmp.ArpEntry
alias Towerops.Snmp.Device, as: SnmpDevice
alias Towerops.Snmp.Interface
alias Towerops.Snmp.MacAddress
alias Towerops.Snmp.Neighbor
alias Towerops.Topology.DeviceLink
alias Towerops.Topology.DeviceLinkEvidence
require Logger
@ups_vendors ~w(apc cyberpower eaton)
@firewall_vendors ~w(fortinet fortigate paloalto pfsense sonicwall sophos)
@server_platforms ~w(linux windows vmware esxi proxmox)
@doc """
Builds lookup maps for matching evidence to managed devices in an organization.
Returns a map with `:by_ip`, `:by_name`, and `:by_mac` keys, each mapping
an identifier string to a device ID.
"""
def build_device_lookup(organization_id) do
devices =
Repo.all(
from(d in Device,
where: d.organization_id == ^organization_id,
left_join: sd in assoc(d, :snmp_device),
left_join: i in Interface,
on: sd.id == i.snmp_device_id,
preload: [snmp_device: {sd, interfaces: i}]
)
)
by_ip = Map.new(devices, fn d -> {d.ip_address, d.id} end)
by_name =
devices
|> Enum.filter(&(&1.name != nil))
|> Map.new(&{String.downcase(&1.name), &1.id})
by_mac =
devices
|> Enum.flat_map(fn device ->
case device.snmp_device do
nil ->
[]
sd ->
sd.interfaces
|> Enum.filter(&(&1.if_phys_address != nil))
|> Enum.map(&{&1.if_phys_address, device.id})
end
end)
|> Map.new()
%{by_ip: by_ip, by_name: by_name, by_mac: by_mac}
end
@doc """
Finds a device ID by IP address from the lookup.
"""
def find_device_by_ip(lookup, ip), do: Map.get(lookup.by_ip, ip)
@doc """
Finds a device ID by name (case-insensitive) from the lookup.
"""
def find_device_by_name(lookup, name) when is_binary(name), do: Map.get(lookup.by_name, String.downcase(name))
def find_device_by_name(_lookup, _name), do: nil
@doc """
Finds a device ID by MAC address from the lookup.
"""
def find_device_by_mac(lookup, mac), do: Map.get(lookup.by_mac, mac)
@doc """
Resolves a device ID from evidence fields, trying MAC first, then IP, then name.
"""
def resolve_device(lookup, %{mac: mac, ip: ip, name: name}) do
find_device_by_mac(lookup, mac) ||
find_device_by_ip(lookup, ip) ||
find_device_by_name(lookup, name)
end
def resolve_device(_lookup, _), do: nil
@doc """
Collects topology evidence from LLDP neighbor entries for a device.
Returns a list of evidence maps with confidence 0.95.
"""
def collect_lldp_evidence(device_id) do
collect_neighbor_evidence(device_id, "lldp", "lldp_neighbor", 0.95)
end
@doc """
Collects topology evidence from CDP neighbor entries for a device.
Returns a list of evidence maps with confidence 0.95.
"""
def collect_cdp_evidence(device_id) do
collect_neighbor_evidence(device_id, "cdp", "cdp_neighbor", 0.95)
end
defp collect_neighbor_evidence(device_id, protocol, evidence_type, confidence) do
from(n in Neighbor,
join: i in Interface,
on: n.interface_id == i.id,
where: n.device_id == ^device_id and n.protocol == ^protocol,
select: %{
neighbor_id: n.id,
source_interface_id: i.id,
remote_chassis_id: n.remote_chassis_id,
remote_system_name: n.remote_system_name,
remote_address: n.remote_address,
remote_port_id: n.remote_port_id,
remote_capabilities: n.remote_capabilities,
last_discovered_at: n.last_discovered_at
}
)
|> Repo.all()
|> Enum.map(fn row ->
%{
evidence_type: evidence_type,
confidence: confidence,
source_interface_id: row.source_interface_id,
remote_mac: row.remote_chassis_id,
remote_ip: row.remote_address,
remote_name: row.remote_system_name,
remote_port: row.remote_port_id,
remote_capabilities: row.remote_capabilities,
evidence_data: %{
"neighbor_id" => row.neighbor_id,
"remote_chassis_id" => row.remote_chassis_id,
"remote_port_id" => row.remote_port_id
}
}
end)
end
@doc """
Collects topology evidence from MAC address table entries for a device.
Uses the device lookup to match MACs to known devices. Excludes entries
that match the device itself (self-links). Returns evidence maps with
confidence 0.7.
"""
def collect_mac_evidence(device_id, lookup) do
from(m in MacAddress,
where: m.device_id == ^device_id,
select: %{
interface_id: m.interface_id,
mac_address: m.mac_address,
vlan_id: m.vlan_id,
last_seen_at: m.last_seen_at
}
)
|> Repo.all()
|> Enum.map(fn row ->
matched_id = find_device_by_mac(lookup, row.mac_address)
%{
evidence_type: "mac_on_interface",
confidence: 0.7,
source_interface_id: row.interface_id,
remote_mac: row.mac_address,
remote_ip: nil,
remote_name: nil,
remote_port: nil,
remote_capabilities: nil,
matched_device_id: matched_id,
evidence_data: %{
"mac_address" => row.mac_address,
"vlan_id" => row.vlan_id
}
}
end)
|> Enum.reject(fn ev -> ev.matched_device_id == device_id end)
end
@doc """
Collects topology evidence from ARP table entries for a device.
Uses the device lookup to match entries by MAC first, then IP. Excludes
entries that match the device itself (self-links). Returns evidence maps
with confidence 0.6.
"""
def collect_arp_evidence(device_id, lookup) do
from(a in ArpEntry,
where: a.device_id == ^device_id,
select: %{
interface_id: a.interface_id,
ip_address: a.ip_address,
mac_address: a.mac_address,
last_seen_at: a.last_seen_at
}
)
|> Repo.all()
|> Enum.map(fn row ->
matched_id =
find_device_by_mac(lookup, row.mac_address) ||
find_device_by_ip(lookup, row.ip_address)
%{
evidence_type: "arp_entry",
confidence: 0.6,
source_interface_id: row.interface_id,
remote_mac: row.mac_address,
remote_ip: row.ip_address,
remote_name: nil,
remote_port: nil,
remote_capabilities: nil,
matched_device_id: matched_id,
evidence_data: %{
"ip_address" => row.ip_address,
"mac_address" => row.mac_address
}
}
end)
|> Enum.reject(fn ev -> ev.matched_device_id == device_id end)
end
@doc "Merge two independent confidence scores: 1 - (1-a)(1-b)"
def merge_confidence(a, b), do: 1.0 - (1.0 - a) * (1.0 - b)
@doc """
Create or update a device link. Keyed on source_device_id + source_interface_id +
discovered_remote_mac. If existing, merges confidence and appends evidence.
Always updates last_confirmed_at.
"""
def upsert_link(attrs) do
evidence_attrs = Map.get(attrs, :evidence, [])
link_attrs = Map.delete(attrs, :evidence)
existing =
Repo.one(
from(l in DeviceLink,
where:
l.source_device_id == ^link_attrs.source_device_id and
l.source_interface_id == ^link_attrs[:source_interface_id] and
l.discovered_remote_mac == ^link_attrs[:discovered_remote_mac]
)
)
result =
case existing do
nil ->
%DeviceLink{}
|> DeviceLink.changeset(link_attrs)
|> Repo.insert()
link ->
merged_confidence = merge_confidence(link.confidence, link_attrs.confidence)
link
|> DeviceLink.changeset(%{
confidence: merged_confidence,
last_confirmed_at: link_attrs.last_confirmed_at,
target_device_id: link_attrs[:target_device_id] || link.target_device_id,
metadata: Map.merge(link.metadata || %{}, link_attrs[:metadata] || %{})
})
|> Repo.update()
end
with {:ok, link} <- result do
Enum.each(evidence_attrs, fn ev ->
%DeviceLinkEvidence{}
|> DeviceLinkEvidence.changeset(Map.put(ev, :device_link_id, link.id))
|> Repo.insert!()
end)
{:ok, Repo.reload!(link)}
end
end
@doc """
Infer the role of a device from its SNMP data and neighbor reports.
Returns a role string. Does not write to the database.
"""
def infer_device_role(device) do
device = Repo.preload(device, snmp_device: :interfaces)
snmp = device.snmp_device
{manufacturer, sys_descr, iface_count} = extract_snmp_hints(snmp)
capabilities = get_reported_capabilities(snmp)
infer_role_from_evidence(capabilities, iface_count, manufacturer, sys_descr)
end
@doc """
Apply inferred role to a device. Only updates when device_role_source is not "manual".
"""
def maybe_update_device_role(device) do
if device.device_role_source == "manual" do
{:ok, device}
else
role = infer_device_role(device)
apply_inferred_role(device, role)
end
end
@doc """
Main topology inference pipeline for a device. Called after each poll cycle.
Collects evidence from all sources, groups by remote identifier, upserts links,
broadcasts changes. Returns {:ok, :changed} or {:ok, :unchanged}.
"""
def process_device(device, organization_id) do
lookup = build_device_lookup(organization_id)
now = DateTime.utc_now()
# Collect all evidence
all_evidence =
collect_lldp_evidence(device.id) ++
collect_cdp_evidence(device.id) ++
collect_mac_evidence(device.id, lookup) ++
collect_arp_evidence(device.id, lookup)
if Enum.empty?(all_evidence) do
{:ok, :unchanged}
else
grouped = group_evidence_by_remote(all_evidence)
results =
grouped
|> Enum.map(&upsert_grouped_evidence(&1, device.id, lookup, now))
|> Enum.reject(&(&1 == :skip))
has_changes = Enum.any?(results, &match?({:ok, _}, &1))
# Re-evaluate device role after collecting new evidence
maybe_update_device_role(device)
if has_changes do
Phoenix.PubSub.broadcast(
Towerops.PubSub,
"topology:#{organization_id}",
{:topology_updated, organization_id}
)
{:ok, :changed}
else
{:ok, :unchanged}
end
end
end
@doc """
Clean up stale links. Called periodically (e.g. hourly).
- Links not confirmed in 24h: set confidence to 0.1
- Links not confirmed in 72h: delete entirely
Returns {stale_count, deleted_count}.
"""
def cleanup_stale_links do
now = DateTime.utc_now()
stale_cutoff = DateTime.add(now, -24, :hour)
delete_cutoff = DateTime.add(now, -72, :hour)
# Delete very old links first
{deleted_count, _} =
Repo.delete_all(from(l in DeviceLink, where: l.last_confirmed_at < ^delete_cutoff))
# Mark stale links (24h+ but not yet 72h)
{stale_count, _} =
Repo.update_all(
from(l in DeviceLink,
where: l.last_confirmed_at < ^stale_cutoff and l.confidence > 0.1
),
set: [confidence: 0.1]
)
{stale_count, deleted_count}
end
@doc """
Build topology data for the network map. Returns %{nodes, edges, stats, last_updated}.
Tab "added" = managed devices only. Tab "all" = managed + discovered.
"""
def get_topology_for_map(organization_id, tab \\ "added") do
devices = list_managed_devices(organization_id)
device_ids = Enum.map(devices, & &1.id)
links = list_links_for_devices(device_ids)
managed_nodes = Enum.map(devices, &device_to_node/1)
discovered_nodes = build_discovered_nodes(links, tab)
all_nodes = managed_nodes ++ discovered_nodes
node_ids = MapSet.new(Enum.map(all_nodes, & &1.id))
edges = build_edges_from_links(links, node_ids)
%{
nodes: all_nodes,
edges: edges,
stats: %{
total_devices: length(all_nodes),
added_devices: length(managed_nodes),
discovered_devices: length(discovered_nodes),
total_links: length(edges)
},
last_updated: DateTime.utc_now()
}
end
@doc "List all device links where this device is the source, ordered by confidence."
def list_device_links(device_id) do
Repo.all(
from(l in DeviceLink,
where: l.source_device_id == ^device_id,
order_by: [desc: l.confidence]
)
)
end
@doc """
List all devices connected to this device via device_links.
Returns links where the device is either the source or target,
with preloaded associations for display.
"""
def list_connected_devices(device_id) do
Repo.all(
from(l in DeviceLink,
where: l.source_device_id == ^device_id or l.target_device_id == ^device_id,
preload: [:source_device, :target_device, :source_interface, :target_interface],
order_by: [desc: l.confidence]
)
)
end
# --- Network map helpers ---
defp list_managed_devices(organization_id) do
Repo.all(
from(d in Device,
where: d.organization_id == ^organization_id,
left_join: s in assoc(d, :site),
left_join: sd in assoc(d, :snmp_device),
preload: [site: s, snmp_device: sd]
)
)
end
defp list_links_for_devices(device_ids) do
Repo.all(
from(l in DeviceLink,
where: l.source_device_id in ^device_ids,
preload: [:source_interface, :target_interface]
)
)
end
defp device_to_node(device) do
%{
id: device.id,
label: device.name,
type: role_to_type_atom(device.device_role),
device_role: device.device_role,
status: device.status,
site_id: device.site_id,
site_name: if(device.site, do: device.site.name),
ip_address: device.ip_address,
discovered: false,
manufacturer: if(device.snmp_device, do: device.snmp_device.manufacturer)
}
end
defp build_discovered_nodes(links, "all") do
links
|> Enum.filter(&is_nil(&1.target_device_id))
|> Enum.uniq_by(& &1.discovered_remote_mac)
|> Enum.map(fn link ->
%{
id: "discovered_#{link.discovered_remote_mac}",
label: link.discovered_remote_name || link.discovered_remote_mac,
type: :unknown,
device_role: nil,
status: :unknown,
site_id: nil,
site_name: nil,
ip_address: link.discovered_remote_ip,
discovered: true,
manufacturer: nil,
parent_device_id: link.source_device_id
}
end)
end
defp build_discovered_nodes(_links, _tab), do: []
defp build_edges_from_links(links, node_ids) do
links
|> Enum.map(fn link ->
target_id =
if link.target_device_id,
do: link.target_device_id,
else: "discovered_#{link.discovered_remote_mac}"
%{
id: link.id,
source: link.source_device_id,
target: target_id,
source_interface: if(link.source_interface, do: link.source_interface.if_name),
target_interface: if(link.target_interface, do: link.target_interface.if_name),
link_type: link.link_type,
confidence: link.confidence,
metadata: link.metadata
}
end)
|> Enum.filter(fn edge ->
MapSet.member?(node_ids, edge.source) and MapSet.member?(node_ids, edge.target)
end)
end
defp role_to_type_atom("core_router"), do: :router
defp role_to_type_atom("distribution_switch"), do: :switch
defp role_to_type_atom("access_switch"), do: :switch
defp role_to_type_atom("access_point"), do: :wireless
defp role_to_type_atom("cpe"), do: :wireless
defp role_to_type_atom("backhaul_radio"), do: :wireless
defp role_to_type_atom("ups"), do: :server
defp role_to_type_atom("firewall"), do: :firewall
defp role_to_type_atom("server"), do: :server
defp role_to_type_atom(_), do: :unknown
# --- Evidence processing helpers ---
defp upsert_grouped_evidence({_key, evidence_list}, device_id, lookup, now) do
best = Enum.max_by(evidence_list, & &1.confidence)
matched_id =
resolve_device(lookup, %{
mac: best.remote_mac,
ip: best.remote_ip,
name: best.remote_name
})
if matched_id == device_id do
:skip
else
upsert_link(%{
source_device_id: device_id,
target_device_id: matched_id,
source_interface_id: best.source_interface_id,
link_type: evidence_type_to_link_type(best.evidence_type),
confidence: combine_evidence_confidence(evidence_list),
discovered_remote_mac: best.remote_mac,
discovered_remote_ip: best.remote_ip,
discovered_remote_name: best.remote_name,
metadata: %{},
last_confirmed_at: now,
evidence:
Enum.map(evidence_list, fn ev ->
%{
evidence_type: ev.evidence_type,
evidence_data: ev.evidence_data,
observed_at: now
}
end)
})
end
end
defp group_evidence_by_remote(evidence_list) do
Enum.group_by(evidence_list, fn ev ->
ev.remote_mac || ev.remote_ip || ev.remote_name || "unknown"
end)
end
defp combine_evidence_confidence(evidence_list) do
evidence_list
|> Enum.map(& &1.confidence)
|> Enum.reduce(0.0, &merge_confidence/2)
end
defp evidence_type_to_link_type("lldp_neighbor"), do: "lldp"
defp evidence_type_to_link_type("cdp_neighbor"), do: "cdp"
defp evidence_type_to_link_type("mac_on_interface"), do: "mac_match"
defp evidence_type_to_link_type("arp_entry"), do: "arp_inference"
defp evidence_type_to_link_type("wireless_registration"), do: "wireless_association"
defp evidence_type_to_link_type(_), do: "mac_match"
# --- Device role inference helpers ---
defp extract_snmp_hints(nil), do: {"", "", 0}
defp extract_snmp_hints(%SnmpDevice{} = snmp) do
manufacturer = String.downcase(snmp.manufacturer || "")
sys_descr = String.downcase(snmp.sys_descr || "")
iface_count = length(snmp.interfaces)
{manufacturer, sys_descr, iface_count}
end
defp get_reported_capabilities(nil), do: []
defp get_reported_capabilities(%SnmpDevice{} = snmp) do
snmp.interfaces
|> Enum.map(& &1.if_phys_address)
|> Enum.reject(&is_nil/1)
|> fetch_capabilities_for_macs()
end
defp fetch_capabilities_for_macs([]), do: []
defp fetch_capabilities_for_macs(mac_addresses) do
from(n in Neighbor,
where: n.remote_chassis_id in ^mac_addresses,
select: n.remote_capabilities
)
|> Repo.all()
|> List.flatten()
|> Enum.uniq()
end
defp infer_role_from_evidence(capabilities, iface_count, manufacturer, sys_descr) do
infer_from_capabilities(capabilities, iface_count) ||
infer_from_vendor(manufacturer, sys_descr) ||
"unknown"
end
defp infer_from_capabilities(capabilities, iface_count) do
cond do
"wlan-ap" in capabilities -> "access_point"
"router" in capabilities and iface_count >= 10 -> "core_router"
"router" in capabilities -> "distribution_switch"
"bridge" in capabilities -> "access_switch"
true -> nil
end
end
defp infer_from_vendor(manufacturer, sys_descr) do
cond do
vendor_match?(manufacturer, @ups_vendors) -> "ups"
vendor_match?(manufacturer, @firewall_vendors) -> "firewall"
vendor_match?(sys_descr, @firewall_vendors) -> "firewall"
platform_match?(sys_descr, @server_platforms) -> "server"
true -> nil
end
end
defp apply_inferred_role(device, role) when role == device.device_role, do: {:ok, device}
defp apply_inferred_role(device, role) do
device
|> Ecto.Changeset.change(%{device_role: role, device_role_source: "inferred"})
|> Repo.update()
end
defp vendor_match?(value, vendors), do: Enum.any?(vendors, &String.contains?(value, &1))
defp platform_match?(description, platforms), do: Enum.any?(platforms, &String.contains?(description, &1))
end