diff --git a/config/test.exs b/config/test.exs index 1bbfa887..9b7a9ca6 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,5 +1,7 @@ import Config +alias Ecto.Adapters.SQL.Sandbox + # Only in tests, remove the complexity from the password hashing algorithm config :argon2_elixir, t_cost: 1, @@ -40,13 +42,23 @@ config :towerops, Towerops.Mailer, adapter: Swoosh.Adapters.Test # The MIX_TEST_PARTITION environment variable can be used # to provide built-in test partitioning in CI environment. # Run `mix help test` for more information. -config :towerops, Towerops.Repo, - username: "postgres", - password: "postgres", - hostname: "localhost", - database: "towerops_test#{System.get_env("MIX_TEST_PARTITION")}", - pool: Ecto.Adapters.SQL.Sandbox, - pool_size: System.schedulers_online() * 2 +# +# In CI, DATABASE_URL is set and will be used automatically by Ecto. +# Locally, fall back to individual connection parameters. +if database_url = System.get_env("DATABASE_URL") do + config :towerops, Towerops.Repo, + url: database_url, + pool: Sandbox, + pool_size: System.schedulers_online() * 2 +else + config :towerops, Towerops.Repo, + username: "postgres", + password: "postgres", + hostname: "localhost", + database: "towerops_test#{System.get_env("MIX_TEST_PARTITION")}", + pool: Sandbox, + pool_size: System.schedulers_online() * 2 +end # Configure Cloak encryption for testing # Use a fixed test key (same as dev for simplicity) diff --git a/lib/towerops/topology.ex b/lib/towerops/topology.ex index aa987a6a..c91a6e09 100644 --- a/lib/towerops/topology.ex +++ b/lib/towerops/topology.ex @@ -6,6 +6,7 @@ defmodule Towerops.Topology do import Ecto.Query + alias Towerops.Devices alias Towerops.Devices.Device alias Towerops.Repo alias Towerops.Snmp.ArpEntry @@ -15,6 +16,8 @@ defmodule Towerops.Topology do alias Towerops.Snmp.Neighbor alias Towerops.Topology.DeviceLink alias Towerops.Topology.DeviceLinkEvidence + alias Towerops.Topology.DeviceNeighbor + alias Towerops.Topology.Lldp require Logger @@ -655,4 +658,139 @@ defmodule Towerops.Topology do defp vendor_match?(value, vendors), do: Enum.any?(vendors, &String.contains?(value, &1)) defp platform_match?(description, platforms), do: Enum.any?(platforms, &String.contains?(description, &1)) + + # --- LLDP Neighbor Discovery (via SNMP) --- + + @doc """ + Discovers LLDP neighbors for a device via SNMP and stores them in device_neighbors. + + Returns {:ok, count} where count is the number of neighbors discovered. + """ + @spec discover_lldp_neighbors(String.t()) :: {:ok, integer()} | {:error, term()} + def discover_lldp_neighbors(device_id) do + case Lldp.discover_neighbors(device_id) do + {:ok, %{neighbors: neighbors}} -> + now = DateTime.utc_now() + results = Enum.map(neighbors, &upsert_device_neighbor(device_id, &1, now)) + success_count = Enum.count(results, &match?({:ok, _}, &1)) + {:ok, success_count} + + {:error, reason} = error -> + Logger.error("Failed to discover LLDP neighbors for device #{device_id}: #{inspect(reason)}") + error + end + end + + @doc """ + Lists all LLDP neighbors for a device. + """ + @spec list_lldp_neighbors(String.t()) :: [DeviceNeighbor.t()] + def list_lldp_neighbors(device_id) do + Repo.all( + from(n in DeviceNeighbor, + where: n.device_id == ^device_id, + order_by: [asc: n.local_port, asc: n.neighbor_name], + preload: [:neighbor_device] + ) + ) + end + + @doc """ + Lists all LLDP neighbors for a site. + """ + @spec list_site_lldp_neighbors(String.t()) :: [DeviceNeighbor.t()] + def list_site_lldp_neighbors(site_id) do + Repo.all( + from(n in DeviceNeighbor, + join: d in assoc(n, :device), + where: d.site_id == ^site_id, + order_by: [asc: d.name, asc: n.local_port], + preload: [device: d, neighbor_device: :site] + ) + ) + end + + @doc """ + Removes stale LLDP neighbors not seen in the specified hours. + + Default: 24 hours. + """ + @spec remove_stale_lldp_neighbors(integer()) :: {integer(), nil} + def remove_stale_lldp_neighbors(hours_ago \\ 24) do + cutoff = DateTime.add(DateTime.utc_now(), -hours_ago * 3600, :second) + + {count, _} = + Repo.delete_all( + from(n in DeviceNeighbor, + where: n.last_seen_at < ^cutoff + ) + ) + + {count, nil} + end + + # Upserts a single device neighbor record + defp upsert_device_neighbor(device_id, neighbor, now) do + neighbor_device_id = find_device_by_lldp_neighbor(device_id, neighbor) + + attrs = %{ + device_id: device_id, + neighbor_device_id: neighbor_device_id, + neighbor_name: neighbor.neighbor_name, + local_port: neighbor.local_port, + remote_port: neighbor.remote_port, + remote_port_id: neighbor.remote_port_id, + management_addresses: neighbor.management_addresses, + discovered_at: now, + last_seen_at: now + } + + case Repo.get_by(DeviceNeighbor, + device_id: device_id, + local_port: neighbor.local_port, + neighbor_name: neighbor.neighbor_name + ) do + nil -> + %DeviceNeighbor{} + |> DeviceNeighbor.changeset(attrs) + |> Repo.insert() + + existing -> + existing + |> DeviceNeighbor.changeset( + Map.take(attrs, [ + :last_seen_at, + :remote_port, + :remote_port_id, + :management_addresses, + :neighbor_device_id + ]) + ) + |> Repo.update() + end + end + + # Attempts to find an existing device that matches the LLDP neighbor + defp find_device_by_lldp_neighbor(discovering_device_id, neighbor) do + discovering_device = Devices.get_device(discovering_device_id) + + if discovering_device do + # Search within the same organization for a device matching: + # 1. Name matches neighbor_name + # 2. IP address matches one of the management_addresses + + query = + from(d in Device, + where: d.organization_id == ^discovering_device.organization_id, + where: + d.name == ^neighbor.neighbor_name or + d.ip_address in ^neighbor.management_addresses + ) + + case Repo.one(query) do + nil -> nil + device -> device.id + end + end + end end diff --git a/lib/towerops/topology/device_neighbor.ex b/lib/towerops/topology/device_neighbor.ex new file mode 100644 index 00000000..09e2f7e2 --- /dev/null +++ b/lib/towerops/topology/device_neighbor.ex @@ -0,0 +1,79 @@ +defmodule Towerops.Topology.DeviceNeighbor do + @moduledoc """ + Represents an LLDP neighbor relationship discovered between network devices. + + A neighbor relationship is discovered via LLDP (Link Layer Discovery Protocol) + and represents a physical connection between two devices. The relationship + includes port information and management addresses for recursive discovery. + """ + use Ecto.Schema + + import Ecto.Changeset + + alias Ecto.Association.NotLoaded + alias Towerops.Devices.Device + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "device_neighbors" do + belongs_to :device, Device + belongs_to :neighbor_device, Device + + field :neighbor_name, :string + field :local_port, :string + field :remote_port, :string + field :remote_port_id, :string + field :management_addresses, {:array, :string}, default: [] + + field :discovered_at, :utc_datetime + field :last_seen_at, :utc_datetime + + timestamps(type: :utc_datetime) + end + + @type t :: %__MODULE__{ + id: Ecto.UUID.t(), + device_id: Ecto.UUID.t(), + device: Device.t() | NotLoaded.t(), + neighbor_device_id: Ecto.UUID.t() | nil, + neighbor_device: Device.t() | NotLoaded.t() | nil, + neighbor_name: String.t(), + local_port: String.t(), + remote_port: String.t() | nil, + remote_port_id: String.t() | nil, + management_addresses: [String.t()], + discovered_at: DateTime.t(), + last_seen_at: DateTime.t(), + inserted_at: DateTime.t(), + updated_at: DateTime.t() + } + + @doc false + def changeset(neighbor, attrs) do + neighbor + |> cast(attrs, [ + :device_id, + :neighbor_device_id, + :neighbor_name, + :local_port, + :remote_port, + :remote_port_id, + :management_addresses, + :discovered_at, + :last_seen_at + ]) + |> validate_required([ + :device_id, + :neighbor_name, + :local_port, + :discovered_at, + :last_seen_at + ]) + |> foreign_key_constraint(:device_id) + |> foreign_key_constraint(:neighbor_device_id) + |> unique_constraint([:device_id, :local_port, :neighbor_name], + name: :device_neighbors_unique_link + ) + end +end diff --git a/lib/towerops/topology/lldp.ex b/lib/towerops/topology/lldp.ex new file mode 100644 index 00000000..1f93a930 --- /dev/null +++ b/lib/towerops/topology/lldp.ex @@ -0,0 +1,313 @@ +defmodule Towerops.Topology.Lldp do + @moduledoc """ + LLDP (Link Layer Discovery Protocol) topology discovery via SNMP. + + Walks LLDP-MIB (IEEE 802.1AB) tables to discover network neighbors, + their ports, and management addresses for recursive discovery. + + ## LLDP-MIB OIDs + + Local system: + - 1.0.8802.1.1.2.1.3.3.0 lldpLocSysName (device hostname) + - 1.0.8802.1.1.2.1.3.7.1.4 lldpLocPortDesc (local port descriptions) + + Remote neighbors (indexed by timeMark.portNum.remIndex): + - 1.0.8802.1.1.2.1.4.1.1.7 lldpRemPortId (remote port ID) + - 1.0.8802.1.1.2.1.4.1.1.8 lldpRemPortDesc (remote port description) + - 1.0.8802.1.1.2.1.4.1.1.9 lldpRemSysName (neighbor hostname) + - 1.0.8802.1.1.2.1.4.2.1.3 lldpRemManAddrIfId (management addresses) + + ## Example + + iex> Lldp.discover_neighbors(device_id) + {:ok, %{ + sys_name: "core-sw01", + neighbors: [ + %{ + neighbor_name: "access-sw02", + local_port: "ge-0/0/1", + remote_port: "ge-0/0/24", + remote_port_id: "525", + management_addresses: ["192.168.1.10"] + } + ] + }} + """ + + alias Towerops.Devices + alias Towerops.Snmp.Client + + require Logger + + # LLDP-MIB OIDs (IEEE 802.1AB) + @oid_loc_sys_name "1.0.8802.1.1.2.1.3.3.0" + @oid_loc_port_desc "1.0.8802.1.1.2.1.3.7.1.4" + @oid_rem_port_id "1.0.8802.1.1.2.1.4.1.1.7" + @oid_rem_port_desc "1.0.8802.1.1.2.1.4.1.1.8" + @oid_rem_sys_name "1.0.8802.1.1.2.1.4.1.1.9" + @oid_rem_man_addr "1.0.8802.1.1.2.1.4.2.1.3" + + @type neighbor :: %{ + neighbor_name: String.t(), + local_port: String.t(), + remote_port: String.t() | nil, + remote_port_id: String.t() | nil, + management_addresses: [String.t()] + } + + @type discovery_result :: %{ + sys_name: String.t() | nil, + neighbors: [neighbor()] + } + + @doc """ + Discovers LLDP neighbors for a device by walking LLDP-MIB tables. + + Returns the local system name and a list of discovered neighbors with + port information and management addresses. + + ## Options + + - `:timeout` - SNMP timeout in milliseconds (default: 5000) + - `:retries` - SNMP retries (default: 2) + """ + @spec discover_neighbors(String.t(), keyword()) :: + {:ok, discovery_result()} | {:error, term()} + def discover_neighbors(device_id, opts \\ []) do + device = Devices.get_device(device_id) + + if device == nil do + {:error, :device_not_found} + else + walk_lldp_tables(device, opts) + end + end + + # Walks LLDP-MIB tables and assembles neighbor information + defp walk_lldp_tables(device, _opts) do + client_opts = build_client_opts(device) + + sys_name = get_local_system_name(client_opts) + local_ports = walk_local_ports(client_opts) + sys_names = walk_remote_sys_names(client_opts, device.name) + + # If no neighbors found, return early + if map_size(sys_names) == 0 do + {:ok, %{sys_name: sys_name, neighbors: []}} + else + remote_ports = walk_remote_ports(client_opts) + remote_port_ids = walk_remote_port_ids(client_opts) + mgmt_addrs = walk_management_addresses(client_opts) + + neighbors = assemble_neighbors(sys_names, local_ports, remote_ports, remote_port_ids, mgmt_addrs) + + {:ok, %{sys_name: sys_name, neighbors: neighbors}} + end + end + + # Gets the local system name + defp get_local_system_name(client_opts) do + case Client.get(client_opts, @oid_loc_sys_name) do + {:ok, value} -> clean_string(value) + _ -> nil + end + end + + # Walks local port descriptions + defp walk_local_ports(client_opts) do + case Client.walk(client_opts, @oid_loc_port_desc) do + {:ok, results} -> + Map.new(results, fn {oid, value} -> + port_num = extract_suffix(oid, @oid_loc_port_desc) + {port_num, clean_string(value)} + end) + + _ -> + %{} + end + end + + # Walks remote system names + defp walk_remote_sys_names(client_opts, device_name) do + case Client.walk(client_opts, @oid_rem_sys_name) do + {:ok, results} -> + results + |> Enum.map(fn {oid, value} -> + key = parse_remote_key(oid, @oid_rem_sys_name) + {key, clean_string(value)} + end) + |> Enum.reject(fn {key, _} -> is_nil(key) end) + |> Map.new() + + {:error, reason} -> + Logger.warning("Failed to walk LLDP remote sys names for #{device_name}: #{inspect(reason)}") + %{} + end + end + + # Walks remote port descriptions + defp walk_remote_ports(client_opts) do + case Client.walk(client_opts, @oid_rem_port_desc) do + {:ok, results} -> + results + |> Enum.map(fn {oid, value} -> + key = parse_remote_key(oid, @oid_rem_port_desc) + {key, clean_string(value)} + end) + |> Enum.reject(fn {key, _} -> is_nil(key) end) + |> Map.new() + + _ -> + %{} + end + end + + # Walks remote port IDs + defp walk_remote_port_ids(client_opts) do + case Client.walk(client_opts, @oid_rem_port_id) do + {:ok, results} -> + results + |> Enum.map(fn {oid, value} -> + key = parse_remote_key(oid, @oid_rem_port_id) + {key, clean_string(value)} + end) + |> Enum.reject(fn {key, _} -> is_nil(key) end) + |> Map.new() + + _ -> + %{} + end + end + + # Walks management addresses + defp walk_management_addresses(client_opts) do + case Client.walk(client_opts, @oid_rem_man_addr) do + {:ok, results} -> + results + |> Enum.map(fn {oid, _value} -> parse_management_address(oid) end) + |> Enum.reject(&is_nil/1) + |> Enum.group_by(fn {key, _ip} -> key end, fn {_key, ip} -> ip end) + + _ -> + %{} + end + end + + # Assembles the neighbor list from all collected data + defp assemble_neighbors(sys_names, local_ports, remote_ports, remote_port_ids, mgmt_addrs) do + sys_names + |> Enum.map(fn {key, neighbor_name} -> + {_time_mark, port_num, _rem_index} = key + + %{ + neighbor_name: neighbor_name, + local_port: Map.get(local_ports, port_num, "port-#{port_num}"), + remote_port: Map.get(remote_ports, key), + remote_port_id: Map.get(remote_port_ids, key), + management_addresses: Map.get(mgmt_addrs, key, []) + } + end) + |> Enum.reject(fn n -> n.neighbor_name == "" end) + end + + # Extracts the suffix after the base OID + defp extract_suffix(oid, base) do + prefix = base <> "." + + if String.starts_with?(oid, prefix) do + String.replace_prefix(oid, prefix, "") + end + end + + # Parses remote table key from OID: timeMark.portNum.remIndex + defp parse_remote_key(oid, base) do + case extract_suffix(oid, base) do + nil -> + nil + + suffix -> + case String.split(suffix, ".", parts: 4) do + [time_mark, port_num, rem_index | _] -> + {time_mark, port_num, rem_index} + + _ -> + nil + end + end + end + + # Parses management address from OID index + # Format: timeMark.portNum.remIndex.addrSubtype.addrLen.addr[bytes] + # addrSubtype: 1=IPv4, 2=IPv6 + defp parse_management_address(oid) do + with suffix when not is_nil(suffix) <- extract_suffix(oid, @oid_rem_man_addr), + parts = String.split(suffix, "."), + true <- length(parts) >= 9, + [time_mark, port_num, rem_index, addr_subtype | rest] <- parts, + ip when not is_nil(ip) <- parse_ip_from_addr_subtype(addr_subtype, rest) do + {{time_mark, port_num, rem_index}, ip} + else + _ -> nil + end + end + + # Parses IP address based on address subtype + defp parse_ip_from_addr_subtype("1", rest), do: parse_ipv4(Enum.slice(rest, 1, 4)) + defp parse_ip_from_addr_subtype("2", rest), do: parse_ipv6(Enum.slice(rest, 1, 16)) + defp parse_ip_from_addr_subtype(_, _), do: nil + + # Parse IPv4 address from OID octets + defp parse_ipv4(octets) when length(octets) == 4 do + Enum.join(octets, ".") + end + + defp parse_ipv4(_), do: nil + + # Parse IPv6 address from OID octets + defp parse_ipv6(octets) when length(octets) == 16 do + octets + |> Enum.map(&String.to_integer/1) + |> Enum.chunk_every(2) + |> Enum.map_join(":", fn [a, b] -> Integer.to_string(a * 256 + b, 16) end) + |> String.downcase() + end + + defp parse_ipv6(_), do: nil + + # Builds SNMP client options from device configuration + defp build_client_opts(device) do + snmp_config = Devices.get_snmp_config(device) + + base_opts = [ + ip: device.ip_address, + version: snmp_config.version, + port: device.snmp_port || 161 + ] + + # Add version-specific credentials + if snmp_config.version == "3" do + v3_config = Devices.get_snmpv3_config(device) + + base_opts ++ + [ + security_name: v3_config.username, + security_level: v3_config.security_level, + auth_protocol: v3_config.auth_protocol, + auth_password: v3_config.auth_password, + priv_protocol: v3_config.priv_protocol, + priv_password: v3_config.priv_password + ] + else + base_opts ++ [community: snmp_config.community] + end + end + + # Cleans string values from SNMP (removes control characters, trims whitespace) + defp clean_string(value) when is_binary(value) do + value + |> String.trim() + |> String.replace(~r/[\x00-\x1F\x7F]/, "") + end + + defp clean_string(value), do: to_string(value) +end diff --git a/priv/repo/migrations/20260305164021_create_device_neighbors.exs b/priv/repo/migrations/20260305164021_create_device_neighbors.exs new file mode 100644 index 00000000..1d39990a --- /dev/null +++ b/priv/repo/migrations/20260305164021_create_device_neighbors.exs @@ -0,0 +1,34 @@ +defmodule Towerops.Repo.Migrations.CreateDeviceNeighbors do + use Ecto.Migration + + def change do + create table(:device_neighbors, primary_key: false) do + add :id, :binary_id, primary_key: true + + add :device_id, references(:devices, type: :binary_id, on_delete: :delete_all), null: false + + add :neighbor_device_id, references(:devices, type: :binary_id, on_delete: :nilify_all) + + add :neighbor_name, :string, null: false + add :local_port, :string, null: false + add :remote_port, :string + add :remote_port_id, :string + + # Management addresses discovered via LLDP (array of IPs) + add :management_addresses, {:array, :string}, default: [] + + add :discovered_at, :utc_datetime, null: false + add :last_seen_at, :utc_datetime, null: false + + timestamps(type: :utc_datetime) + end + + create index(:device_neighbors, [:device_id]) + create index(:device_neighbors, [:neighbor_device_id]) + create index(:device_neighbors, [:last_seen_at]) + + create unique_index(:device_neighbors, [:device_id, :local_port, :neighbor_name], + name: :device_neighbors_unique_link + ) + end +end