towerops/lib/towerops/snmp.ex

defmodule Towerops.Snmp do
  @moduledoc """
  The SNMP context.

  Provides the public API for SNMP discovery and monitoring functionality.
  """

  import Ecto.Query

  alias Towerops.Devices.Device, as: DeviceSchema
  alias Towerops.Repo
  alias Towerops.Snmp.ArpEntry
  alias Towerops.Snmp.Client
  alias Towerops.Snmp.Device
  alias Towerops.Snmp.Discovery
  alias Towerops.Snmp.Interface
  alias Towerops.Snmp.InterfaceStat
  alias Towerops.Snmp.IpAddress
  alias Towerops.Snmp.MacAddress
  alias Towerops.Snmp.Neighbor
  alias Towerops.Snmp.Processor
  alias Towerops.Snmp.ProcessorReading
  alias Towerops.Snmp.Sensor
  alias Towerops.Snmp.SensorReading
  alias Towerops.Snmp.StateSensor
  alias Towerops.Snmp.Storage
  alias Towerops.Snmp.StorageReading
  alias Towerops.Snmp.Vlan

  @doc """
  Tests SNMP connectivity to a device.

  ## Examples

      iex> test_connection("192.168.1.1", "public", "2c")
      {:ok, "Connection successful"}

      iex> test_connection("192.168.1.99", "wrong", "2c")
      {:error, :timeout}
  """
  def test_connection(ip, community, version, port \\ 161) do
    Client.test_connection(
      ip: ip,
      community: community,
      version: version,
      port: port
    )
  end

  @doc """
  Runs SNMP discovery for a piece of device.

  Discovers:
  - Device information (manufacturer, model, firmware)
  - Network interfaces
  - Sensors (temperature, power, fans, etc.)

  Updates the database with discovered data.

  ## Examples

      iex> discover_device(device)
      {:ok, %Device{}}
  """
  def discover_device(%DeviceSchema{} = device) do
    Discovery.discover_device(device)
  end

  @doc """
  Runs SNMP discovery for all SNMP-enabled devices in an organization.

  Returns a summary of successful and failed discoveries.

  ## Examples

      iex> discover_all_for_org(org_id)
      {:ok, %{success: 10, failed: 2, errors: [:timeout, :no_response]}}
  """
  def discover_all_for_org(org_id) do
    Discovery.discover_all(org_id)
  end

  # Device queries

  @doc """
  Gets the SNMP device for a piece of device.

  ## Examples

      iex> get_device(device_id)
      %Device{}

      iex> get_device(nonexistent_id)
      nil
  """
  def get_device(device_id) do
    Repo.get_by(Device, device_id: device_id)
  end

  @doc """
  Gets the SNMP device for a device with preloaded associations.
  """
  def get_device_with_associations(device_id) do
    Device
    |> where([d], d.device_id == ^device_id)
    |> preload([:interfaces, :sensors, :state_sensors, :processors, :storage])
    |> Repo.one()
  end

  # Interface queries

  @doc """
  Lists all interfaces for a device.
  """
  def list_interfaces(device_id) do
    Interface
    |> where([i], i.snmp_device_id == ^device_id)
    |> order_by([i], i.if_index)
    |> Repo.all()
  end

  @doc """
  Lists only monitored interfaces for a device.
  """
  def list_monitored_interfaces(device_id) do
    Interface
    |> where([i], i.snmp_device_id == ^device_id and i.monitored == true)
    |> order_by([i], i.if_index)
    |> Repo.all()
  end

  @doc """
  Gets a specific interface.
  """
  def get_interface(interface_id) do
    Repo.get(Interface, interface_id)
  end

  @doc """
  Updates an interface.
  """
  def update_interface(%Interface{} = interface, attrs) do
    interface
    |> Interface.changeset(attrs)
    |> Repo.update()
  end

  # Sensor queries

  @doc """
  Lists all sensors for a device.
  """
  def list_sensors(device_id) do
    Sensor
    |> where([s], s.snmp_device_id == ^device_id)
    |> order_by([s], [s.sensor_type, s.sensor_index])
    |> Repo.all()
  end

  @doc """
  Lists only monitored sensors for a device.
  """
  def list_monitored_sensors(device_id) do
    Sensor
    |> where([s], s.snmp_device_id == ^device_id and s.monitored == true)
    |> order_by([s], [s.sensor_type, s.sensor_index])
    |> Repo.all()
  end

  @doc """
  Lists sensors grouped by type.
  """
  def list_sensors_by_type(device_id) do
    sensors =
      Sensor
      |> where([s], s.snmp_device_id == ^device_id)
      |> order_by([s], [s.sensor_type, s.sensor_index])
      |> Repo.all()

    Enum.group_by(sensors, & &1.sensor_type)
  end

  @doc """
  Gets a specific sensor.
  """
  def get_sensor(sensor_id) do
    Repo.get(Sensor, sensor_id)
  end

  @doc """
  Updates a sensor.
  """
  def update_sensor(%Sensor{} = sensor, attrs) do
    sensor
    |> Sensor.changeset(attrs)
    |> Repo.update()
  end

  # State sensor queries

  @doc """
  Updates a state sensor.
  """
  def update_state_sensor(%StateSensor{} = state_sensor, attrs) do
    state_sensor
    |> StateSensor.changeset(attrs)
    |> Repo.update()
  end

  @doc """
  Lists all state sensors for a device.
  """
  def list_state_sensors(device_id) do
    StateSensor
    |> where([s], s.snmp_device_id == ^device_id)
    |> order_by([s], s.sensor_index)
    |> Repo.all()
  end

  # Sensor readings queries

  @doc """
  Gets recent sensor readings for a sensor.

  ## Options

  - `:limit` - Maximum number of readings to return (default: 100)
  - `:since` - Only return readings after this datetime
  """
  def get_sensor_readings(sensor_id, opts \\ []) do
    limit = Keyword.get(opts, :limit, 100)
    since = Keyword.get(opts, :since)

    query =
      SensorReading
      |> where([r], r.sensor_id == ^sensor_id)
      |> order_by([r], desc: r.checked_at)
      |> limit(^limit)

    query =
      if since do
        where(query, [r], r.checked_at >= ^since)
      else
        query
      end

    Repo.all(query)
  end

  @doc """
  Gets the latest sensor reading for a sensor.
  """
  def get_latest_sensor_reading(sensor_id) do
    SensorReading
    |> where([r], r.sensor_id == ^sensor_id)
    |> order_by([r], desc: r.checked_at)
    |> limit(1)
    |> Repo.one()
  end

  @doc """
  Gets the latest sensor readings for multiple sensors in a single query.

  Returns a map of %{sensor_id => reading} for efficient batch loading.
  Sensors without readings will not be present in the map.

  ## Examples

      iex> get_latest_sensor_readings_batch([sensor_id1, sensor_id2])
      %{sensor_id1 => %SensorReading{}, sensor_id2 => %SensorReading{}}
  """
  def get_latest_sensor_readings_batch(sensor_ids) when is_list(sensor_ids) do
    if Enum.empty?(sensor_ids) do
      %{}
    else
      # Use DISTINCT ON to get only the latest reading per sensor
      query =
        from(r in SensorReading,
          where: r.sensor_id in ^sensor_ids,
          distinct: r.sensor_id,
          order_by: [asc: r.sensor_id, desc: r.checked_at]
        )

      query
      |> Repo.all()
      |> Map.new(fn reading -> {reading.sensor_id, reading} end)
    end
  end

  # Interface stats queries

  @doc """
  Gets recent interface statistics for an interface.

  ## Options

  - `:limit` - Maximum number of stats to return (default: 100)
  - `:since` - Only return stats after this datetime
  """
  def get_interface_stats(interface_id, opts \\ []) do
    limit = Keyword.get(opts, :limit, 100)
    since = Keyword.get(opts, :since)

    query =
      InterfaceStat
      |> where([s], s.interface_id == ^interface_id)
      |> order_by([s], desc: s.checked_at)
      |> limit(^limit)

    query =
      if since do
        where(query, [s], s.checked_at >= ^since)
      else
        query
      end

    Repo.all(query)
  end

  @doc """
  Gets the latest interface stat for an interface.
  """
  def get_latest_interface_stat(interface_id) do
    InterfaceStat
    |> where([s], s.interface_id == ^interface_id)
    |> order_by([s], desc: s.checked_at)
    |> limit(1)
    |> Repo.one()
  end

  @doc """
  Gets the latest interface stats for multiple interfaces in a single query.

  Returns a map of %{interface_id => stat} for efficient batch loading.
  Interfaces without stats will not be present in the map.

  ## Examples

      iex> get_latest_interface_stats_batch([if_id1, if_id2])
      %{if_id1 => %InterfaceStat{}, if_id2 => %InterfaceStat{}}
  """
  def get_latest_interface_stats_batch(interface_ids) when is_list(interface_ids) do
    if Enum.empty?(interface_ids) do
      %{}
    else
      # Use DISTINCT ON to get only the latest stat per interface
      query =
        from(s in InterfaceStat,
          where: s.interface_id in ^interface_ids,
          distinct: s.interface_id,
          order_by: [asc: s.interface_id, desc: s.checked_at]
        )

      query
      |> Repo.all()
      |> Map.new(fn stat -> {stat.interface_id, stat} end)
    end
  end

  @doc """
  Records a new sensor reading.
  """
  def create_sensor_reading(attrs) do
    %SensorReading{}
    |> SensorReading.changeset(attrs)
    |> Repo.insert()
  end

  @doc """
  Records a new interface stat.
  """
  def create_interface_stat(attrs) do
    %InterfaceStat{}
    |> InterfaceStat.changeset(attrs)
    |> Repo.insert()
  end

  # Neighbor queries

  @doc """
  Lists all neighbors for a piece of device.
  """
  def list_neighbors(device_id) do
    Neighbor
    |> where([n], n.device_id == ^device_id)
    |> preload(:interface)
    |> order_by([n], [n.protocol, n.remote_system_name])
    |> Repo.all()
  end

  @doc """
  Lists all neighbors for device with enriched data linking to known device.
  Adds a `matched_equipment` field to each neighbor if we can identify the remote device.
  """
  def list_neighbors_with_equipment(device_id) do
    neighbors = list_neighbors(device_id)

    # device to search within the same org
    device = Towerops.Devices.get_device!(device_id)

    Enum.map(neighbors, fn neighbor ->
      matched_device = find_matching_equipment(neighbor, device.site.organization_id)
      Map.put(neighbor, :matched_device, matched_device)
    end)
  end

  # device in the organization
  defp find_matching_equipment(neighbor, organization_id) do
    # Try matching strategies in order: chassis ID, port ID, system name
    try_match_by_chassis(neighbor, organization_id) ||
      try_match_by_port(neighbor, organization_id) ||
      try_match_by_name(neighbor, organization_id)
  end

  defp try_match_by_chassis(neighbor, organization_id) do
    if neighbor.remote_chassis_id && mac_address?(neighbor.remote_chassis_id) do
      find_device_by_mac(neighbor.remote_chassis_id, organization_id)
    end
  end

  defp try_match_by_port(neighbor, organization_id) do
    if neighbor.remote_port_id && mac_address?(neighbor.remote_port_id) do
      find_device_by_mac(neighbor.remote_port_id, organization_id)
    end
  end

  defp try_match_by_name(neighbor, organization_id) do
    if neighbor.remote_system_name do
      find_device_by_name(neighbor.remote_system_name, organization_id)
    end
  end

  # Check if a string looks like a MAC address
  defp mac_address?(str) when is_binary(str) do
    # Match patterns like "AA:BB:CC:DD:EE:FF" or "aa-bb-cc-dd-ee-ff"
    String.match?(str, ~r/^([0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}$/)
  end

  defp mac_address?(_), do: false

  defp find_device_by_mac(mac_address, organization_id) do
    # MAC address might be in format "aa:bb:cc:dd:ee:ff", "aa-bb-cc-dd-ee-ff", or "aabbccddeeff"
    # Normalize to just the hex digits for comparison
    normalized_mac = mac_address |> String.downcase() |> String.replace(~r/[:-]/, "")

    Repo.one(
      from(i in Interface,
        join: d in Device,
        on: i.snmp_device_id == d.id,
        join: e in DeviceSchema,
        on: d.device_id == e.id,
        join: s in assoc(e, :site),
        where: s.organization_id == ^organization_id,
        where:
          fragment("REPLACE(REPLACE(LOWER(?), ':', ''), '-', '')", i.if_phys_address) ==
            ^normalized_mac,
        select: e,
        limit: 1
      )
    )
  end

  defp find_device_by_name(system_name, organization_id) do
    # Try exact match first, then partial match
    normalized_name = String.downcase(system_name)

    from(e in DeviceSchema,
      join: s in assoc(e, :site),
      where: s.organization_id == ^organization_id,
      where: fragment("LOWER(?)", e.name) == ^normalized_name,
      select: e,
      limit: 1
    )
    |> Repo.one()
    |> case do
      nil ->
        # Try partial match as fallback
        Repo.one(
          from(e in DeviceSchema,
            join: s in assoc(e, :site),
            where: s.organization_id == ^organization_id,
            where: fragment("LOWER(?) LIKE ?", e.name, ^"%#{normalized_name}%"),
            select: e,
            limit: 1
          )
        )

      device ->
        device
    end
  end

  @doc """
  Gets a specific neighbor.
  """
  def get_neighbor(neighbor_id) do
    Repo.get(Neighbor, neighbor_id)
  end

  @doc """
  Creates or updates a neighbor record.
  """
  def upsert_neighbor(attrs) do
    case Repo.get_by(Neighbor,
           interface_id: attrs.interface_id,
           remote_chassis_id: attrs[:remote_chassis_id],
           protocol: attrs.protocol
         ) do
      nil ->
        %Neighbor{}
        |> Neighbor.changeset(attrs)
        |> Repo.insert()

      neighbor ->
        neighbor
        |> Neighbor.changeset(attrs)
        |> Repo.update()
    end
  end

  @doc """
  Deletes stale neighbors that haven't been seen since the given datetime.
  """
  def delete_stale_neighbors(device_id, cutoff_datetime) do
    Neighbor
    |> where([n], n.device_id == ^device_id)
    |> where([n], n.last_discovered_at < ^cutoff_datetime)
    |> Repo.delete_all()
  end

  @doc """
  Lists all discovered devices across an organization that haven't been added yet.

  Aggregates data from LLDP/CDP neighbors, ARP entries, and MAC addresses.
  Returns a list of discovered device structs with merged data.

  ## Returns

  List of maps with:
  - `identifier`: %{type: :mac | :ip | :hostname, value: string}
  - `hostname`: string | nil
  - `ip_addresses`: [string]
  - `mac_addresses`: [string]
  - `device_type`: :router | :switch | :wireless | :server | :workstation | :unknown
  - `capabilities`: [string] (from LLDP/CDP)
  - `platform`: string | nil
  - `discovered_by`: [%{device_id: uuid, device_name: string, interface: string}]
  - `source_count`: integer
  - `last_seen`: DateTime.t()
  - `protocols_used`: [:lldp | :cdp | :arp | :mac]
  """
  def list_discovered_devices_for_organization(organization_id) do
    # Get all devices in organization
    device_ids = query_organization_device_ids(organization_id)

    # Fetch all discovery data
    neighbors = query_neighbors_for_devices(device_ids)
    arp_entries = query_arp_for_devices(device_ids)
    mac_addresses = query_mac_for_devices(device_ids)

    # Get existing devices to filter out
    existing_ips = get_existing_device_ips(organization_id)
    existing_macs = get_existing_interface_macs(organization_id)

    # Build identifier index by aggregating all sources
    discovered_map =
      %{}
      |> merge_neighbors_into_map(neighbors)
      |> merge_arp_into_map(arp_entries)
      |> merge_mac_into_map(mac_addresses)

    # Filter out existing devices and build final structs
    discovered_map
    |> Enum.reject(fn {_key, entry} ->
      device_already_exists?(entry, existing_ips, existing_macs)
    end)
    |> Enum.map(fn {_key, entry} -> build_discovered_device_struct(entry) end)
    |> Enum.sort(fn a, b ->
      # Sort by source_count descending, then by last_seen descending
      cond do
        a.source_count > b.source_count -> true
        a.source_count < b.source_count -> false
        true -> DateTime.after?(a.last_seen, b.last_seen)
      end
    end)
  end

  # Query all device IDs in organization
  defp query_organization_device_ids(organization_id) do
    Repo.all(
      from(d in DeviceSchema, join: s in assoc(d, :site), where: s.organization_id == ^organization_id, select: d.id)
    )
  end

  # Batch query neighbors for all devices in org
  defp query_neighbors_for_devices(device_ids) do
    cutoff = DateTime.add(DateTime.utc_now(), -7, :day)

    Repo.all(
      from(n in Neighbor,
        join: d in DeviceSchema,
        on: n.device_id == d.id,
        left_join: i in Interface,
        on: n.interface_id == i.id,
        where: n.device_id in ^device_ids,
        where: n.last_discovered_at > ^cutoff,
        preload: [device: d, interface: i],
        order_by: [desc: n.last_discovered_at]
      )
    )
  end

  # Batch query ARP entries for all devices in org
  defp query_arp_for_devices(device_ids) do
    cutoff = DateTime.add(DateTime.utc_now(), -7, :day)

    Repo.all(
      from(a in ArpEntry,
        join: d in DeviceSchema,
        on: a.device_id == d.id,
        left_join: i in Interface,
        on: a.interface_id == i.id,
        where: a.device_id in ^device_ids,
        where: a.last_seen_at > ^cutoff,
        preload: [device: d, interface: i],
        order_by: [desc: a.last_seen_at]
      )
    )
  end

  # Batch query MAC addresses for all devices in org
  defp query_mac_for_devices(device_ids) do
    cutoff = DateTime.add(DateTime.utc_now(), -7, :day)

    Repo.all(
      from(m in MacAddress,
        join: d in DeviceSchema,
        on: m.device_id == d.id,
        left_join: i in Interface,
        on: m.interface_id == i.id,
        where: m.device_id in ^device_ids,
        where: m.last_seen_at > ^cutoff,
        preload: [device: d, interface: i],
        order_by: [desc: m.last_seen_at]
      )
    )
  end

  # Get existing device IPs in organization
  defp get_existing_device_ips(organization_id) do
    from(d in DeviceSchema,
      join: s in assoc(d, :site),
      where: s.organization_id == ^organization_id,
      where: not is_nil(d.ip_address),
      select: d.ip_address
    )
    |> Repo.all()
    |> MapSet.new()
  end

  # Get existing interface MAC addresses in organization
  defp get_existing_interface_macs(organization_id) do
    from(i in Interface,
      join: sd in Device,
      on: i.snmp_device_id == sd.id,
      join: d in DeviceSchema,
      on: sd.device_id == d.id,
      join: s in assoc(d, :site),
      where: s.organization_id == ^organization_id,
      where: not is_nil(i.if_phys_address),
      select: i.if_phys_address
    )
    |> Repo.all()
    |> MapSet.new(&normalize_mac/1)
  end

  # Normalize MAC address to "aa:bb:cc:dd:ee:ff" format
  defp normalize_mac(nil), do: nil

  defp normalize_mac(mac) when is_binary(mac) do
    mac
    |> String.downcase()
    |> String.replace(~r/[:-]/, "")
    |> String.replace(~r/(.{2})/, "\\1:")
    |> String.trim_trailing(":")
  end

  # Merge LLDP/CDP neighbor data into the discovery map
  defp merge_neighbors_into_map(map, neighbors) do
    Enum.reduce(neighbors, map, fn neighbor, acc ->
      identifiers = extract_neighbor_identifiers(neighbor)
      # Use best available identifier as the key (MAC > IP > hostname)
      {id_type, id_value} = choose_best_identifier_from_list(identifiers)
      key = identifier_key(id_type, id_value)

      entry =
        Map.get(acc, key, %{
          identifiers: [],
          hostnames: [],
          ips: [],
          macs: [],
          capabilities: [],
          platforms: [],
          discovered_by: [],
          sources: %{neighbors: [], arp: [], mac: []}
        })

      updated_entry = %{
        entry
        | identifiers: Enum.uniq(identifiers ++ entry.identifiers),
          hostnames: add_if_present(entry.hostnames, neighbor.remote_system_name),
          ips: add_if_present(entry.ips, neighbor.remote_address),
          macs: add_if_present(entry.macs, normalize_mac(neighbor.remote_chassis_id)),
          capabilities: Enum.uniq(entry.capabilities ++ (neighbor.remote_capabilities || [])),
          platforms: add_if_present(entry.platforms, neighbor.remote_platform),
          discovered_by:
            Enum.uniq_by(
              [
                %{
                  device_id: neighbor.device.id,
                  device_name: neighbor.device.name,
                  interface: interface_name(neighbor.interface),
                  timestamp: neighbor.last_discovered_at
                }
                | entry.discovered_by
              ],
              & &1.device_id
            ),
          sources: %{entry.sources | neighbors: [neighbor | entry.sources.neighbors]}
      }

      Map.put(acc, key, updated_entry)
    end)
  end

  # Merge ARP entry data into the discovery map
  defp merge_arp_into_map(map, arp_entries) do
    Enum.reduce(arp_entries, map, fn arp, acc ->
      identifiers = extract_arp_identifiers(arp)
      # Use best available identifier as the key (MAC > IP)
      {id_type, id_value} = choose_best_identifier_from_list(identifiers)
      key = identifier_key(id_type, id_value)

      entry =
        Map.get(acc, key, %{
          identifiers: [],
          hostnames: [],
          ips: [],
          macs: [],
          capabilities: [],
          platforms: [],
          discovered_by: [],
          sources: %{neighbors: [], arp: [], mac: []}
        })

      updated_entry = %{
        entry
        | identifiers: Enum.uniq(identifiers ++ entry.identifiers),
          ips: add_if_present(entry.ips, arp.ip_address),
          macs: add_if_present(entry.macs, normalize_mac(arp.mac_address)),
          discovered_by:
            Enum.uniq_by(
              [
                %{
                  device_id: arp.device.id,
                  device_name: arp.device.name,
                  interface: interface_name(arp.interface),
                  timestamp: arp.last_seen_at
                }
                | entry.discovered_by
              ],
              & &1.device_id
            ),
          sources: %{entry.sources | arp: [arp | entry.sources.arp]}
      }

      Map.put(acc, key, updated_entry)
    end)
  end

  # Merge MAC address table data into the discovery map
  defp merge_mac_into_map(map, mac_addresses) do
    Enum.reduce(mac_addresses, map, fn mac_entry, acc ->
      mac_normalized = normalize_mac(mac_entry.mac_address)
      if is_nil(mac_normalized), do: acc, else: do_merge_mac(acc, mac_entry, mac_normalized)
    end)
  end

  defp do_merge_mac(acc, mac_entry, mac_normalized) do
    key = identifier_key(:mac, mac_normalized)

    entry =
      Map.get(acc, key, %{
        identifiers: [],
        hostnames: [],
        ips: [],
        macs: [],
        capabilities: [],
        platforms: [],
        discovered_by: [],
        sources: %{neighbors: [], arp: [], mac: []}
      })

    updated_entry = %{
      entry
      | identifiers: Enum.uniq([{:mac, mac_normalized} | entry.identifiers]),
        macs: Enum.uniq([mac_normalized | entry.macs]),
        discovered_by:
          Enum.uniq_by(
            [
              %{
                device_id: mac_entry.device.id,
                device_name: mac_entry.device.name,
                interface: interface_name(mac_entry.interface),
                timestamp: mac_entry.last_seen_at
              }
              | entry.discovered_by
            ],
            & &1.device_id
          ),
        sources: %{entry.sources | mac: [mac_entry | entry.sources.mac]}
    }

    Map.put(acc, key, updated_entry)
  end

  # Choose best identifier from a list of {type, value} tuples
  # Priority: MAC > IP > hostname
  defp choose_best_identifier_from_list(identifiers) do
    Enum.find(identifiers, fn {type, _value} -> type == :mac end) ||
      Enum.find(identifiers, fn {type, _value} -> type == :ip end) ||
      Enum.find(identifiers, fn {type, _value} -> type == :hostname end) ||
      {:unknown, "unknown"}
  end

  # Extract identifiers from neighbor record
  defp extract_neighbor_identifiers(neighbor) do
    Enum.reject(
      [
        if(neighbor.remote_chassis_id && mac_address?(neighbor.remote_chassis_id),
          do: {:mac, normalize_mac(neighbor.remote_chassis_id)}
        ),
        if(neighbor.remote_address, do: {:ip, neighbor.remote_address}),
        if(neighbor.remote_system_name, do: {:hostname, neighbor.remote_system_name})
      ],
      &is_nil/1
    )
  end

  # Extract identifiers from ARP entry
  defp extract_arp_identifiers(arp) do
    Enum.reject(
      [if(arp.mac_address, do: {:mac, normalize_mac(arp.mac_address)}), if(arp.ip_address, do: {:ip, arp.ip_address})],
      &is_nil/1
    )
  end

  # Create a consistent key for the identifier
  defp identifier_key(type, value), do: "#{type}:#{value}"

  # Add value to list if not nil or empty
  defp add_if_present(list, nil), do: list
  defp add_if_present(list, ""), do: list
  defp add_if_present(list, value), do: Enum.uniq([value | list])

  # Get interface name or description
  defp interface_name(nil), do: "unknown"
  defp interface_name(interface), do: interface.if_name || interface.if_descr || "if#{interface.if_index}"

  # Check if device already exists in organization
  defp device_already_exists?(entry, existing_ips, existing_macs) do
    Enum.any?(entry.ips, &MapSet.member?(existing_ips, &1)) ||
      Enum.any?(entry.macs, &MapSet.member?(existing_macs, &1))
  end

  # Build final discovered device struct from aggregated entry
  defp build_discovered_device_struct(entry) do
    identifier = choose_best_identifier(entry)
    device_type = infer_device_type(entry)
    manufacturer = infer_manufacturer(entry)

    %{
      identifier: identifier,
      hostname: List.first(entry.hostnames),
      ip_addresses: Enum.uniq(entry.ips),
      mac_addresses: Enum.uniq(entry.macs),
      device_type: device_type,
      manufacturer: manufacturer,
      capabilities: Enum.uniq(entry.capabilities),
      platform: List.first(entry.platforms),
      discovered_by: Enum.sort_by(entry.discovered_by, & &1.timestamp, {:desc, DateTime}),
      source_count: count_sources(entry),
      last_seen: get_most_recent_timestamp(entry),
      protocols_used: get_protocols_used(entry.sources)
    }
  end

  # Choose best identifier: prefer MAC > IP > hostname
  defp choose_best_identifier(entry) do
    cond do
      entry.macs != [] ->
        %{type: :mac, value: List.first(entry.macs)}

      entry.ips != [] ->
        %{type: :ip, value: List.first(entry.ips)}

      entry.hostnames != [] ->
        %{type: :hostname, value: List.first(entry.hostnames)}

      true ->
        %{type: :unknown, value: "unknown"}
    end
  end

  # Infer device type from capabilities, platform, or system name
  defp infer_device_type(entry) do
    capabilities = Enum.map(entry.capabilities, &String.downcase/1)
    platform = String.downcase(List.first(entry.platforms) || "")
    hostname = String.downcase(List.first(entry.hostnames) || "")

    infer_from_capabilities(capabilities) || infer_from_platform(platform) ||
      infer_from_hostname(hostname) || :unknown
  end

  defp infer_from_capabilities(capabilities) do
    cond do
      "router" in capabilities -> :router
      "bridge" in capabilities or "switch" in capabilities -> :switch
      "wlan-ap" in capabilities or "wlan" in capabilities -> :wireless
      true -> nil
    end
  end

  defp infer_from_platform(platform) do
    cond do
      String.contains?(platform, "ios") -> :router
      String.contains?(platform, "catalyst") -> :switch
      String.contains?(platform, "linux") or String.contains?(platform, "windows") -> :server
      true -> nil
    end
  end

  defp infer_from_hostname(hostname) do
    cond do
      hostname_suggests_router?(hostname) -> :router
      hostname_suggests_switch?(hostname) -> :switch
      hostname_suggests_wireless?(hostname) -> :wireless
      hostname_suggests_server?(hostname) -> :server
      hostname_suggests_workstation?(hostname) -> :workstation
      true -> nil
    end
  end

  defp hostname_suggests_router?(hostname) do
    String.contains?(hostname, "router") or String.contains?(hostname, "rtr") or
      String.contains?(hostname, "gateway") or String.contains?(hostname, "gw")
  end

  defp hostname_suggests_switch?(hostname) do
    String.contains?(hostname, "switch") or String.contains?(hostname, "sw")
  end

  defp hostname_suggests_wireless?(hostname) do
    String.contains?(hostname, "ap") or String.contains?(hostname, "access") or
      String.contains?(hostname, "wireless") or String.contains?(hostname, "wifi") or
      String.contains?(hostname, "wap")
  end

  defp hostname_suggests_server?(hostname) do
    String.contains?(hostname, "server") or String.contains?(hostname, "srv")
  end

  defp hostname_suggests_workstation?(hostname) do
    String.contains?(hostname, "workstation") or String.contains?(hostname, "desktop") or
      String.contains?(hostname, "laptop") or String.contains?(hostname, "pc") or
      String.contains?(hostname, "ws")
  end

  # Infer manufacturer from MAC OUI (first 3 bytes)
  defp infer_manufacturer(entry) do
    case entry.macs do
      [] ->
        nil

      [mac | _] ->
        lookup_mac_oui(mac)
    end
  end

  # Lookup manufacturer by MAC OUI
  defp lookup_mac_oui(mac) when is_binary(mac) do
    # Extract first 3 bytes (OUI) from MAC address
    oui =
      mac
      |> String.replace(":", "")
      |> String.upcase()
      |> String.slice(0, 6)

    Map.get(mac_oui_database(), oui)
  end

  defp lookup_mac_oui(_), do: nil

  # Common MAC OUI to manufacturer mappings
  # This is a subset of the IEEE OUI database focused on common network equipment vendors
  defp mac_oui_database do
    %{
      # Cisco
      "000142" => "Cisco",
      "000143" => "Cisco",
      "00010D" => "Cisco",
      "000164" => "Cisco",
      "000196" => "Cisco",
      "000197" => "Cisco",
      "0001C7" => "Cisco",
      "0001C9" => "Cisco",
      "001818" => "Cisco",
      "001B0C" => "Cisco",
      "001B2A" => "Cisco",
      "001D45" => "Cisco",
      "001D70" => "Cisco",
      "001E13" => "Cisco",
      "001E4A" => "Cisco",
      "001E7A" => "Cisco",
      "001EBD" => "Cisco",
      "001F6C" => "Cisco",
      "001FCA" => "Cisco",
      "0022BD" => "Cisco",
      "002555" => "Cisco",
      "0026CA" => "Cisco",
      "002713" => "Cisco",
      "00D097" => "Cisco",
      "00E014" => "Cisco",
      "04C5A4" => "Cisco",
      "0C8527" => "Cisco",
      "1C6A7A" => "Cisco",
      "40F4EC" => "Cisco",
      "5C50F7" => "Cisco",
      "6CAB31" => "Cisco",
      "6C41" => "Cisco",
      "74A2E6" => "Cisco",
      "D4A928" => "Cisco",
      "E84F25" => "Cisco",
      "F46E95" => "Cisco",

      # Ubiquiti
      "00156D" => "Ubiquiti",
      "001B2F" => "Ubiquiti",
      "001DD7" => "Ubiquiti",
      "0027BA" => "Ubiquiti",
      "04189A" => "Ubiquiti",
      "0418D6" => "Ubiquiti",
      "247F20" => "Ubiquiti",
      "248A07" => "Ubiquiti",
      "44D9E7" => "Ubiquiti",
      "74ACB9" => "Ubiquiti",
      "788A20" => "Ubiquiti",
      "80EA96" => "Ubiquiti",
      "A42BB0" => "Ubiquiti",
      "B4FBE4" => "Ubiquiti",
      "DC9FDB" => "Ubiquiti",
      "E063DA" => "Ubiquiti",
      "F09FC2" => "Ubiquiti",
      "F4E2C6" => "Ubiquiti",
      "FCECDA" => "Ubiquiti",

      # MikroTik
      "000456" => "MikroTik",
      "001D2E" => "MikroTik",
      "00272D" => "MikroTik",
      "0C424C" => "MikroTik",
      "2C3952" => "MikroTik",
      "48A9D2" => "MikroTik",
      "4C5E0C" => "MikroTik",
      "6C3B6B" => "MikroTik",
      "D49443" => "MikroTik",
      "D4CA6D" => "MikroTik",
      "E748B8" => "MikroTik",

      # TP-Link
      "000E8F" => "TP-Link",
      "001DD9" => "TP-Link",
      "0C8268" => "TP-Link",
      "1027F5" => "TP-Link",
      "148824" => "TP-Link",
      "1C3BF3" => "TP-Link",
      "300108" => "TP-Link",
      "3C468A" => "TP-Link",
      "50C7BF" => "TP-Link",
      "641C67" => "TP-Link",
      "7C8BCA" => "TP-Link",
      "98DAC4" => "TP-Link",
      "A0F3C1" => "TP-Link",
      "C46E1F" => "TP-Link",
      "E8DE27" => "TP-Link",
      "EC086B" => "TP-Link",
      "F0B429" => "TP-Link",

      # Juniper
      "001517" => "Juniper",
      "001B1F" => "Juniper",
      "002163" => "Juniper",
      "002281" => "Juniper",
      "0026DE" => "Juniper",
      "0026DC" => "Juniper",
      "002642" => "Juniper",
      "009E1E" => "Juniper",
      "285024" => "Juniper",
      "28A24B" => "Juniper",
      "3C8AB6" => "Juniper",
      "5C5E" => "Juniper",
      "842B2B" => "Juniper",

      # Aruba (HPE)
      "001A1E" => "Aruba",
      "00242C" => "Aruba",
      "04BD88" => "Aruba",
      "0C5496" => "Aruba",
      "200E52" => "Aruba",
      "24DE52" => "Aruba",
      "40E3D6" => "Aruba",
      "6C72E7" => "Aruba",
      "94B4C4" => "Aruba",
      "9C1C12" => "Aruba",
      "B8D9CE" => "Aruba",
      "D8C7C8" => "Aruba",

      # Dell
      "000874" => "Dell",
      "000BDB" => "Dell",
      "000C7B" => "Dell",
      "000D56" => "Dell",
      "000E0C" => "Dell",
      "000F1F" => "Dell",
      "001111" => "Dell",
      "00123F" => "Dell",
      "0013C3" => "Dell",
      "001372" => "Dell",
      "001485" => "Dell",
      "0015C5" => "Dell",
      "0016CE" => "Dell",
      "001731" => "Dell",
      "001871" => "Dell",
      "0019B9" => "Dell",
      "001A4B" => "Dell",
      "001C23" => "Dell",
      "001D09" => "Dell",
      "001E4F" => "Dell",
      "0021F6" => "Dell",
      "002219" => "Dell",
      "002264" => "Dell",
      "0023AE" => "Dell",
      "002564" => "Dell",
      "5C260A" => "Dell",

      # HP/HPE
      "001279" => "HP",
      "001438" => "HP",
      "001560" => "HP",
      "0016B9" => "HP",
      "001708" => "HP",
      "001B78" => "HP",
      "001C2E" => "HP",
      "001E0B" => "HP",
      "001F29" => "HP",
      "00237D" => "HP",
      "002481" => "HP",
      "00248D" => "HP",
      "002655" => "HP",
      "3863BB" => "HP",

      # Netgear
      "00095B" => "Netgear",
      "000FB5" => "Netgear",
      "001E2A" => "Netgear",
      "001F33" => "Netgear",
      "00223F" => "Netgear",
      "00260B" => "Netgear",
      "04A151" => "Netgear",
      "0846A0" => "Netgear",
      "2C3033" => "Netgear",
      "3448ED" => "Netgear",
      "4C9EFF" => "Netgear",
      "74DA38" => "Netgear",
      "84C9B2" => "Netgear",
      "A0040A" => "Netgear",
      "C43DC7" => "Netgear",

      # Fortinet
      "0009819" => "Fortinet",
      "001910" => "Fortinet",
      "00E04C" => "Fortinet",
      "080027" => "Fortinet",
      "0C1D36" => "Fortinet",
      "246C05" => "Fortinet",
      "708BCD" => "Fortinet",
      "900B29" => "Fortinet",

      # Palo Alto
      "009C33" => "Palo Alto",
      "001B17" => "Palo Alto",
      "5C5B35" => "Palo Alto",
      "8C89A5" => "Palo Alto",
      "D4F46F" => "Palo Alto"
    }
  end

  # Count distinct discovery sources
  defp count_sources(entry) do
    count = 0
    count = if entry.sources.neighbors == [], do: count, else: count + 1
    count = if entry.sources.arp == [], do: count, else: count + 1
    if entry.sources.mac == [], do: count, else: count + 1
  end

  # Get most recent timestamp from all sources
  defp get_most_recent_timestamp(entry) do
    timestamps =
      Enum.reject(
        Enum.map(entry.sources.neighbors, & &1.last_discovered_at) ++
          Enum.map(entry.sources.arp, & &1.last_seen_at) ++ Enum.map(entry.sources.mac, & &1.last_seen_at),
        &is_nil/1
      )

    case timestamps do
      [] -> DateTime.utc_now()
      _ -> Enum.max(timestamps, DateTime)
    end
  end

  # Get list of protocols used across all sources
  defp get_protocols_used(sources) do
    protocols = []

    protocols =
      case sources.neighbors do
        [] ->
          protocols

        neighbors ->
          neighbor_protocols =
            neighbors
            |> Enum.map(& &1.protocol)
            |> Enum.map(&String.to_atom/1)
            |> Enum.uniq()

          protocols ++ neighbor_protocols
      end

    protocols = if sources.arp == [], do: protocols, else: [:arp | protocols]
    protocols = if sources.mac == [], do: protocols, else: [:mac | protocols]

    Enum.uniq(protocols)
  end

  # VLAN queries

  @doc """
  Lists all VLANs for a device.
  """
  def list_vlans(snmp_device_id) do
    Vlan
    |> where([v], v.snmp_device_id == ^snmp_device_id)
    |> order_by([v], v.vlan_id)
    |> Repo.all()
  end

  @doc """
  Gets a specific VLAN.
  """
  def get_vlan(vlan_id) do
    Repo.get(Vlan, vlan_id)
  end

  # IP Address queries

  @doc """
  Lists all IP addresses for a device (across all its interfaces).
  """
  def list_ip_addresses(snmp_device_id) do
    IpAddress
    |> join(:inner, [ip], i in Interface, on: ip.snmp_interface_id == i.id)
    |> where([ip, i], i.snmp_device_id == ^snmp_device_id)
    |> preload(:snmp_interface)
    |> order_by([ip], [ip.ip_type, ip.ip_address])
    |> Repo.all()
  end

  @doc """
  Lists IP addresses for a specific interface.
  """
  def list_interface_ip_addresses(interface_id) do
    IpAddress
    |> where([ip], ip.snmp_interface_id == ^interface_id)
    |> order_by([ip], [ip.ip_type, ip.ip_address])
    |> Repo.all()
  end

  # Processor queries

  @doc """
  Lists all processors for a device.
  """
  def list_processors(snmp_device_id) do
    Processor
    |> where([p], p.snmp_device_id == ^snmp_device_id)
    |> order_by([p], p.processor_index)
    |> Repo.all()
  end

  @doc """
  Gets a specific processor.
  """
  def get_processor(processor_id) do
    Repo.get(Processor, processor_id)
  end

  @doc """
  Updates a processor.
  """
  def update_processor(processor, attrs) do
    processor
    |> Processor.changeset(attrs)
    |> Repo.update()
  end

  # Processor reading queries

  @doc """
  Gets recent processor readings for a processor.

  ## Options

  - `:limit` - Maximum number of readings to return (default: 100)
  - `:since` - Only return readings after this datetime
  """
  def get_processor_readings(processor_id, opts \\ []) do
    limit = Keyword.get(opts, :limit, 100)
    since = Keyword.get(opts, :since)

    query =
      ProcessorReading
      |> where([r], r.processor_id == ^processor_id)
      |> order_by([r], desc: r.checked_at)
      |> limit(^limit)

    query =
      if since do
        where(query, [r], r.checked_at >= ^since)
      else
        query
      end

    Repo.all(query)
  end

  @doc """
  Records a new processor reading.
  """
  def create_processor_reading(attrs) do
    %ProcessorReading{}
    |> ProcessorReading.changeset(attrs)
    |> Repo.insert()
  end

  # Storage queries

  @doc """
  Lists all storage entries for a device.
  """
  def list_storage(snmp_device_id) do
    Storage
    |> where([s], s.snmp_device_id == ^snmp_device_id)
    |> order_by([s], s.storage_index)
    |> Repo.all()
  end

  @doc """
  Gets a specific storage entry.
  """
  def get_storage(storage_id) do
    Repo.get(Storage, storage_id)
  end

  @doc """
  Updates a storage entry.
  """
  def update_storage(storage, attrs) do
    storage
    |> Storage.changeset(attrs)
    |> Repo.update()
  end

  # Storage reading queries

  @doc """
  Gets recent storage readings for a storage entry.

  ## Options

  - `:limit` - Maximum number of readings to return (default: 100)
  - `:since` - Only return readings after this datetime
  """
  def get_storage_readings(storage_id, opts \\ []) do
    limit = Keyword.get(opts, :limit, 100)
    since = Keyword.get(opts, :since)

    query =
      StorageReading
      |> where([r], r.storage_id == ^storage_id)
      |> order_by([r], desc: r.checked_at)
      |> limit(^limit)

    query =
      if since do
        where(query, [r], r.checked_at >= ^since)
      else
        query
      end

    Repo.all(query)
  end

  @doc """
  Gets the latest storage reading for a storage entry.
  """
  def get_latest_storage_reading(storage_id) do
    StorageReading
    |> where([r], r.storage_id == ^storage_id)
    |> order_by([r], desc: r.checked_at)
    |> limit(1)
    |> Repo.one()
  end

  @doc """
  Gets the latest storage readings for multiple storage entries in a single query.

  Returns a map of %{storage_id => reading} for efficient batch loading.
  Storage entries without readings will not be present in the map.

  ## Examples

      iex> get_latest_storage_readings_batch([storage_id1, storage_id2])
      %{storage_id1 => %StorageReading{}, storage_id2 => %StorageReading{}}
  """
  def get_latest_storage_readings_batch(storage_ids) when is_list(storage_ids) do
    if Enum.empty?(storage_ids) do
      %{}
    else
      # Use DISTINCT ON to get only the latest reading per storage
      query =
        from(r in StorageReading,
          where: r.storage_id in ^storage_ids,
          distinct: r.storage_id,
          order_by: [asc: r.storage_id, desc: r.checked_at]
        )

      query
      |> Repo.all()
      |> Map.new(fn reading -> {reading.storage_id, reading} end)
    end
  end

  @doc """
  Records a new storage reading.
  """
  def create_storage_reading(attrs) do
    %StorageReading{}
    |> StorageReading.changeset(attrs)
    |> Repo.insert()
  end

  # ARP Entry queries

  @doc """
  Lists all ARP entries for a device.
  """
  def list_arp_entries(device_id) do
    ArpEntry
    |> where([a], a.device_id == ^device_id)
    |> order_by([a], a.ip_address)
    |> preload(:interface)
    |> Repo.all()
  end

  @doc """
  Gets a specific ARP entry.
  """
  def get_arp_entry(arp_entry_id) do
    Repo.get(ArpEntry, arp_entry_id)
  end

  @doc """
  Creates or updates an ARP entry.
  """
  def upsert_arp_entry(attrs) do
    case Repo.get_by(ArpEntry,
           device_id: attrs.device_id,
           ip_address: attrs.ip_address,
           mac_address: attrs.mac_address
         ) do
      nil ->
        %ArpEntry{}
        |> ArpEntry.changeset(attrs)
        |> Repo.insert()

      arp_entry ->
        arp_entry
        |> ArpEntry.changeset(attrs)
        |> Repo.update()
    end
  end

  @doc """
  Deletes stale ARP entries that haven't been seen since the given datetime.
  """
  def delete_stale_arp_entries(device_id, cutoff_datetime) do
    ArpEntry
    |> where([a], a.device_id == ^device_id)
    |> where([a], a.last_seen_at < ^cutoff_datetime)
    |> Repo.delete_all()
  end

  @doc """
  Bulk upserts ARP entries for a device.
  Returns the number of entries processed.
  """
  def upsert_arp_entries(device_id, arp_entries, interfaces) do
    interface_map = Map.new(interfaces, fn i -> {i.if_index, i.id} end)

    Enum.each(arp_entries, fn entry ->
      interface_id = Map.get(interface_map, entry.if_index)

      attrs =
        entry
        |> Map.put(:device_id, device_id)
        |> Map.put(:interface_id, interface_id)

      upsert_arp_entry(attrs)
    end)

    length(arp_entries)
  end

  # MAC Address queries

  @doc """
  Lists all MAC address entries for a device.
  """
  def list_mac_addresses(device_id) do
    MacAddress
    |> where([m], m.device_id == ^device_id)
    |> order_by([m], [m.vlan_id, m.mac_address])
    |> preload(:interface)
    |> Repo.all()
  end

  @doc """
  Creates or updates a MAC address entry.
  """
  def upsert_mac_address(attrs) do
    case Repo.get_by(MacAddress,
           device_id: attrs.device_id,
           mac_address: attrs.mac_address,
           vlan_id: attrs[:vlan_id]
         ) do
      nil ->
        %MacAddress{}
        |> MacAddress.changeset(attrs)
        |> Repo.insert()

      mac_address ->
        mac_address
        |> MacAddress.changeset(attrs)
        |> Repo.update()
    end
  end

  @doc """
  Deletes stale MAC address entries that haven't been seen since the given datetime.
  """
  def delete_stale_mac_addresses(device_id, cutoff_datetime) do
    MacAddress
    |> where([m], m.device_id == ^device_id)
    |> where([m], m.last_seen_at < ^cutoff_datetime)
    |> Repo.delete_all()
  end

  @doc """
  Bulk upserts MAC address entries for a device.
  Returns the number of entries processed.
  """
  def upsert_mac_addresses(device_id, mac_entries, interfaces) do
    # Build a map from bridge port index to interface ID
    # Note: port_index from BRIDGE-MIB is not the same as if_index
    # For now, we'll use a simple mapping if they're available
    interface_map = Map.new(interfaces, fn i -> {i.if_index, i.id} end)

    Enum.each(mac_entries, fn entry ->
      interface_id = Map.get(interface_map, entry.port_index)

      attrs =
        entry
        |> Map.put(:device_id, device_id)
        |> Map.put(:interface_id, interface_id)

      upsert_mac_address(attrs)
    end)

    length(mac_entries)
  end

  # Network Topology Functions

  @doc """
  Builds a network topology map for an organization showing devices and their connections.

  Returns a graph structure with:
  - nodes: All devices (both added and discovered)
  - edges: Physical connections between interfaces
  - subnets: Logical groupings of devices on same networks
  """
  def get_network_topology(organization_id) do
    # Get all devices with their neighbors and interfaces
    devices = get_devices_with_topology(organization_id)

    # Get discovered devices not yet added
    discovered = list_discovered_devices_for_organization(organization_id)

    # Build topology structure
    nodes = build_topology_nodes(devices, discovered)
    edges = build_topology_edges(devices, nodes)
    subnets = build_subnet_groups(devices)

    %{
      nodes: nodes,
      edges: edges,
      subnets: subnets,
      stats: %{
        total_devices: length(nodes),
        added_devices: Enum.count(nodes, &(!&1.discovered)),
        discovered_devices: Enum.count(nodes, & &1.discovered),
        total_links: length(edges),
        total_subnets: length(subnets)
      },
      last_updated: DateTime.utc_now()
    }
  end

  # Get devices with all topology-related data preloaded
  defp get_devices_with_topology(organization_id) do
    Repo.all(
      from(d in DeviceSchema,
        join: s in assoc(d, :site),
        where: s.organization_id == ^organization_id,
        left_join: sd in Device,
        on: d.id == sd.device_id,
        left_join: i in Interface,
        on: sd.id == i.snmp_device_id,
        left_join: n in Neighbor,
        on: i.id == n.interface_id,
        left_join: ip in IpAddress,
        on: i.id == ip.snmp_interface_id,
        preload: [
          site: s,
          snmp_device: {sd, [interfaces: {i, [neighbors: n, ip_addresses: ip]}]}
        ]
      )
    )
  end

  # Build node structures for the graph
  defp build_topology_nodes(devices, discovered) do
    # Nodes for added devices
    device_nodes =
      Enum.map(devices, fn device ->
        %{
          id: device.id,
          label: device.name,
          type: device_type_atom(device),
          status: device.status,
          site_id: device.site_id,
          site_name: device.site.name,
          ip_address: device.ip_address,
          discovered: false,
          snmp_enabled: device.snmp_enabled,
          monitoring_enabled: device.monitoring_enabled
        }
      end)

    # Nodes for discovered but not added devices
    discovered_nodes =
      Enum.map(discovered, fn disc ->
        %{
          id: "discovered_#{disc.identifier.type}_#{disc.identifier.value}",
          label: disc.hostname || disc.identifier.value,
          type: disc.device_type,
          status: :unknown,
          site_id: nil,
          site_name: nil,
          ip_address: List.first(disc.ip_addresses),
          discovered: true,
          manufacturer: disc.manufacturer
        }
      end)

    device_nodes ++ discovered_nodes
  end

  # Build edge structures for the graph (physical connections)
  defp build_topology_edges(devices, nodes) do
    # Build lookup maps for matching neighbors to nodes
    node_lookup = build_node_lookup(devices, nodes)

    devices
    |> Enum.flat_map(&build_device_edges(&1, node_lookup))
    |> Enum.uniq_by(& &1.id)
  end

  # Build lookup maps to match neighbors to nodes using multiple strategies
  defp build_node_lookup(devices, nodes) do
    # Create maps for matching by different attributes
    by_id = Map.new(nodes, &{&1.id, &1})
    by_name = Map.new(nodes, &{String.downcase(&1.label), &1.id})
    by_ip = Map.new(nodes, fn node -> {node.ip_address, node.id} end)

    # Build MAC address lookup from SNMP devices
    by_mac =
      devices
      |> Enum.flat_map(fn device ->
        case device.snmp_device do
          nil ->
            []

          snmp_device ->
            snmp_device.interfaces
            |> Enum.filter(&(&1.if_phys_address != nil))
            |> Enum.map(&{&1.if_phys_address, device.id})
        end
      end)
      |> Map.new()

    %{
      by_id: by_id,
      by_name: by_name,
      by_ip: by_ip,
      by_mac: by_mac,
      node_ids: MapSet.new(Map.keys(by_id))
    }
  end

  defp build_device_edges(device, node_lookup) do
    case device.snmp_device do
      nil -> []
      snmp_device -> Enum.flat_map(snmp_device.interfaces, &build_interface_edges(device, &1, node_lookup))
    end
  end

  defp build_interface_edges(device, interface, node_lookup) do
    interface.neighbors
    |> Enum.map(fn neighbor ->
      # Try to match neighbor to existing node using multiple strategies
      target_id = find_target_node_id(neighbor, node_lookup)

      # Get IP addresses for source interface
      source_ips = format_interface_ips(interface.ip_addresses)

      # Create bidirectional edge ID (alphabetically sorted to deduplicate)
      edge_parts = Enum.sort([device.id, target_id || "unknown"])
      edge_id = Enum.join(edge_parts, "_")

      %{
        id: edge_id,
        source: device.id,
        target: target_id,
        source_interface: interface.if_name || interface.if_descr,
        target_interface: neighbor.remote_port_id,
        source_interface_id: interface.id,
        source_ips: source_ips,
        target_ip: neighbor.remote_address,
        protocol: neighbor.protocol,
        last_discovered: neighbor.last_discovered_at
      }
    end)
    |> Enum.filter(&(&1.target != nil))
  end

  # Format IP addresses with CIDR notation (IPv4 only)
  defp format_interface_ips(ip_addresses) do
    ip_addresses
    |> Enum.filter(&ipv4?(&1.ip_address))
    |> Enum.map(fn ip ->
      subnet = if ip.subnet_mask, do: "/#{calculate_prefix_length(ip.subnet_mask)}", else: ""
      "#{ip.ip_address}#{subnet}"
    end)
  end

  # Check if an IP address is IPv4
  defp ipv4?(ip_address) when is_binary(ip_address) do
    case :inet.parse_address(String.to_charlist(ip_address)) do
      {:ok, {_, _, _, _}} -> true
      _ -> false
    end
  end

  defp ipv4?(_), do: false

  # Calculate prefix length from subnet mask
  defp calculate_prefix_length(nil), do: ""

  defp calculate_prefix_length(subnet_mask) do
    subnet_mask
    |> String.split(".")
    |> Enum.map(&String.to_integer/1)
    |> Enum.map(&Integer.to_string(&1, 2))
    |> Enum.map_join(&String.pad_leading(&1, 8, "0"))
    |> String.graphemes()
    |> Enum.count(&(&1 == "1"))
  end

  # Find the node ID that matches the neighbor using multiple strategies
  defp find_target_node_id(neighbor, node_lookup) do
    # Strategy 1: Match by system name to device name
    name_match =
      if neighbor.remote_system_name do
        Map.get(node_lookup.by_name, String.downcase(neighbor.remote_system_name))
      end

    # Strategy 2: Match by IP address
    ip_match =
      if neighbor.remote_address do
        Map.get(node_lookup.by_ip, neighbor.remote_address)
      end

    # Strategy 3: Match by chassis ID (MAC address)
    mac_match =
      if neighbor.remote_chassis_id do
        Map.get(node_lookup.by_mac, neighbor.remote_chassis_id)
      end

    # Strategy 4: Check if it's a discovered node
    discovered_match =
      if neighbor.remote_chassis_id do
        discovered_id = "discovered_mac_#{neighbor.remote_chassis_id}"
        if MapSet.member?(node_lookup.node_ids, discovered_id), do: discovered_id
      end

    # Return first successful match
    name_match || ip_match || mac_match || discovered_match
  end

  # Build subnet groupings
  defp build_subnet_groups(devices) do
    devices
    |> Enum.flat_map(fn device ->
      case device.snmp_device do
        nil ->
          []

        snmp_device ->
          Enum.flat_map(snmp_device.interfaces, fn interface ->
            Enum.map(interface.ip_addresses, fn ip ->
              {device.id, ip.ip_address, ip.subnet_mask}
            end)
          end)
      end
    end)
    |> Enum.group_by(fn {_device_id, ip, mask} ->
      # Calculate CIDR notation
      case mask do
        nil -> nil
        _ -> calculate_cidr(ip, mask)
      end
    end)
    |> Enum.reject(fn {cidr, _} -> is_nil(cidr) end)
    |> Enum.map(fn {cidr, entries} ->
      device_ids = entries |> Enum.map(fn {device_id, _, _} -> device_id end) |> Enum.uniq()

      %{
        cidr: cidr,
        device_ids: device_ids,
        device_count: length(device_ids)
      }
    end)
    |> Enum.sort_by(& &1.device_count, :desc)
  end

  # Convert device type to atom (handles both string and atom)
  defp device_type_atom(device) when is_atom(device), do: device
  defp device_type_atom(device) when is_binary(device), do: String.to_atom(device)
  defp device_type_atom(_), do: :unknown

  # Calculate CIDR notation from IP and mask
  defp calculate_cidr(ip, mask) do
    case {parse_ip(ip), parse_ip(mask)} do
      {{:ok, _ip_tuple}, {:ok, mask_tuple}} ->
        prefix_length = count_mask_bits(mask_tuple)
        network = calculate_network(ip, mask)
        "#{network}/#{prefix_length}"

      _ ->
        nil
    end
  end

  defp parse_ip(ip) when is_binary(ip) do
    case :inet.parse_address(String.to_charlist(ip)) do
      {:ok, tuple} -> {:ok, tuple}
      {:error, _} -> :error
    end
  end

  defp parse_ip(_), do: :error

  defp count_mask_bits({a, b, c, d}) do
    [a, b, c, d]
    |> Enum.map(&Integer.to_string(&1, 2))
    |> Enum.map_join(&String.pad_leading(&1, 8, "0"))
    |> String.graphemes()
    |> Enum.count(&(&1 == "1"))
  end

  defp calculate_network(ip, mask) do
    {:ok, ip_tuple} = parse_ip(ip)
    {:ok, mask_tuple} = parse_ip(mask)

    {a1, b1, c1, d1} = ip_tuple
    {a2, b2, c2, d2} = mask_tuple

    a = Bitwise.band(a1, a2)
    b = Bitwise.band(b1, b2)
    c = Bitwise.band(c1, c2)
    d = Bitwise.band(d1, d2)

    "#{a}.#{b}.#{c}.#{d}"
  end
end