towerops/lib/towerops/topology.ex

defmodule Towerops.Topology do
  @moduledoc """
  Topology inference engine. Builds and maintains a persistent model of
  network device relationships by analyzing SNMP polling data.
  """

  import Ecto.Query

  alias Towerops.Capacity
  alias Towerops.Devices
  alias Towerops.Devices.Device
  alias Towerops.Repo
  alias Towerops.Result
  alias Towerops.Snmp.ArpEntry
  alias Towerops.Snmp.Device, as: SnmpDevice
  alias Towerops.Snmp.Interface
  alias Towerops.Snmp.IpAddress
  alias Towerops.Snmp.MacAddress
  alias Towerops.Snmp.Neighbor
  alias Towerops.Snmp.Sensor
  alias Towerops.Snmp.WirelessClient
  alias Towerops.Topology.DeviceLink
  alias Towerops.Topology.DeviceLinkEvidence
  alias Towerops.Topology.DeviceNeighbor
  alias Towerops.Topology.Identifier
  alias Towerops.Topology.InferenceEngine
  alias Towerops.Topology.Lldp

  require Logger

  @doc """
  Builds lookup maps for matching evidence to managed devices in an organization.

  Returns a map with `:by_ip`, `:by_name`, and `:by_mac` keys, each mapping
  an identifier string to a device ID.
  """
  def build_device_lookup(organization_id) do
    devices = fetch_lookup_devices(organization_id)

    %{
      by_ip: map_ips_to_ids(devices),
      by_name: map_names_to_ids(devices),
      by_mac: map_macs_to_ids(devices)
    }
  end

  defp fetch_lookup_devices(organization_id) do
    Repo.all(
      from(d in Device,
        where: d.organization_id == ^organization_id,
        left_join: sd in assoc(d, :snmp_device),
        left_join: i in Interface,
        on: sd.id == i.snmp_device_id,
        left_join: ip in IpAddress,
        on: i.id == ip.snmp_interface_id,
        preload: [snmp_device: {sd, interfaces: {i, ip_addresses: ip}}]
      )
    )
  end

  defp map_ips_to_ids(devices) do
    devices
    |> Enum.flat_map(fn device ->
      [device.ip_address | device_interface_ips(device)]
      |> Enum.map(&Identifier.normalize_ip/1)
      |> Enum.reject(&is_nil/1)
      |> Enum.map(&{&1, device.id})
    end)
    |> Map.new()
  end

  defp map_names_to_ids(devices) do
    devices
    |> Enum.flat_map(fn device ->
      [device.name, device.snmp_device && device.snmp_device.sys_name]
      |> Enum.flat_map(&Identifier.candidate_names/1)
      |> Enum.map(&{&1, device.id})
    end)
    |> Map.new()
  end

  defp map_macs_to_ids(devices) do
    devices
    |> Enum.flat_map(fn device ->
      case device.snmp_device do
        nil ->
          []

        sd ->
          sd.interfaces
          |> Enum.filter(&(&1.if_phys_address != nil))
          |> Enum.map(&Identifier.normalize_mac(&1.if_phys_address))
          |> Enum.reject(&is_nil/1)
          |> Enum.map(&{&1, device.id})
      end
    end)
    |> Map.new()
  end

  @doc """
  Finds a device ID by IP address from the lookup.
  """
  def find_device_by_ip(lookup, ip), do: Map.get(lookup.by_ip, Identifier.normalize_ip(ip))

  @doc """
  Finds a device ID by name (case-insensitive) from the lookup.
  """
  def find_device_by_name(lookup, name) when is_binary(name) do
    name
    |> Identifier.candidate_names()
    |> Enum.find_value(&Map.get(lookup.by_name, &1))
  end

  def find_device_by_name(_lookup, _name), do: nil

  @doc """
  Finds a device ID by MAC address from the lookup.
  """
  def find_device_by_mac(lookup, mac), do: Map.get(lookup.by_mac, Identifier.normalize_mac(mac))

  @doc """
  Resolves a device ID from evidence fields, trying MAC first, then IP, then name.
  """
  def resolve_device(lookup, %{mac: mac, ip: ip, name: name}) do
    find_device_by_mac(lookup, mac) ||
      find_device_by_ip(lookup, ip) ||
      find_device_by_name(lookup, name)
  end

  def resolve_device(_lookup, _), do: nil

  @doc """
  Collects topology evidence from LLDP neighbor entries for a device.

  Returns a list of evidence maps with confidence 0.95.
  """
  def collect_lldp_evidence(device_id) do
    collect_neighbor_evidence(device_id, "lldp", "lldp_neighbor", 0.95)
  end

  @doc """
  Collects topology evidence from CDP neighbor entries for a device.

  Returns a list of evidence maps with confidence 0.95.
  """
  def collect_cdp_evidence(device_id) do
    collect_neighbor_evidence(device_id, "cdp", "cdp_neighbor", 0.95)
  end

  defp collect_neighbor_evidence(device_id, protocol, evidence_type, confidence) do
    from(n in Neighbor,
      join: i in Interface,
      on: n.interface_id == i.id,
      where: n.device_id == ^device_id and n.protocol == ^protocol,
      select: %{
        neighbor_id: n.id,
        source_interface_id: i.id,
        remote_chassis_id: n.remote_chassis_id,
        remote_system_name: n.remote_system_name,
        remote_address: n.remote_address,
        remote_port_id: n.remote_port_id,
        remote_capabilities: n.remote_capabilities,
        last_discovered_at: n.last_discovered_at
      }
    )
    |> Repo.all()
    |> Enum.map(fn row ->
      %{
        evidence_type: evidence_type,
        confidence: confidence,
        source_interface_id: row.source_interface_id,
        remote_mac: Identifier.normalize_mac(row.remote_chassis_id),
        remote_ip: Identifier.normalize_ip(row.remote_address),
        remote_name: Identifier.normalize_name(row.remote_system_name),
        remote_port: row.remote_port_id,
        remote_capabilities: row.remote_capabilities,
        evidence_data: %{
          "neighbor_id" => row.neighbor_id,
          "remote_chassis_id" => row.remote_chassis_id,
          "remote_port_id" => row.remote_port_id
        }
      }
    end)
  end

  @doc """
  Collects topology evidence from MAC address table entries for a device.

  Uses the device lookup to match MACs to known devices. Excludes entries
  that match the device itself (self-links). Returns evidence maps with
  confidence 0.7.
  """
  def collect_mac_evidence(device_id, lookup) do
    query =
      from(m in MacAddress,
        where: m.device_id == ^device_id,
        select: %{
          interface_id: m.interface_id,
          mac_address: m.mac_address,
          vlan_id: m.vlan_id,
          last_seen_at: m.last_seen_at
        }
      )

    collect_lookup_evidence(device_id, query, &mac_row_to_evidence(&1, lookup))
  end

  defp mac_row_to_evidence(row, lookup) do
    %{
      evidence_type: "mac_on_interface",
      confidence: 0.7,
      source_interface_id: row.interface_id,
      remote_mac: Identifier.normalize_mac(row.mac_address),
      remote_ip: nil,
      remote_name: nil,
      remote_port: nil,
      remote_capabilities: nil,
      matched_device_id: find_device_by_mac(lookup, row.mac_address),
      evidence_data: %{
        "mac_address" => row.mac_address,
        "vlan_id" => row.vlan_id
      }
    }
  end

  @doc """
  Collects topology evidence from ARP table entries for a device.

  Uses the device lookup to match entries by MAC first, then IP. Excludes
  entries that match the device itself (self-links). Returns evidence maps
  with confidence 0.6.
  """
  def collect_arp_evidence(device_id, lookup) do
    query =
      from(a in ArpEntry,
        where: a.device_id == ^device_id,
        select: %{
          interface_id: a.interface_id,
          ip_address: a.ip_address,
          mac_address: a.mac_address,
          last_seen_at: a.last_seen_at
        }
      )

    collect_lookup_evidence(device_id, query, &arp_row_to_evidence(&1, lookup))
  end

  defp arp_row_to_evidence(row, lookup) do
    %{
      evidence_type: "arp_entry",
      confidence: 0.6,
      source_interface_id: row.interface_id,
      remote_mac: Identifier.normalize_mac(row.mac_address),
      remote_ip: Identifier.normalize_ip(row.ip_address),
      remote_name: nil,
      remote_port: nil,
      remote_capabilities: nil,
      matched_device_id:
        find_device_by_mac(lookup, row.mac_address) ||
          find_device_by_ip(lookup, row.ip_address),
      evidence_data: %{
        "ip_address" => row.ip_address,
        "mac_address" => row.mac_address
      }
    }
  end

  # Common pipeline for lookup-based evidence collectors (MAC, ARP):
  # query the source table, map each row to an evidence map, drop self-links.
  defp collect_lookup_evidence(device_id, query, build_fn) do
    query
    |> Repo.all()
    |> Enum.map(build_fn)
    |> Enum.reject(&(&1.matched_device_id == device_id))
  end

  defdelegate merge_confidence(a, b), to: InferenceEngine
  defdelegate infer_role_from_capabilities(capabilities), to: InferenceEngine

  @doc """
  Create or update a device link. Keyed on source_device_id + source_interface_id +
  discovered_remote_mac. If existing, merges confidence and appends evidence.
  Always updates last_confirmed_at.
  """
  def upsert_link(attrs) do
    evidence_attrs = Map.get(attrs, :evidence, [])
    link_attrs = Map.delete(attrs, :evidence)

    existing = find_existing_link(link_attrs)

    result =
      case existing do
        nil ->
          %DeviceLink{}
          |> DeviceLink.changeset(link_attrs)
          |> Repo.insert()

        link ->
          merged_confidence = merge_confidence(link.confidence, link_attrs.confidence)

          link
          |> DeviceLink.changeset(%{
            confidence: merged_confidence,
            last_confirmed_at: link_attrs.last_confirmed_at,
            target_device_id: link_attrs[:target_device_id] || link.target_device_id,
            metadata: Map.merge(link.metadata || %{}, link_attrs[:metadata] || %{})
          })
          |> Repo.update()
      end

    with {:ok, link} <- result do
      _ = insert_link_evidence_batch(link.id, evidence_attrs)
      {:ok, Repo.reload!(link)}
    end
  end

  @valid_evidence_types ~w(lldp_neighbor cdp_neighbor mac_on_interface arp_entry wireless_registration subnet_match)

  # Batch-inserts all evidence rows for a link in a single round-trip.
  # Skips rows missing required fields and logs them; bypasses changeset
  # validation since evidence is built internally, not from user input.
  defp insert_link_evidence_batch(_link_id, []), do: :ok

  defp insert_link_evidence_batch(link_id, evidence_attrs) do
    now = DateTime.truncate(DateTime.utc_now(), :second)

    {valid, invalid} =
      Enum.split_with(evidence_attrs, fn ev ->
        Map.has_key?(ev, :evidence_type) and ev.evidence_type in @valid_evidence_types and
          Map.has_key?(ev, :observed_at)
      end)

    Enum.each(invalid, fn ev ->
      Logger.error("Dropping invalid device link evidence: #{inspect(ev)}")
    end)

    rows =
      Enum.map(valid, fn ev ->
        %{
          id: Ecto.UUID.generate(),
          device_link_id: link_id,
          evidence_type: ev.evidence_type,
          evidence_data: Map.get(ev, :evidence_data, %{}),
          observed_at: DateTime.truncate(ev.observed_at, :second),
          inserted_at: now,
          updated_at: now
        }
      end)

    case rows do
      [] -> :ok
      _ -> Repo.insert_all(DeviceLinkEvidence, rows)
    end
  end

  defdelegate infer_device_role(device), to: InferenceEngine

  @doc """
  Deprecated: Auto-inference is disabled. All device types are now manually set.
  This function is kept for backwards compatibility but does nothing.
  """
  def maybe_update_device_role(device) do
    {:ok, device}
  end

  @doc """
  Main topology inference pipeline for a device. Called after each poll cycle.
  Collects evidence from all sources, groups by remote identifier, upserts links,
  broadcasts changes. Returns {:ok, :changed} or {:ok, :unchanged}.
  """
  def process_device(device, organization_id) do
    lookup = build_device_lookup(organization_id)
    now = DateTime.utc_now()

    # Collect all evidence
    all_evidence =
      collect_lldp_evidence(device.id) ++
        collect_cdp_evidence(device.id) ++
        collect_mac_evidence(device.id, lookup) ++
        collect_arp_evidence(device.id, lookup)

    if Enum.empty?(all_evidence) do
      {:ok, :unchanged}
    else
      grouped = group_evidence_by_remote(all_evidence)

      results =
        grouped
        |> Enum.map(&upsert_grouped_evidence(&1, device.id, lookup, now))
        |> Enum.reject(&(&1 == :skip))

      has_changes = Enum.any?(results, &Result.ok?/1)

      # Auto-inference disabled - device type is now manual-only
      # maybe_update_device_role(device)

      if has_changes do
        _ =
          Phoenix.PubSub.broadcast(
            Towerops.PubSub,
            "topology:#{organization_id}",
            {:topology_updated, organization_id}
          )

        {:ok, :changed}
      else
        {:ok, :unchanged}
      end
    end
  end

  @doc """
  Clean up stale links. Called periodically (e.g. hourly).
  - Links not confirmed in 24h: set confidence to 0.1
  - Links not confirmed in 72h: delete entirely
  Returns {stale_count, deleted_count}.
  """
  def cleanup_stale_links do
    now = DateTime.utc_now()
    stale_cutoff = DateTime.add(now, -24, :hour)
    delete_cutoff = DateTime.add(now, -72, :hour)

    # Delete very old links first
    {deleted_count, _} =
      Repo.delete_all(from(l in DeviceLink, where: l.last_confirmed_at < ^delete_cutoff))

    # Mark stale links (24h+ but not yet 72h)
    {stale_count, _} =
      Repo.update_all(
        from(l in DeviceLink,
          where: l.last_confirmed_at < ^stale_cutoff and l.confidence > 0.1
        ),
        set: [confidence: 0.1]
      )

    {stale_count, deleted_count}
  end

  @doc """
  Build topology data for the network map. Returns %{nodes, edges, stats, last_updated}.
  Tab "added" = managed devices only. Tab "all" = managed + discovered.
  """
  def get_topology_for_map(organization_id, tab \\ "added") do
    devices = list_managed_devices(organization_id)
    device_ids = Enum.map(devices, & &1.id)
    links = list_links_for_devices(device_ids)

    # Compute wireless stats (client counts + signal health) per device
    wireless_stats = compute_wireless_stats(device_ids)

    # Compute RF signal health per link for edge enrichment
    rf_link_stats = compute_rf_link_stats(device_ids)

    managed_nodes = Enum.map(devices, &device_to_node(&1, wireless_stats))
    discovered_nodes = build_discovered_nodes(links, tab)

    # Build site compound nodes from devices that have sites
    site_nodes =
      devices
      |> Enum.filter(& &1.site)
      |> Enum.uniq_by(& &1.site_id)
      |> Enum.map(fn device ->
        %{
          id: "site_#{device.site_id}",
          label: device.site.name,
          type: :site,
          device_role: nil,
          status: nil,
          site_id: device.site_id,
          site_name: device.site.name,
          ip_address: nil,
          discovered: false,
          manufacturer: nil,
          parent: nil,
          latitude: device.site.latitude,
          longitude: device.site.longitude
        }
      end)

    all_nodes = site_nodes ++ managed_nodes ++ discovered_nodes
    node_ids = MapSet.new(Enum.map(all_nodes, & &1.id))

    edges =
      links
      |> build_edges_from_links(node_ids)
      |> merge_bidirectional_edges()
      |> enrich_edges_with_rf(rf_link_stats)

    # Compute whether geographic layout is available
    has_geo =
      Enum.any?(all_nodes, fn n ->
        n[:latitude] != nil and n[:longitude] != nil
      end)

    %{
      nodes: all_nodes,
      edges: edges,
      stats: %{
        total_devices: length(managed_nodes) + length(discovered_nodes),
        added_devices: length(managed_nodes),
        discovered_devices: length(discovered_nodes),
        total_links: length(edges),
        degraded_links: Enum.count(edges, fn e -> e[:signal_health] == "degraded" or e[:signal_health] == "critical" end),
        sites_with_alerts:
          devices
          |> Enum.filter(&(&1.status == :down))
          |> Enum.map(& &1.site_id)
          |> Enum.reject(&is_nil/1)
          |> Enum.uniq()
          |> length()
      },
      has_geo: has_geo,
      last_updated: DateTime.utc_now()
    }
  end

  @doc """
  Build topology data for the network weathermap with edge utilization info.
  Returns %{nodes, edges, stats, last_updated} with utilization data.
  Tab "added" = managed devices only. Tab "all" = managed + discovered.
  """
  def get_topology_for_weathermap(organization_id, tab \\ "added") do
    devices = list_managed_devices(organization_id)
    device_ids = Enum.map(devices, & &1.id)
    links = list_links_for_devices(device_ids)

    # Compute wireless stats (client counts + signal health) per device
    wireless_stats = compute_wireless_stats(device_ids)

    # Compute RF signal health per link for edge enrichment
    rf_link_stats = compute_rf_link_stats(device_ids)

    managed_nodes = Enum.map(devices, &device_to_node(&1, wireless_stats))
    discovered_nodes = build_discovered_nodes(links, tab)

    # Build site compound nodes from devices that have sites
    site_nodes =
      devices
      |> Enum.filter(& &1.site)
      |> Enum.uniq_by(& &1.site_id)
      |> Enum.map(fn device ->
        %{
          id: "site_#{device.site_id}",
          label: device.site.name,
          type: :site,
          device_role: nil,
          status: nil,
          site_id: device.site_id,
          site_name: device.site.name,
          ip_address: nil,
          discovered: false,
          manufacturer: nil,
          parent: nil,
          latitude: device.site.latitude,
          longitude: device.site.longitude
        }
      end)

    all_nodes = site_nodes ++ managed_nodes ++ discovered_nodes
    node_ids = MapSet.new(Enum.map(all_nodes, & &1.id))

    # Build edges with utilization data
    edges =
      links
      |> build_edges_from_links(node_ids)
      |> merge_bidirectional_edges()
      |> enrich_edges_with_rf(rf_link_stats)
      |> enrich_edges_with_utilization(device_ids)

    # Compute utilization stats for the dashboard
    utilization_stats = compute_utilization_stats(edges)

    # Compute whether geographic layout is available
    has_geo =
      Enum.any?(all_nodes, fn n ->
        n[:latitude] != nil and n[:longitude] != nil
      end)

    %{
      nodes: all_nodes,
      edges: edges,
      stats: %{
        total_devices: length(managed_nodes) + length(discovered_nodes),
        added_devices: length(managed_nodes),
        discovered_devices: length(discovered_nodes),
        total_links: length(edges),
        low_utilization_links: utilization_stats.low_utilization_links,
        medium_utilization_links: utilization_stats.medium_utilization_links,
        high_utilization_links: utilization_stats.high_utilization_links,
        overutilized_links: utilization_stats.overutilized_links,
        degraded_links: Enum.count(edges, fn e -> e[:signal_health] == "degraded" or e[:signal_health] == "critical" end),
        sites_with_alerts:
          devices
          |> Enum.filter(&(&1.status == :down))
          |> Enum.map(& &1.site_id)
          |> Enum.reject(&is_nil/1)
          |> Enum.uniq()
          |> length()
      },
      has_geo: has_geo,
      last_updated: DateTime.utc_now()
    }
  end

  @doc """
  Get detail information for a node (managed or discovered) for the detail panel.

  For managed devices: returns device with site, connections.
  For discovered nodes (id starts with "discovered_"): returns info from link metadata.
  Returns nil if node not found or not in organization.
  """
  def get_node_detail("discovered_" <> _suffix = node_id, organization_id) do
    # Find the link that created this discovered node
    device_ids =
      Repo.all(
        from(d in Device,
          where: d.organization_id == ^organization_id,
          select: d.id
        )
      )

    links =
      Repo.all(
        from(l in DeviceLink,
          where: l.source_device_id in ^device_ids and is_nil(l.target_device_id),
          preload: [:source_device, :source_interface, :target_interface]
        )
      )

    # Find the link matching this discovered node ID
    link =
      Enum.find(links, fn l ->
        discovered_node_id(l) == node_id
      end)

    case link do
      nil ->
        nil

      link ->
        capabilities = get_in(link.metadata, ["remote_capabilities"])

        %{
          id: node_id,
          name: link.discovered_remote_name || link.discovered_remote_ip || link.discovered_remote_mac,
          ip_address: link.discovered_remote_ip,
          mac_address: link.discovered_remote_mac,
          type: :discovered,
          role: infer_role_from_capabilities(capabilities),
          status: :unknown,
          site_name: nil,
          manufacturer: nil,
          connections: [
            %{
              device_id: link.source_device_id,
              device_name: link.source_device.name,
              interface: if(link.source_interface, do: link.source_interface.if_name),
              confidence: link.confidence,
              link_type: link.link_type
            }
          ]
        }
    end
  end

  def get_node_detail("site_" <> _rest, _organization_id), do: nil

  def get_node_detail(device_id, organization_id) do
    device =
      Repo.one(
        from(d in Device,
          where: d.id == ^device_id and d.organization_id == ^organization_id,
          left_join: s in assoc(d, :site),
          left_join: sd in assoc(d, :snmp_device),
          preload: [site: s, snmp_device: sd]
        )
      )

    case device do
      nil ->
        nil

      device ->
        connections =
          device_id
          |> list_connected_devices()
          |> Enum.map(&link_to_connection(&1, device_id))

        # Wireless stats for detail panel
        ws = compute_wireless_stats([device_id])
        device_ws = Map.get(ws, device_id, %{client_count: 0, signal_health: nil})

        rf = compute_rf_link_stats([device_id])
        device_rf = Map.get(rf, device_id)

        %{
          id: device.id,
          name: device.name,
          ip_address: device.ip_address,
          type: :managed,
          role: role_to_type_atom(device.device_role),
          device_role: device.device_role,
          status: device.status,
          site_name: if(device.site, do: device.site.name),
          manufacturer: if(device.snmp_device, do: device.snmp_device.manufacturer),
          connections: connections,
          client_count: device_ws.client_count,
          signal_health: device_ws.signal_health,
          snr: if(device_rf, do: device_rf[:snr]),
          is_wireless: device.device_role in ~w(access_point backhaul cpe backhaul_radio)
        }
    end
  end

  defp link_to_connection(link, device_id) do
    {connected_device, interface} =
      if link.source_device_id == device_id do
        {link.target_device, link.target_interface}
      else
        {link.source_device, link.source_interface}
      end

    %{
      device_id: if(connected_device, do: connected_device.id),
      device_name: connected_device_name(connected_device, link),
      interface: if(interface, do: interface.if_name),
      confidence: link.confidence,
      link_type: link.link_type
    }
  end

  defp connected_device_name(device, _link) when not is_nil(device), do: device.name

  defp connected_device_name(nil, link),
    do: link.discovered_remote_name || link.discovered_remote_ip || link.discovered_remote_mac

  @doc "List all device links where this device is the source, ordered by confidence."
  def list_device_links(device_id) do
    Repo.all(
      from(l in DeviceLink,
        where: l.source_device_id == ^device_id,
        order_by: [desc: l.confidence]
      )
    )
  end

  @doc """
  List all devices connected to this device via device_links.
  Returns links where the device is either the source or target,
  with preloaded associations for display.
  """
  def list_connected_devices(device_id) do
    Repo.all(
      from(l in DeviceLink,
        where: l.source_device_id == ^device_id or l.target_device_id == ^device_id,
        preload: [:source_device, :target_device, :source_interface, :target_interface],
        order_by: [desc: l.confidence]
      )
    )
  end

  # --- Network map helpers ---

  defp list_managed_devices(organization_id) do
    Repo.all(
      from(d in Device,
        where: d.organization_id == ^organization_id,
        left_join: s in assoc(d, :site),
        left_join: sd in assoc(d, :snmp_device),
        preload: [site: s, snmp_device: sd]
      )
    )
  end

  defp list_links_for_devices(device_ids) do
    Repo.all(
      from(l in DeviceLink,
        where: l.source_device_id in ^device_ids or l.target_device_id in ^device_ids,
        preload: [:source_interface, :target_interface]
      )
    )
  end

  defp device_to_node(device, wireless_stats) do
    stats = Map.get(wireless_stats, device.id, %{client_count: 0, signal_health: nil})

    %{
      id: device.id,
      label: device.name || device.ip_address || "Unknown",
      type: role_to_type_atom(device.device_role),
      device_role: device.device_role,
      status: device.status,
      site_id: device.site_id,
      site_name: if(device.site, do: device.site.name),
      ip_address: device.ip_address,
      discovered: false,
      manufacturer: if(device.snmp_device, do: device.snmp_device.manufacturer),
      parent: if(device.site_id, do: "site_#{device.site_id}"),
      client_count: stats.client_count,
      signal_health: stats.signal_health,
      latitude: if(device.site, do: device.site.latitude),
      longitude: if(device.site, do: device.site.longitude),
      has_alerts: device.status == :down
    }
  end

  defp build_discovered_nodes(links, "all") do
    links
    |> Enum.filter(&is_nil(&1.target_device_id))
    |> Enum.uniq_by(&discovered_node_id/1)
    |> Enum.map(fn link ->
      capabilities = get_in(link.metadata, ["remote_capabilities"])
      inferred_type = infer_role_from_capabilities(capabilities)

      %{
        id: discovered_node_id(link),
        label: link.discovered_remote_name || link.discovered_remote_ip || link.discovered_remote_mac,
        type: inferred_type,
        device_role: nil,
        status: :unknown,
        site_id: nil,
        site_name: nil,
        ip_address: link.discovered_remote_ip,
        discovered: true,
        manufacturer: nil,
        parent_device_id: link.source_device_id
      }
    end)
  end

  defp build_discovered_nodes(_links, _tab), do: []

  defp merge_bidirectional_edges(edges) do
    edges
    |> Enum.group_by(fn edge ->
      {min(edge.source, edge.target), max(edge.source, edge.target)}
    end)
    |> Enum.map(fn {_key, group} -> merge_edge_group(group) end)
  end

  defp merge_edge_group([single]), do: single

  defp merge_edge_group([first | rest] = group) do
    merged_confidence =
      Enum.reduce(rest, first.confidence, fn edge, acc ->
        merge_confidence(acc, edge.confidence)
      end)

    all_interfaces =
      group
      |> Enum.flat_map(&[&1.source_interface, &1.target_interface])
      |> Enum.reject(&is_nil/1)
      |> Enum.uniq()

    best_link_type =
      group
      |> Enum.map(& &1.link_type)
      |> Enum.min_by(&link_type_priority/1)

    if_speed =
      group
      |> Enum.map(&Map.get(&1, :if_speed))
      |> Enum.reject(&is_nil/1)
      |> Enum.max(fn -> nil end)

    %{
      first
      | confidence: merged_confidence,
        link_type: best_link_type,
        source_interface: Enum.at(all_interfaces, 0),
        target_interface: Enum.at(all_interfaces, 1),
        if_speed: if_speed
    }
  end

  defp link_type_priority("lldp"), do: 0
  defp link_type_priority("cdp"), do: 1
  defp link_type_priority("mac_match"), do: 2
  defp link_type_priority("arp_inference"), do: 3
  defp link_type_priority(_), do: 4

  defp build_edges_from_links(links, node_ids) do
    links
    |> Enum.map(fn link ->
      target_id =
        if link.target_device_id,
          do: link.target_device_id,
          else: discovered_node_id(link)

      %{
        id: link.id,
        source: link.source_device_id,
        target: target_id,
        source_interface: if(link.source_interface, do: link.source_interface.if_name),
        target_interface: if(link.target_interface, do: link.target_interface.if_name),
        source_interface_id: link.source_interface_id,
        target_interface_id: if(link.target_interface, do: link.target_interface.id),
        link_type: link.link_type,
        confidence: link.confidence,
        metadata: link.metadata,
        if_speed: if(link.source_interface, do: link.source_interface.if_speed)
      }
    end)
    |> Enum.filter(fn edge ->
      MapSet.member?(node_ids, edge.source) and MapSet.member?(node_ids, edge.target)
    end)
  end

  # Map device roles to visual types for network topology
  defp role_to_type_atom("router"), do: :router
  defp role_to_type_atom("core_router"), do: :router
  defp role_to_type_atom("switch"), do: :switch
  defp role_to_type_atom("distribution_switch"), do: :switch
  defp role_to_type_atom("access_switch"), do: :switch
  defp role_to_type_atom("access_point"), do: :wireless
  defp role_to_type_atom("cpe"), do: :wireless
  defp role_to_type_atom("backhaul"), do: :wireless
  defp role_to_type_atom("backhaul_radio"), do: :wireless
  defp role_to_type_atom("server"), do: :server
  defp role_to_type_atom("firewall"), do: :firewall
  defp role_to_type_atom("ups"), do: :server
  defp role_to_type_atom("other"), do: :unknown
  defp role_to_type_atom(_), do: :unknown

  # --- Evidence processing helpers ---

  defp upsert_grouped_evidence({_key, evidence_list}, device_id, lookup, now) do
    best = Enum.max_by(evidence_list, & &1.confidence)
    remote_identity = summarize_remote_identity(evidence_list)

    matched_id =
      resolve_device(lookup, %{
        mac: remote_identity.remote_mac,
        ip: remote_identity.remote_ip,
        name: remote_identity.remote_name
      })

    if matched_id == device_id do
      :skip
    else
      upsert_link(%{
        source_device_id: device_id,
        target_device_id: matched_id,
        source_interface_id: best.source_interface_id,
        link_type: evidence_type_to_link_type(best.evidence_type),
        confidence: combine_evidence_confidence(evidence_list),
        discovered_remote_mac: remote_identity.remote_mac,
        discovered_remote_ip: remote_identity.remote_ip,
        discovered_remote_name: remote_identity.remote_name,
        metadata: build_link_metadata(evidence_list),
        last_confirmed_at: now,
        evidence:
          Enum.map(evidence_list, fn ev ->
            %{
              evidence_type: ev.evidence_type,
              evidence_data: ev.evidence_data,
              observed_at: now
            }
          end)
      })
    end
  end

  defp build_link_metadata(evidence_list) do
    capabilities =
      evidence_list
      |> Enum.flat_map(fn ev -> ev[:remote_capabilities] || [] end)
      |> Enum.uniq()

    if Enum.empty?(capabilities) do
      %{}
    else
      %{"remote_capabilities" => capabilities}
    end
  end

  defp group_evidence_by_remote(evidence_list) do
    Enum.group_by(evidence_list, &evidence_group_key/1)
  end

  defp combine_evidence_confidence(evidence_list) do
    evidence_list
    |> Enum.map(& &1.confidence)
    |> Enum.reduce(0.0, &merge_confidence/2)
  end

  defp evidence_type_to_link_type("lldp_neighbor"), do: "lldp"
  defp evidence_type_to_link_type("cdp_neighbor"), do: "cdp"
  defp evidence_type_to_link_type("mac_on_interface"), do: "mac_match"
  defp evidence_type_to_link_type("arp_entry"), do: "arp_inference"
  defp evidence_type_to_link_type("wireless_registration"), do: "wireless_association"
  defp evidence_type_to_link_type(_), do: "mac_match"

  defp device_interface_ips(%Device{snmp_device: nil}), do: []

  defp device_interface_ips(%Device{snmp_device: %SnmpDevice{} = snmp_device}) do
    Enum.flat_map(snmp_device.interfaces, fn interface ->
      Enum.map(interface.ip_addresses, & &1.ip_address)
    end)
  end

  defp find_existing_link(link_attrs) do
    source_device_id = link_attrs.source_device_id
    source_interface_id = Map.get(link_attrs, :source_interface_id)

    base_query =
      if is_nil(source_interface_id) do
        from(l in DeviceLink,
          where: l.source_device_id == ^source_device_id and is_nil(l.source_interface_id)
        )
      else
        from(l in DeviceLink,
          where: l.source_device_id == ^source_device_id and l.source_interface_id == ^source_interface_id
        )
      end

    link_attrs
    |> existing_link_query(base_query)
    |> Repo.one()
  end

  defp existing_link_query(%{discovered_remote_mac: remote_mac}, base_query) when is_binary(remote_mac) do
    from(l in base_query, where: l.discovered_remote_mac == ^remote_mac)
  end

  defp existing_link_query(%{discovered_remote_ip: remote_ip} = link_attrs, base_query) when is_binary(remote_ip) do
    remote_name = Map.get(link_attrs, :discovered_remote_name)

    case remote_name do
      name when is_binary(name) ->
        from(l in base_query,
          where:
            is_nil(l.discovered_remote_mac) and l.discovered_remote_ip == ^remote_ip and
              l.discovered_remote_name == ^name
        )

      _ ->
        from(l in base_query,
          where: is_nil(l.discovered_remote_mac) and l.discovered_remote_ip == ^remote_ip
        )
    end
  end

  defp existing_link_query(%{discovered_remote_name: remote_name}, base_query) when is_binary(remote_name) do
    from(l in base_query,
      where:
        is_nil(l.discovered_remote_mac) and is_nil(l.discovered_remote_ip) and
          l.discovered_remote_name == ^remote_name
    )
  end

  defp existing_link_query(%{link_type: link_type}, base_query) do
    from(l in base_query,
      where:
        is_nil(l.discovered_remote_mac) and is_nil(l.discovered_remote_ip) and
          is_nil(l.discovered_remote_name) and l.link_type == ^link_type
    )
  end

  defp evidence_group_key(ev) do
    remote_mac = Identifier.normalize_mac(ev.remote_mac)
    remote_ip = Identifier.normalize_ip(ev.remote_ip)
    remote_name = Identifier.normalize_name(ev.remote_name)

    case remote_mac || remote_ip || remote_name do
      nil ->
        {"anonymous", ev.source_interface_id, ev.evidence_type, inspect(ev.evidence_data)}

      _ ->
        {ev.source_interface_id, remote_mac, remote_ip, remote_name}
    end
  end

  defp summarize_remote_identity(evidence_list) do
    %{
      remote_mac: select_remote_field(evidence_list, :remote_mac),
      remote_ip: select_remote_field(evidence_list, :remote_ip),
      remote_name: select_remote_field(evidence_list, :remote_name)
    }
  end

  defp select_remote_field(evidence_list, field) do
    evidence_list
    |> Enum.filter(&(is_binary(Map.get(&1, field)) and Map.get(&1, field) != ""))
    |> Enum.max_by(& &1.confidence, fn -> nil end)
    |> case do
      nil -> nil
      ev -> Map.get(ev, field)
    end
  end

  defp discovered_node_id(link) do
    suffix =
      link.discovered_remote_mac ||
        link.discovered_remote_ip ||
        link.discovered_remote_name ||
        "anonymous_#{link.id}"

    "discovered_#{suffix}"
  end

  # --- LLDP Neighbor Discovery (via SNMP) ---

  @doc """
  Discovers LLDP neighbors for a device via SNMP and stores them in device_neighbors.

  Returns {:ok, count} where count is the number of neighbors discovered.
  """
  @spec discover_lldp_neighbors(String.t()) :: {:ok, non_neg_integer()} | {:error, :device_not_found}
  def discover_lldp_neighbors(device_id) do
    case Lldp.discover_neighbors(device_id) do
      {:ok, %{neighbors: neighbors}} ->
        now = DateTime.utc_now()
        results = Enum.map(neighbors, &upsert_device_neighbor(device_id, &1, now))
        success_count = Enum.count(results, &Result.ok?/1)
        {:ok, success_count}

      {:error, reason} = error ->
        Logger.error("Failed to discover LLDP neighbors for device #{device_id}: #{inspect(reason)}")
        error
    end
  end

  @doc """
  Lists all LLDP neighbors for a device.
  """
  @spec list_lldp_neighbors(String.t()) :: [DeviceNeighbor.t()]
  def list_lldp_neighbors(device_id) do
    Repo.all(
      from(n in DeviceNeighbor,
        where: n.device_id == ^device_id,
        order_by: [asc: n.local_port, asc: n.neighbor_name],
        preload: [:neighbor_device]
      )
    )
  end

  @doc """
  Lists all LLDP neighbors for a site.
  """
  @spec list_site_lldp_neighbors(String.t()) :: [DeviceNeighbor.t()]
  def list_site_lldp_neighbors(site_id) do
    Repo.all(
      from(n in DeviceNeighbor,
        join: d in assoc(n, :device),
        where: d.site_id == ^site_id,
        order_by: [asc: d.name, asc: n.local_port],
        preload: [device: d, neighbor_device: :site]
      )
    )
  end

  @doc """
  Removes stale LLDP neighbors not seen in the specified hours.

  Default: 24 hours.
  """
  @spec remove_stale_lldp_neighbors(integer()) :: {integer(), nil}
  def remove_stale_lldp_neighbors(hours_ago \\ 24) do
    cutoff = DateTime.add(DateTime.utc_now(), -hours_ago * 3600, :second)

    {count, _} =
      Repo.delete_all(
        from(n in DeviceNeighbor,
          where: n.last_seen_at < ^cutoff
        )
      )

    {count, nil}
  end

  @doc """
  Upserts a single LLDP neighbor record from agent discovery.

  Public wrapper for use by AgentChannel when processing LLDP topology results.
  """
  @spec upsert_neighbor(String.t(), map(), DateTime.t()) :: {:ok, DeviceNeighbor.t()} | {:error, Ecto.Changeset.t()}
  def upsert_neighbor(device_id, neighbor, timestamp) do
    upsert_device_neighbor(device_id, neighbor, timestamp)
  end

  # Upserts a single device neighbor record
  defp upsert_device_neighbor(device_id, neighbor, now) do
    neighbor_device_id = find_device_by_lldp_neighbor(device_id, neighbor)

    attrs = %{
      device_id: device_id,
      neighbor_device_id: neighbor_device_id,
      neighbor_name: neighbor.neighbor_name,
      local_port: neighbor.local_port,
      remote_port: neighbor.remote_port,
      remote_port_id: neighbor.remote_port_id,
      management_addresses: neighbor.management_addresses,
      discovered_at: now,
      last_seen_at: now
    }

    case Repo.get_by(DeviceNeighbor,
           device_id: device_id,
           local_port: neighbor.local_port,
           neighbor_name: neighbor.neighbor_name
         ) do
      nil ->
        %DeviceNeighbor{}
        |> DeviceNeighbor.changeset(attrs)
        |> Repo.insert()

      existing ->
        existing
        |> DeviceNeighbor.changeset(
          Map.take(attrs, [
            :last_seen_at,
            :remote_port,
            :remote_port_id,
            :management_addresses,
            :neighbor_device_id
          ])
        )
        |> Repo.update()
    end
  end

  # Attempts to find an existing device that matches the LLDP neighbor
  defp find_device_by_lldp_neighbor(discovering_device_id, neighbor) do
    discovering_device = Devices.get_device(discovering_device_id)

    if discovering_device do
      # Search within the same organization for a device matching:
      # 1. Name matches neighbor_name
      # 2. IP address matches one of the management_addresses

      query =
        from(d in Device,
          where: d.organization_id == ^discovering_device.organization_id,
          where:
            d.name == ^neighbor.neighbor_name or
              d.ip_address in ^neighbor.management_addresses
        )

      case Repo.one(query) do
        nil -> nil
        device -> device.id
      end
    end
  end

  # --- WISP wireless stats helpers ---

  @doc """
  Compute wireless client counts and average signal health per device.
  Returns a map of device_id => %{client_count: int, signal_health: "good"|"degraded"|"critical"|nil}
  """
  def compute_wireless_stats(device_ids) when is_list(device_ids) do
    if Enum.empty?(device_ids) do
      %{}
    else
      # Count wireless clients and average signal per device
      results =
        Repo.all(
          from(wc in WirelessClient,
            where: wc.device_id in ^device_ids,
            group_by: wc.device_id,
            select: {wc.device_id, count(wc.id), avg(wc.signal_strength)}
          )
        )

      Map.new(results, fn {device_id, count, avg_signal} ->
        health = classify_signal_health(avg_signal)
        {device_id, %{client_count: count, signal_health: health}}
      end)
    end
  end

  @doc """
  Compute RF link stats for edges. Uses SNR sensors on devices to
  determine signal health for PTP/backhaul links.
  Returns a map of device_id => %{signal_dbm: float, snr: float, signal_health: string}
  """
  def compute_rf_link_stats(device_ids) when is_list(device_ids) do
    if Enum.empty?(device_ids) do
      %{}
    else
      # Get latest SNR sensor readings per device
      results =
        Repo.all(
          from(s in Sensor,
            join: sd in SnmpDevice,
            on: s.snmp_device_id == sd.id,
            where: sd.device_id in ^device_ids and s.sensor_type == "snr" and not is_nil(s.last_value),
            select: {sd.device_id, s.last_value, s.sensor_descr}
          )
        )

      results
      |> Enum.group_by(fn {device_id, _val, _descr} -> device_id end)
      |> Map.new(&compute_device_snr_stats/1)
    end
  end

  defp compute_device_snr_stats({device_id, entries}) do
    vals = Enum.map(entries, fn {_, val, _} -> val end)
    avg_snr = Enum.sum(vals) / max(length(vals), 1)

    {device_id, %{snr: Float.round(avg_snr, 1), signal_health: classify_snr_health(avg_snr)}}
  end

  defp classify_signal_health(nil), do: nil

  defp classify_signal_health(avg_signal) do
    avg = if is_float(avg_signal), do: avg_signal, else: Decimal.to_float(avg_signal)

    cond do
      avg >= -65 -> "good"
      avg >= -75 -> "degraded"
      true -> "critical"
    end
  end

  defp classify_snr_health(snr) when snr >= 25, do: "good"
  defp classify_snr_health(snr) when snr >= 15, do: "degraded"
  defp classify_snr_health(_snr), do: "critical"

  defp enrich_edges_with_rf(edges, rf_stats) do
    Enum.map(edges, &enrich_single_edge(&1, rf_stats))
  end

  defp enrich_single_edge(edge, rf_stats) do
    source_rf = Map.get(rf_stats, edge.source)
    target_rf = Map.get(rf_stats, edge.target)

    signal_health = compute_edge_signal_health(source_rf, target_rf)
    snr = compute_edge_snr(source_rf, target_rf)

    Map.merge(edge, %{signal_health: signal_health, snr: snr})
  end

  defp compute_edge_signal_health(source_rf, target_rf) do
    case {source_rf, target_rf} do
      {nil, nil} -> nil
      {s, nil} -> s.signal_health
      {nil, t} -> t.signal_health
      {s, t} -> worse_health(s.signal_health, t.signal_health)
    end
  end

  defp compute_edge_snr(source_rf, target_rf) do
    case {source_rf, target_rf} do
      {nil, nil} -> nil
      {s, nil} -> s[:snr]
      {nil, t} -> t[:snr]
      {s, t} -> min(s[:snr] || 0, t[:snr] || 0)
    end
  end

  defp worse_health(a, b) do
    priority = %{"good" => 0, "degraded" => 1, "critical" => 2}
    if (priority[a] || 0) >= (priority[b] || 0), do: a, else: b
  end

  # Enriches edges with bandwidth utilization data by calculating interface throughput
  # and comparing it to configured capacity.
  defp enrich_edges_with_utilization(edges, device_ids) do
    # Get all interfaces with configured capacity for these devices
    interface_utilizations = get_interface_utilizations(device_ids)

    Enum.map(edges, fn edge ->
      # Try to find utilization data for the source or target interface
      source_util = get_edge_utilization(edge, :source, interface_utilizations)
      target_util = get_edge_utilization(edge, :target, interface_utilizations)

      # Use the higher utilization of the two interfaces
      utilization = combine_interface_utilizations(source_util, target_util)

      Map.merge(edge, utilization)
    end)
  end

  defp get_interface_utilizations(device_ids) do
    # Query interfaces with capacity configuration and recent stats
    from(i in Interface,
      join: sd in SnmpDevice,
      on: i.snmp_device_id == sd.id,
      join: d in Device,
      on: sd.device_id == d.id,
      where: d.id in ^device_ids and not is_nil(i.configured_capacity_bps),
      select: %{
        interface_id: i.id,
        device_id: d.id,
        interface_name: i.if_name,
        capacity_bps: i.configured_capacity_bps,
        if_index: i.if_index
      }
    )
    |> Repo.all()
    |> Enum.map(fn interface ->
      # Calculate current utilization using the Capacity module
      interface_struct = %Interface{
        id: interface.interface_id,
        configured_capacity_bps: interface.capacity_bps
      }

      utilization = Capacity.get_utilization(interface_struct)

      Map.put(interface, :utilization_data, utilization)
    end)
    |> Map.new(fn interface -> {interface.interface_id, interface} end)
  end

  defp get_edge_utilization(edge, direction, interface_utilizations) do
    interface_id =
      case direction do
        :source -> edge[:source_interface_id]
        :target -> edge[:target_interface_id]
      end

    case interface_id do
      nil -> nil
      id -> Map.get(interface_utilizations, id)
    end
  end

  defp combine_interface_utilizations(source_util, target_util) do
    case {source_util, target_util} do
      {nil, nil} ->
        %{
          utilization_pct: nil,
          utilization_level: nil,
          throughput_bps: nil,
          capacity_bps: nil,
          utilization_text: nil
        }

      {util, nil} ->
        process_single_utilization(util)

      {nil, util} ->
        process_single_utilization(util)

      {source, target} ->
        # Use the higher utilization of the two interfaces
        source_data = process_single_utilization(source)
        target_data = process_single_utilization(target)

        if (source_data.utilization_pct || 0) >= (target_data.utilization_pct || 0) do
          source_data
        else
          target_data
        end
    end
  end

  defp process_single_utilization(util_data) do
    case util_data.utilization_data do
      nil ->
        %{
          utilization_pct: nil,
          utilization_level: nil,
          throughput_bps: nil,
          capacity_bps: util_data.capacity_bps,
          utilization_text: nil
        }

      util ->
        utilization_pct = Float.round(util.utilization_pct, 1)
        level = classify_utilization_level(utilization_pct)

        %{
          utilization_pct: utilization_pct,
          utilization_level: level,
          throughput_bps: util.throughput.max_bps,
          capacity_bps: util_data.capacity_bps,
          utilization_text: format_utilization_text(util.throughput.max_bps, util_data.capacity_bps),
          interface_name: util_data.interface_name
        }
    end
  end

  defp classify_utilization_level(pct) when pct < 30, do: :low
  defp classify_utilization_level(pct) when pct < 60, do: :medium
  defp classify_utilization_level(pct) when pct < 80, do: :high
  defp classify_utilization_level(_pct), do: :critical

  defp format_utilization_text(throughput_bps, capacity_bps) do
    throughput_text = format_bps(throughput_bps)
    capacity_text = format_bps(capacity_bps)
    "#{throughput_text} / #{capacity_text}"
  end

  defp format_bps(nil), do: "N/A"

  defp format_bps(bps) when bps >= 1_000_000_000 do
    "#{Float.round(bps / 1_000_000_000, 1)} Gbps"
  end

  defp format_bps(bps) when bps >= 1_000_000 do
    "#{Float.round(bps / 1_000_000, 0)} Mbps"
  end

  defp format_bps(bps) when bps >= 1_000 do
    "#{Float.round(bps / 1_000, 0)} Kbps"
  end

  defp format_bps(bps) do
    "#{Float.round(bps, 0)} bps"
  end

  # Compute utilization statistics for the weathermap dashboard.
  defp compute_utilization_stats(edges) do
    utilization_counts =
      edges
      |> Enum.map(& &1[:utilization_level])
      |> Enum.frequencies()

    %{
      low_utilization_links: Map.get(utilization_counts, :low, 0),
      medium_utilization_links: Map.get(utilization_counts, :medium, 0),
      high_utilization_links: Map.get(utilization_counts, :high, 0),
      overutilized_links: Map.get(utilization_counts, :critical, 0)
    }
  end
end