diff --git a/config/dev.exs b/config/dev.exs index 2d4036d4..fcdf2ebc 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -112,6 +112,8 @@ config :towerops, Oban, {"*/5 * * * *", Towerops.Workers.SystemInsightWorker}, # Gaiia reconciliation insights nightly at 4:30 AM {"30 4 * * *", Towerops.Workers.GaiiaInsightWorker}, + # Backhaul capacity utilization insights every 15 minutes + {"*/15 * * * *", Towerops.Workers.CapacityInsightWorker}, # Sync device usage to Stripe daily at 3 AM UTC {"0 3 * * *", Towerops.Workers.BillingSyncWorker} ]}, diff --git a/config/runtime.exs b/config/runtime.exs index 4f543bdc..fc36aee8 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -220,6 +220,8 @@ if config_env() == :prod do {"*/5 * * * *", Towerops.Workers.SystemInsightWorker}, # Gaiia reconciliation insights nightly at 4:30 AM {"30 4 * * *", Towerops.Workers.GaiiaInsightWorker}, + # Backhaul capacity utilization insights every 15 minutes + {"*/15 * * * *", Towerops.Workers.CapacityInsightWorker}, # Sync device usage to Stripe daily at 3 AM UTC {"0 3 * * *", Towerops.Workers.BillingSyncWorker} ]}, diff --git a/lib/towerops/capacity.ex b/lib/towerops/capacity.ex index b57a8510..6ea78ed5 100644 --- a/lib/towerops/capacity.ex +++ b/lib/towerops/capacity.ex @@ -114,6 +114,7 @@ defmodule Towerops.Capacity do total_throughput_bps: summary.total_throughput_bps, utilization_pct: summary.utilization_pct, status: summary.status, + interfaces: summary.interfaces, headroom_bps: max(0, summary.total_capacity_bps - summary.total_throughput_bps) } end) diff --git a/lib/towerops/preseem/insight.ex b/lib/towerops/preseem/insight.ex index 3ca98be4..09060957 100644 --- a/lib/towerops/preseem/insight.ex +++ b/lib/towerops/preseem/insight.ex @@ -10,7 +10,7 @@ defmodule Towerops.Preseem.Insight do @primary_key {:id, :binary_id, autogenerate: true} @foreign_key_type :binary_id - @valid_types ~w(qoe_degradation capacity_saturation firmware_opportunity model_underperforming subscriber_growth config_drift snmp_cpu_high snmp_memory_high snmp_disk_high device_poll_gap agent_offline firmware_mismatch reconciliation_finding subscriber_growth_trend suspect_config_change) + @valid_types ~w(qoe_degradation capacity_saturation firmware_opportunity model_underperforming subscriber_growth config_drift snmp_cpu_high snmp_memory_high snmp_disk_high device_poll_gap agent_offline firmware_mismatch reconciliation_finding subscriber_growth_trend suspect_config_change backhaul_over_capacity backhaul_near_capacity) @valid_urgencies ~w(critical warning info) @valid_statuses ~w(active dismissed resolved) @valid_channels ~w(proactive contextual passive) diff --git a/lib/towerops/workers/capacity_insight_worker.ex b/lib/towerops/workers/capacity_insight_worker.ex new file mode 100644 index 00000000..2e8d1d99 --- /dev/null +++ b/lib/towerops/workers/capacity_insight_worker.ex @@ -0,0 +1,143 @@ +defmodule Towerops.Workers.CapacityInsightWorker do + @moduledoc """ + Evaluates backhaul capacity utilization and generates insights when + interfaces approach or exceed their configured capacity. + + Runs every 15 minutes. Generates: + - `backhaul_over_capacity` (critical) when utilization >= 90% + - `backhaul_near_capacity` (warning) when utilization >= 75% + + Auto-resolves insights when utilization drops below 70%. + """ + + use Oban.Worker, queue: :maintenance + + import Ecto.Query + + alias Towerops.Capacity + alias Towerops.Devices.Device + alias Towerops.Organizations.Organization + alias Towerops.Preseem.Insight + alias Towerops.Preseem.Insights + alias Towerops.Repo + alias Towerops.Snmp.Interface + + require Logger + + @critical_threshold 90 + @warning_threshold 75 + @resolve_threshold 70 + + @impl Oban.Worker + def perform(%Oban.Job{}) do + org_ids = Repo.all(from(o in Organization, select: o.id)) + + Enum.each(org_ids, &evaluate_organization/1) + + :ok + end + + defp evaluate_organization(organization_id) do + interfaces = list_capacity_interfaces(organization_id) + + evaluated = + Enum.map(interfaces, fn {interface, device} -> + utilization = Capacity.get_utilization(interface) + {interface, device, utilization} + end) + + # Generate insights for high utilization + Enum.each(evaluated, fn {interface, device, utilization} -> + generate_capacity_insight(organization_id, interface, device, utilization) + end) + + # Auto-resolve insights for recovered interfaces + evaluated_ids = MapSet.new(evaluated, fn {iface, _, _} -> iface.id end) + auto_resolve_recovered(organization_id, evaluated, evaluated_ids) + end + + defp generate_capacity_insight(_org_id, _interface, _device, nil), do: :ok + + defp generate_capacity_insight(organization_id, interface, device, utilization) do + pct = utilization.utilization_pct + + cond do + pct >= @critical_threshold -> + insert_insight(organization_id, interface, device, utilization, "backhaul_over_capacity", "critical") + + pct >= @warning_threshold -> + insert_insight(organization_id, interface, device, utilization, "backhaul_near_capacity", "warning") + + true -> + :ok + end + end + + defp insert_insight(organization_id, interface, device, utilization, type, urgency) do + Insights.insert_insight_if_new(%{ + organization_id: organization_id, + device_id: device.id, + type: type, + urgency: urgency, + channel: "proactive", + source: "snmp", + title: "#{interface.if_name} on #{device.name} #{humanize_type(type)}", + description: + "Interface #{interface.if_name} is at #{Float.round(utilization.utilization_pct, 1)}% of its #{format_capacity(interface.configured_capacity_bps)} capacity.", + dedup_key: interface.id, + metadata: %{ + "dedup_key" => interface.id, + "interface_id" => interface.id, + "interface_name" => interface.if_name, + "capacity_bps" => interface.configured_capacity_bps, + "throughput_bps" => utilization.throughput.max_bps, + "utilization_pct" => Float.round(utilization.utilization_pct, 1), + "device_name" => device.name + } + }) + end + + defp auto_resolve_recovered(organization_id, evaluated, _evaluated_ids) do + active_insights = + Insight + |> where([i], i.organization_id == ^organization_id) + |> where([i], i.type in ["backhaul_over_capacity", "backhaul_near_capacity"]) + |> where([i], i.status == "active") + |> Repo.all() + + Enum.each(active_insights, fn insight -> + interface_id = insight.metadata["interface_id"] + match = Enum.find(evaluated, fn {iface, _, _} -> iface.id == interface_id end) + maybe_resolve_insight(insight, match) + end) + end + + defp maybe_resolve_insight(_insight, nil), do: :ok + defp maybe_resolve_insight(_insight, {_iface, _device, nil}), do: :ok + + defp maybe_resolve_insight(insight, {_iface, _device, utilization}) do + if utilization.utilization_pct < @resolve_threshold do + insight + |> Insight.changeset(%{status: "resolved"}) + |> Repo.update() + end + end + + defp list_capacity_interfaces(organization_id) do + Interface + |> join(:inner, [i], sd in Towerops.Snmp.Device, on: i.snmp_device_id == sd.id) + |> join(:inner, [_i, sd], d in Device, on: sd.device_id == d.id) + |> where([_i, _sd, d], d.organization_id == ^organization_id) + |> where([i], not is_nil(i.configured_capacity_bps)) + |> select([i, _sd, d], {i, d}) + |> Repo.all() + end + + defp humanize_type("backhaul_over_capacity"), do: "over capacity" + defp humanize_type("backhaul_near_capacity"), do: "approaching capacity" + + defp format_capacity(bps) when bps >= 1_000_000_000, do: "#{Float.round(bps / 1_000_000_000, 1)} Gbps" + defp format_capacity(bps) when bps >= 1_000_000, do: "#{Float.round(bps / 1_000_000, 1)} Mbps" + defp format_capacity(bps) when bps >= 1_000, do: "#{Float.round(bps / 1_000, 1)} Kbps" + defp format_capacity(bps), do: "#{bps} bps" +end diff --git a/lib/towerops_web/components/layouts.ex b/lib/towerops_web/components/layouts.ex index 374daf5e..648dd0b6 100644 --- a/lib/towerops_web/components/layouts.ex +++ b/lib/towerops_web/components/layouts.ex @@ -306,7 +306,7 @@ defmodule ToweropsWeb.Layouts do } class={[ "inline-flex items-center gap-1 h-14 sm:h-16 px-2 text-sm font-medium border-b-2 transition-colors", - if(@active_page in ["network-map", "insights", "trace", "activity"], + if(@active_page in ["network-map", "insights", "trace", "activity", "capacity"], do: "border-gray-900 text-gray-900 font-semibold dark:border-white dark:text-white", else: @@ -379,6 +379,19 @@ defmodule ToweropsWeb.Layouts do > <.icon name="hero-clock" class="h-4 w-4 inline mr-2" />{t("Activity")} + <.link + navigate={~p"/capacity"} + role="menuitem" + class={[ + "block px-4 py-2 text-sm hover:bg-gray-100 dark:hover:bg-white/5", + if(@active_page == "capacity", + do: "text-gray-900 font-medium dark:text-white", + else: "text-gray-700 dark:text-gray-300" + ) + ]} + > + <.icon name="hero-signal" class="h-4 w-4 inline mr-2" />{t("Capacity")} + @@ -588,6 +601,9 @@ defmodule ToweropsWeb.Layouts do <.mobile_nav_link navigate={~p"/activity"} active={@active_page == "activity"}> <.icon name="hero-clock" class="size-5" /> {t("Activity")} + <.mobile_nav_link navigate={~p"/capacity"} active={@active_page == "capacity"}> + <.icon name="hero-signal" class="size-5" /> {t("Capacity")} + diff --git a/lib/towerops_web/live/capacity_live.ex b/lib/towerops_web/live/capacity_live.ex new file mode 100644 index 00000000..21620e36 --- /dev/null +++ b/lib/towerops_web/live/capacity_live.ex @@ -0,0 +1,56 @@ +defmodule ToweropsWeb.CapacityLive do + @moduledoc false + use ToweropsWeb, :live_view + + alias Towerops.Capacity + + @impl true + def mount(_params, _session, socket) do + organization = socket.assigns.current_scope.organization + summaries = Capacity.get_organization_capacity_summary(organization.id) + + total_capacity = Enum.reduce(summaries, 0, fn s, acc -> acc + s.total_capacity_bps end) + total_throughput = Enum.reduce(summaries, 0.0, fn s, acc -> acc + s.total_throughput_bps end) + + overall_utilization = + if total_capacity > 0, do: Float.round(total_throughput / total_capacity * 100, 1), else: 0.0 + + sites_at_risk = Enum.count(summaries, fn s -> s.utilization_pct >= 75 end) + sites_with_headroom = Enum.count(summaries, fn s -> s.utilization_pct < 70 and s.total_capacity_bps > 0 end) + + {:ok, + socket + |> assign(:page_title, "Capacity") + |> assign(:summaries, summaries) + |> assign(:total_capacity, total_capacity) + |> assign(:total_throughput, total_throughput) + |> assign(:overall_utilization, overall_utilization) + |> assign(:sites_at_risk, sites_at_risk) + |> assign(:sites_with_headroom, sites_with_headroom)} + end + + defp format_capacity(bps) when is_number(bps) and bps >= 1_000_000_000, + do: "#{Float.round(bps / 1_000_000_000, 1)} Gbps" + + defp format_capacity(bps) when is_number(bps) and bps >= 1_000_000, do: "#{Float.round(bps / 1_000_000, 1)} Mbps" + + defp format_capacity(bps) when is_number(bps) and bps >= 1_000, do: "#{Float.round(bps / 1_000, 1)} Kbps" + + defp format_capacity(bps) when is_number(bps), do: "#{bps} bps" + defp format_capacity(_), do: "-" + + defp utilization_color(pct) when pct >= 90, do: "bg-red-500" + defp utilization_color(pct) when pct >= 70, do: "bg-yellow-500" + defp utilization_color(_pct), do: "bg-green-500" + + defp utilization_text_color(pct) when pct >= 90, do: "text-red-600 dark:text-red-400" + defp utilization_text_color(pct) when pct >= 70, do: "text-yellow-600 dark:text-yellow-400" + defp utilization_text_color(_pct), do: "text-green-600 dark:text-green-400" + + defp status_badge(pct) when pct >= 90, do: {"Critical", "bg-red-100 text-red-700 dark:bg-red-900/50 dark:text-red-300"} + + defp status_badge(pct) when pct >= 70, + do: {"Warning", "bg-yellow-100 text-yellow-700 dark:bg-yellow-900/50 dark:text-yellow-300"} + + defp status_badge(_pct), do: {"Healthy", "bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300"} +end diff --git a/lib/towerops_web/live/capacity_live.html.heex b/lib/towerops_web/live/capacity_live.html.heex new file mode 100644 index 00000000..0450264f --- /dev/null +++ b/lib/towerops_web/live/capacity_live.html.heex @@ -0,0 +1,137 @@ + + <.breadcrumb items={[ + %{label: "Dashboard", navigate: ~p"/dashboard"}, + %{label: "Capacity"} + ]} /> + +
+

{t("Network Capacity")}

+

+ {t("Backhaul capacity utilization across all sites")} +

+
+ + +
+
+
+ {t("Total Capacity")} +
+
+ {format_capacity(@total_capacity)} +
+
+
+
+ {t("Current Throughput")} +
+
+ {format_capacity(@total_throughput)} +
+
+
+
+ {t("Sites at Risk")} +
+
+ {@sites_at_risk} +
+
+
+
+ {t("Sites with Headroom")} +
+
+ {@sites_with_headroom} +
+
+
+ + + <%= if @summaries != [] do %> +
+
+

+ {t("Site Capacity")} +

+
+
+ + + + + + + + + + + + + <%= for summary <- Enum.sort_by(@summaries, & &1.utilization_pct, :desc) do %> + + + + + + + + + <% end %> + +
{t("Site")}{t("Capacity")}{t("Throughput")}{t("Utilization")}{t("Status")}{t("Interfaces")}
+ <.link + navigate={~p"/sites/#{summary.site_id}"} + class="font-medium text-gray-900 dark:text-white hover:text-blue-600 dark:hover:text-blue-400 hover:underline" + > + {summary.site_name} + + + {format_capacity(summary.total_capacity_bps)} + + {format_capacity(summary.total_throughput_bps)} + +
+
+
+
+
+ + {Float.round(summary.utilization_pct, 1)}% + +
+
+ <% {label, classes} = status_badge(summary.utilization_pct) %> + + {label} + + + {length(summary.interfaces)} +
+
+
+ <% else %> +
+ <.icon name="hero-signal" class="h-12 w-12 mx-auto text-gray-300 dark:text-gray-600" /> +

+ {t("No capacity data")} +

+

+ {t("Configure capacity on device interfaces to see utilization data here.")} +

+
+ <% end %> +
diff --git a/lib/towerops_web/live/device_live/show.ex b/lib/towerops_web/live/device_live/show.ex index 1d3be52f..5378182a 100644 --- a/lib/towerops_web/live/device_live/show.ex +++ b/lib/towerops_web/live/device_live/show.ex @@ -444,11 +444,21 @@ defmodule ToweropsWeb.DeviceLive.Show do end end - # Ports tab: interfaces grouped by type. - # Uses snmp_interfaces already loaded in base data. + # Ports tab: interfaces grouped by type, enriched with utilization data. defp assign_ports_data(socket) do interfaces = socket.assigns.snmp_interfaces - interfaces_by_type = group_interfaces_by_type(interfaces) + + interfaces_with_utilization = + Enum.map(interfaces, fn interface -> + utilization = + if interface.configured_capacity_bps do + Towerops.Capacity.get_utilization(interface) + end + + Map.put(interface, :utilization, utilization) + end) + + interfaces_by_type = group_interfaces_by_type(interfaces_with_utilization) assign(socket, :interfaces_by_type, interfaces_by_type) end @@ -726,6 +736,26 @@ defmodule ToweropsWeb.DeviceLive.Show do defp format_speed(_), do: "-" + defp capacity_source_label("manual"), do: "Manual" + defp capacity_source_label("sensor"), do: "Sensor" + defp capacity_source_label("if_speed"), do: "Link" + defp capacity_source_label("peak"), do: "Peak" + defp capacity_source_label(_), do: "" + + defp capacity_source_class("manual"), do: "bg-blue-100 text-blue-700 dark:bg-blue-900/50 dark:text-blue-300" + defp capacity_source_class("sensor"), do: "bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300" + defp capacity_source_class("if_speed"), do: "bg-gray-100 text-gray-600 dark:bg-gray-700 dark:text-gray-300" + defp capacity_source_class("peak"), do: "bg-purple-100 text-purple-700 dark:bg-purple-900/50 dark:text-purple-300" + defp capacity_source_class(_), do: "bg-gray-100 text-gray-600 dark:bg-gray-700 dark:text-gray-300" + + defp utilization_color(pct) when pct >= 90, do: "bg-red-500" + defp utilization_color(pct) when pct >= 70, do: "bg-yellow-500" + defp utilization_color(_pct), do: "bg-green-500" + + defp utilization_text_color(pct) when pct >= 90, do: "text-red-600 dark:text-red-400" + defp utilization_text_color(pct) when pct >= 70, do: "text-yellow-600 dark:text-yellow-400" + defp utilization_text_color(_pct), do: "text-green-600 dark:text-green-400" + defp format_bytes(nil), do: "-" defp format_bytes(0), do: "0 B" @@ -1283,6 +1313,38 @@ defmodule ToweropsWeb.DeviceLive.Show do ] end + @impl true + def handle_event("set_capacity", %{"interface_id" => id, "capacity_mbps" => mbps_str}, socket) do + case Float.parse(mbps_str) do + {mbps, _} when mbps > 0 -> + bps = round(mbps * 1_000_000) + + case Snmp.set_manual_capacity(id, bps) do + {:ok, _} -> + socket = reload_snmp_and_ports(socket) + {:noreply, put_flash(socket, :info, t("Capacity updated"))} + + {:error, _} -> + {:noreply, put_flash(socket, :error, t("Failed to update capacity"))} + end + + _ -> + {:noreply, put_flash(socket, :error, t("Invalid capacity value"))} + end + end + + @impl true + def handle_event("clear_capacity", %{"interface_id" => id}, socket) do + case Snmp.clear_manual_capacity(id) do + {:ok, _} -> + socket = reload_snmp_and_ports(socket) + {:noreply, put_flash(socket, :info, t("Capacity cleared"))} + + {:error, _} -> + {:noreply, put_flash(socket, :error, t("Failed to clear capacity"))} + end + end + @impl true def handle_event("add_check", _params, socket) do {:noreply, assign(socket, :show_check_form, true)} @@ -1401,6 +1463,15 @@ defmodule ToweropsWeb.DeviceLive.Show do end end + defp reload_snmp_and_ports(socket) do + device_id = socket.assigns.device.id + snmp_data = load_snmp_data(device_id) + + socket + |> assign(:snmp_interfaces, snmp_data.interfaces) + |> assign_ports_data() + end + defp trigger_manual_backup(socket, device, agent_token_id) do alias Towerops.Agent.AgentJob alias Towerops.Agent.MikrotikCommand diff --git a/lib/towerops_web/live/device_live/show.html.heex b/lib/towerops_web/live/device_live/show.html.heex index 0aff8183..3f51b9d5 100644 --- a/lib/towerops_web/live/device_live/show.html.heex +++ b/lib/towerops_web/live/device_live/show.html.heex @@ -1294,6 +1294,8 @@ Speed IP Address MAC + Capacity + Utilization In Out @@ -1392,6 +1394,71 @@ - + + <%= if interface.configured_capacity_bps do %> +
+ + {format_speed(interface.configured_capacity_bps)} + + + {capacity_source_label(interface.capacity_source)} + + +
+ <% else %> + + <% end %> + + + <%= if interface.utilization do %> +
+
+
+
+
+ + {Float.round(interface.utilization.utilization_pct, 1)}% + +
+ <% else %> + - + <% end %> + {if interface.latest_stat && interface.latest_stat.if_in_octets, do: format_bytes(interface.latest_stat.if_in_octets), diff --git a/lib/towerops_web/live/site_live/show.ex b/lib/towerops_web/live/site_live/show.ex index 769313eb..7169d4f7 100644 --- a/lib/towerops_web/live/site_live/show.ex +++ b/lib/towerops_web/live/site_live/show.ex @@ -3,6 +3,7 @@ defmodule ToweropsWeb.SiteLive.Show do use ToweropsWeb, :live_view alias Towerops.Alerts + alias Towerops.Capacity alias Towerops.ConfigChanges alias Towerops.Dashboard alias Towerops.Devices @@ -70,7 +71,8 @@ defmodule ToweropsWeb.SiteLive.Show do |> assign(:config_changes, config_changes) |> assign(:active_alerts, active_alerts) |> assign(:insights, insights) - |> assign(:response_times, response_times)} + |> assign(:response_times, response_times) + |> assign(:capacity_summary, Capacity.get_site_capacity_summary(site.id))} end @impl true @@ -224,6 +226,24 @@ defmodule ToweropsWeb.SiteLive.Show do defp alert_severity(:device_up), do: "info" defp alert_severity(_), do: "warning" + defp format_capacity(bps) when is_number(bps) and bps >= 1_000_000_000, + do: "#{Float.round(bps / 1_000_000_000, 1)} Gbps" + + defp format_capacity(bps) when is_number(bps) and bps >= 1_000_000, do: "#{Float.round(bps / 1_000_000, 1)} Mbps" + + defp format_capacity(bps) when is_number(bps) and bps >= 1_000, do: "#{Float.round(bps / 1_000, 1)} Kbps" + + defp format_capacity(bps) when is_number(bps), do: "#{bps} bps" + defp format_capacity(_), do: "-" + + defp utilization_bar_color(pct) when pct >= 90, do: "bg-red-500" + defp utilization_bar_color(pct) when pct >= 70, do: "bg-yellow-500" + defp utilization_bar_color(_pct), do: "bg-green-500" + + defp utilization_text_color(pct) when pct >= 90, do: "text-red-600 dark:text-red-400" + defp utilization_text_color(pct) when pct >= 70, do: "text-yellow-600 dark:text-yellow-400" + defp utilization_text_color(_pct), do: "text-green-600 dark:text-green-400" + # Enqueue discovery job - safe to call in test environment defp enqueue_discovery(device_id) do if Application.get_env(:towerops, :env) == :test do diff --git a/lib/towerops_web/live/site_live/show.html.heex b/lib/towerops_web/live/site_live/show.html.heex index 814fb294..0f3dd7db 100644 --- a/lib/towerops_web/live/site_live/show.html.heex +++ b/lib/towerops_web/live/site_live/show.html.heex @@ -128,6 +128,63 @@ + <%!-- Backhaul Capacity Card --%> + <%= if @capacity_summary.interfaces != [] do %> +
+
+

+ <.icon name="hero-signal" class="h-4 w-4 inline mr-1" />{t("Backhaul Capacity")} +

+ + {format_capacity(@capacity_summary.total_capacity_bps)} {t("total")} + +
+
+
+
+
+
+ + {Float.round(@capacity_summary.utilization_pct, 1)}% + +
+
+
+ {t("Throughput")} +

+ {format_capacity(@capacity_summary.total_throughput_bps)} +

+
+
+ {t("Headroom")} +

+ {format_capacity( + max( + 0, + @capacity_summary.total_capacity_bps - @capacity_summary.total_throughput_bps + ) + )} +

+
+
+ {t("Interfaces")} +

+ {length(@capacity_summary.interfaces)} +

+
+
+
+ <% end %> + <%!-- QoE Summary Card --%> <%= if @qoe_summary do %>
diff --git a/lib/towerops_web/router.ex b/lib/towerops_web/router.ex index 8c5862a3..e65190a2 100644 --- a/lib/towerops_web/router.ex +++ b/lib/towerops_web/router.ex @@ -400,6 +400,9 @@ defmodule ToweropsWeb.Router do # Alert routes live "/alerts", AlertLive.Index, :index + # Capacity route + live "/capacity", CapacityLive, :index + # Agent routes live "/agents", AgentLive.Index, :index live "/agents/:id/edit", AgentLive.Edit, :edit diff --git a/test/towerops/workers/capacity_insight_worker_test.exs b/test/towerops/workers/capacity_insight_worker_test.exs new file mode 100644 index 00000000..949f4fa8 --- /dev/null +++ b/test/towerops/workers/capacity_insight_worker_test.exs @@ -0,0 +1,220 @@ +defmodule Towerops.Workers.CapacityInsightWorkerTest do + use Towerops.DataCase + + import Towerops.AccountsFixtures + + alias Towerops.Preseem.Insight + alias Towerops.Snmp.Device, as: SnmpDevice + alias Towerops.Snmp.Interface + alias Towerops.Snmp.InterfaceStat + alias Towerops.Workers.CapacityInsightWorker + + setup do + user = user_fixture() + {:ok, organization} = Towerops.Organizations.create_organization(%{name: "Test ISP"}, user.id) + + {:ok, site} = + Towerops.Sites.create_site(%{ + name: "Tower Alpha", + organization_id: organization.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Backhaul Radio", + ip_address: "10.0.0.1", + snmp_enabled: true, + snmp_version: "2c", + snmp_community: "public", + site_id: site.id, + organization_id: organization.id + }) + + snmp_device = + %SnmpDevice{} + |> SnmpDevice.changeset(%{device_id: device.id, sys_name: "radio", sys_descr: "PTP"}) + |> Repo.insert!() + + interface = + %Interface{} + |> Interface.changeset(%{ + snmp_device_id: snmp_device.id, + if_index: 1, + if_name: "eth0", + if_speed: 1_000_000_000, + configured_capacity_bps: 100_000_000, + capacity_source: "manual" + }) + |> Repo.insert!() + + %{ + organization: organization, + site: site, + device: device, + snmp_device: snmp_device, + interface: interface + } + end + + describe "perform/1" do + test "generates critical insight when utilization >= 90%", %{interface: interface} do + # 100 Mbps capacity, sending at 95 Mbps = 95% utilization + # 95 Mbps = 11_875_000 bytes/sec * 60 sec = 712_500_000 bytes + now = DateTime.utc_now() + + insert_stat(interface.id, %{ + if_in_octets: 0, + if_out_octets: 0, + checked_at: DateTime.add(now, -120, :second) + }) + + insert_stat(interface.id, %{ + if_in_octets: 0, + if_out_octets: 712_500_000, + checked_at: DateTime.add(now, -60, :second) + }) + + assert :ok = CapacityInsightWorker.perform(%Oban.Job{}) + + insights = Repo.all(from(i in Insight, where: i.type == "backhaul_over_capacity")) + assert [insight] = insights + assert insight.urgency == "critical" + assert insight.status == "active" + end + + test "generates warning insight when utilization >= 75%", %{interface: interface} do + # 100 Mbps capacity, sending at 80 Mbps = 80% utilization + # 80 Mbps = 10_000_000 bytes/sec * 60 sec = 600_000_000 bytes + now = DateTime.utc_now() + + insert_stat(interface.id, %{ + if_in_octets: 0, + if_out_octets: 0, + checked_at: DateTime.add(now, -120, :second) + }) + + insert_stat(interface.id, %{ + if_in_octets: 0, + if_out_octets: 600_000_000, + checked_at: DateTime.add(now, -60, :second) + }) + + assert :ok = CapacityInsightWorker.perform(%Oban.Job{}) + + insights = Repo.all(from(i in Insight, where: i.type == "backhaul_near_capacity")) + assert [insight] = insights + assert insight.urgency == "warning" + end + + test "does not generate insight when utilization < 75%", %{interface: interface} do + # 100 Mbps capacity, sending at 50 Mbps = 50% utilization + now = DateTime.utc_now() + + insert_stat(interface.id, %{ + if_in_octets: 0, + if_out_octets: 0, + checked_at: DateTime.add(now, -120, :second) + }) + + insert_stat(interface.id, %{ + if_in_octets: 0, + if_out_octets: 375_000_000, + checked_at: DateTime.add(now, -60, :second) + }) + + assert :ok = CapacityInsightWorker.perform(%Oban.Job{}) + + assert [] = Repo.all(from(i in Insight, where: i.type in ["backhaul_over_capacity", "backhaul_near_capacity"])) + end + + test "skips interfaces without configured capacity", %{snmp_device: snmp_device} do + # Create an interface without capacity + %Interface{} + |> Interface.changeset(%{ + snmp_device_id: snmp_device.id, + if_index: 99, + if_name: "lo0" + }) + |> Repo.insert!() + + assert :ok = CapacityInsightWorker.perform(%Oban.Job{}) + + # Only insights from the interface WITH capacity (if any threshold met) + all_insights = + Repo.all(from(i in Insight, where: i.type in ["backhaul_over_capacity", "backhaul_near_capacity"])) + + refute Enum.any?(all_insights, fn i -> i.metadata["interface_name"] == "lo0" end) + end + + test "auto-resolves insight when utilization drops below 70%", %{ + interface: interface, + device: device, + organization: organization + } do + # First create an active insight + %Insight{} + |> Insight.changeset(%{ + organization_id: organization.id, + device_id: device.id, + type: "backhaul_over_capacity", + urgency: "critical", + channel: "proactive", + source: "snmp", + title: "eth0 over capacity", + metadata: %{"dedup_key" => interface.id, "interface_id" => interface.id} + }) + |> Repo.insert!() + + # Now set up stats showing low utilization (30 Mbps on 100 Mbps = 30%) + now = DateTime.utc_now() + + insert_stat(interface.id, %{ + if_in_octets: 0, + if_out_octets: 0, + checked_at: DateTime.add(now, -120, :second) + }) + + insert_stat(interface.id, %{ + if_in_octets: 0, + if_out_octets: 225_000_000, + checked_at: DateTime.add(now, -60, :second) + }) + + assert :ok = CapacityInsightWorker.perform(%Oban.Job{}) + + # Insight should be resolved + [insight] = Repo.all(from(i in Insight, where: i.type == "backhaul_over_capacity")) + assert insight.status == "resolved" + end + + test "deduplicates insights for same interface", %{interface: interface} do + now = DateTime.utc_now() + + insert_stat(interface.id, %{ + if_in_octets: 0, + if_out_octets: 0, + checked_at: DateTime.add(now, -120, :second) + }) + + insert_stat(interface.id, %{ + if_in_octets: 0, + if_out_octets: 712_500_000, + checked_at: DateTime.add(now, -60, :second) + }) + + # Run twice + assert :ok = CapacityInsightWorker.perform(%Oban.Job{}) + assert :ok = CapacityInsightWorker.perform(%Oban.Job{}) + + # Should still only have one insight + insights = Repo.all(from(i in Insight, where: i.type == "backhaul_over_capacity")) + assert length(insights) == 1 + end + end + + defp insert_stat(interface_id, attrs) do + %InterfaceStat{} + |> InterfaceStat.changeset(Map.put(attrs, :interface_id, interface_id)) + |> Repo.insert!() + end +end diff --git a/test/towerops_web/live/capacity_live_test.exs b/test/towerops_web/live/capacity_live_test.exs new file mode 100644 index 00000000..951003bc --- /dev/null +++ b/test/towerops_web/live/capacity_live_test.exs @@ -0,0 +1,97 @@ +defmodule ToweropsWeb.CapacityLiveTest do + use ToweropsWeb.ConnCase, async: true + + import Phoenix.LiveViewTest + import Towerops.AccountsFixtures + + alias Towerops.Snmp.Device, as: SnmpDevice + alias Towerops.Snmp.Interface + alias Towerops.Snmp.InterfaceStat + + setup do + user = user_fixture(enable_totp: true) + {:ok, organization} = Towerops.Organizations.create_organization(%{name: "Test ISP"}, user.id) + + {:ok, site} = + Towerops.Sites.create_site(%{ + name: "Tower Alpha", + organization_id: organization.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Backhaul Radio", + ip_address: "10.0.0.1", + snmp_enabled: true, + snmp_version: "2c", + snmp_community: "public", + site_id: site.id, + organization_id: organization.id + }) + + snmp_device = + %SnmpDevice{} + |> SnmpDevice.changeset(%{device_id: device.id, sys_name: "radio", sys_descr: "PTP"}) + |> Towerops.Repo.insert!() + + interface = + %Interface{} + |> Interface.changeset(%{ + snmp_device_id: snmp_device.id, + if_index: 1, + if_name: "eth0", + if_speed: 1_000_000_000, + if_type: 6, + configured_capacity_bps: 300_000_000, + capacity_source: "manual" + }) + |> Towerops.Repo.insert!() + + now = DateTime.utc_now() + + %InterfaceStat{} + |> InterfaceStat.changeset(%{ + interface_id: interface.id, + if_in_octets: 0, + if_out_octets: 0, + checked_at: DateTime.add(now, -120, :second) + }) + |> Towerops.Repo.insert!() + + %InterfaceStat{} + |> InterfaceStat.changeset(%{ + interface_id: interface.id, + if_in_octets: 0, + if_out_octets: 1_125_000_000, + checked_at: DateTime.add(now, -60, :second) + }) + |> Towerops.Repo.insert!() + + %{user: user, organization: organization, site: site, device: device, interface: interface} + end + + describe "capacity index page" do + test "redirects to login when not authenticated", %{conn: conn} do + assert {:error, redirect} = live(conn, ~p"/capacity") + assert {:redirect, %{to: to}} = redirect + assert to =~ "/users/log-in" + end + + test "displays capacity page with site data", %{conn: conn, user: user} do + conn = log_in_user(conn, user) + {:ok, _view, html} = live(conn, ~p"/capacity") + + assert html =~ "Capacity" + assert html =~ "Tower Alpha" + assert html =~ "300.0 Mbps" + end + + test "shows utilization percentage for site", %{conn: conn, user: user} do + conn = log_in_user(conn, user) + {:ok, _view, html} = live(conn, ~p"/capacity") + + # Site has 300 Mbps capacity with ~150 Mbps throughput = ~50% + assert html =~ "%" + end + end +end diff --git a/test/towerops_web/live/device_live/show_test.exs b/test/towerops_web/live/device_live/show_test.exs index 346c00a2..4de3ac13 100644 --- a/test/towerops_web/live/device_live/show_test.exs +++ b/test/towerops_web/live/device_live/show_test.exs @@ -280,4 +280,102 @@ defmodule ToweropsWeb.DeviceLive.ShowTest do assert render(view) end end + + describe "ports tab capacity" do + setup %{device: device} do + alias Towerops.Snmp.Device, as: SnmpDevice + alias Towerops.Snmp.Interface + alias Towerops.Snmp.InterfaceStat + + snmp_device = + %SnmpDevice{} + |> SnmpDevice.changeset(%{device_id: device.id, sys_name: "radio", sys_descr: "PTP"}) + |> Towerops.Repo.insert!() + + interface = + %Interface{} + |> Interface.changeset(%{ + snmp_device_id: snmp_device.id, + if_index: 1, + if_name: "eth0", + if_speed: 1_000_000_000, + if_type: 6, + configured_capacity_bps: 300_000_000, + capacity_source: "manual" + }) + |> Towerops.Repo.insert!() + + # Insert stats for utilization calculation + now = DateTime.utc_now() + + %InterfaceStat{} + |> InterfaceStat.changeset(%{ + interface_id: interface.id, + if_in_octets: 0, + if_out_octets: 0, + checked_at: DateTime.add(now, -120, :second) + }) + |> Towerops.Repo.insert!() + + # ~150 Mbps = 50% of 300 Mbps capacity + %InterfaceStat{} + |> InterfaceStat.changeset(%{ + interface_id: interface.id, + if_in_octets: 0, + if_out_octets: 1_125_000_000, + checked_at: DateTime.add(now, -60, :second) + }) + |> Towerops.Repo.insert!() + + %{snmp_device: snmp_device, interface: interface} + end + + test "displays capacity column on ports tab", %{conn: conn, user: user, device: device} do + conn = log_in_user(conn, user) + {:ok, _view, html} = live(conn, ~p"/devices/#{device.id}?tab=ports") + + assert html =~ "Capacity" + assert html =~ "300.0 Mbps" + end + + test "displays utilization bar on ports tab", %{conn: conn, user: user, device: device} do + conn = log_in_user(conn, user) + {:ok, _view, html} = live(conn, ~p"/devices/#{device.id}?tab=ports") + + assert html =~ "Utilization" + end + + test "set capacity via event handler", %{ + conn: conn, + user: user, + device: device, + interface: interface + } do + conn = log_in_user(conn, user) + {:ok, view, _html} = live(conn, ~p"/devices/#{device.id}?tab=ports") + + html = + render_click(view, "set_capacity", %{ + "interface_id" => interface.id, + "capacity_mbps" => "500" + }) + + assert html =~ "500.0 Mbps" + end + + test "clear capacity via event handler", %{ + conn: conn, + user: user, + device: device, + interface: interface + } do + conn = log_in_user(conn, user) + {:ok, view, _html} = live(conn, ~p"/devices/#{device.id}?tab=ports") + + html = render_click(view, "clear_capacity", %{"interface_id" => interface.id}) + + # After clearing, the capacity badge should not show 300 Mbps anymore + refute html =~ "300.0 Mbps" + end + end end