diff --git a/lib/towerops/agents.ex b/lib/towerops/agents.ex index d9ee78a1..e63d96eb 100644 --- a/lib/towerops/agents.ex +++ b/lib/towerops/agents.ex @@ -357,6 +357,7 @@ defmodule Towerops.Agents do 1. Directly assigned to this agent 2. At a site that has this agent as default 3. In an organization that has this agent as default (and not overridden by site or device) + 4. This agent is the global default cloud poller (and device has no assignment at any level) Returns device records with preloaded associations, filtered to only SNMP-enabled devices. @@ -368,26 +369,39 @@ defmodule Towerops.Agents do """ @spec list_agent_polling_targets(Ecto.UUID.t()) :: [Device.t()] def list_agent_polling_targets(agent_token_id) do - # Use SQL to filter devices by effective agent assignment - # This avoids loading all devices into memory and filtering in application code - Repo.all( + global_default = Towerops.Settings.get_global_default_cloud_poller() + is_global_default = agent_token_id == global_default + + query = from(e in Device, join: s in assoc(e, :site), join: o in assoc(s, :organization), left_join: aa in AgentAssignment, on: aa.device_id == e.id and aa.enabled == true, - where: - e.snmp_enabled == true and - (aa.agent_token_id == ^agent_token_id or - (is_nil(aa.agent_token_id) and s.agent_token_id == ^agent_token_id) or - (is_nil(aa.agent_token_id) and is_nil(s.agent_token_id) and - o.default_agent_token_id == ^agent_token_id)), + where: e.snmp_enabled == true, preload: [ :agent_assignments, site: :organization, snmp_device: [:sensors, :interfaces] ] ) + + query + |> where_agent_matches(agent_token_id, is_global_default) + |> Repo.all() + end + + # Adds where clause to match devices assigned to this agent + # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity + defp where_agent_matches(query, agent_token_id, is_global_default) do + from([e, s, o, aa] in query, + where: + aa.agent_token_id == ^agent_token_id or + (is_nil(aa.agent_token_id) and s.agent_token_id == ^agent_token_id) or + (is_nil(aa.agent_token_id) and is_nil(s.agent_token_id) and + o.default_agent_token_id == ^agent_token_id) or + (^is_global_default and is_nil(aa.agent_token_id) and is_nil(s.agent_token_id) and + is_nil(o.default_agent_token_id)) ) end @@ -470,7 +484,9 @@ defmodule Towerops.Agents do Resolves the agent token using the following hierarchy: 1. Device's direct agent assignment (highest priority) 2. Site's agent_token_id - 3. Organization's default_agent_token_id (lowest priority) + 3. Organization's default_agent_token_id + 4. Global default cloud poller (application-wide fallback) + 5. Cloud polling (nil - lowest priority) Returns the agent_token_id if found, nil otherwise. @@ -487,27 +503,28 @@ defmodule Towerops.Agents do """ def get_effective_agent_token(%Device{} = device) do - # device assignment + # 1. Check device assignment case get_device_assignment(device.id) do %AgentAssignment{agent_token_id: agent_token_id} when not is_nil(agent_token_id) -> agent_token_id _ -> - # 2. Check site's agent token - site = device.site + get_fallback_agent_token(device.site) + end + end - case site do - %{agent_token_id: agent_token_id} when not is_nil(agent_token_id) -> - agent_token_id + # Checks site, organization, and global default in sequence + defp get_fallback_agent_token(site) do + case site do + %{agent_token_id: agent_token_id} when not is_nil(agent_token_id) -> + agent_token_id - %{organization: %{default_agent_token_id: agent_token_id}} - when not is_nil(agent_token_id) -> - # 3. Use organization's default agent token - agent_token_id + %{organization: %{default_agent_token_id: agent_token_id}} + when not is_nil(agent_token_id) -> + agent_token_id - _ -> - nil - end + _ -> + Towerops.Settings.get_global_default_cloud_poller() end end @@ -518,6 +535,7 @@ defmodule Towerops.Agents do - :device - Direct device assignment - :site - Inherited from site - :organization - Inherited from organization + - :global - Global default cloud poller - :none - No agent assigned ## Examples @@ -530,23 +548,30 @@ defmodule Towerops.Agents do """ def get_effective_agent_token_with_source(%Device{} = device) do - # device assignment + # 1. Check device assignment case get_device_assignment(device.id) do %AgentAssignment{agent_token_id: agent_token_id} when not is_nil(agent_token_id) -> {agent_token_id, :device} _ -> - # 2. Check site's agent token - site = device.site + get_fallback_agent_token_with_source(device.site) + end + end - case site do - %{agent_token_id: agent_token_id} when not is_nil(agent_token_id) -> - {agent_token_id, :site} + # Checks site, organization, and global default in sequence with source tracking + defp get_fallback_agent_token_with_source(site) do + case site do + %{agent_token_id: agent_token_id} when not is_nil(agent_token_id) -> + {agent_token_id, :site} - %{organization: %{default_agent_token_id: agent_token_id}} - when not is_nil(agent_token_id) -> - # 3. Use organization's default agent token - {agent_token_id, :organization} + %{organization: %{default_agent_token_id: agent_token_id}} + when not is_nil(agent_token_id) -> + {agent_token_id, :organization} + + _ -> + case Towerops.Settings.get_global_default_cloud_poller() do + agent_token_id when not is_nil(agent_token_id) -> + {agent_token_id, :global} _ -> {nil, :none} diff --git a/lib/towerops/agents/stats.ex b/lib/towerops/agents/stats.ex index 6dbfb805..90880119 100644 --- a/lib/towerops/agents/stats.ex +++ b/lib/towerops/agents/stats.ex @@ -10,6 +10,7 @@ defmodule Towerops.Agents.Stats do alias Towerops.Agents.AgentAssignment alias Towerops.Agents.AgentToken alias Towerops.Devices.Device + alias Towerops.Monitoring.Check alias Towerops.Repo alias Towerops.Snmp.SensorReading @@ -298,8 +299,218 @@ defmodule Towerops.Agents.Stats do ) end + @doc """ + Get average latency for a device grouped by agent over the last 24 hours. + + Returns a list of agents with their average response times for the given device, + filtered to successful checks only and requiring minimum check count for statistical validity. + + ## Examples + + iex> get_device_latency_by_agent(device_id, min_checks: 10) + [ + %{agent_token_id: "uuid-1", avg_latency_ms: 12.5, check_count: 120}, + %{agent_token_id: "uuid-2", avg_latency_ms: 45.2, check_count: 115} + ] + """ + def get_device_latency_by_agent(device_id, opts \\ []) do + min_checks = Keyword.get(opts, :min_checks, 10) + hours_ago = Keyword.get(opts, :hours_ago, 24) + time_threshold = DateTime.add(DateTime.utc_now(), -hours_ago, :hour) + + Repo.all( + from(c in Check, + where: + c.device_id == ^device_id and + c.status == :success and + not is_nil(c.agent_token_id) and + c.checked_at > ^time_threshold, + group_by: c.agent_token_id, + having: count(c.id) >= ^min_checks, + select: %{ + agent_token_id: c.agent_token_id, + avg_latency_ms: avg(c.response_time_ms), + check_count: count(c.id) + }, + order_by: [asc: avg(c.response_time_ms)] + ) + ) + end + + @doc """ + Get device assignment information combined with latency statistics. + + Returns device assignment details with current agent and latency data for potential reassignment. + + ## Examples + + iex> get_device_assignment_with_latency(device_id) + %{ + device_id: "uuid", + current_agent_token_id: "uuid-1", + assignment_source: :site, + latency_stats: [ + %{agent_token_id: "uuid-1", avg_latency_ms: 12.5, check_count: 120}, + %{agent_token_id: "uuid-2", avg_latency_ms: 8.3, check_count: 115} + ] + } + """ + def get_device_assignment_with_latency(device_id) do + device = + Device + |> Repo.get!(device_id) + |> Repo.preload(site: [organization: :default_agent_token]) + + {current_agent_id, source} = Towerops.Agents.get_effective_agent_token_with_source(device) + + latency_stats = get_device_latency_by_agent(device_id) + + %{ + device_id: device_id, + current_agent_token_id: current_agent_id, + assignment_source: source, + latency_stats: latency_stats + } + end + + @doc """ + Find devices that are candidates for latency-based reassignment. + + Identifies devices where an alternative agent has significantly better latency (default: 20% improvement). + Only considers devices with automatic assignments (site, organization, global, or none). + + ## Examples + + iex> find_reassignment_candidates(min_improvement_percent: 20, min_checks_per_agent: 10) + [ + %{ + device_id: "uuid", + device_name: "Switch-01", + current_agent_token_id: "uuid-1", + current_latency_ms: 45.2, + best_agent_token_id: "uuid-2", + best_latency_ms: 28.3, + improvement_percent: 37.4, + assignment_source: :site + } + ] + """ + def find_reassignment_candidates(opts \\ []) do + min_improvement_percent = Keyword.get(opts, :min_improvement_percent, 20) + min_checks = Keyword.get(opts, :min_checks_per_agent, 10) + hours_ago = Keyword.get(opts, :hours_ago, 24) + time_threshold = DateTime.add(DateTime.utc_now(), -hours_ago, :hour) + + # Get all devices with multiple agent latency measurements + devices_with_agents = + Repo.all( + from(c in Check, + where: c.status == :success and not is_nil(c.agent_token_id) and c.checked_at > ^time_threshold, + group_by: [c.device_id, c.agent_token_id], + having: count(c.id) >= ^min_checks, + select: %{ + device_id: c.device_id, + agent_token_id: c.agent_token_id, + avg_latency_ms: avg(c.response_time_ms), + check_count: count(c.id) + } + ) + ) + + # Group by device and evaluate candidates + devices_with_agents + |> Enum.group_by(& &1.device_id) + |> Enum.filter(fn {_device_id, agents} -> length(agents) >= 2 end) + |> Enum.flat_map(fn {device_id, agents} -> + evaluate_device_reassignment(device_id, agents, min_improvement_percent) + end) + end + # Private helper functions + # Evaluates if a device should be reassigned based on latency comparison + defp evaluate_device_reassignment(device_id, agents, min_improvement_percent) do + device = + Device + |> Repo.get!(device_id) + |> Repo.preload(site: [organization: :default_agent_token]) + + {current_agent_id, source} = Towerops.Agents.get_effective_agent_token_with_source(device) + + # Only consider automatic assignments for reassignment + if source in [:site, :organization, :global, :none] do + build_reassignment_candidate(device, current_agent_id, source, agents, min_improvement_percent) + else + [] + end + end + + # Builds reassignment candidate if improvement threshold is met + defp build_reassignment_candidate(device, current_agent_id, source, agents, min_improvement_percent) do + # Special case: if no current agent (:none or :global with no agent), + # just pick the best agent without requiring improvement + if is_nil(current_agent_id) do + build_initial_assignment_candidate(device, source, agents) + else + build_reassignment_with_improvement(device, current_agent_id, source, agents, min_improvement_percent) + end + end + + defp build_initial_assignment_candidate(device, source, agents) do + best = Enum.min_by(agents, & &1.avg_latency_ms, fn -> nil end) + + if best do + [ + %{ + device_id: device.id, + device_name: device.name, + current_agent_token_id: nil, + current_latency_ms: nil, + best_agent_token_id: best.agent_token_id, + best_latency_ms: Float.round(best.avg_latency_ms, 2), + improvement_percent: 100.0, + assignment_source: source + } + ] + else + [] + end + end + + defp build_reassignment_with_improvement(device, current_agent_id, source, agents, min_improvement_percent) do + current_latency = Enum.find(agents, fn a -> a.agent_token_id == current_agent_id end) + + with %{} <- current_latency, + best when not is_nil(best) <- find_best_alternative(agents, current_agent_id), + improvement when improvement >= min_improvement_percent <- + calculate_improvement_percent(current_latency, best) do + [ + %{ + device_id: device.id, + device_name: device.name, + current_agent_token_id: current_agent_id, + current_latency_ms: Float.round(current_latency.avg_latency_ms, 2), + best_agent_token_id: best.agent_token_id, + best_latency_ms: Float.round(best.avg_latency_ms, 2), + improvement_percent: improvement, + assignment_source: source + } + ] + else + _ -> [] + end + end + + defp find_best_alternative(agents, current_agent_id) do + agents + |> Enum.reject(fn a -> a.agent_token_id == current_agent_id end) + |> Enum.min_by(& &1.avg_latency_ms, fn -> nil end) + end + + defp calculate_improvement_percent(current, best) do + Float.round((current.avg_latency_ms - best.avg_latency_ms) / current.avg_latency_ms * 100, 1) + end + defp polling_targets_subquery(agent_token_id) do # This is a simplified version - in production you'd want to use # Agents.list_agent_polling_targets/1 logic here diff --git a/lib/towerops/application.ex b/lib/towerops/application.ex index 46285b8d..d69399d9 100644 --- a/lib/towerops/application.ex +++ b/lib/towerops/application.ex @@ -65,7 +65,9 @@ defmodule Towerops.Application do # Start monitoring supervisor (includes NeighborCleanupWorker) Towerops.Monitoring.Supervisor, # Start stale agent detection worker - Towerops.Workers.StaleAgentWorker + Towerops.Workers.StaleAgentWorker, + # Start latency-based agent reassignment worker + Towerops.Workers.AgentLatencyEvaluator ] end end diff --git a/lib/towerops/monitoring/check.ex b/lib/towerops/monitoring/check.ex index e629345b..df7faa0a 100644 --- a/lib/towerops/monitoring/check.ex +++ b/lib/towerops/monitoring/check.ex @@ -2,7 +2,8 @@ defmodule Towerops.Monitoring.Check do @moduledoc """ Monitoring check schema for device reachability status. - Records the result of each monitoring check (success/failure) with latency. + Records the result of each monitoring check (success/failure) with latency + and tracks which agent performed the check for intelligent routing. """ use Ecto.Schema @@ -16,6 +17,7 @@ defmodule Towerops.Monitoring.Check do field :checked_at, :utc_datetime belongs_to :device, Towerops.Devices.Device + belongs_to :agent_token, Towerops.Agents.AgentToken timestamps(type: :utc_datetime, updated_at: false) end @@ -23,8 +25,9 @@ defmodule Towerops.Monitoring.Check do @doc false def changeset(check, attrs) do check - |> cast(attrs, [:device_id, :status, :response_time_ms, :checked_at]) + |> cast(attrs, [:device_id, :agent_token_id, :status, :response_time_ms, :checked_at]) |> validate_required([:device_id, :status, :checked_at]) |> foreign_key_constraint(:device_id, match: :suffix) + |> foreign_key_constraint(:agent_token_id, match: :suffix) end end diff --git a/lib/towerops/settings.ex b/lib/towerops/settings.ex new file mode 100644 index 00000000..2c68995d --- /dev/null +++ b/lib/towerops/settings.ex @@ -0,0 +1,117 @@ +defmodule Towerops.Settings do + @moduledoc """ + The Settings context for application-wide configuration. + + Provides functions for reading and updating global settings that affect + the entire application. Only accessible by superadmins. + """ + + import Ecto.Query, warn: false + + alias Towerops.Repo + alias Towerops.Settings.ApplicationSetting + + @doc """ + Gets a setting by key and returns the parsed value. + + Returns nil if the setting doesn't exist or has a nil value. + + ## Examples + + iex> get_setting("global_default_cloud_poller_id") + "abc-123-uuid" + + iex> get_setting("nonexistent_key") + nil + """ + def get_setting(key) when is_binary(key) do + case Repo.get_by(ApplicationSetting, key: key) do + nil -> nil + setting -> ApplicationSetting.parse_value(setting) + end + end + + @doc """ + Gets the raw ApplicationSetting record by key. + + ## Examples + + iex> get_setting_record("global_default_cloud_poller_id") + %ApplicationSetting{key: "global_default_cloud_poller_id", value: "abc-123"} + """ + def get_setting_record(key) when is_binary(key) do + Repo.get_by(ApplicationSetting, key: key) + end + + @doc """ + Updates a setting value. + + ## Examples + + iex> update_setting("global_default_cloud_poller_id", "new-uuid") + {:ok, %ApplicationSetting{}} + + iex> update_setting("global_default_cloud_poller_id", nil) + {:ok, %ApplicationSetting{value: nil}} + """ + def update_setting(key, value) when is_binary(key) do + case get_setting_record(key) do + nil -> + {:error, :setting_not_found} + + setting -> + setting + |> ApplicationSetting.changeset(%{value: to_string_or_nil(value)}) + |> Repo.update() + end + end + + @doc """ + Gets the global default cloud poller agent token ID. + + Returns nil if not configured. + + ## Examples + + iex> get_global_default_cloud_poller() + "abc-123-uuid" + + iex> get_global_default_cloud_poller() + nil + """ + def get_global_default_cloud_poller do + get_setting("global_default_cloud_poller_id") + end + + @doc """ + Sets the global default cloud poller agent token ID. + + ## Examples + + iex> set_global_default_cloud_poller("agent-token-uuid") + {:ok, %ApplicationSetting{}} + + iex> set_global_default_cloud_poller(nil) + {:ok, %ApplicationSetting{value: nil}} + """ + def set_global_default_cloud_poller(agent_token_id) do + update_setting("global_default_cloud_poller_id", agent_token_id) + end + + @doc """ + Lists all application settings. + + ## Examples + + iex> list_all_settings() + [%ApplicationSetting{}, ...] + """ + def list_all_settings do + Repo.all(ApplicationSetting) + end + + # Private helpers + + defp to_string_or_nil(nil), do: nil + defp to_string_or_nil(value), do: to_string(value) +end diff --git a/lib/towerops/settings/application_setting.ex b/lib/towerops/settings/application_setting.ex new file mode 100644 index 00000000..e5c5d2fa --- /dev/null +++ b/lib/towerops/settings/application_setting.ex @@ -0,0 +1,75 @@ +defmodule Towerops.Settings.ApplicationSetting do + @moduledoc """ + Application-wide settings schema. + + Stores global configuration values that affect the entire application, + regardless of organization. Only accessible by superadmins. + """ + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + schema "application_settings" do + field :key, :string + field :value, :string + field :value_type, :string, default: "string" + field :description, :string + + timestamps(type: :utc_datetime) + end + + @type t :: %__MODULE__{ + id: Ecto.UUID.t(), + key: String.t(), + value: String.t() | nil, + value_type: String.t(), + description: String.t() | nil, + inserted_at: DateTime.t(), + updated_at: DateTime.t() + } + + @doc false + def changeset(setting, attrs) do + setting + |> cast(attrs, [:key, :value, :value_type, :description]) + |> validate_required([:key, :value_type]) + |> validate_inclusion(:value_type, ["string", "integer", "uuid", "boolean", "json"]) + |> unique_constraint(:key) + end + + @doc """ + Parses the value field according to value_type. + + ## Examples + + iex> parse_value(%ApplicationSetting{value: "abc-123", value_type: "uuid"}) + "abc-123" + + iex> parse_value(%ApplicationSetting{value: nil, value_type: "uuid"}) + nil + """ + def parse_value(%__MODULE__{value: nil}), do: nil + + def parse_value(%__MODULE__{value: value, value_type: "string"}), do: value + def parse_value(%__MODULE__{value: value, value_type: "uuid"}), do: value + + def parse_value(%__MODULE__{value: value, value_type: "integer"}) do + case Integer.parse(value) do + {int, _} -> int + :error -> nil + end + end + + def parse_value(%__MODULE__{value: value, value_type: "boolean"}) do + value in ["true", "1", "yes"] + end + + def parse_value(%__MODULE__{value: value, value_type: "json"}) do + case Jason.decode(value) do + {:ok, decoded} -> decoded + {:error, _} -> nil + end + end +end diff --git a/lib/towerops/workers/agent_latency_evaluator.ex b/lib/towerops/workers/agent_latency_evaluator.ex new file mode 100644 index 00000000..40001bc1 --- /dev/null +++ b/lib/towerops/workers/agent_latency_evaluator.ex @@ -0,0 +1,173 @@ +defmodule Towerops.Workers.AgentLatencyEvaluator do + @moduledoc """ + GenServer that periodically evaluates device-to-agent latency and performs automatic reassignments. + + Runs every 5 minutes to: + 1. Find devices where an alternative agent has significantly better latency (20%+ improvement) + 2. Reassign devices based on their assignment source (site, organization, global, or device-level) + 3. Broadcast PubSub events for UI updates + 4. Log all reassignment decisions with improvement metrics + + Only considers devices with automatic assignments (site, organization, global, or none). + Manual device-level assignments are never automatically changed. + """ + use GenServer + + alias Towerops.Agents + alias Towerops.Agents.Stats + alias Towerops.Devices.Device + alias Towerops.Organizations + alias Towerops.Repo + alias Towerops.Sites + + require Logger + + # Run latency evaluation every 5 minutes + @check_interval to_timeout(minute: 5) + + # Minimum improvement required for reassignment (20%) + @min_improvement_percent 20 + + # Minimum checks per agent for statistical validity + @min_checks_per_agent 10 + + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @doc """ + Manually trigger a latency evaluation. + + Useful for testing and manual triggering from admin UI. + """ + def trigger_evaluation do + GenServer.cast(__MODULE__, :evaluate_latency) + end + + @impl true + def init(_opts) do + # Schedule first evaluation shortly after startup (30 seconds) + schedule_evaluation(30_000) + {:ok, %{last_evaluation_at: nil, total_reassignments: 0}} + end + + @impl true + def handle_cast(:evaluate_latency, state) do + new_state = evaluate_and_reassign(state) + {:noreply, new_state} + end + + @impl true + def handle_info(:evaluate_latency, state) do + new_state = evaluate_and_reassign(state) + schedule_evaluation(@check_interval) + {:noreply, new_state} + end + + defp evaluate_and_reassign(state) do + start_time = System.monotonic_time(:millisecond) + + candidates = + Stats.find_reassignment_candidates( + min_improvement_percent: @min_improvement_percent, + min_checks_per_agent: @min_checks_per_agent + ) + + reassignment_count = + candidates + |> Enum.map(&reassign_device/1) + |> Enum.count(&(&1 == :ok)) + + duration_ms = System.monotonic_time(:millisecond) - start_time + + Logger.info( + "Latency evaluation completed: #{length(candidates)} candidate(s) found, " <> + "#{reassignment_count} device(s) reassigned in #{duration_ms}ms" + ) + + %{ + state + | last_evaluation_at: DateTime.utc_now(), + total_reassignments: state.total_reassignments + reassignment_count + } + end + + defp reassign_device(candidate) do + Logger.info( + "Reassigning device '#{candidate.device_name}' " <> + "(#{candidate.improvement_percent}% improvement: " <> + "#{candidate.current_latency_ms}ms → #{candidate.best_latency_ms}ms)", + device_id: candidate.device_id, + device_name: candidate.device_name, + old_agent: candidate.current_agent_token_id, + new_agent: candidate.best_agent_token_id, + improvement_percent: candidate.improvement_percent, + assignment_source: candidate.assignment_source + ) + + result = + case candidate.assignment_source do + :site -> + reassign_at_site_level(candidate) + + :organization -> + reassign_at_organization_level(candidate) + + source when source in [:global, :none] -> + reassign_at_device_level(candidate) + end + + case result do + {:ok, _} -> + broadcast_reassignment(candidate) + :ok + + {:error, reason} -> + Logger.error( + "Failed to reassign device '#{candidate.device_name}': #{inspect(reason)}", + device_id: candidate.device_id, + error: reason + ) + + :error + end + end + + defp reassign_at_site_level(candidate) do + device = Device |> Repo.get!(candidate.device_id) |> Repo.preload(:site) + + Sites.update_site(device.site, %{agent_token_id: candidate.best_agent_token_id}) + end + + defp reassign_at_organization_level(candidate) do + device = + Device + |> Repo.get!(candidate.device_id) + |> Repo.preload(site: :organization) + + organization = device.site.organization + + Organizations.update_organization(organization, %{ + default_agent_token_id: candidate.best_agent_token_id + }) + end + + defp reassign_at_device_level(candidate) do + Agents.update_device_assignment( + candidate.device_id, + candidate.best_agent_token_id + ) + end + + defp broadcast_reassignment(candidate) do + Phoenix.PubSub.broadcast( + Towerops.PubSub, + "devices:latency", + {:device_reassigned, candidate} + ) + end + + defp schedule_evaluation(delay) do + Process.send_after(self(), :evaluate_latency, delay) + end +end diff --git a/lib/towerops_web/channels/agent_channel.ex b/lib/towerops_web/channels/agent_channel.ex index 8880a5db..f09d33de 100644 --- a/lib/towerops_web/channels/agent_channel.ex +++ b/lib/towerops_web/channels/agent_channel.ex @@ -136,7 +136,14 @@ defmodule ToweropsWeb.AgentChannel do def handle_in("monitoring_check", %{"binary" => binary_b64}, socket) do binary = Base.decode64!(binary_b64) check = MonitoringCheck.decode(binary) - _ = process_monitoring_check(socket.assigns.organization_id, check) + + _ = + process_monitoring_check( + socket.assigns.organization_id, + socket.assigns.agent_token_id, + check + ) + {:noreply, socket} end @@ -383,7 +390,7 @@ defmodule ToweropsWeb.AgentChannel do # since it requires complex LLDP/CDP parsing end - defp process_monitoring_check(organization_id, check) do + defp process_monitoring_check(organization_id, agent_token_id, check) do with {:ok, device} <- fetch_device(check.device_id), :ok <- verify_device_organization(device, organization_id) do # Parse status from protobuf string ("success" or "failure") @@ -396,9 +403,10 @@ defmodule ToweropsWeb.AgentChannel do timestamp = DateTime.from_unix!(check.timestamp, :second) - # Create monitoring check record + # Create monitoring check record with agent tracking Monitoring.create_check(%{ device_id: device.id, + agent_token_id: agent_token_id, status: status, response_time_ms: check.response_time_ms, checked_at: timestamp diff --git a/lib/towerops_web/live/agent_live/index.ex b/lib/towerops_web/live/agent_live/index.ex index 0258c92a..452efbc4 100644 --- a/lib/towerops_web/live/agent_live/index.ex +++ b/lib/towerops_web/live/agent_live/index.ex @@ -6,6 +6,7 @@ defmodule ToweropsWeb.AgentLive.Index do alias Towerops.Agents alias Towerops.Agents.Stats + alias Towerops.Settings @impl true def mount(_params, _session, socket) do @@ -13,12 +14,12 @@ defmodule ToweropsWeb.AgentLive.Index do current_user = socket.assigns.current_scope.user agent_tokens = Agents.list_organization_agent_tokens(organization.id) - # If superadmin, also load cloud pollers - cloud_pollers = + # If superadmin, also load cloud pollers and global default + {cloud_pollers, global_default_cloud_poller_id} = if current_user.is_superuser do - Agents.list_cloud_pollers() + {Agents.list_cloud_pollers(), Settings.get_global_default_cloud_poller()} else - [] + {[], nil} end # device counts for each agent (both direct and total with inheritance) @@ -42,6 +43,7 @@ defmodule ToweropsWeb.AgentLive.Index do |> assign(:page_title, "Remote Agents") |> assign(:agent_tokens, agent_tokens) |> assign(:cloud_pollers, cloud_pollers) + |> assign(:global_default_cloud_poller_id, global_default_cloud_poller_id) |> assign(:device_counts, equipment_counts) |> assign(:agent_health_stats, agent_health_stats) |> assign(:assignment_breakdown, assignment_breakdown) @@ -156,6 +158,26 @@ defmodule ToweropsWeb.AgentLive.Index do end end + @impl true + def handle_event("set_global_default", %{"agent_token_id" => agent_token_id}, socket) do + current_user = socket.assigns.current_scope.user + + if current_user.is_superuser do + # Handle empty string as nil + agent_token_id = if agent_token_id == "", do: nil, else: agent_token_id + + case Settings.set_global_default_cloud_poller(agent_token_id) do + {:ok, _} -> + handle_global_default_success(socket, agent_token_id) + + {:error, _} -> + {:noreply, put_flash(socket, :error, "Failed to update global default cloud poller")} + end + else + {:noreply, put_flash(socket, :error, "Only superadmins can set the global default cloud poller")} + end + end + defp apply_action(socket, :index, _params) do socket end @@ -234,4 +256,16 @@ defmodule ToweropsWeb.AgentLive.Index do do: "Cloud poller created successfully", else: "Agent created successfully" end + + defp handle_global_default_success(socket, agent_token_id) do + message = + if agent_token_id, + do: "Global default cloud poller set successfully", + else: "Global default cloud poller cleared" + + {:noreply, + socket + |> assign(:global_default_cloud_poller_id, agent_token_id) + |> put_flash(:info, message)} + end end diff --git a/lib/towerops_web/live/agent_live/index.html.heex b/lib/towerops_web/live/agent_live/index.html.heex index 39e41b49..84b711b7 100644 --- a/lib/towerops_web/live/agent_live/index.html.heex +++ b/lib/towerops_web/live/agent_live/index.html.heex @@ -261,6 +261,61 @@ <% end %> + + <%!-- Global Default Cloud Poller (Superadmin Only) --%> + <%= if @current_scope.user.is_superuser do %> +
+
+

+ Global Default Cloud Poller +

+

+ Fallback agent for organizations without a default agent configured. Devices with no assignment at any level will use this agent. +

+ + <.form for={%{}} phx-change="set_global_default" class="max-w-md"> +
+ + + + <%= if @global_default_cloud_poller_id do %> + <% selected_poller = + Enum.find(@cloud_pollers, &(&1.id == @global_default_cloud_poller_id)) %> + <%= if selected_poller do %> +

+ <.icon name="hero-check-circle" class="h-4 w-4 inline" /> Currently using: + {selected_poller.name} +

+ <% end %> + <% else %> +

+ <.icon name="hero-exclamation-triangle" class="h-4 w-4 inline" /> + No global default configured. Devices without assignments will use cloud polling (nil). +

+ <% end %> +
+ +
+
+ <% end %> <%= if @show_token_modal && @new_token do %> diff --git a/priv/repo/migrations/20260124003501_add_agent_token_id_to_monitoring_checks.exs b/priv/repo/migrations/20260124003501_add_agent_token_id_to_monitoring_checks.exs new file mode 100644 index 00000000..3296482d --- /dev/null +++ b/priv/repo/migrations/20260124003501_add_agent_token_id_to_monitoring_checks.exs @@ -0,0 +1,25 @@ +defmodule Towerops.Repo.Migrations.AddAgentTokenIdToMonitoringChecks do + use Ecto.Migration + + def up do + # Add agent_token_id column to track which agent performed the check + alter table(:monitoring_checks) do + add :agent_token_id, references(:agent_tokens, type: :binary_id, on_delete: :nilify_all) + end + + # Add index for efficient latency queries (agent + device + time range) + create index(:monitoring_checks, [:agent_token_id, :device_id, :checked_at]) + + # Add index for device-specific latency analysis + create index(:monitoring_checks, [:device_id, :agent_token_id, :status, :checked_at]) + end + + def down do + drop index(:monitoring_checks, [:device_id, :agent_token_id, :status, :checked_at]) + drop index(:monitoring_checks, [:agent_token_id, :device_id, :checked_at]) + + alter table(:monitoring_checks) do + remove :agent_token_id + end + end +end diff --git a/priv/repo/migrations/20260124003545_create_application_settings.exs b/priv/repo/migrations/20260124003545_create_application_settings.exs new file mode 100644 index 00000000..1ec72606 --- /dev/null +++ b/priv/repo/migrations/20260124003545_create_application_settings.exs @@ -0,0 +1,36 @@ +defmodule Towerops.Repo.Migrations.CreateApplicationSettings do + use Ecto.Migration + + def up do + create table(:application_settings, primary_key: false) do + add :id, :binary_id, primary_key: true + add :key, :string, null: false + add :value, :text + add :value_type, :string, null: false, default: "string" + add :description, :text + + timestamps(type: :utc_datetime) + end + + # Enforce unique keys + create unique_index(:application_settings, [:key]) + + # Insert default global cloud poller setting (nil = not configured) + execute """ + INSERT INTO application_settings (id, key, value, value_type, description, inserted_at, updated_at) + VALUES ( + gen_random_uuid(), + 'global_default_cloud_poller_id', + NULL, + 'uuid', + 'Default cloud poller agent used as fallback when no organization-level agent is configured', + NOW(), + NOW() + ) + """ + end + + def down do + drop table(:application_settings) + end +end diff --git a/test/towerops/agents/stats_test.exs b/test/towerops/agents/stats_test.exs index 0c43f253..1469b129 100644 --- a/test/towerops/agents/stats_test.exs +++ b/test/towerops/agents/stats_test.exs @@ -7,6 +7,7 @@ defmodule Towerops.Agents.StatsTest do alias Towerops.Agents alias Towerops.Agents.Stats alias Towerops.Devices.Device + alias Towerops.Monitoring alias Towerops.Repo alias Towerops.Sites alias Towerops.Snmp.Device @@ -452,4 +453,536 @@ defmodule Towerops.Agents.StatsTest do assert stats.last_submission end end + + describe "get_device_latency_by_agent/2" do + test "returns empty list when no monitoring checks exist" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + latency_by_agent = Stats.get_device_latency_by_agent(device.id) + + assert latency_by_agent == [] + end + + test "calculates average latency per agent for a device" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, agent1, _} = Agents.create_agent_token(org.id, "Agent 1") + {:ok, agent2, _} = Agents.create_agent_token(org.id, "Agent 2") + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create checks from agent1 with avg 50ms + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent1.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Create checks from agent2 with avg 100ms + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent2.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + end + + latency_by_agent = Stats.get_device_latency_by_agent(device.id) + + assert length(latency_by_agent) == 2 + + agent1_latency = Enum.find(latency_by_agent, &(&1.agent_token_id == agent1.id)) + agent2_latency = Enum.find(latency_by_agent, &(&1.agent_token_id == agent2.id)) + + assert agent1_latency.avg_latency_ms == 50.0 + assert agent1_latency.check_count == 15 + assert agent2_latency.avg_latency_ms == 100.0 + assert agent2_latency.check_count == 15 + end + + test "filters by successful checks only" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, agent, _} = Agents.create_agent_token(org.id, "Agent") + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create successful checks with 50ms + for _ <- 1..10 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Create failed checks (should be ignored) + for _ <- 1..5 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent.id, + status: :error, + response_time_ms: nil, + checked_at: DateTime.utc_now() + }) + end + + latency_by_agent = Stats.get_device_latency_by_agent(device.id) + + assert length(latency_by_agent) == 1 + agent_latency = hd(latency_by_agent) + assert agent_latency.avg_latency_ms == 50.0 + assert agent_latency.check_count == 10 + end + + test "respects minimum check count filter" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, agent1, _} = Agents.create_agent_token(org.id, "Agent 1") + {:ok, agent2, _} = Agents.create_agent_token(org.id, "Agent 2") + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Agent1 has 15 checks (above min) + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent1.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Agent2 has only 5 checks (below min) + for _ <- 1..5 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent2.id, + status: :success, + response_time_ms: 30, + checked_at: DateTime.utc_now() + }) + end + + latency_by_agent = Stats.get_device_latency_by_agent(device.id, min_checks: 10) + + assert length(latency_by_agent) == 1 + assert hd(latency_by_agent).agent_token_id == agent1.id + end + + test "respects time window parameter" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, agent, _} = Agents.create_agent_token(org.id, "Agent") + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create recent checks + for _ <- 1..10 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Create old checks (25 hours ago, outside default 24h window) + old_time = DateTime.add(DateTime.utc_now(), -25, :hour) + + for _ <- 1..5 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent.id, + status: :success, + response_time_ms: 100, + checked_at: old_time + }) + end + + # Default 24h window should only include recent checks + latency_by_agent = Stats.get_device_latency_by_agent(device.id) + + assert length(latency_by_agent) == 1 + agent_latency = hd(latency_by_agent) + assert agent_latency.avg_latency_ms == 50.0 + assert agent_latency.check_count == 10 + + # 48h window should include both + latency_by_agent_48h = Stats.get_device_latency_by_agent(device.id, hours_ago: 48) + + assert length(latency_by_agent_48h) == 1 + agent_latency_48h = hd(latency_by_agent_48h) + assert agent_latency_48h.check_count == 15 + end + end + + describe "get_device_assignment_with_latency/1" do + test "returns assignment details with latency stats" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, agent1, _} = Agents.create_agent_token(org.id, "Fast Agent") + {:ok, agent2, _} = Agents.create_agent_token(org.id, "Slow Agent") + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Assign to agent2 + {:ok, _} = Agents.assign_device_to_agent(agent2.id, device.id) + + # Create checks from both agents + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent1.id, + status: :success, + response_time_ms: 30, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent2.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + end + + result = Stats.get_device_assignment_with_latency(device.id) + + assert result.device_id == device.id + assert result.current_agent_token_id == agent2.id + assert result.assignment_source == :device + assert length(result.latency_stats) == 2 + + fast_agent = Enum.find(result.latency_stats, &(&1.agent_token_id == agent1.id)) + slow_agent = Enum.find(result.latency_stats, &(&1.agent_token_id == agent2.id)) + + assert fast_agent.avg_latency_ms == 30.0 + assert slow_agent.avg_latency_ms == 100.0 + end + + test "returns nil current_agent_id when using cloud polling" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + result = Stats.get_device_assignment_with_latency(device.id) + + assert result.device_id == device.id + assert is_nil(result.current_agent_token_id) + assert result.assignment_source == :none + end + end + + describe "find_reassignment_candidates/1" do + test "returns empty list when no devices have better options" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, agent, _} = Agents.create_agent_token(org.id, "Agent") + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + {:ok, _} = Agents.assign_device_to_agent(agent.id, device.id) + + # Create checks showing this is the only/best agent + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + candidates = Stats.find_reassignment_candidates() + + assert candidates == [] + end + + test "identifies devices with 20%+ improvement available" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent") + {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent") + + {:ok, site} = + Sites.create_site(%{ + name: "Test Site", + organization_id: org.id, + agent_token_id: slow_agent.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create checks: slow agent = 100ms, fast agent = 50ms (50% improvement) + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: slow_agent.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: fast_agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + candidates = Stats.find_reassignment_candidates(min_improvement_percent: 20) + + assert length(candidates) == 1 + candidate = hd(candidates) + + assert candidate.device_id == device.id + assert candidate.current_agent_token_id == slow_agent.id + assert candidate.best_agent_token_id == fast_agent.id + assert candidate.current_latency_ms == 100.0 + assert candidate.best_latency_ms == 50.0 + assert candidate.improvement_percent == 50.0 + end + + test "only considers automatic assignments" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent") + {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent") + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Direct device assignment (should be excluded from automatic reassignment) + {:ok, _} = Agents.assign_device_to_agent(slow_agent.id, device.id) + + # Create checks showing fast agent is better + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: slow_agent.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: fast_agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Should not reassign direct assignments + candidates = Stats.find_reassignment_candidates() + + assert candidates == [] + end + + test "includes site-level assignments for automatic reassignment" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent") + {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent") + + {:ok, site} = + Sites.create_site(%{ + name: "Test Site", + organization_id: org.id, + agent_token_id: slow_agent.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create checks showing fast agent is better + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: slow_agent.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: fast_agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + candidates = Stats.find_reassignment_candidates() + + assert length(candidates) == 1 + candidate = hd(candidates) + assert candidate.assignment_source == :site + assert candidate.best_agent_token_id == fast_agent.id + end + + test "requires minimum check count" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent") + {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent") + + {:ok, site} = + Sites.create_site(%{ + name: "Test Site", + organization_id: org.id, + agent_token_id: slow_agent.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create only 5 checks (below min of 10) + for _ <- 1..5 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: slow_agent.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: fast_agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Should not reassign with insufficient data + candidates = Stats.find_reassignment_candidates(min_checks_per_agent: 10) + + assert candidates == [] + end + end end diff --git a/test/towerops/agents_test.exs b/test/towerops/agents_test.exs index 78b956db..818d6bfa 100644 --- a/test/towerops/agents_test.exs +++ b/test/towerops/agents_test.exs @@ -691,6 +691,60 @@ defmodule Towerops.AgentsTest do assert Agents.get_effective_agent_token(device) == nil end + + test "returns global default cloud poller when no other assignment exists" do + # Create a cloud poller (application-wide agent) + {:ok, cloud_poller, _} = Agents.create_cloud_poller("Global Cloud Poller") + + # Set it as the global default + {:ok, _} = Towerops.Settings.set_global_default_cloud_poller(cloud_poller.id) + + # Create a fresh organization and device with no assignments + user = user_fixture() + {:ok, org} = Towerops.Organizations.create_organization(%{name: "Test Org 2"}, user.id) + + {:ok, site} = + Towerops.Sites.create_site(%{ + name: "Test Site 2", + organization_id: org.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Test Equipment 2", + ip_address: "192.168.2.1", + site_id: site.id + }) + + device = Repo.preload(device, site: [organization: :default_agent_token]) + + # Should use global default cloud poller + assert Agents.get_effective_agent_token(device) == cloud_poller.id + + # Clean up + {:ok, _} = Towerops.Settings.set_global_default_cloud_poller(nil) + end + + test "organization default takes precedence over global default", %{ + organization: org, + agent1: agent1, + device: device + } do + # Create a cloud poller as global default + {:ok, cloud_poller, _} = Agents.create_cloud_poller("Global Cloud Poller") + {:ok, _} = Towerops.Settings.set_global_default_cloud_poller(cloud_poller.id) + + # Set organization default + {:ok, _org} = Towerops.Organizations.update_organization(org, %{default_agent_token_id: agent1.id}) + + device = Repo.preload(device, site: [organization: :default_agent_token]) + + # Should use organization default, not global default + assert Agents.get_effective_agent_token(device) == agent1.id + + # Clean up + {:ok, _} = Towerops.Settings.set_global_default_cloud_poller(nil) + end end describe "get_effective_agent_token_with_source/1" do @@ -764,6 +818,37 @@ defmodule Towerops.AgentsTest do assert {nil, :none} = Agents.get_effective_agent_token_with_source(device) end + + test "returns global source for global default cloud poller" do + # Create a cloud poller and set as global default + {:ok, cloud_poller, _} = Agents.create_cloud_poller("Global Cloud Poller") + {:ok, _} = Towerops.Settings.set_global_default_cloud_poller(cloud_poller.id) + + # Create a fresh organization and device with no assignments + user = user_fixture() + {:ok, org} = Towerops.Organizations.create_organization(%{name: "Test Org 3"}, user.id) + + {:ok, site} = + Towerops.Sites.create_site(%{ + name: "Test Site 3", + organization_id: org.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Test Equipment 3", + ip_address: "192.168.3.1", + site_id: site.id + }) + + device = Repo.preload(device, site: [organization: :default_agent_token]) + + assert {agent_id, :global} = Agents.get_effective_agent_token_with_source(device) + assert agent_id == cloud_poller.id + + # Clean up + {:ok, _} = Towerops.Settings.set_global_default_cloud_poller(nil) + end end describe "count_assigned_devices/1" do diff --git a/test/towerops/settings_test.exs b/test/towerops/settings_test.exs new file mode 100644 index 00000000..c96bb33c --- /dev/null +++ b/test/towerops/settings_test.exs @@ -0,0 +1,152 @@ +defmodule Towerops.SettingsTest do + use Towerops.DataCase + + alias Towerops.Settings + alias Towerops.Settings.ApplicationSetting + + describe "get_setting/1" do + test "returns parsed value for existing setting" do + # The migration already created the global_default_cloud_poller_id setting + assert Settings.get_setting("global_default_cloud_poller_id") == nil + end + + test "returns nil for non-existent setting" do + assert Settings.get_setting("nonexistent_key") == nil + end + + test "parses string values correctly" do + {:ok, _setting} = create_setting("test_string", "hello", "string") + assert Settings.get_setting("test_string") == "hello" + end + + test "parses integer values correctly" do + {:ok, _setting} = create_setting("test_integer", "42", "integer") + assert Settings.get_setting("test_integer") == 42 + end + + test "parses boolean values correctly" do + {:ok, _setting} = create_setting("test_bool_true", "true", "boolean") + {:ok, _setting} = create_setting("test_bool_false", "false", "boolean") + {:ok, _setting} = create_setting("test_bool_one", "1", "boolean") + {:ok, _setting} = create_setting("test_bool_zero", "0", "boolean") + + assert Settings.get_setting("test_bool_true") == true + assert Settings.get_setting("test_bool_false") == false + assert Settings.get_setting("test_bool_one") == true + assert Settings.get_setting("test_bool_zero") == false + end + + test "parses json values correctly" do + {:ok, _setting} = create_setting("test_json", ~s({"key": "value"}), "json") + assert Settings.get_setting("test_json") == %{"key" => "value"} + end + + test "returns nil for invalid integer" do + {:ok, _setting} = create_setting("test_invalid_int", "not_a_number", "integer") + assert Settings.get_setting("test_invalid_int") == nil + end + + test "returns nil for invalid json" do + {:ok, _setting} = create_setting("test_invalid_json", "not json", "json") + assert Settings.get_setting("test_invalid_json") == nil + end + end + + describe "get_setting_record/1" do + test "returns the raw ApplicationSetting record" do + record = Settings.get_setting_record("global_default_cloud_poller_id") + assert %ApplicationSetting{} = record + assert record.key == "global_default_cloud_poller_id" + assert record.value_type == "uuid" + end + + test "returns nil for non-existent setting" do + assert Settings.get_setting_record("nonexistent") == nil + end + end + + describe "update_setting/2" do + test "updates an existing setting value" do + agent_token_id = Ecto.UUID.generate() + {:ok, updated} = Settings.update_setting("global_default_cloud_poller_id", agent_token_id) + + assert updated.value == agent_token_id + assert Settings.get_setting("global_default_cloud_poller_id") == agent_token_id + end + + test "updates setting to nil" do + agent_token_id = Ecto.UUID.generate() + {:ok, _} = Settings.update_setting("global_default_cloud_poller_id", agent_token_id) + {:ok, updated} = Settings.update_setting("global_default_cloud_poller_id", nil) + + assert updated.value == nil + assert Settings.get_setting("global_default_cloud_poller_id") == nil + end + + test "returns error for non-existent setting" do + assert {:error, :setting_not_found} = + Settings.update_setting("nonexistent_key", "value") + end + + test "converts non-string values to strings" do + {:ok, _setting} = create_setting("test_number", "0", "integer") + {:ok, updated} = Settings.update_setting("test_number", 123) + + assert updated.value == "123" + assert Settings.get_setting("test_number") == 123 + end + end + + describe "get_global_default_cloud_poller/0" do + test "returns nil when not configured" do + assert Settings.get_global_default_cloud_poller() == nil + end + + test "returns agent token ID when configured" do + agent_token_id = Ecto.UUID.generate() + {:ok, _} = Settings.set_global_default_cloud_poller(agent_token_id) + + assert Settings.get_global_default_cloud_poller() == agent_token_id + end + end + + describe "set_global_default_cloud_poller/1" do + test "sets the global default cloud poller" do + agent_token_id = Ecto.UUID.generate() + {:ok, setting} = Settings.set_global_default_cloud_poller(agent_token_id) + + assert setting.value == agent_token_id + assert Settings.get_global_default_cloud_poller() == agent_token_id + end + + test "clears the global default cloud poller when set to nil" do + agent_token_id = Ecto.UUID.generate() + {:ok, _} = Settings.set_global_default_cloud_poller(agent_token_id) + {:ok, setting} = Settings.set_global_default_cloud_poller(nil) + + assert setting.value == nil + assert Settings.get_global_default_cloud_poller() == nil + end + end + + describe "list_all_settings/0" do + test "returns all application settings" do + settings = Settings.list_all_settings() + + assert is_list(settings) + refute Enum.empty?(settings) + assert Enum.any?(settings, &(&1.key == "global_default_cloud_poller_id")) + end + end + + # Helper function to create test settings + defp create_setting(key, value, value_type) do + %ApplicationSetting{} + |> ApplicationSetting.changeset(%{ + key: key, + value: value, + value_type: value_type + }) + |> Repo.insert() + end +end diff --git a/test/towerops/workers/agent_latency_evaluator_test.exs b/test/towerops/workers/agent_latency_evaluator_test.exs new file mode 100644 index 00000000..e7ccaa7e --- /dev/null +++ b/test/towerops/workers/agent_latency_evaluator_test.exs @@ -0,0 +1,517 @@ +defmodule Towerops.Workers.AgentLatencyEvaluatorTest do + use Towerops.DataCase, async: false + + import Towerops.AccountsFixtures + import Towerops.OrganizationsFixtures + + alias Towerops.Agents + alias Towerops.Monitoring + alias Towerops.Sites + alias Towerops.Workers.AgentLatencyEvaluator + + setup do + # Start the worker for testing + {:ok, pid} = start_supervised(AgentLatencyEvaluator) + %{worker_pid: pid} + end + + describe "evaluate_latency/0" do + test "reassigns device to faster agent when 20%+ improvement available" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent") + {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent") + + {:ok, site} = + Sites.create_site(%{ + name: "Test Site", + organization_id: org.id, + agent_token_id: slow_agent.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create checks: slow = 100ms, fast = 50ms (50% improvement) + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: slow_agent.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: fast_agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Trigger evaluation + AgentLatencyEvaluator.trigger_evaluation() + + # Wait for async processing + Process.sleep(100) + + # Verify site was reassigned + updated_site = Sites.get_site!(site.id) + assert updated_site.agent_token_id == fast_agent.id + end + + test "does not reassign when improvement is below 20% threshold" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, agent1, _} = Agents.create_agent_token(org.id, "Agent 1") + {:ok, agent2, _} = Agents.create_agent_token(org.id, "Agent 2") + + {:ok, site} = + Sites.create_site(%{ + name: "Test Site", + organization_id: org.id, + agent_token_id: agent1.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create checks: agent1 = 100ms, agent2 = 85ms (15% improvement, below 20%) + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent1.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: agent2.id, + status: :success, + response_time_ms: 85, + checked_at: DateTime.utc_now() + }) + end + + # Trigger evaluation + AgentLatencyEvaluator.trigger_evaluation() + + # Wait for async processing + Process.sleep(100) + + # Verify site was NOT reassigned + updated_site = Sites.get_site!(site.id) + assert updated_site.agent_token_id == agent1.id + end + + test "reassigns organization-level assignment" do + user = user_fixture() + + {:ok, slow_agent, _} = Agents.create_cloud_poller("Slow Cloud") + {:ok, fast_agent, _} = Agents.create_cloud_poller("Fast Cloud") + + org = + organization_fixture(user.id, %{ + default_agent_token_id: slow_agent.id + }) + + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create checks showing fast agent is better + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: slow_agent.id, + status: :success, + response_time_ms: 150, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: fast_agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Trigger evaluation + AgentLatencyEvaluator.trigger_evaluation() + + # Wait for async processing + Process.sleep(100) + + # Verify organization was reassigned + updated_org = Towerops.Organizations.get_organization!(org.id) + assert updated_org.default_agent_token_id == fast_agent.id + end + + test "creates device assignment for global default assignment" do + user = user_fixture() + org = organization_fixture(user.id) + + {:ok, slow_cloud, _} = Agents.create_cloud_poller("Slow Global") + {:ok, fast_cloud, _} = Agents.create_cloud_poller("Fast Global") + + # Set slow cloud as global default + {:ok, _} = Towerops.Settings.set_global_default_cloud_poller(slow_cloud.id) + + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create checks showing fast cloud is better + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: slow_cloud.id, + status: :success, + response_time_ms: 200, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: fast_cloud.id, + status: :success, + response_time_ms: 60, + checked_at: DateTime.utc_now() + }) + end + + # Trigger evaluation + AgentLatencyEvaluator.trigger_evaluation() + + # Wait for async processing + Process.sleep(100) + + # Verify device assignment was created + assignment = Agents.get_device_assignment(device.id) + assert assignment.agent_token_id == fast_cloud.id + + # Cleanup global default + {:ok, _} = Towerops.Settings.set_global_default_cloud_poller(nil) + end + + test "creates device assignment for cloud polling (no assignment)" do + user = user_fixture() + org = organization_fixture(user.id) + + {:ok, slow_cloud, _} = Agents.create_cloud_poller("Slow Cloud") + {:ok, fast_cloud, _} = Agents.create_cloud_poller("Fast Cloud") + + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create checks from both cloud agents + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: slow_cloud.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: fast_cloud.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Trigger evaluation + AgentLatencyEvaluator.trigger_evaluation() + + # Wait for async processing + Process.sleep(500) + + # Verify device assignment was created to the fast agent + assignment = Agents.get_device_assignment(device.id) + assert assignment != nil, "Expected device assignment to be created" + assert assignment.agent_token_id == fast_cloud.id + end + + test "does not reassign direct device assignments" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent") + {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent") + + {:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id}) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create direct device assignment + {:ok, _} = Agents.assign_device_to_agent(slow_agent.id, device.id) + + # Create checks showing fast agent is better + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: slow_agent.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: fast_agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Trigger evaluation + AgentLatencyEvaluator.trigger_evaluation() + + # Wait for async processing + Process.sleep(100) + + # Verify device assignment was NOT changed + assignment = Agents.get_device_assignment(device.id) + assert assignment.agent_token_id == slow_agent.id + end + + test "requires minimum check count" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent") + {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent") + + {:ok, site} = + Sites.create_site(%{ + name: "Test Site", + organization_id: org.id, + agent_token_id: slow_agent.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create only 5 checks (below default minimum of 10) + for _ <- 1..5 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: slow_agent.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: fast_agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Trigger evaluation + AgentLatencyEvaluator.trigger_evaluation() + + # Wait for async processing + Process.sleep(100) + + # Verify site was NOT reassigned (insufficient data) + updated_site = Sites.get_site!(site.id) + assert updated_site.agent_token_id == slow_agent.id + end + + test "handles multiple devices in single evaluation" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent") + {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent") + + {:ok, site1} = + Sites.create_site(%{ + name: "Site 1", + organization_id: org.id, + agent_token_id: slow_agent.id + }) + + {:ok, site2} = + Sites.create_site(%{ + name: "Site 2", + organization_id: org.id, + agent_token_id: slow_agent.id + }) + + {:ok, device1} = + Towerops.Devices.create_device(%{ + name: "Router 1", + ip_address: "192.168.1.1", + site_id: site1.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + {:ok, device2} = + Towerops.Devices.create_device(%{ + name: "Router 2", + ip_address: "192.168.2.1", + site_id: site2.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create checks for both devices + for device_id <- [device1.id, device2.id] do + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device_id, + agent_token_id: slow_agent.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device_id, + agent_token_id: fast_agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + end + + # Trigger evaluation + AgentLatencyEvaluator.trigger_evaluation() + + # Wait for async processing + Process.sleep(100) + + # Verify both sites were reassigned + updated_site1 = Sites.get_site!(site1.id) + updated_site2 = Sites.get_site!(site2.id) + + assert updated_site1.agent_token_id == fast_agent.id + assert updated_site2.agent_token_id == fast_agent.id + end + end + + describe "handle_info(:evaluate_latency)" do + test "worker processes evaluation on schedule" do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent") + {:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent") + + {:ok, site} = + Sites.create_site(%{ + name: "Test Site", + organization_id: org.id, + agent_token_id: slow_agent.id + }) + + {:ok, device} = + Towerops.Devices.create_device(%{ + name: "Router", + ip_address: "192.168.1.1", + site_id: site.id, + snmp_enabled: true, + snmp_community: "public", + snmp_version: "2c" + }) + + # Create checks + for _ <- 1..15 do + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: slow_agent.id, + status: :success, + response_time_ms: 100, + checked_at: DateTime.utc_now() + }) + + Monitoring.create_check(%{ + device_id: device.id, + agent_token_id: fast_agent.id, + status: :success, + response_time_ms: 50, + checked_at: DateTime.utc_now() + }) + end + + # Send the message directly to the worker + send(Process.whereis(AgentLatencyEvaluator), :evaluate_latency) + + # Wait for async processing + Process.sleep(100) + + # Verify site was reassigned + updated_site = Sites.get_site!(site.id) + assert updated_site.agent_token_id == fast_agent.id + end + end +end