remote agent/cloud improvements
This commit is contained in:
parent
9697596e99
commit
e6054129b2
16 changed files with 2094 additions and 43 deletions
|
|
@ -357,6 +357,7 @@ defmodule Towerops.Agents do
|
|||
1. Directly assigned to this agent
|
||||
2. At a site that has this agent as default
|
||||
3. In an organization that has this agent as default (and not overridden by site or device)
|
||||
4. This agent is the global default cloud poller (and device has no assignment at any level)
|
||||
|
||||
Returns device records with preloaded associations, filtered to only SNMP-enabled devices.
|
||||
|
||||
|
|
@ -368,26 +369,39 @@ defmodule Towerops.Agents do
|
|||
"""
|
||||
@spec list_agent_polling_targets(Ecto.UUID.t()) :: [Device.t()]
|
||||
def list_agent_polling_targets(agent_token_id) do
|
||||
# Use SQL to filter devices by effective agent assignment
|
||||
# This avoids loading all devices into memory and filtering in application code
|
||||
Repo.all(
|
||||
global_default = Towerops.Settings.get_global_default_cloud_poller()
|
||||
is_global_default = agent_token_id == global_default
|
||||
|
||||
query =
|
||||
from(e in Device,
|
||||
join: s in assoc(e, :site),
|
||||
join: o in assoc(s, :organization),
|
||||
left_join: aa in AgentAssignment,
|
||||
on: aa.device_id == e.id and aa.enabled == true,
|
||||
where:
|
||||
e.snmp_enabled == true and
|
||||
(aa.agent_token_id == ^agent_token_id or
|
||||
(is_nil(aa.agent_token_id) and s.agent_token_id == ^agent_token_id) or
|
||||
(is_nil(aa.agent_token_id) and is_nil(s.agent_token_id) and
|
||||
o.default_agent_token_id == ^agent_token_id)),
|
||||
where: e.snmp_enabled == true,
|
||||
preload: [
|
||||
:agent_assignments,
|
||||
site: :organization,
|
||||
snmp_device: [:sensors, :interfaces]
|
||||
]
|
||||
)
|
||||
|
||||
query
|
||||
|> where_agent_matches(agent_token_id, is_global_default)
|
||||
|> Repo.all()
|
||||
end
|
||||
|
||||
# Adds where clause to match devices assigned to this agent
|
||||
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
|
||||
defp where_agent_matches(query, agent_token_id, is_global_default) do
|
||||
from([e, s, o, aa] in query,
|
||||
where:
|
||||
aa.agent_token_id == ^agent_token_id or
|
||||
(is_nil(aa.agent_token_id) and s.agent_token_id == ^agent_token_id) or
|
||||
(is_nil(aa.agent_token_id) and is_nil(s.agent_token_id) and
|
||||
o.default_agent_token_id == ^agent_token_id) or
|
||||
(^is_global_default and is_nil(aa.agent_token_id) and is_nil(s.agent_token_id) and
|
||||
is_nil(o.default_agent_token_id))
|
||||
)
|
||||
end
|
||||
|
||||
|
|
@ -470,7 +484,9 @@ defmodule Towerops.Agents do
|
|||
Resolves the agent token using the following hierarchy:
|
||||
1. Device's direct agent assignment (highest priority)
|
||||
2. Site's agent_token_id
|
||||
3. Organization's default_agent_token_id (lowest priority)
|
||||
3. Organization's default_agent_token_id
|
||||
4. Global default cloud poller (application-wide fallback)
|
||||
5. Cloud polling (nil - lowest priority)
|
||||
|
||||
Returns the agent_token_id if found, nil otherwise.
|
||||
|
||||
|
|
@ -487,27 +503,28 @@ defmodule Towerops.Agents do
|
|||
|
||||
"""
|
||||
def get_effective_agent_token(%Device{} = device) do
|
||||
# device assignment
|
||||
# 1. Check device assignment
|
||||
case get_device_assignment(device.id) do
|
||||
%AgentAssignment{agent_token_id: agent_token_id} when not is_nil(agent_token_id) ->
|
||||
agent_token_id
|
||||
|
||||
_ ->
|
||||
# 2. Check site's agent token
|
||||
site = device.site
|
||||
get_fallback_agent_token(device.site)
|
||||
end
|
||||
end
|
||||
|
||||
case site do
|
||||
%{agent_token_id: agent_token_id} when not is_nil(agent_token_id) ->
|
||||
agent_token_id
|
||||
# Checks site, organization, and global default in sequence
|
||||
defp get_fallback_agent_token(site) do
|
||||
case site do
|
||||
%{agent_token_id: agent_token_id} when not is_nil(agent_token_id) ->
|
||||
agent_token_id
|
||||
|
||||
%{organization: %{default_agent_token_id: agent_token_id}}
|
||||
when not is_nil(agent_token_id) ->
|
||||
# 3. Use organization's default agent token
|
||||
agent_token_id
|
||||
%{organization: %{default_agent_token_id: agent_token_id}}
|
||||
when not is_nil(agent_token_id) ->
|
||||
agent_token_id
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
_ ->
|
||||
Towerops.Settings.get_global_default_cloud_poller()
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -518,6 +535,7 @@ defmodule Towerops.Agents do
|
|||
- :device - Direct device assignment
|
||||
- :site - Inherited from site
|
||||
- :organization - Inherited from organization
|
||||
- :global - Global default cloud poller
|
||||
- :none - No agent assigned
|
||||
|
||||
## Examples
|
||||
|
|
@ -530,23 +548,30 @@ defmodule Towerops.Agents do
|
|||
|
||||
"""
|
||||
def get_effective_agent_token_with_source(%Device{} = device) do
|
||||
# device assignment
|
||||
# 1. Check device assignment
|
||||
case get_device_assignment(device.id) do
|
||||
%AgentAssignment{agent_token_id: agent_token_id} when not is_nil(agent_token_id) ->
|
||||
{agent_token_id, :device}
|
||||
|
||||
_ ->
|
||||
# 2. Check site's agent token
|
||||
site = device.site
|
||||
get_fallback_agent_token_with_source(device.site)
|
||||
end
|
||||
end
|
||||
|
||||
case site do
|
||||
%{agent_token_id: agent_token_id} when not is_nil(agent_token_id) ->
|
||||
{agent_token_id, :site}
|
||||
# Checks site, organization, and global default in sequence with source tracking
|
||||
defp get_fallback_agent_token_with_source(site) do
|
||||
case site do
|
||||
%{agent_token_id: agent_token_id} when not is_nil(agent_token_id) ->
|
||||
{agent_token_id, :site}
|
||||
|
||||
%{organization: %{default_agent_token_id: agent_token_id}}
|
||||
when not is_nil(agent_token_id) ->
|
||||
# 3. Use organization's default agent token
|
||||
{agent_token_id, :organization}
|
||||
%{organization: %{default_agent_token_id: agent_token_id}}
|
||||
when not is_nil(agent_token_id) ->
|
||||
{agent_token_id, :organization}
|
||||
|
||||
_ ->
|
||||
case Towerops.Settings.get_global_default_cloud_poller() do
|
||||
agent_token_id when not is_nil(agent_token_id) ->
|
||||
{agent_token_id, :global}
|
||||
|
||||
_ ->
|
||||
{nil, :none}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ defmodule Towerops.Agents.Stats do
|
|||
alias Towerops.Agents.AgentAssignment
|
||||
alias Towerops.Agents.AgentToken
|
||||
alias Towerops.Devices.Device
|
||||
alias Towerops.Monitoring.Check
|
||||
alias Towerops.Repo
|
||||
alias Towerops.Snmp.SensorReading
|
||||
|
||||
|
|
@ -298,8 +299,218 @@ defmodule Towerops.Agents.Stats do
|
|||
)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Get average latency for a device grouped by agent over the last 24 hours.
|
||||
|
||||
Returns a list of agents with their average response times for the given device,
|
||||
filtered to successful checks only and requiring minimum check count for statistical validity.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> get_device_latency_by_agent(device_id, min_checks: 10)
|
||||
[
|
||||
%{agent_token_id: "uuid-1", avg_latency_ms: 12.5, check_count: 120},
|
||||
%{agent_token_id: "uuid-2", avg_latency_ms: 45.2, check_count: 115}
|
||||
]
|
||||
"""
|
||||
def get_device_latency_by_agent(device_id, opts \\ []) do
|
||||
min_checks = Keyword.get(opts, :min_checks, 10)
|
||||
hours_ago = Keyword.get(opts, :hours_ago, 24)
|
||||
time_threshold = DateTime.add(DateTime.utc_now(), -hours_ago, :hour)
|
||||
|
||||
Repo.all(
|
||||
from(c in Check,
|
||||
where:
|
||||
c.device_id == ^device_id and
|
||||
c.status == :success and
|
||||
not is_nil(c.agent_token_id) and
|
||||
c.checked_at > ^time_threshold,
|
||||
group_by: c.agent_token_id,
|
||||
having: count(c.id) >= ^min_checks,
|
||||
select: %{
|
||||
agent_token_id: c.agent_token_id,
|
||||
avg_latency_ms: avg(c.response_time_ms),
|
||||
check_count: count(c.id)
|
||||
},
|
||||
order_by: [asc: avg(c.response_time_ms)]
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Get device assignment information combined with latency statistics.
|
||||
|
||||
Returns device assignment details with current agent and latency data for potential reassignment.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> get_device_assignment_with_latency(device_id)
|
||||
%{
|
||||
device_id: "uuid",
|
||||
current_agent_token_id: "uuid-1",
|
||||
assignment_source: :site,
|
||||
latency_stats: [
|
||||
%{agent_token_id: "uuid-1", avg_latency_ms: 12.5, check_count: 120},
|
||||
%{agent_token_id: "uuid-2", avg_latency_ms: 8.3, check_count: 115}
|
||||
]
|
||||
}
|
||||
"""
|
||||
def get_device_assignment_with_latency(device_id) do
|
||||
device =
|
||||
Device
|
||||
|> Repo.get!(device_id)
|
||||
|> Repo.preload(site: [organization: :default_agent_token])
|
||||
|
||||
{current_agent_id, source} = Towerops.Agents.get_effective_agent_token_with_source(device)
|
||||
|
||||
latency_stats = get_device_latency_by_agent(device_id)
|
||||
|
||||
%{
|
||||
device_id: device_id,
|
||||
current_agent_token_id: current_agent_id,
|
||||
assignment_source: source,
|
||||
latency_stats: latency_stats
|
||||
}
|
||||
end
|
||||
|
||||
@doc """
|
||||
Find devices that are candidates for latency-based reassignment.
|
||||
|
||||
Identifies devices where an alternative agent has significantly better latency (default: 20% improvement).
|
||||
Only considers devices with automatic assignments (site, organization, global, or none).
|
||||
|
||||
## Examples
|
||||
|
||||
iex> find_reassignment_candidates(min_improvement_percent: 20, min_checks_per_agent: 10)
|
||||
[
|
||||
%{
|
||||
device_id: "uuid",
|
||||
device_name: "Switch-01",
|
||||
current_agent_token_id: "uuid-1",
|
||||
current_latency_ms: 45.2,
|
||||
best_agent_token_id: "uuid-2",
|
||||
best_latency_ms: 28.3,
|
||||
improvement_percent: 37.4,
|
||||
assignment_source: :site
|
||||
}
|
||||
]
|
||||
"""
|
||||
def find_reassignment_candidates(opts \\ []) do
|
||||
min_improvement_percent = Keyword.get(opts, :min_improvement_percent, 20)
|
||||
min_checks = Keyword.get(opts, :min_checks_per_agent, 10)
|
||||
hours_ago = Keyword.get(opts, :hours_ago, 24)
|
||||
time_threshold = DateTime.add(DateTime.utc_now(), -hours_ago, :hour)
|
||||
|
||||
# Get all devices with multiple agent latency measurements
|
||||
devices_with_agents =
|
||||
Repo.all(
|
||||
from(c in Check,
|
||||
where: c.status == :success and not is_nil(c.agent_token_id) and c.checked_at > ^time_threshold,
|
||||
group_by: [c.device_id, c.agent_token_id],
|
||||
having: count(c.id) >= ^min_checks,
|
||||
select: %{
|
||||
device_id: c.device_id,
|
||||
agent_token_id: c.agent_token_id,
|
||||
avg_latency_ms: avg(c.response_time_ms),
|
||||
check_count: count(c.id)
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# Group by device and evaluate candidates
|
||||
devices_with_agents
|
||||
|> Enum.group_by(& &1.device_id)
|
||||
|> Enum.filter(fn {_device_id, agents} -> length(agents) >= 2 end)
|
||||
|> Enum.flat_map(fn {device_id, agents} ->
|
||||
evaluate_device_reassignment(device_id, agents, min_improvement_percent)
|
||||
end)
|
||||
end
|
||||
|
||||
# Private helper functions
|
||||
|
||||
# Evaluates if a device should be reassigned based on latency comparison
|
||||
defp evaluate_device_reassignment(device_id, agents, min_improvement_percent) do
|
||||
device =
|
||||
Device
|
||||
|> Repo.get!(device_id)
|
||||
|> Repo.preload(site: [organization: :default_agent_token])
|
||||
|
||||
{current_agent_id, source} = Towerops.Agents.get_effective_agent_token_with_source(device)
|
||||
|
||||
# Only consider automatic assignments for reassignment
|
||||
if source in [:site, :organization, :global, :none] do
|
||||
build_reassignment_candidate(device, current_agent_id, source, agents, min_improvement_percent)
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
# Builds reassignment candidate if improvement threshold is met
|
||||
defp build_reassignment_candidate(device, current_agent_id, source, agents, min_improvement_percent) do
|
||||
# Special case: if no current agent (:none or :global with no agent),
|
||||
# just pick the best agent without requiring improvement
|
||||
if is_nil(current_agent_id) do
|
||||
build_initial_assignment_candidate(device, source, agents)
|
||||
else
|
||||
build_reassignment_with_improvement(device, current_agent_id, source, agents, min_improvement_percent)
|
||||
end
|
||||
end
|
||||
|
||||
defp build_initial_assignment_candidate(device, source, agents) do
|
||||
best = Enum.min_by(agents, & &1.avg_latency_ms, fn -> nil end)
|
||||
|
||||
if best do
|
||||
[
|
||||
%{
|
||||
device_id: device.id,
|
||||
device_name: device.name,
|
||||
current_agent_token_id: nil,
|
||||
current_latency_ms: nil,
|
||||
best_agent_token_id: best.agent_token_id,
|
||||
best_latency_ms: Float.round(best.avg_latency_ms, 2),
|
||||
improvement_percent: 100.0,
|
||||
assignment_source: source
|
||||
}
|
||||
]
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp build_reassignment_with_improvement(device, current_agent_id, source, agents, min_improvement_percent) do
|
||||
current_latency = Enum.find(agents, fn a -> a.agent_token_id == current_agent_id end)
|
||||
|
||||
with %{} <- current_latency,
|
||||
best when not is_nil(best) <- find_best_alternative(agents, current_agent_id),
|
||||
improvement when improvement >= min_improvement_percent <-
|
||||
calculate_improvement_percent(current_latency, best) do
|
||||
[
|
||||
%{
|
||||
device_id: device.id,
|
||||
device_name: device.name,
|
||||
current_agent_token_id: current_agent_id,
|
||||
current_latency_ms: Float.round(current_latency.avg_latency_ms, 2),
|
||||
best_agent_token_id: best.agent_token_id,
|
||||
best_latency_ms: Float.round(best.avg_latency_ms, 2),
|
||||
improvement_percent: improvement,
|
||||
assignment_source: source
|
||||
}
|
||||
]
|
||||
else
|
||||
_ -> []
|
||||
end
|
||||
end
|
||||
|
||||
defp find_best_alternative(agents, current_agent_id) do
|
||||
agents
|
||||
|> Enum.reject(fn a -> a.agent_token_id == current_agent_id end)
|
||||
|> Enum.min_by(& &1.avg_latency_ms, fn -> nil end)
|
||||
end
|
||||
|
||||
defp calculate_improvement_percent(current, best) do
|
||||
Float.round((current.avg_latency_ms - best.avg_latency_ms) / current.avg_latency_ms * 100, 1)
|
||||
end
|
||||
|
||||
defp polling_targets_subquery(agent_token_id) do
|
||||
# This is a simplified version - in production you'd want to use
|
||||
# Agents.list_agent_polling_targets/1 logic here
|
||||
|
|
|
|||
|
|
@ -65,7 +65,9 @@ defmodule Towerops.Application do
|
|||
# Start monitoring supervisor (includes NeighborCleanupWorker)
|
||||
Towerops.Monitoring.Supervisor,
|
||||
# Start stale agent detection worker
|
||||
Towerops.Workers.StaleAgentWorker
|
||||
Towerops.Workers.StaleAgentWorker,
|
||||
# Start latency-based agent reassignment worker
|
||||
Towerops.Workers.AgentLatencyEvaluator
|
||||
]
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ defmodule Towerops.Monitoring.Check do
|
|||
@moduledoc """
|
||||
Monitoring check schema for device reachability status.
|
||||
|
||||
Records the result of each monitoring check (success/failure) with latency.
|
||||
Records the result of each monitoring check (success/failure) with latency
|
||||
and tracks which agent performed the check for intelligent routing.
|
||||
"""
|
||||
use Ecto.Schema
|
||||
|
||||
|
|
@ -16,6 +17,7 @@ defmodule Towerops.Monitoring.Check do
|
|||
field :checked_at, :utc_datetime
|
||||
|
||||
belongs_to :device, Towerops.Devices.Device
|
||||
belongs_to :agent_token, Towerops.Agents.AgentToken
|
||||
|
||||
timestamps(type: :utc_datetime, updated_at: false)
|
||||
end
|
||||
|
|
@ -23,8 +25,9 @@ defmodule Towerops.Monitoring.Check do
|
|||
@doc false
|
||||
def changeset(check, attrs) do
|
||||
check
|
||||
|> cast(attrs, [:device_id, :status, :response_time_ms, :checked_at])
|
||||
|> cast(attrs, [:device_id, :agent_token_id, :status, :response_time_ms, :checked_at])
|
||||
|> validate_required([:device_id, :status, :checked_at])
|
||||
|> foreign_key_constraint(:device_id, match: :suffix)
|
||||
|> foreign_key_constraint(:agent_token_id, match: :suffix)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
117
lib/towerops/settings.ex
Normal file
117
lib/towerops/settings.ex
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
defmodule Towerops.Settings do
|
||||
@moduledoc """
|
||||
The Settings context for application-wide configuration.
|
||||
|
||||
Provides functions for reading and updating global settings that affect
|
||||
the entire application. Only accessible by superadmins.
|
||||
"""
|
||||
|
||||
import Ecto.Query, warn: false
|
||||
|
||||
alias Towerops.Repo
|
||||
alias Towerops.Settings.ApplicationSetting
|
||||
|
||||
@doc """
|
||||
Gets a setting by key and returns the parsed value.
|
||||
|
||||
Returns nil if the setting doesn't exist or has a nil value.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> get_setting("global_default_cloud_poller_id")
|
||||
"abc-123-uuid"
|
||||
|
||||
iex> get_setting("nonexistent_key")
|
||||
nil
|
||||
"""
|
||||
def get_setting(key) when is_binary(key) do
|
||||
case Repo.get_by(ApplicationSetting, key: key) do
|
||||
nil -> nil
|
||||
setting -> ApplicationSetting.parse_value(setting)
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Gets the raw ApplicationSetting record by key.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> get_setting_record("global_default_cloud_poller_id")
|
||||
%ApplicationSetting{key: "global_default_cloud_poller_id", value: "abc-123"}
|
||||
"""
|
||||
def get_setting_record(key) when is_binary(key) do
|
||||
Repo.get_by(ApplicationSetting, key: key)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Updates a setting value.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> update_setting("global_default_cloud_poller_id", "new-uuid")
|
||||
{:ok, %ApplicationSetting{}}
|
||||
|
||||
iex> update_setting("global_default_cloud_poller_id", nil)
|
||||
{:ok, %ApplicationSetting{value: nil}}
|
||||
"""
|
||||
def update_setting(key, value) when is_binary(key) do
|
||||
case get_setting_record(key) do
|
||||
nil ->
|
||||
{:error, :setting_not_found}
|
||||
|
||||
setting ->
|
||||
setting
|
||||
|> ApplicationSetting.changeset(%{value: to_string_or_nil(value)})
|
||||
|> Repo.update()
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Gets the global default cloud poller agent token ID.
|
||||
|
||||
Returns nil if not configured.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> get_global_default_cloud_poller()
|
||||
"abc-123-uuid"
|
||||
|
||||
iex> get_global_default_cloud_poller()
|
||||
nil
|
||||
"""
|
||||
def get_global_default_cloud_poller do
|
||||
get_setting("global_default_cloud_poller_id")
|
||||
end
|
||||
|
||||
@doc """
|
||||
Sets the global default cloud poller agent token ID.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> set_global_default_cloud_poller("agent-token-uuid")
|
||||
{:ok, %ApplicationSetting{}}
|
||||
|
||||
iex> set_global_default_cloud_poller(nil)
|
||||
{:ok, %ApplicationSetting{value: nil}}
|
||||
"""
|
||||
def set_global_default_cloud_poller(agent_token_id) do
|
||||
update_setting("global_default_cloud_poller_id", agent_token_id)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Lists all application settings.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> list_all_settings()
|
||||
[%ApplicationSetting{}, ...]
|
||||
"""
|
||||
def list_all_settings do
|
||||
Repo.all(ApplicationSetting)
|
||||
end
|
||||
|
||||
# Private helpers
|
||||
|
||||
defp to_string_or_nil(nil), do: nil
|
||||
defp to_string_or_nil(value), do: to_string(value)
|
||||
end
|
||||
75
lib/towerops/settings/application_setting.ex
Normal file
75
lib/towerops/settings/application_setting.ex
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
defmodule Towerops.Settings.ApplicationSetting do
|
||||
@moduledoc """
|
||||
Application-wide settings schema.
|
||||
|
||||
Stores global configuration values that affect the entire application,
|
||||
regardless of organization. Only accessible by superadmins.
|
||||
"""
|
||||
use Ecto.Schema
|
||||
|
||||
import Ecto.Changeset
|
||||
|
||||
@primary_key {:id, :binary_id, autogenerate: true}
|
||||
@foreign_key_type :binary_id
|
||||
schema "application_settings" do
|
||||
field :key, :string
|
||||
field :value, :string
|
||||
field :value_type, :string, default: "string"
|
||||
field :description, :string
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
@type t :: %__MODULE__{
|
||||
id: Ecto.UUID.t(),
|
||||
key: String.t(),
|
||||
value: String.t() | nil,
|
||||
value_type: String.t(),
|
||||
description: String.t() | nil,
|
||||
inserted_at: DateTime.t(),
|
||||
updated_at: DateTime.t()
|
||||
}
|
||||
|
||||
@doc false
|
||||
def changeset(setting, attrs) do
|
||||
setting
|
||||
|> cast(attrs, [:key, :value, :value_type, :description])
|
||||
|> validate_required([:key, :value_type])
|
||||
|> validate_inclusion(:value_type, ["string", "integer", "uuid", "boolean", "json"])
|
||||
|> unique_constraint(:key)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Parses the value field according to value_type.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> parse_value(%ApplicationSetting{value: "abc-123", value_type: "uuid"})
|
||||
"abc-123"
|
||||
|
||||
iex> parse_value(%ApplicationSetting{value: nil, value_type: "uuid"})
|
||||
nil
|
||||
"""
|
||||
def parse_value(%__MODULE__{value: nil}), do: nil
|
||||
|
||||
def parse_value(%__MODULE__{value: value, value_type: "string"}), do: value
|
||||
def parse_value(%__MODULE__{value: value, value_type: "uuid"}), do: value
|
||||
|
||||
def parse_value(%__MODULE__{value: value, value_type: "integer"}) do
|
||||
case Integer.parse(value) do
|
||||
{int, _} -> int
|
||||
:error -> nil
|
||||
end
|
||||
end
|
||||
|
||||
def parse_value(%__MODULE__{value: value, value_type: "boolean"}) do
|
||||
value in ["true", "1", "yes"]
|
||||
end
|
||||
|
||||
def parse_value(%__MODULE__{value: value, value_type: "json"}) do
|
||||
case Jason.decode(value) do
|
||||
{:ok, decoded} -> decoded
|
||||
{:error, _} -> nil
|
||||
end
|
||||
end
|
||||
end
|
||||
173
lib/towerops/workers/agent_latency_evaluator.ex
Normal file
173
lib/towerops/workers/agent_latency_evaluator.ex
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
defmodule Towerops.Workers.AgentLatencyEvaluator do
|
||||
@moduledoc """
|
||||
GenServer that periodically evaluates device-to-agent latency and performs automatic reassignments.
|
||||
|
||||
Runs every 5 minutes to:
|
||||
1. Find devices where an alternative agent has significantly better latency (20%+ improvement)
|
||||
2. Reassign devices based on their assignment source (site, organization, global, or device-level)
|
||||
3. Broadcast PubSub events for UI updates
|
||||
4. Log all reassignment decisions with improvement metrics
|
||||
|
||||
Only considers devices with automatic assignments (site, organization, global, or none).
|
||||
Manual device-level assignments are never automatically changed.
|
||||
"""
|
||||
use GenServer
|
||||
|
||||
alias Towerops.Agents
|
||||
alias Towerops.Agents.Stats
|
||||
alias Towerops.Devices.Device
|
||||
alias Towerops.Organizations
|
||||
alias Towerops.Repo
|
||||
alias Towerops.Sites
|
||||
|
||||
require Logger
|
||||
|
||||
# Run latency evaluation every 5 minutes
|
||||
@check_interval to_timeout(minute: 5)
|
||||
|
||||
# Minimum improvement required for reassignment (20%)
|
||||
@min_improvement_percent 20
|
||||
|
||||
# Minimum checks per agent for statistical validity
|
||||
@min_checks_per_agent 10
|
||||
|
||||
def start_link(opts \\ []) do
|
||||
GenServer.start_link(__MODULE__, opts, name: __MODULE__)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Manually trigger a latency evaluation.
|
||||
|
||||
Useful for testing and manual triggering from admin UI.
|
||||
"""
|
||||
def trigger_evaluation do
|
||||
GenServer.cast(__MODULE__, :evaluate_latency)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def init(_opts) do
|
||||
# Schedule first evaluation shortly after startup (30 seconds)
|
||||
schedule_evaluation(30_000)
|
||||
{:ok, %{last_evaluation_at: nil, total_reassignments: 0}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast(:evaluate_latency, state) do
|
||||
new_state = evaluate_and_reassign(state)
|
||||
{:noreply, new_state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:evaluate_latency, state) do
|
||||
new_state = evaluate_and_reassign(state)
|
||||
schedule_evaluation(@check_interval)
|
||||
{:noreply, new_state}
|
||||
end
|
||||
|
||||
defp evaluate_and_reassign(state) do
|
||||
start_time = System.monotonic_time(:millisecond)
|
||||
|
||||
candidates =
|
||||
Stats.find_reassignment_candidates(
|
||||
min_improvement_percent: @min_improvement_percent,
|
||||
min_checks_per_agent: @min_checks_per_agent
|
||||
)
|
||||
|
||||
reassignment_count =
|
||||
candidates
|
||||
|> Enum.map(&reassign_device/1)
|
||||
|> Enum.count(&(&1 == :ok))
|
||||
|
||||
duration_ms = System.monotonic_time(:millisecond) - start_time
|
||||
|
||||
Logger.info(
|
||||
"Latency evaluation completed: #{length(candidates)} candidate(s) found, " <>
|
||||
"#{reassignment_count} device(s) reassigned in #{duration_ms}ms"
|
||||
)
|
||||
|
||||
%{
|
||||
state
|
||||
| last_evaluation_at: DateTime.utc_now(),
|
||||
total_reassignments: state.total_reassignments + reassignment_count
|
||||
}
|
||||
end
|
||||
|
||||
defp reassign_device(candidate) do
|
||||
Logger.info(
|
||||
"Reassigning device '#{candidate.device_name}' " <>
|
||||
"(#{candidate.improvement_percent}% improvement: " <>
|
||||
"#{candidate.current_latency_ms}ms → #{candidate.best_latency_ms}ms)",
|
||||
device_id: candidate.device_id,
|
||||
device_name: candidate.device_name,
|
||||
old_agent: candidate.current_agent_token_id,
|
||||
new_agent: candidate.best_agent_token_id,
|
||||
improvement_percent: candidate.improvement_percent,
|
||||
assignment_source: candidate.assignment_source
|
||||
)
|
||||
|
||||
result =
|
||||
case candidate.assignment_source do
|
||||
:site ->
|
||||
reassign_at_site_level(candidate)
|
||||
|
||||
:organization ->
|
||||
reassign_at_organization_level(candidate)
|
||||
|
||||
source when source in [:global, :none] ->
|
||||
reassign_at_device_level(candidate)
|
||||
end
|
||||
|
||||
case result do
|
||||
{:ok, _} ->
|
||||
broadcast_reassignment(candidate)
|
||||
:ok
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.error(
|
||||
"Failed to reassign device '#{candidate.device_name}': #{inspect(reason)}",
|
||||
device_id: candidate.device_id,
|
||||
error: reason
|
||||
)
|
||||
|
||||
:error
|
||||
end
|
||||
end
|
||||
|
||||
defp reassign_at_site_level(candidate) do
|
||||
device = Device |> Repo.get!(candidate.device_id) |> Repo.preload(:site)
|
||||
|
||||
Sites.update_site(device.site, %{agent_token_id: candidate.best_agent_token_id})
|
||||
end
|
||||
|
||||
defp reassign_at_organization_level(candidate) do
|
||||
device =
|
||||
Device
|
||||
|> Repo.get!(candidate.device_id)
|
||||
|> Repo.preload(site: :organization)
|
||||
|
||||
organization = device.site.organization
|
||||
|
||||
Organizations.update_organization(organization, %{
|
||||
default_agent_token_id: candidate.best_agent_token_id
|
||||
})
|
||||
end
|
||||
|
||||
defp reassign_at_device_level(candidate) do
|
||||
Agents.update_device_assignment(
|
||||
candidate.device_id,
|
||||
candidate.best_agent_token_id
|
||||
)
|
||||
end
|
||||
|
||||
defp broadcast_reassignment(candidate) do
|
||||
Phoenix.PubSub.broadcast(
|
||||
Towerops.PubSub,
|
||||
"devices:latency",
|
||||
{:device_reassigned, candidate}
|
||||
)
|
||||
end
|
||||
|
||||
defp schedule_evaluation(delay) do
|
||||
Process.send_after(self(), :evaluate_latency, delay)
|
||||
end
|
||||
end
|
||||
|
|
@ -136,7 +136,14 @@ defmodule ToweropsWeb.AgentChannel do
|
|||
def handle_in("monitoring_check", %{"binary" => binary_b64}, socket) do
|
||||
binary = Base.decode64!(binary_b64)
|
||||
check = MonitoringCheck.decode(binary)
|
||||
_ = process_monitoring_check(socket.assigns.organization_id, check)
|
||||
|
||||
_ =
|
||||
process_monitoring_check(
|
||||
socket.assigns.organization_id,
|
||||
socket.assigns.agent_token_id,
|
||||
check
|
||||
)
|
||||
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
||||
|
|
@ -383,7 +390,7 @@ defmodule ToweropsWeb.AgentChannel do
|
|||
# since it requires complex LLDP/CDP parsing
|
||||
end
|
||||
|
||||
defp process_monitoring_check(organization_id, check) do
|
||||
defp process_monitoring_check(organization_id, agent_token_id, check) do
|
||||
with {:ok, device} <- fetch_device(check.device_id),
|
||||
:ok <- verify_device_organization(device, organization_id) do
|
||||
# Parse status from protobuf string ("success" or "failure")
|
||||
|
|
@ -396,9 +403,10 @@ defmodule ToweropsWeb.AgentChannel do
|
|||
|
||||
timestamp = DateTime.from_unix!(check.timestamp, :second)
|
||||
|
||||
# Create monitoring check record
|
||||
# Create monitoring check record with agent tracking
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent_token_id,
|
||||
status: status,
|
||||
response_time_ms: check.response_time_ms,
|
||||
checked_at: timestamp
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ defmodule ToweropsWeb.AgentLive.Index do
|
|||
|
||||
alias Towerops.Agents
|
||||
alias Towerops.Agents.Stats
|
||||
alias Towerops.Settings
|
||||
|
||||
@impl true
|
||||
def mount(_params, _session, socket) do
|
||||
|
|
@ -13,12 +14,12 @@ defmodule ToweropsWeb.AgentLive.Index do
|
|||
current_user = socket.assigns.current_scope.user
|
||||
agent_tokens = Agents.list_organization_agent_tokens(organization.id)
|
||||
|
||||
# If superadmin, also load cloud pollers
|
||||
cloud_pollers =
|
||||
# If superadmin, also load cloud pollers and global default
|
||||
{cloud_pollers, global_default_cloud_poller_id} =
|
||||
if current_user.is_superuser do
|
||||
Agents.list_cloud_pollers()
|
||||
{Agents.list_cloud_pollers(), Settings.get_global_default_cloud_poller()}
|
||||
else
|
||||
[]
|
||||
{[], nil}
|
||||
end
|
||||
|
||||
# device counts for each agent (both direct and total with inheritance)
|
||||
|
|
@ -42,6 +43,7 @@ defmodule ToweropsWeb.AgentLive.Index do
|
|||
|> assign(:page_title, "Remote Agents")
|
||||
|> assign(:agent_tokens, agent_tokens)
|
||||
|> assign(:cloud_pollers, cloud_pollers)
|
||||
|> assign(:global_default_cloud_poller_id, global_default_cloud_poller_id)
|
||||
|> assign(:device_counts, equipment_counts)
|
||||
|> assign(:agent_health_stats, agent_health_stats)
|
||||
|> assign(:assignment_breakdown, assignment_breakdown)
|
||||
|
|
@ -156,6 +158,26 @@ defmodule ToweropsWeb.AgentLive.Index do
|
|||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_event("set_global_default", %{"agent_token_id" => agent_token_id}, socket) do
|
||||
current_user = socket.assigns.current_scope.user
|
||||
|
||||
if current_user.is_superuser do
|
||||
# Handle empty string as nil
|
||||
agent_token_id = if agent_token_id == "", do: nil, else: agent_token_id
|
||||
|
||||
case Settings.set_global_default_cloud_poller(agent_token_id) do
|
||||
{:ok, _} ->
|
||||
handle_global_default_success(socket, agent_token_id)
|
||||
|
||||
{:error, _} ->
|
||||
{:noreply, put_flash(socket, :error, "Failed to update global default cloud poller")}
|
||||
end
|
||||
else
|
||||
{:noreply, put_flash(socket, :error, "Only superadmins can set the global default cloud poller")}
|
||||
end
|
||||
end
|
||||
|
||||
defp apply_action(socket, :index, _params) do
|
||||
socket
|
||||
end
|
||||
|
|
@ -234,4 +256,16 @@ defmodule ToweropsWeb.AgentLive.Index do
|
|||
do: "Cloud poller created successfully",
|
||||
else: "Agent created successfully"
|
||||
end
|
||||
|
||||
defp handle_global_default_success(socket, agent_token_id) do
|
||||
message =
|
||||
if agent_token_id,
|
||||
do: "Global default cloud poller set successfully",
|
||||
else: "Global default cloud poller cleared"
|
||||
|
||||
{:noreply,
|
||||
socket
|
||||
|> assign(:global_default_cloud_poller_id, agent_token_id)
|
||||
|> put_flash(:info, message)}
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -261,6 +261,61 @@
|
|||
</.table>
|
||||
</div>
|
||||
<% end %>
|
||||
|
||||
<%!-- Global Default Cloud Poller (Superadmin Only) --%>
|
||||
<%= if @current_scope.user.is_superuser do %>
|
||||
<div class="mt-8">
|
||||
<div class="bg-amber-50 dark:bg-amber-900/10 border border-amber-200 dark:border-amber-800/30 rounded-lg p-6">
|
||||
<h3 class="text-lg font-semibold text-amber-900 dark:text-amber-200 mb-2">
|
||||
Global Default Cloud Poller
|
||||
</h3>
|
||||
<p class="text-sm text-amber-700 dark:text-amber-300 mb-4">
|
||||
Fallback agent for organizations without a default agent configured. Devices with no assignment at any level will use this agent.
|
||||
</p>
|
||||
|
||||
<.form for={%{}} phx-change="set_global_default" class="max-w-md">
|
||||
<div class="space-y-2">
|
||||
<label class="block text-sm font-medium text-amber-900 dark:text-amber-200">
|
||||
Select Global Default
|
||||
</label>
|
||||
<select
|
||||
name="agent_token_id"
|
||||
class="block w-full rounded-md border-amber-300 dark:border-amber-700 bg-white dark:bg-gray-900 text-gray-900 dark:text-white shadow-sm focus:border-amber-500 focus:ring-amber-500 sm:text-sm"
|
||||
>
|
||||
<option value="">No global default (cloud polling only)</option>
|
||||
<%= for cloud_poller <- @cloud_pollers do %>
|
||||
<option
|
||||
value={cloud_poller.id}
|
||||
selected={cloud_poller.id == @global_default_cloud_poller_id}
|
||||
>
|
||||
{cloud_poller.name}
|
||||
<%= if cloud_poller.id == @global_default_cloud_poller_id do %>
|
||||
(Current)
|
||||
<% end %>
|
||||
</option>
|
||||
<% end %>
|
||||
</select>
|
||||
|
||||
<%= if @global_default_cloud_poller_id do %>
|
||||
<% selected_poller =
|
||||
Enum.find(@cloud_pollers, &(&1.id == @global_default_cloud_poller_id)) %>
|
||||
<%= if selected_poller do %>
|
||||
<p class="text-xs text-amber-600 dark:text-amber-400 mt-2">
|
||||
<.icon name="hero-check-circle" class="h-4 w-4 inline" /> Currently using:
|
||||
<span class="font-medium">{selected_poller.name}</span>
|
||||
</p>
|
||||
<% end %>
|
||||
<% else %>
|
||||
<p class="text-xs text-amber-600 dark:text-amber-400 mt-2">
|
||||
<.icon name="hero-exclamation-triangle" class="h-4 w-4 inline" />
|
||||
No global default configured. Devices without assignments will use cloud polling (nil).
|
||||
</p>
|
||||
<% end %>
|
||||
</div>
|
||||
</.form>
|
||||
</div>
|
||||
</div>
|
||||
<% end %>
|
||||
|
||||
<!-- Token Display Modal -->
|
||||
<%= if @show_token_modal && @new_token do %>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
defmodule Towerops.Repo.Migrations.AddAgentTokenIdToMonitoringChecks do
|
||||
use Ecto.Migration
|
||||
|
||||
def up do
|
||||
# Add agent_token_id column to track which agent performed the check
|
||||
alter table(:monitoring_checks) do
|
||||
add :agent_token_id, references(:agent_tokens, type: :binary_id, on_delete: :nilify_all)
|
||||
end
|
||||
|
||||
# Add index for efficient latency queries (agent + device + time range)
|
||||
create index(:monitoring_checks, [:agent_token_id, :device_id, :checked_at])
|
||||
|
||||
# Add index for device-specific latency analysis
|
||||
create index(:monitoring_checks, [:device_id, :agent_token_id, :status, :checked_at])
|
||||
end
|
||||
|
||||
def down do
|
||||
drop index(:monitoring_checks, [:device_id, :agent_token_id, :status, :checked_at])
|
||||
drop index(:monitoring_checks, [:agent_token_id, :device_id, :checked_at])
|
||||
|
||||
alter table(:monitoring_checks) do
|
||||
remove :agent_token_id
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
defmodule Towerops.Repo.Migrations.CreateApplicationSettings do
|
||||
use Ecto.Migration
|
||||
|
||||
def up do
|
||||
create table(:application_settings, primary_key: false) do
|
||||
add :id, :binary_id, primary_key: true
|
||||
add :key, :string, null: false
|
||||
add :value, :text
|
||||
add :value_type, :string, null: false, default: "string"
|
||||
add :description, :text
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
# Enforce unique keys
|
||||
create unique_index(:application_settings, [:key])
|
||||
|
||||
# Insert default global cloud poller setting (nil = not configured)
|
||||
execute """
|
||||
INSERT INTO application_settings (id, key, value, value_type, description, inserted_at, updated_at)
|
||||
VALUES (
|
||||
gen_random_uuid(),
|
||||
'global_default_cloud_poller_id',
|
||||
NULL,
|
||||
'uuid',
|
||||
'Default cloud poller agent used as fallback when no organization-level agent is configured',
|
||||
NOW(),
|
||||
NOW()
|
||||
)
|
||||
"""
|
||||
end
|
||||
|
||||
def down do
|
||||
drop table(:application_settings)
|
||||
end
|
||||
end
|
||||
|
|
@ -7,6 +7,7 @@ defmodule Towerops.Agents.StatsTest do
|
|||
alias Towerops.Agents
|
||||
alias Towerops.Agents.Stats
|
||||
alias Towerops.Devices.Device
|
||||
alias Towerops.Monitoring
|
||||
alias Towerops.Repo
|
||||
alias Towerops.Sites
|
||||
alias Towerops.Snmp.Device
|
||||
|
|
@ -452,4 +453,536 @@ defmodule Towerops.Agents.StatsTest do
|
|||
assert stats.last_submission
|
||||
end
|
||||
end
|
||||
|
||||
describe "get_device_latency_by_agent/2" do
|
||||
test "returns empty list when no monitoring checks exist" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
latency_by_agent = Stats.get_device_latency_by_agent(device.id)
|
||||
|
||||
assert latency_by_agent == []
|
||||
end
|
||||
|
||||
test "calculates average latency per agent for a device" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, agent1, _} = Agents.create_agent_token(org.id, "Agent 1")
|
||||
{:ok, agent2, _} = Agents.create_agent_token(org.id, "Agent 2")
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create checks from agent1 with avg 50ms
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent1.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Create checks from agent2 with avg 100ms
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent2.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
latency_by_agent = Stats.get_device_latency_by_agent(device.id)
|
||||
|
||||
assert length(latency_by_agent) == 2
|
||||
|
||||
agent1_latency = Enum.find(latency_by_agent, &(&1.agent_token_id == agent1.id))
|
||||
agent2_latency = Enum.find(latency_by_agent, &(&1.agent_token_id == agent2.id))
|
||||
|
||||
assert agent1_latency.avg_latency_ms == 50.0
|
||||
assert agent1_latency.check_count == 15
|
||||
assert agent2_latency.avg_latency_ms == 100.0
|
||||
assert agent2_latency.check_count == 15
|
||||
end
|
||||
|
||||
test "filters by successful checks only" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, agent, _} = Agents.create_agent_token(org.id, "Agent")
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create successful checks with 50ms
|
||||
for _ <- 1..10 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Create failed checks (should be ignored)
|
||||
for _ <- 1..5 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent.id,
|
||||
status: :error,
|
||||
response_time_ms: nil,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
latency_by_agent = Stats.get_device_latency_by_agent(device.id)
|
||||
|
||||
assert length(latency_by_agent) == 1
|
||||
agent_latency = hd(latency_by_agent)
|
||||
assert agent_latency.avg_latency_ms == 50.0
|
||||
assert agent_latency.check_count == 10
|
||||
end
|
||||
|
||||
test "respects minimum check count filter" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, agent1, _} = Agents.create_agent_token(org.id, "Agent 1")
|
||||
{:ok, agent2, _} = Agents.create_agent_token(org.id, "Agent 2")
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Agent1 has 15 checks (above min)
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent1.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Agent2 has only 5 checks (below min)
|
||||
for _ <- 1..5 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent2.id,
|
||||
status: :success,
|
||||
response_time_ms: 30,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
latency_by_agent = Stats.get_device_latency_by_agent(device.id, min_checks: 10)
|
||||
|
||||
assert length(latency_by_agent) == 1
|
||||
assert hd(latency_by_agent).agent_token_id == agent1.id
|
||||
end
|
||||
|
||||
test "respects time window parameter" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, agent, _} = Agents.create_agent_token(org.id, "Agent")
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create recent checks
|
||||
for _ <- 1..10 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Create old checks (25 hours ago, outside default 24h window)
|
||||
old_time = DateTime.add(DateTime.utc_now(), -25, :hour)
|
||||
|
||||
for _ <- 1..5 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: old_time
|
||||
})
|
||||
end
|
||||
|
||||
# Default 24h window should only include recent checks
|
||||
latency_by_agent = Stats.get_device_latency_by_agent(device.id)
|
||||
|
||||
assert length(latency_by_agent) == 1
|
||||
agent_latency = hd(latency_by_agent)
|
||||
assert agent_latency.avg_latency_ms == 50.0
|
||||
assert agent_latency.check_count == 10
|
||||
|
||||
# 48h window should include both
|
||||
latency_by_agent_48h = Stats.get_device_latency_by_agent(device.id, hours_ago: 48)
|
||||
|
||||
assert length(latency_by_agent_48h) == 1
|
||||
agent_latency_48h = hd(latency_by_agent_48h)
|
||||
assert agent_latency_48h.check_count == 15
|
||||
end
|
||||
end
|
||||
|
||||
describe "get_device_assignment_with_latency/1" do
|
||||
test "returns assignment details with latency stats" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, agent1, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
||||
{:ok, agent2, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Assign to agent2
|
||||
{:ok, _} = Agents.assign_device_to_agent(agent2.id, device.id)
|
||||
|
||||
# Create checks from both agents
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent1.id,
|
||||
status: :success,
|
||||
response_time_ms: 30,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent2.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
result = Stats.get_device_assignment_with_latency(device.id)
|
||||
|
||||
assert result.device_id == device.id
|
||||
assert result.current_agent_token_id == agent2.id
|
||||
assert result.assignment_source == :device
|
||||
assert length(result.latency_stats) == 2
|
||||
|
||||
fast_agent = Enum.find(result.latency_stats, &(&1.agent_token_id == agent1.id))
|
||||
slow_agent = Enum.find(result.latency_stats, &(&1.agent_token_id == agent2.id))
|
||||
|
||||
assert fast_agent.avg_latency_ms == 30.0
|
||||
assert slow_agent.avg_latency_ms == 100.0
|
||||
end
|
||||
|
||||
test "returns nil current_agent_id when using cloud polling" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
result = Stats.get_device_assignment_with_latency(device.id)
|
||||
|
||||
assert result.device_id == device.id
|
||||
assert is_nil(result.current_agent_token_id)
|
||||
assert result.assignment_source == :none
|
||||
end
|
||||
end
|
||||
|
||||
describe "find_reassignment_candidates/1" do
|
||||
test "returns empty list when no devices have better options" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, agent, _} = Agents.create_agent_token(org.id, "Agent")
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
{:ok, _} = Agents.assign_device_to_agent(agent.id, device.id)
|
||||
|
||||
# Create checks showing this is the only/best agent
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
candidates = Stats.find_reassignment_candidates()
|
||||
|
||||
assert candidates == []
|
||||
end
|
||||
|
||||
test "identifies devices with 20%+ improvement available" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
||||
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
||||
|
||||
{:ok, site} =
|
||||
Sites.create_site(%{
|
||||
name: "Test Site",
|
||||
organization_id: org.id,
|
||||
agent_token_id: slow_agent.id
|
||||
})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create checks: slow agent = 100ms, fast agent = 50ms (50% improvement)
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: slow_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: fast_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
candidates = Stats.find_reassignment_candidates(min_improvement_percent: 20)
|
||||
|
||||
assert length(candidates) == 1
|
||||
candidate = hd(candidates)
|
||||
|
||||
assert candidate.device_id == device.id
|
||||
assert candidate.current_agent_token_id == slow_agent.id
|
||||
assert candidate.best_agent_token_id == fast_agent.id
|
||||
assert candidate.current_latency_ms == 100.0
|
||||
assert candidate.best_latency_ms == 50.0
|
||||
assert candidate.improvement_percent == 50.0
|
||||
end
|
||||
|
||||
test "only considers automatic assignments" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
||||
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Direct device assignment (should be excluded from automatic reassignment)
|
||||
{:ok, _} = Agents.assign_device_to_agent(slow_agent.id, device.id)
|
||||
|
||||
# Create checks showing fast agent is better
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: slow_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: fast_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Should not reassign direct assignments
|
||||
candidates = Stats.find_reassignment_candidates()
|
||||
|
||||
assert candidates == []
|
||||
end
|
||||
|
||||
test "includes site-level assignments for automatic reassignment" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
||||
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
||||
|
||||
{:ok, site} =
|
||||
Sites.create_site(%{
|
||||
name: "Test Site",
|
||||
organization_id: org.id,
|
||||
agent_token_id: slow_agent.id
|
||||
})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create checks showing fast agent is better
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: slow_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: fast_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
candidates = Stats.find_reassignment_candidates()
|
||||
|
||||
assert length(candidates) == 1
|
||||
candidate = hd(candidates)
|
||||
assert candidate.assignment_source == :site
|
||||
assert candidate.best_agent_token_id == fast_agent.id
|
||||
end
|
||||
|
||||
test "requires minimum check count" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
||||
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
||||
|
||||
{:ok, site} =
|
||||
Sites.create_site(%{
|
||||
name: "Test Site",
|
||||
organization_id: org.id,
|
||||
agent_token_id: slow_agent.id
|
||||
})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create only 5 checks (below min of 10)
|
||||
for _ <- 1..5 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: slow_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: fast_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Should not reassign with insufficient data
|
||||
candidates = Stats.find_reassignment_candidates(min_checks_per_agent: 10)
|
||||
|
||||
assert candidates == []
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -691,6 +691,60 @@ defmodule Towerops.AgentsTest do
|
|||
|
||||
assert Agents.get_effective_agent_token(device) == nil
|
||||
end
|
||||
|
||||
test "returns global default cloud poller when no other assignment exists" do
|
||||
# Create a cloud poller (application-wide agent)
|
||||
{:ok, cloud_poller, _} = Agents.create_cloud_poller("Global Cloud Poller")
|
||||
|
||||
# Set it as the global default
|
||||
{:ok, _} = Towerops.Settings.set_global_default_cloud_poller(cloud_poller.id)
|
||||
|
||||
# Create a fresh organization and device with no assignments
|
||||
user = user_fixture()
|
||||
{:ok, org} = Towerops.Organizations.create_organization(%{name: "Test Org 2"}, user.id)
|
||||
|
||||
{:ok, site} =
|
||||
Towerops.Sites.create_site(%{
|
||||
name: "Test Site 2",
|
||||
organization_id: org.id
|
||||
})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Test Equipment 2",
|
||||
ip_address: "192.168.2.1",
|
||||
site_id: site.id
|
||||
})
|
||||
|
||||
device = Repo.preload(device, site: [organization: :default_agent_token])
|
||||
|
||||
# Should use global default cloud poller
|
||||
assert Agents.get_effective_agent_token(device) == cloud_poller.id
|
||||
|
||||
# Clean up
|
||||
{:ok, _} = Towerops.Settings.set_global_default_cloud_poller(nil)
|
||||
end
|
||||
|
||||
test "organization default takes precedence over global default", %{
|
||||
organization: org,
|
||||
agent1: agent1,
|
||||
device: device
|
||||
} do
|
||||
# Create a cloud poller as global default
|
||||
{:ok, cloud_poller, _} = Agents.create_cloud_poller("Global Cloud Poller")
|
||||
{:ok, _} = Towerops.Settings.set_global_default_cloud_poller(cloud_poller.id)
|
||||
|
||||
# Set organization default
|
||||
{:ok, _org} = Towerops.Organizations.update_organization(org, %{default_agent_token_id: agent1.id})
|
||||
|
||||
device = Repo.preload(device, site: [organization: :default_agent_token])
|
||||
|
||||
# Should use organization default, not global default
|
||||
assert Agents.get_effective_agent_token(device) == agent1.id
|
||||
|
||||
# Clean up
|
||||
{:ok, _} = Towerops.Settings.set_global_default_cloud_poller(nil)
|
||||
end
|
||||
end
|
||||
|
||||
describe "get_effective_agent_token_with_source/1" do
|
||||
|
|
@ -764,6 +818,37 @@ defmodule Towerops.AgentsTest do
|
|||
|
||||
assert {nil, :none} = Agents.get_effective_agent_token_with_source(device)
|
||||
end
|
||||
|
||||
test "returns global source for global default cloud poller" do
|
||||
# Create a cloud poller and set as global default
|
||||
{:ok, cloud_poller, _} = Agents.create_cloud_poller("Global Cloud Poller")
|
||||
{:ok, _} = Towerops.Settings.set_global_default_cloud_poller(cloud_poller.id)
|
||||
|
||||
# Create a fresh organization and device with no assignments
|
||||
user = user_fixture()
|
||||
{:ok, org} = Towerops.Organizations.create_organization(%{name: "Test Org 3"}, user.id)
|
||||
|
||||
{:ok, site} =
|
||||
Towerops.Sites.create_site(%{
|
||||
name: "Test Site 3",
|
||||
organization_id: org.id
|
||||
})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Test Equipment 3",
|
||||
ip_address: "192.168.3.1",
|
||||
site_id: site.id
|
||||
})
|
||||
|
||||
device = Repo.preload(device, site: [organization: :default_agent_token])
|
||||
|
||||
assert {agent_id, :global} = Agents.get_effective_agent_token_with_source(device)
|
||||
assert agent_id == cloud_poller.id
|
||||
|
||||
# Clean up
|
||||
{:ok, _} = Towerops.Settings.set_global_default_cloud_poller(nil)
|
||||
end
|
||||
end
|
||||
|
||||
describe "count_assigned_devices/1" do
|
||||
|
|
|
|||
152
test/towerops/settings_test.exs
Normal file
152
test/towerops/settings_test.exs
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
defmodule Towerops.SettingsTest do
|
||||
use Towerops.DataCase
|
||||
|
||||
alias Towerops.Settings
|
||||
alias Towerops.Settings.ApplicationSetting
|
||||
|
||||
describe "get_setting/1" do
|
||||
test "returns parsed value for existing setting" do
|
||||
# The migration already created the global_default_cloud_poller_id setting
|
||||
assert Settings.get_setting("global_default_cloud_poller_id") == nil
|
||||
end
|
||||
|
||||
test "returns nil for non-existent setting" do
|
||||
assert Settings.get_setting("nonexistent_key") == nil
|
||||
end
|
||||
|
||||
test "parses string values correctly" do
|
||||
{:ok, _setting} = create_setting("test_string", "hello", "string")
|
||||
assert Settings.get_setting("test_string") == "hello"
|
||||
end
|
||||
|
||||
test "parses integer values correctly" do
|
||||
{:ok, _setting} = create_setting("test_integer", "42", "integer")
|
||||
assert Settings.get_setting("test_integer") == 42
|
||||
end
|
||||
|
||||
test "parses boolean values correctly" do
|
||||
{:ok, _setting} = create_setting("test_bool_true", "true", "boolean")
|
||||
{:ok, _setting} = create_setting("test_bool_false", "false", "boolean")
|
||||
{:ok, _setting} = create_setting("test_bool_one", "1", "boolean")
|
||||
{:ok, _setting} = create_setting("test_bool_zero", "0", "boolean")
|
||||
|
||||
assert Settings.get_setting("test_bool_true") == true
|
||||
assert Settings.get_setting("test_bool_false") == false
|
||||
assert Settings.get_setting("test_bool_one") == true
|
||||
assert Settings.get_setting("test_bool_zero") == false
|
||||
end
|
||||
|
||||
test "parses json values correctly" do
|
||||
{:ok, _setting} = create_setting("test_json", ~s({"key": "value"}), "json")
|
||||
assert Settings.get_setting("test_json") == %{"key" => "value"}
|
||||
end
|
||||
|
||||
test "returns nil for invalid integer" do
|
||||
{:ok, _setting} = create_setting("test_invalid_int", "not_a_number", "integer")
|
||||
assert Settings.get_setting("test_invalid_int") == nil
|
||||
end
|
||||
|
||||
test "returns nil for invalid json" do
|
||||
{:ok, _setting} = create_setting("test_invalid_json", "not json", "json")
|
||||
assert Settings.get_setting("test_invalid_json") == nil
|
||||
end
|
||||
end
|
||||
|
||||
describe "get_setting_record/1" do
|
||||
test "returns the raw ApplicationSetting record" do
|
||||
record = Settings.get_setting_record("global_default_cloud_poller_id")
|
||||
assert %ApplicationSetting{} = record
|
||||
assert record.key == "global_default_cloud_poller_id"
|
||||
assert record.value_type == "uuid"
|
||||
end
|
||||
|
||||
test "returns nil for non-existent setting" do
|
||||
assert Settings.get_setting_record("nonexistent") == nil
|
||||
end
|
||||
end
|
||||
|
||||
describe "update_setting/2" do
|
||||
test "updates an existing setting value" do
|
||||
agent_token_id = Ecto.UUID.generate()
|
||||
{:ok, updated} = Settings.update_setting("global_default_cloud_poller_id", agent_token_id)
|
||||
|
||||
assert updated.value == agent_token_id
|
||||
assert Settings.get_setting("global_default_cloud_poller_id") == agent_token_id
|
||||
end
|
||||
|
||||
test "updates setting to nil" do
|
||||
agent_token_id = Ecto.UUID.generate()
|
||||
{:ok, _} = Settings.update_setting("global_default_cloud_poller_id", agent_token_id)
|
||||
{:ok, updated} = Settings.update_setting("global_default_cloud_poller_id", nil)
|
||||
|
||||
assert updated.value == nil
|
||||
assert Settings.get_setting("global_default_cloud_poller_id") == nil
|
||||
end
|
||||
|
||||
test "returns error for non-existent setting" do
|
||||
assert {:error, :setting_not_found} =
|
||||
Settings.update_setting("nonexistent_key", "value")
|
||||
end
|
||||
|
||||
test "converts non-string values to strings" do
|
||||
{:ok, _setting} = create_setting("test_number", "0", "integer")
|
||||
{:ok, updated} = Settings.update_setting("test_number", 123)
|
||||
|
||||
assert updated.value == "123"
|
||||
assert Settings.get_setting("test_number") == 123
|
||||
end
|
||||
end
|
||||
|
||||
describe "get_global_default_cloud_poller/0" do
|
||||
test "returns nil when not configured" do
|
||||
assert Settings.get_global_default_cloud_poller() == nil
|
||||
end
|
||||
|
||||
test "returns agent token ID when configured" do
|
||||
agent_token_id = Ecto.UUID.generate()
|
||||
{:ok, _} = Settings.set_global_default_cloud_poller(agent_token_id)
|
||||
|
||||
assert Settings.get_global_default_cloud_poller() == agent_token_id
|
||||
end
|
||||
end
|
||||
|
||||
describe "set_global_default_cloud_poller/1" do
|
||||
test "sets the global default cloud poller" do
|
||||
agent_token_id = Ecto.UUID.generate()
|
||||
{:ok, setting} = Settings.set_global_default_cloud_poller(agent_token_id)
|
||||
|
||||
assert setting.value == agent_token_id
|
||||
assert Settings.get_global_default_cloud_poller() == agent_token_id
|
||||
end
|
||||
|
||||
test "clears the global default cloud poller when set to nil" do
|
||||
agent_token_id = Ecto.UUID.generate()
|
||||
{:ok, _} = Settings.set_global_default_cloud_poller(agent_token_id)
|
||||
{:ok, setting} = Settings.set_global_default_cloud_poller(nil)
|
||||
|
||||
assert setting.value == nil
|
||||
assert Settings.get_global_default_cloud_poller() == nil
|
||||
end
|
||||
end
|
||||
|
||||
describe "list_all_settings/0" do
|
||||
test "returns all application settings" do
|
||||
settings = Settings.list_all_settings()
|
||||
|
||||
assert is_list(settings)
|
||||
refute Enum.empty?(settings)
|
||||
assert Enum.any?(settings, &(&1.key == "global_default_cloud_poller_id"))
|
||||
end
|
||||
end
|
||||
|
||||
# Helper function to create test settings
|
||||
defp create_setting(key, value, value_type) do
|
||||
%ApplicationSetting{}
|
||||
|> ApplicationSetting.changeset(%{
|
||||
key: key,
|
||||
value: value,
|
||||
value_type: value_type
|
||||
})
|
||||
|> Repo.insert()
|
||||
end
|
||||
end
|
||||
517
test/towerops/workers/agent_latency_evaluator_test.exs
Normal file
517
test/towerops/workers/agent_latency_evaluator_test.exs
Normal file
|
|
@ -0,0 +1,517 @@
|
|||
defmodule Towerops.Workers.AgentLatencyEvaluatorTest do
|
||||
use Towerops.DataCase, async: false
|
||||
|
||||
import Towerops.AccountsFixtures
|
||||
import Towerops.OrganizationsFixtures
|
||||
|
||||
alias Towerops.Agents
|
||||
alias Towerops.Monitoring
|
||||
alias Towerops.Sites
|
||||
alias Towerops.Workers.AgentLatencyEvaluator
|
||||
|
||||
setup do
|
||||
# Start the worker for testing
|
||||
{:ok, pid} = start_supervised(AgentLatencyEvaluator)
|
||||
%{worker_pid: pid}
|
||||
end
|
||||
|
||||
describe "evaluate_latency/0" do
|
||||
test "reassigns device to faster agent when 20%+ improvement available" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
||||
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
||||
|
||||
{:ok, site} =
|
||||
Sites.create_site(%{
|
||||
name: "Test Site",
|
||||
organization_id: org.id,
|
||||
agent_token_id: slow_agent.id
|
||||
})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create checks: slow = 100ms, fast = 50ms (50% improvement)
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: slow_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: fast_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Trigger evaluation
|
||||
AgentLatencyEvaluator.trigger_evaluation()
|
||||
|
||||
# Wait for async processing
|
||||
Process.sleep(100)
|
||||
|
||||
# Verify site was reassigned
|
||||
updated_site = Sites.get_site!(site.id)
|
||||
assert updated_site.agent_token_id == fast_agent.id
|
||||
end
|
||||
|
||||
test "does not reassign when improvement is below 20% threshold" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, agent1, _} = Agents.create_agent_token(org.id, "Agent 1")
|
||||
{:ok, agent2, _} = Agents.create_agent_token(org.id, "Agent 2")
|
||||
|
||||
{:ok, site} =
|
||||
Sites.create_site(%{
|
||||
name: "Test Site",
|
||||
organization_id: org.id,
|
||||
agent_token_id: agent1.id
|
||||
})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create checks: agent1 = 100ms, agent2 = 85ms (15% improvement, below 20%)
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent1.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: agent2.id,
|
||||
status: :success,
|
||||
response_time_ms: 85,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Trigger evaluation
|
||||
AgentLatencyEvaluator.trigger_evaluation()
|
||||
|
||||
# Wait for async processing
|
||||
Process.sleep(100)
|
||||
|
||||
# Verify site was NOT reassigned
|
||||
updated_site = Sites.get_site!(site.id)
|
||||
assert updated_site.agent_token_id == agent1.id
|
||||
end
|
||||
|
||||
test "reassigns organization-level assignment" do
|
||||
user = user_fixture()
|
||||
|
||||
{:ok, slow_agent, _} = Agents.create_cloud_poller("Slow Cloud")
|
||||
{:ok, fast_agent, _} = Agents.create_cloud_poller("Fast Cloud")
|
||||
|
||||
org =
|
||||
organization_fixture(user.id, %{
|
||||
default_agent_token_id: slow_agent.id
|
||||
})
|
||||
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create checks showing fast agent is better
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: slow_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 150,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: fast_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Trigger evaluation
|
||||
AgentLatencyEvaluator.trigger_evaluation()
|
||||
|
||||
# Wait for async processing
|
||||
Process.sleep(100)
|
||||
|
||||
# Verify organization was reassigned
|
||||
updated_org = Towerops.Organizations.get_organization!(org.id)
|
||||
assert updated_org.default_agent_token_id == fast_agent.id
|
||||
end
|
||||
|
||||
test "creates device assignment for global default assignment" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
|
||||
{:ok, slow_cloud, _} = Agents.create_cloud_poller("Slow Global")
|
||||
{:ok, fast_cloud, _} = Agents.create_cloud_poller("Fast Global")
|
||||
|
||||
# Set slow cloud as global default
|
||||
{:ok, _} = Towerops.Settings.set_global_default_cloud_poller(slow_cloud.id)
|
||||
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create checks showing fast cloud is better
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: slow_cloud.id,
|
||||
status: :success,
|
||||
response_time_ms: 200,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: fast_cloud.id,
|
||||
status: :success,
|
||||
response_time_ms: 60,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Trigger evaluation
|
||||
AgentLatencyEvaluator.trigger_evaluation()
|
||||
|
||||
# Wait for async processing
|
||||
Process.sleep(100)
|
||||
|
||||
# Verify device assignment was created
|
||||
assignment = Agents.get_device_assignment(device.id)
|
||||
assert assignment.agent_token_id == fast_cloud.id
|
||||
|
||||
# Cleanup global default
|
||||
{:ok, _} = Towerops.Settings.set_global_default_cloud_poller(nil)
|
||||
end
|
||||
|
||||
test "creates device assignment for cloud polling (no assignment)" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
|
||||
{:ok, slow_cloud, _} = Agents.create_cloud_poller("Slow Cloud")
|
||||
{:ok, fast_cloud, _} = Agents.create_cloud_poller("Fast Cloud")
|
||||
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create checks from both cloud agents
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: slow_cloud.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: fast_cloud.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Trigger evaluation
|
||||
AgentLatencyEvaluator.trigger_evaluation()
|
||||
|
||||
# Wait for async processing
|
||||
Process.sleep(500)
|
||||
|
||||
# Verify device assignment was created to the fast agent
|
||||
assignment = Agents.get_device_assignment(device.id)
|
||||
assert assignment != nil, "Expected device assignment to be created"
|
||||
assert assignment.agent_token_id == fast_cloud.id
|
||||
end
|
||||
|
||||
test "does not reassign direct device assignments" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
||||
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
||||
|
||||
{:ok, site} = Sites.create_site(%{name: "Test Site", organization_id: org.id})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create direct device assignment
|
||||
{:ok, _} = Agents.assign_device_to_agent(slow_agent.id, device.id)
|
||||
|
||||
# Create checks showing fast agent is better
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: slow_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: fast_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Trigger evaluation
|
||||
AgentLatencyEvaluator.trigger_evaluation()
|
||||
|
||||
# Wait for async processing
|
||||
Process.sleep(100)
|
||||
|
||||
# Verify device assignment was NOT changed
|
||||
assignment = Agents.get_device_assignment(device.id)
|
||||
assert assignment.agent_token_id == slow_agent.id
|
||||
end
|
||||
|
||||
test "requires minimum check count" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
||||
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
||||
|
||||
{:ok, site} =
|
||||
Sites.create_site(%{
|
||||
name: "Test Site",
|
||||
organization_id: org.id,
|
||||
agent_token_id: slow_agent.id
|
||||
})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create only 5 checks (below default minimum of 10)
|
||||
for _ <- 1..5 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: slow_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: fast_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Trigger evaluation
|
||||
AgentLatencyEvaluator.trigger_evaluation()
|
||||
|
||||
# Wait for async processing
|
||||
Process.sleep(100)
|
||||
|
||||
# Verify site was NOT reassigned (insufficient data)
|
||||
updated_site = Sites.get_site!(site.id)
|
||||
assert updated_site.agent_token_id == slow_agent.id
|
||||
end
|
||||
|
||||
test "handles multiple devices in single evaluation" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
||||
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
||||
|
||||
{:ok, site1} =
|
||||
Sites.create_site(%{
|
||||
name: "Site 1",
|
||||
organization_id: org.id,
|
||||
agent_token_id: slow_agent.id
|
||||
})
|
||||
|
||||
{:ok, site2} =
|
||||
Sites.create_site(%{
|
||||
name: "Site 2",
|
||||
organization_id: org.id,
|
||||
agent_token_id: slow_agent.id
|
||||
})
|
||||
|
||||
{:ok, device1} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router 1",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site1.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
{:ok, device2} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router 2",
|
||||
ip_address: "192.168.2.1",
|
||||
site_id: site2.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create checks for both devices
|
||||
for device_id <- [device1.id, device2.id] do
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device_id,
|
||||
agent_token_id: slow_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device_id,
|
||||
agent_token_id: fast_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
# Trigger evaluation
|
||||
AgentLatencyEvaluator.trigger_evaluation()
|
||||
|
||||
# Wait for async processing
|
||||
Process.sleep(100)
|
||||
|
||||
# Verify both sites were reassigned
|
||||
updated_site1 = Sites.get_site!(site1.id)
|
||||
updated_site2 = Sites.get_site!(site2.id)
|
||||
|
||||
assert updated_site1.agent_token_id == fast_agent.id
|
||||
assert updated_site2.agent_token_id == fast_agent.id
|
||||
end
|
||||
end
|
||||
|
||||
describe "handle_info(:evaluate_latency)" do
|
||||
test "worker processes evaluation on schedule" do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
{:ok, slow_agent, _} = Agents.create_agent_token(org.id, "Slow Agent")
|
||||
{:ok, fast_agent, _} = Agents.create_agent_token(org.id, "Fast Agent")
|
||||
|
||||
{:ok, site} =
|
||||
Sites.create_site(%{
|
||||
name: "Test Site",
|
||||
organization_id: org.id,
|
||||
agent_token_id: slow_agent.id
|
||||
})
|
||||
|
||||
{:ok, device} =
|
||||
Towerops.Devices.create_device(%{
|
||||
name: "Router",
|
||||
ip_address: "192.168.1.1",
|
||||
site_id: site.id,
|
||||
snmp_enabled: true,
|
||||
snmp_community: "public",
|
||||
snmp_version: "2c"
|
||||
})
|
||||
|
||||
# Create checks
|
||||
for _ <- 1..15 do
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: slow_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 100,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
|
||||
Monitoring.create_check(%{
|
||||
device_id: device.id,
|
||||
agent_token_id: fast_agent.id,
|
||||
status: :success,
|
||||
response_time_ms: 50,
|
||||
checked_at: DateTime.utc_now()
|
||||
})
|
||||
end
|
||||
|
||||
# Send the message directly to the worker
|
||||
send(Process.whereis(AgentLatencyEvaluator), :evaluate_latency)
|
||||
|
||||
# Wait for async processing
|
||||
Process.sleep(100)
|
||||
|
||||
# Verify site was reassigned
|
||||
updated_site = Sites.get_site!(site.id)
|
||||
assert updated_site.agent_token_id == fast_agent.id
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue