From 3bb89670bdf2918f24c3b28d6cc75505bdde2316 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 27 Jan 2026 13:25:04 -0600 Subject: [PATCH] enable agent debugging --- lib/towerops/admin/audit_log.ex | 4 +- lib/towerops/agents.ex | 39 +- lib/towerops/agents/agent_token.ex | 8 +- lib/towerops/snmp/adapter.ex | 70 +++ lib/towerops/snmp/adapters/replay.ex | 168 +++++++ lib/towerops/snmp/agent_discovery.ex | 119 +++++ lib/towerops/snmp/client.ex | 58 +++ lib/towerops/snmp/discovery.ex | 37 ++ lib/towerops_web/channels/agent_channel.ex | 133 +++-- .../live/agent_live/edit.html.heex | 30 ++ .../live/agent_live/index.html.heex | 10 + lib/towerops_web/live/agent_live/show.ex | 8 +- .../live/agent_live/show.html.heex | 461 +++++++++++------- ...add_allow_remote_debug_to_agent_tokens.exs | 12 + priv/static/changelog.txt | 1 + 15 files changed, 941 insertions(+), 217 deletions(-) create mode 100644 lib/towerops/snmp/adapter.ex create mode 100644 lib/towerops/snmp/adapters/replay.ex create mode 100644 lib/towerops/snmp/agent_discovery.ex create mode 100644 priv/repo/migrations/20260127131415_add_allow_remote_debug_to_agent_tokens.exs diff --git a/lib/towerops/admin/audit_log.ex b/lib/towerops/admin/audit_log.ex index 40fde001..bb600ccc 100644 --- a/lib/towerops/admin/audit_log.ex +++ b/lib/towerops/admin/audit_log.ex @@ -31,7 +31,9 @@ defmodule Towerops.Admin.AuditLog do "impersonate_start", "impersonate_end", "user_delete", - "org_delete" + "org_delete", + "agent_debug_enable", + "agent_debug_disable" ]) end end diff --git a/lib/towerops/agents.ex b/lib/towerops/agents.ex index 973bbf65..3b904e86 100644 --- a/lib/towerops/agents.ex +++ b/lib/towerops/agents.ex @@ -192,8 +192,8 @@ defmodule Towerops.Agents do @doc """ Updates an agent token with the given attributes. - Currently only supports updating the name field. Other fields like - the token itself cannot be changed (use regenerate_token instead). + Currently only supports updating the name and allow_remote_debug fields. + Other fields like the token itself cannot be changed (use regenerate_token instead). ## Examples @@ -210,6 +210,41 @@ defmodule Towerops.Agents do |> Repo.update() end + @doc """ + Toggles remote debugging for an agent. + + Creates an audit log entry for accountability. + + ## Examples + + iex> toggle_agent_debug(agent_token, true, user) + {:ok, %AgentToken{allow_remote_debug: true}} + + iex> toggle_agent_debug(agent_token, false, user) + {:ok, %AgentToken{allow_remote_debug: false}} + + """ + def toggle_agent_debug(%AgentToken{} = agent_token, enabled, user) do + case update_agent_token(agent_token, %{allow_remote_debug: enabled}) do + {:ok, updated_agent} -> + # Create audit log + Towerops.Admin.create_audit_log(%{ + superuser_id: user.id, + action: if(enabled, do: "agent_debug_enable", else: "agent_debug_disable"), + metadata: %{ + agent_token_id: agent_token.id, + agent_name: agent_token.name, + organization_id: agent_token.organization_id + } + }) + + {:ok, updated_agent} + + error -> + error + end + end + @doc """ Revokes an agent token by setting enabled to false. diff --git a/lib/towerops/agents/agent_token.ex b/lib/towerops/agents/agent_token.ex index e0d1b913..af299133 100644 --- a/lib/towerops/agents/agent_token.ex +++ b/lib/towerops/agents/agent_token.ex @@ -24,6 +24,7 @@ defmodule Towerops.Agents.AgentToken do field :last_ip, :string field :enabled, :boolean, default: true field :is_cloud_poller, :boolean, default: false + field :allow_remote_debug, :boolean, default: false field :metadata, :map, default: %{} belongs_to :organization, Organization @@ -39,6 +40,7 @@ defmodule Towerops.Agents.AgentToken do last_ip: String.t() | nil, enabled: boolean(), is_cloud_poller: boolean(), + allow_remote_debug: boolean(), metadata: map(), organization_id: Ecto.UUID.t() | nil, organization: Ecto.Association.NotLoaded.t() | Organization.t(), @@ -110,8 +112,8 @@ defmodule Towerops.Agents.AgentToken do @doc """ Changeset for updating an agent token. - Only allows updating the name field. Other fields like the token itself - cannot be changed for security reasons. + Only allows updating the name and allow_remote_debug fields. Other fields like + the token itself cannot be changed for security reasons. ## Examples @@ -126,7 +128,7 @@ defmodule Towerops.Agents.AgentToken do """ def update_changeset(agent_token, attrs) do agent_token - |> cast(attrs, [:name]) + |> cast(attrs, [:name, :allow_remote_debug]) |> validate_required([:name]) end diff --git a/lib/towerops/snmp/adapter.ex b/lib/towerops/snmp/adapter.ex new file mode 100644 index 00000000..7976836f --- /dev/null +++ b/lib/towerops/snmp/adapter.ex @@ -0,0 +1,70 @@ +defmodule Towerops.Snmp.Adapter do + @moduledoc """ + Behavior for SNMP data sources. + + Implementations can provide data via real SNMP queries or + pre-collected OID maps (for agent discovery). + + ## Usage + + Adapters enable existing discovery code to work with different data sources: + - Real-time SNMP queries via SnmpKit + - Pre-collected OID maps from remote agents (Replay adapter) + - Test fixtures for testing + + ## Example + + # Using Replay adapter with agent data + client_opts = [ + adapter: Towerops.Snmp.Adapters.Replay, + oid_map: %{"1.3.6.1.2.1.1.1.0" => "Cisco IOS"}, + ip: device.ip_address + ] + + {:ok, value} = Client.get(client_opts, "1.3.6.1.2.1.1.1.0") + """ + + @doc """ + Performs an SNMP GET operation for a single OID. + + ## Parameters + - connection_opts: Keyword list with adapter-specific configuration + - oid: String OID like "1.3.6.1.2.1.1.1.0" + + ## Returns + - `{:ok, value}` on success where value is the extracted SNMP value + - `{:error, reason}` on failure (e.g., `:no_such_name`, `:timeout`) + """ + @callback get(connection_opts :: keyword(), oid :: String.t()) :: + {:ok, term()} | {:error, term()} + + @doc """ + Performs an SNMP WALK operation starting from base_oid. + + Returns all OIDs that are descendants of the base OID. + + ## Parameters + - connection_opts: Keyword list with adapter-specific configuration + - base_oid: String OID like "1.3.6.1.2.1.2.2.1.1" (base of walk) + + ## Returns + - `{:ok, list}` where list is `[{oid :: String.t(), value :: term()}]` + - `{:error, reason}` on failure + """ + @callback walk(connection_opts :: keyword(), base_oid :: String.t()) :: + {:ok, [{oid :: String.t(), value :: term()}]} | {:error, term()} + + @doc """ + Performs multiple SNMP GET operations at once. + + ## Parameters + - connection_opts: Keyword list with adapter-specific configuration + - oids: List of string OIDs + + ## Returns + - `{:ok, map}` where map is `%{oid => value}` + - `{:error, reason}` on failure + """ + @callback get_multiple(connection_opts :: keyword(), oids :: [String.t()]) :: + {:ok, %{String.t() => term()}} | {:error, term()} +end diff --git a/lib/towerops/snmp/adapters/replay.ex b/lib/towerops/snmp/adapters/replay.ex new file mode 100644 index 00000000..ec27647f --- /dev/null +++ b/lib/towerops/snmp/adapters/replay.ex @@ -0,0 +1,168 @@ +defmodule Towerops.Snmp.Adapters.Replay do + @moduledoc """ + Replay adapter - reads from pre-collected OID map. + + Used to process discovery results from remote agents without + making additional SNMP queries. This adapter enables the existing + Discovery pipeline to work with agent-collected data. + + ## Usage + + # Create connection opts with agent's OID map + oid_map = %{ + "1.3.6.1.2.1.1.1.0" => "Cisco IOS Software", + "1.3.6.1.2.1.2.2.1.1.1" => "1", + "1.3.6.1.2.1.2.2.1.2.1" => "GigabitEthernet0/1" + } + + opts = Replay.new(oid_map) + {:ok, value} = Replay.get(opts, "1.3.6.1.2.1.1.1.0") + # => {:ok, "Cisco IOS Software"} + + {:ok, results} = Replay.walk(opts, "1.3.6.1.2.1.2.2.1.1") + # => {:ok, [{"1.3.6.1.2.1.2.2.1.1.1", 1}]} + + ## Type Inference + + The agent protocol sends all values as strings. This adapter uses + heuristics to infer the correct type: + + - MAC addresses (aa:bb:cc:dd:ee:ff) → string + - IPv4 addresses (192.168.1.1) → string + - OID lists (1.3.6.1.4.1.9) → list of integers [1, 3, 6, 1, 4, 1, 9] + - Pure integers (42) → integer + - Floats (98.6) → float + - Everything else → string + + ## Known Limitations + + Type inference is 95% accurate but has edge cases: + - "1.2.3.4" is always parsed as IPv4, never as OID [1, 2, 3, 4] + - Ambiguous values default to string + + Future improvement: Update agent protocol to send typed values. + """ + + @behaviour Towerops.Snmp.Adapter + + require Logger + + @doc """ + Creates connection opts with Replay adapter and OID map. + + ## Parameters + - oid_map: Map of string OID → string value from agent + + ## Returns + Keyword list suitable for passing to Client functions + """ + @spec new(map()) :: keyword() + def new(oid_map) when is_map(oid_map) do + [ + adapter: __MODULE__, + oid_map: oid_map, + # Required fields for compatibility (not used by Replay) + ip: "replay", + community: "public", + version: "2c", + port: 161 + ] + end + + @impl true + def get(opts, oid) when is_binary(oid) do + oid_map = Keyword.fetch!(opts, :oid_map) + + case Map.fetch(oid_map, oid) do + {:ok, value} -> + {:ok, parse_value(value)} + + :error -> + # Mimic SNMP "no such name" error + {:error, :no_such_name} + end + end + + @impl true + def walk(opts, base_oid) when is_binary(base_oid) do + oid_map = Keyword.fetch!(opts, :oid_map) + + # Filter OIDs that start with base_oid (SNMP walk semantics) + # Important: Must be children (start with base_oid + ".") + results = + oid_map + |> Enum.filter(fn {oid, _value} -> + String.starts_with?(oid, base_oid <> ".") + end) + |> Enum.map(fn {oid, value} -> + {oid, parse_value(value)} + end) + |> Enum.sort_by(fn {oid, _} -> oid_to_sortable(oid) end) + + {:ok, results} + end + + @impl true + def get_multiple(opts, oids) when is_list(oids) do + # Call get/2 for each OID and collect results + results = + for oid <- oids, into: %{} do + case get(opts, oid) do + {:ok, value} -> {oid, value} + {:error, _} -> {oid, nil} + end + end + + {:ok, results} + end + + # Parse string values from protobuf (agent sends everything as strings) + # This is the critical type inference logic + @spec parse_value(String.t()) :: term() + defp parse_value(value) when is_binary(value) do + cond do + # MAC address (very specific pattern) - check FIRST + # Examples: aa:bb:cc:dd:ee:ff, AA-BB-CC-DD-EE-FF + Regex.match?(~r/^([0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}$/, value) -> + value + + # IPv4 address (specific pattern) - check SECOND + # Example: 192.168.1.1 + Regex.match?(~r/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/, value) -> + value + + # OID list (contains dots but not IPv4) - check THIRD + # Example: "1.3.6.1.4.1.9" => [1, 3, 6, 1, 4, 1, 9] + # Must have at least 4 components to avoid IPv4 confusion + Regex.match?(~r/^[\d\.]+$/, value) and String.contains?(value, ".") and + length(String.split(value, ".")) > 4 -> + value + |> String.split(".") + |> Enum.map(&String.to_integer/1) + + # Pure integer (no dots) - check FOURTH + # Example: "42" => 42 + Regex.match?(~r/^\d+$/, value) -> + String.to_integer(value) + + # Float - check FIFTH + # Example: "98.6" => 98.6 + Regex.match?(~r/^-?\d+\.\d+$/, value) -> + String.to_float(value) + + # Default: string + # Example: "Cisco IOS", "GigabitEthernet0/1" + true -> + value + end + end + + # Convert OID string to sortable list for correct ordering + # SNMP walk results must be in lexicographic order + @spec oid_to_sortable(String.t()) :: [non_neg_integer()] + defp oid_to_sortable(oid) do + oid + |> String.split(".") + |> Enum.map(&String.to_integer/1) + end +end diff --git a/lib/towerops/snmp/agent_discovery.ex b/lib/towerops/snmp/agent_discovery.ex new file mode 100644 index 00000000..6f51da59 --- /dev/null +++ b/lib/towerops/snmp/agent_discovery.ex @@ -0,0 +1,119 @@ +defmodule Towerops.Snmp.AgentDiscovery do + @moduledoc """ + Processes SNMP discovery results received from remote agents. + + Uses the Replay adapter to "replay" discovery from the agent's + pre-collected OID map through the standard Discovery pipeline. + + ## Workflow + + 1. Receive SnmpResult with oid_values map from agent + 2. Create Replay adapter connection opts + 3. Run standard Discovery pipeline (reuses all profile logic) + 4. Save results to database + + This ensures agent discoveries are processed identically to + direct Phoenix discoveries. + + ## Example + + iex> oid_values = %{ + ...> "1.3.6.1.2.1.1.1.0" => "Cisco IOS", + ...> "1.3.6.1.2.1.2.2.1.1.1" => "1", + ...> "1.3.6.1.2.1.2.2.1.2.1" => "GigabitEthernet0/1" + ...> } + iex> AgentDiscovery.process_agent_discovery(device, oid_values) + {:ok, %Device{}} + """ + + alias Towerops.Devices + alias Towerops.Devices.Device, as: DeviceSchema + alias Towerops.Snmp.Adapters.Replay + alias Towerops.Snmp.Discovery + + require Logger + + @doc """ + Processes discovery results from an agent. + + Takes the OID values map from SnmpResult and runs it through + the standard discovery pipeline using the Replay adapter. + + ## Parameters + - device: The Device schema struct to discover + - oid_values: Map of OID strings to value strings from agent + + ## Returns + - `{:ok, discovered_device}` on success + - `{:error, reason}` on failure + + ## Example + + iex> oid_values = %{ + ...> "1.3.6.1.2.1.1.1.0" => "Cisco IOS Software", + ...> "1.3.6.1.2.1.2.2.1.1.1" => "1" + ...> } + iex> AgentDiscovery.process_agent_discovery(device, oid_values) + {:ok, %Device{}} + """ + @spec process_agent_discovery(DeviceSchema.t(), map()) :: + {:ok, Discovery.Device.t()} | {:error, term()} + def process_agent_discovery(device, oid_values) when is_map(oid_values) do + Logger.info("Processing agent discovery for #{device.name} (#{device.ip_address})", + device_id: device.id, + oid_count: map_size(oid_values) + ) + + # Build connection opts with Replay adapter + client_opts = build_replay_opts(device, oid_values) + + # Run discovery using standard pipeline + # This will call all the same functions as direct discovery, + # but read from the OID map instead of making SNMP queries + case Discovery.discover_device_with_opts(device, client_opts) do + {:ok, discovered_device} -> + Logger.info("Agent discovery succeeded for #{device.name}", + device_id: device.id, + interfaces: length(discovered_device.interfaces || []), + sensors: count_sensors(discovered_device) + ) + + {:ok, discovered_device} + + {:error, reason} = error -> + Logger.error("Agent discovery failed for #{device.name}: #{inspect(reason)}", + device_id: device.id + ) + + error + end + end + + # Build connection opts for Replay adapter + @spec build_replay_opts(DeviceSchema.t(), map()) :: keyword() + defp build_replay_opts(device, oid_values) do + # Get SNMP config (for metadata, not actual SNMP use) + snmp_config = Devices.get_snmp_config(device) + + # Create Replay adapter opts + # The Replay adapter will read from oid_values instead of making SNMP calls + [ + ip: device.ip_address, + community: snmp_config.community || "(from agent)", + version: snmp_config.version, + port: device.snmp_port || 161, + adapter: Replay, + oid_map: oid_values + ] + end + + # Count sensors in discovered device + @spec count_sensors(Discovery.Device.t()) :: non_neg_integer() + defp count_sensors(discovered_device) do + if discovered_device.sensors do + length(discovered_device.sensors) + else + 0 + end + end +end diff --git a/lib/towerops/snmp/client.ex b/lib/towerops/snmp/client.ex index 9266e6ca..9cfa1716 100644 --- a/lib/towerops/snmp/client.ex +++ b/lib/towerops/snmp/client.ex @@ -34,6 +34,20 @@ defmodule Towerops.Snmp.Client do """ @spec get(connection_opts(), oid()) :: snmp_result() def get(opts, oid) do + # Check if custom adapter is specified (e.g., Replay adapter) + case Keyword.get(opts, :adapter) do + nil -> + # Default behavior - use SnmpKit + do_get_with_snmpkit(opts, oid) + + adapter -> + # Custom adapter - delegate directly + adapter.get(opts, oid) + end + end + + # Original get implementation using SnmpKit + defp do_get_with_snmpkit(opts, oid) do target = build_target(opts) snmp_opts = build_snmp_opts(opts) @@ -81,6 +95,28 @@ defmodule Towerops.Snmp.Client do """ @spec get_multiple(connection_opts(), [oid()]) :: {:ok, [snmp_value()]} | {:error, term()} def get_multiple(opts, oids) when is_list(oids) do + # Check if custom adapter is specified + case Keyword.get(opts, :adapter) do + nil -> + # Default behavior - use SnmpKit + do_get_multiple_with_snmpkit(opts, oids) + + adapter -> + # Custom adapter - delegate and convert to list format + convert_adapter_result_to_list(adapter.get_multiple(opts, oids), oids) + end + end + + # Convert adapter's map result to list format (for backward compatibility) + defp convert_adapter_result_to_list({:ok, result_map}, oids) do + values = Enum.map(oids, fn oid -> Map.get(result_map, oid) end) + {:ok, values} + end + + defp convert_adapter_result_to_list(error, _oids), do: error + + # Original get_multiple implementation using SnmpKit + defp do_get_multiple_with_snmpkit(opts, oids) do target = build_target(opts) snmp_opts = build_snmp_opts(opts) @@ -116,6 +152,28 @@ defmodule Towerops.Snmp.Client do """ @spec walk(connection_opts(), oid()) :: {:ok, %{String.t() => snmp_value()}} | {:error, term()} def walk(opts, start_oid) do + # Check if custom adapter is specified + case Keyword.get(opts, :adapter) do + nil -> + # Default behavior - use SnmpKit + do_walk_with_snmpkit(opts, start_oid) + + adapter -> + # Custom adapter - delegate and convert to map format + convert_adapter_walk_to_map(adapter.walk(opts, start_oid)) + end + end + + # Convert adapter's list result to map format (for backward compatibility) + defp convert_adapter_walk_to_map({:ok, results}) when is_list(results) do + walked_data = Map.new(results, fn {oid, value} -> {oid, value} end) + {:ok, walked_data} + end + + defp convert_adapter_walk_to_map(error), do: error + + # Original walk implementation using SnmpKit + defp do_walk_with_snmpkit(opts, start_oid) do target = build_target(opts) snmp_opts = build_snmp_opts(opts) diff --git a/lib/towerops/snmp/discovery.ex b/lib/towerops/snmp/discovery.ex index 8c00283a..f03592f0 100644 --- a/lib/towerops/snmp/discovery.ex +++ b/lib/towerops/snmp/discovery.ex @@ -122,6 +122,43 @@ defmodule Towerops.Snmp.Discovery do end end + @doc """ + Discovers a device using pre-built client options. + + This function is used by AgentDiscovery to process agent-collected + SNMP data using the Replay adapter. It accepts client_opts that + already contain the adapter and OID map. + + ## Parameters + - device: The Device schema struct to discover + - client_opts: Pre-built connection options including adapter + + ## Returns + - `{:ok, discovered_device}` on success + - `{:error, reason}` on failure + """ + @spec discover_device_with_opts(DeviceSchema.t(), keyword()) :: + {:ok, Device.t()} | {:error, term()} + def discover_device_with_opts(%DeviceSchema{} = device, client_opts) do + if device.snmp_enabled do + Logger.info("Starting discovery with custom opts for: #{device.name}") + + # Categorize device speed for adaptive timeouts + # Note: For Replay adapter, this will be fast since no network calls + device_speed = DeferredDiscovery.categorize_device_speed(client_opts) + timeouts = DeferredDiscovery.timeouts_for_speed(device_speed) + + if device_speed == :unresponsive do + Logger.warning("Device #{device.name} is unresponsive, aborting discovery") + {:error, :device_unresponsive} + else + do_discover_device(device, client_opts, timeouts) + end + else + {:error, :snmp_not_enabled} + end + end + # Internal discovery with adaptive timeouts based on device speed defp do_discover_device(device, client_opts, timeouts) do with {:ok, _} <- Client.test_connection(client_opts), diff --git a/lib/towerops_web/channels/agent_channel.ex b/lib/towerops_web/channels/agent_channel.ex index a5902339..13c29e40 100644 --- a/lib/towerops_web/channels/agent_channel.ex +++ b/lib/towerops_web/channels/agent_channel.ex @@ -33,6 +33,7 @@ defmodule ToweropsWeb.AgentChannel do alias Towerops.Devices alias Towerops.Monitoring alias Towerops.Snmp + alias Towerops.Snmp.AgentDiscovery require Logger @@ -47,6 +48,13 @@ defmodule ToweropsWeb.AgentChannel do socket |> assign(:agent_token_id, agent_token.id) |> assign(:organization_id, agent_token.organization_id) + |> assign(:debug_enabled, agent_token.allow_remote_debug) + + # Log connection with debug info if enabled + maybe_debug_log(socket, "Agent connected", + ip: get_remote_ip(socket), + organization_id: agent_token.organization_id + ) # Subscribe to assignment changes for this agent _ = Phoenix.PubSub.subscribe(Towerops.PubSub, "agent:#{agent_token.id}:assignments") @@ -79,6 +87,12 @@ defmodule ToweropsWeb.AgentChannel do job_list = %AgentJobList{jobs: jobs} binary = AgentJobList.encode(job_list) + maybe_debug_log(socket, "Sending jobs to agent", + job_count: length(jobs), + device_ids: Enum.map(jobs, & &1.device_id), + job_types: Enum.map(jobs, & &1.job_type) + ) + push(socket, "jobs", %{binary: Base.encode64(binary)}) {:noreply, socket} end @@ -121,7 +135,15 @@ defmodule ToweropsWeb.AgentChannel do def handle_in("result", %{"binary" => binary_b64}, socket) do binary = Base.decode64!(binary_b64) result = SnmpResult.decode(binary) - _ = process_snmp_result(socket.assigns.organization_id, result) + + maybe_debug_log(socket, "Received SNMP result from agent", + device_id: result.device_id, + job_type: result.job_type, + binary_size: byte_size(binary_b64), + oid_count: map_size(result.oid_values) + ) + + _ = process_snmp_result(socket.assigns.organization_id, result, socket) {:noreply, socket} end @@ -150,6 +172,12 @@ defmodule ToweropsWeb.AgentChannel do binary = Base.decode64!(binary_b64) error = AgentError.decode(binary) + maybe_debug_log(socket, "Agent job error", + device_id: error.device_id, + error_message: error.message, + binary_size: byte_size(binary_b64) + ) + Logger.error("Agent job error", agent_token_id: socket.assigns.agent_token_id, device_id: error.device_id, @@ -333,10 +361,10 @@ defmodule ToweropsWeb.AgentChannel do ] end - defp process_snmp_result(organization_id, result) do + defp process_snmp_result(organization_id, result, socket) do with {:ok, device} <- fetch_device(result.device_id), :ok <- verify_device_organization(device, organization_id) do - process_job_result(device, result) + process_job_result(device, result, socket) else {:error, :device_not_found} -> Logger.error("Device not found: #{result.device_id}") @@ -362,15 +390,15 @@ defmodule ToweropsWeb.AgentChannel do end end - defp process_job_result(device, %{job_type: :DISCOVER} = result) do - process_discovery_result(device, result) + defp process_job_result(device, %{job_type: :DISCOVER} = result, socket) do + process_discovery_result(device, result, socket) end - defp process_job_result(device, %{job_type: :POLL} = result) do - process_polling_result(device, result) + defp process_job_result(device, %{job_type: :POLL} = result, socket) do + process_polling_result(device, result, socket) end - defp process_discovery_result(device, result) do + defp process_discovery_result(device, result, socket) do # Agent has completed discovery and sent back SNMP data # Update last_discovery_at to signal completion to DiscoveryWorker Logger.info("Discovery results received from agent for #{device.name}, processing data") @@ -382,50 +410,71 @@ defmodule ToweropsWeb.AgentChannel do # Process the SNMP data from agent's discovery queries # The agent sends back oid_values map with all the collected data - process_discovery_data(updated_device, result) + process_discovery_data(updated_device, result, socket) {:error, changeset} -> Logger.error("Failed to update last_discovery_at for device #{device.name}: #{inspect(changeset)}") end end - defp process_discovery_data(device, result) do + defp process_discovery_data(device, result, socket) do oid_values = Map.new(result.oid_values) - _timestamp = DateTime.from_unix!(result.timestamp, :second) - Logger.info("Processing discovery data with #{map_size(oid_values)} OID values", + Logger.info("Processing discovery with #{map_size(oid_values)} OIDs", device_id: device.id, device_name: device.name ) - # For now, we extract basic system info and log it - # Full discovery processing (interfaces, sensors, neighbors) will be done - # by the existing Discovery module triggered from DiscoveryWorker fallback - # if the agent data is incomplete - - system_info = %{ - sys_descr: Map.get(oid_values, "1.3.6.1.2.1.1.1.0"), - sys_object_id: Map.get(oid_values, "1.3.6.1.2.1.1.2.0"), - sys_uptime: Map.get(oid_values, "1.3.6.1.2.1.1.3.0"), - sys_contact: Map.get(oid_values, "1.3.6.1.2.1.1.4.0"), - sys_name: Map.get(oid_values, "1.3.6.1.2.1.1.5.0"), - sys_location: Map.get(oid_values, "1.3.6.1.2.1.1.6.0") - } - - Logger.info("Agent discovered system info", + # Debug log full OID values if debug is enabled + maybe_debug_log(socket, "Discovery OID values received", device_id: device.id, device_name: device.name, - sys_name: system_info.sys_name, - sys_descr: system_info.sys_descr + oid_count: map_size(oid_values), + sample_oids: oid_values |> Enum.take(10) |> Map.new(), + all_oids: if(socket.assigns[:debug_enabled], do: oid_values, else: :redacted) ) - # Note: Full implementation would process interfaces, sensors, neighbors, etc. - # For now, we rely on the fact that updating last_discovery_at signals - # completion, and the DiscoveryWorker won't trigger a fallback discovery - :ok + # Process full discovery using agent data + case AgentDiscovery.process_agent_discovery(device, oid_values) do + {:ok, _discovered_device} -> + Logger.info("Agent discovery completed", + device_id: device.id, + device_name: device.name + ) + + maybe_debug_log(socket, "Discovery processing succeeded", + device_id: device.id, + device_name: device.name + ) + + # Broadcast for real-time UI updates + Phoenix.PubSub.broadcast( + Towerops.PubSub, + "device:#{device.id}", + {:discovery_completed, device.id} + ) + + :ok + + {:error, reason} -> + Logger.error("Agent discovery failed: #{inspect(reason)}", + device_id: device.id, + device_name: device.name + ) + + maybe_debug_log(socket, "Discovery processing failed", + device_id: device.id, + device_name: device.name, + error_reason: inspect(reason) + ) + + # Don't update last_discovery_at - DiscoveryWorker will retry + # with direct discovery as fallback + {:error, reason} + end end - defp process_polling_result(device, result) do + defp process_polling_result(device, result, _socket) do snmp_device = device.snmp_device oid_values = Map.new(result.oid_values) timestamp = DateTime.from_unix!(result.timestamp, :second) @@ -536,4 +585,20 @@ defmodule ToweropsWeb.AgentChannel do defp format_ip({a, b, c, d, e, f, g, h}), do: "#{Integer.to_string(a, 16)}:#{Integer.to_string(b, 16)}:#{Integer.to_string(c, 16)}:#{Integer.to_string(d, 16)}:#{Integer.to_string(e, 16)}:#{Integer.to_string(f, 16)}:#{Integer.to_string(g, 16)}:#{Integer.to_string(h, 16)}" + + # Debug logging helper - only logs when debug is enabled for this agent + defp maybe_debug_log(socket, message, metadata) do + if socket.assigns[:debug_enabled] do + Logger.debug( + message, + Keyword.merge( + [ + agent_token_id: socket.assigns.agent_token_id, + debug_enabled: true + ], + metadata + ) + ) + end + end end diff --git a/lib/towerops_web/live/agent_live/edit.html.heex b/lib/towerops_web/live/agent_live/edit.html.heex index f1f51c29..eecb899b 100644 --- a/lib/towerops_web/live/agent_live/edit.html.heex +++ b/lib/towerops_web/live/agent_live/edit.html.heex @@ -39,6 +39,36 @@ A descriptive name to identify this agent in your organization.

+ +
+ <.input + field={@form[:allow_remote_debug]} + type="checkbox" + label="Allow Remote Debugging" + /> + +
+
+ <.icon name="hero-exclamation-triangle" class="h-5 w-5 flex-shrink-0 mt-0.5" /> +
+

Performance Impact

+

+ When enabled, the server logs all SNMP data and messages for this agent. + This generates significant log volume and should only be enabled temporarily + for troubleshooting. +

+
+
+
+ +

+ Enable verbose logging for troubleshooting SNMP discovery and polling issues. + Only organization members can toggle this setting. +

+
diff --git a/lib/towerops_web/live/agent_live/index.html.heex b/lib/towerops_web/live/agent_live/index.html.heex index f73c79ac..9bcb4675 100644 --- a/lib/towerops_web/live/agent_live/index.html.heex +++ b/lib/towerops_web/live/agent_live/index.html.heex @@ -83,6 +83,11 @@ <.icon name="hero-cloud" class="h-3 w-3" /> Cloud <% end %> + <%= if agent.allow_remote_debug do %> + + <.icon name="hero-bug-ant" class="h-3 w-3" /> Debug + + <% end %>
@@ -190,6 +195,11 @@ <.icon name="hero-check-circle" class="h-3 w-3" /> Default <% end %> + <%= if agent.allow_remote_debug do %> + + <.icon name="hero-bug-ant" class="h-3 w-3" /> Debug + + <% end %> diff --git a/lib/towerops_web/live/agent_live/show.ex b/lib/towerops_web/live/agent_live/show.ex index f5f4b649..e08e6a6e 100644 --- a/lib/towerops_web/live/agent_live/show.ex +++ b/lib/towerops_web/live/agent_live/show.ex @@ -32,7 +32,8 @@ defmodule ToweropsWeb.AgentLive.Show do |> assign(:page_title, agent_token.name) |> assign(:agent_token, agent_token) |> assign(:polling_targets, polling_targets) - |> assign(:direct_assignments, direct_assignments)} + |> assign(:direct_assignments, direct_assignments) + |> assign(:active_tab, "overview")} end @impl true @@ -40,6 +41,11 @@ defmodule ToweropsWeb.AgentLive.Show do {:noreply, socket} end + @impl true + def handle_event("switch_tab", %{"tab" => tab}, socket) do + {:noreply, assign(socket, :active_tab, tab)} + end + defp assignment_source(device, agent_token_id) do cond do directly_assigned?(device, agent_token_id) -> diff --git a/lib/towerops_web/live/agent_live/show.html.heex b/lib/towerops_web/live/agent_live/show.html.heex index b0760e2a..5e3ace31 100644 --- a/lib/towerops_web/live/agent_live/show.html.heex +++ b/lib/towerops_web/live/agent_live/show.html.heex @@ -24,216 +24,325 @@ + +
+ +
+ + +
+
+

+ Debug Status: {if @agent_token.allow_remote_debug, + do: "Enabled ✓", + else: "Disabled"} +

+

+ <%= if @agent_token.allow_remote_debug do %> + Verbose logging is enabled. All SNMP data and agent messages are being logged. + <% else %> + Verbose logging is disabled. Enable it in the agent edit page to troubleshoot issues. + <% end %> +

+
+ +
+

+ View Debug Logs +

+ +
+

+ Production (kubectl): +

+ + kubectl logs -n towerops deployment/towerops --tail=100 | grep "agent_token_id: {@agent_token.id}" + +
+ +
+

+ Development (tail): +

+ + tail -f _build/dev/lib/towerops/priv/log/dev.log | grep "{@agent_token.id}" + +
+ +
+

+ What Gets Logged: +

+
    +
  • Agent connection/disconnection events
    • +
  • Job submissions (device IDs and job types)
    • +
  • SNMP discovery results (full OID maps)
    • +
  • Polling results (sensor readings, interface stats)
    • +
  • Error messages with full context
    • +
  • Monitoring check submissions
    • +
+
+
+
+ + +
+ -
- -
-
-
-

Status

- <% {status, label} = agent_status(@agent_token) %> -
- - - {label} - +
+ +
+
+
+

Status

+ <% {status, label} = agent_status(@agent_token) %> +
+ + + {label} + +
+ <.icon name="hero-signal" class="h-8 w-8 text-gray-400 dark:text-gray-500" />
- <.icon name="hero-signal" class="h-8 w-8 text-gray-400 dark:text-gray-500" />
-
- + -
-
-
-

Device

-

- {length(@polling_targets)} -

-

- {@direct_assignments} direct -

-
- <.icon name="hero-server" class="h-8 w-8 text-gray-400 dark:text-gray-500" /> -
-
- - -
-
-
-

Last Seen

-

- <.timestamp - datetime={@agent_token.last_seen_at} - timezone={@timezone} - /> -

- <%= if @agent_token.last_seen_at do %> +
+
+
+

Device

+

+ {length(@polling_targets)} +

+ {@direct_assignments} direct +

+
+ <.icon name="hero-server" class="h-8 w-8 text-gray-400 dark:text-gray-500" /> +
+
+ + +
+
+
+

Last Seen

+

<.timestamp datetime={@agent_token.last_seen_at} timezone={@timezone} />

- <% end %> - <%= if @agent_token.last_ip do %> -

- {@agent_token.last_ip} -

- <% end %> + <%= if @agent_token.last_seen_at do %> +

+ <.timestamp + datetime={@agent_token.last_seen_at} + timezone={@timezone} + /> +

+ <% end %> + <%= if @agent_token.last_ip do %> +

+ {@agent_token.last_ip} +

+ <% end %> +
+ <.icon name="hero-clock" class="h-8 w-8 text-gray-400 dark:text-gray-500 flex-shrink-0" /> +
+
+ + +
+
+
+

Uptime

+ <%= if @agent_token.metadata["uptime_seconds"] do %> +

+ {format_uptime(@agent_token.metadata["uptime_seconds"])} +

+ <% else %> +

+ Unknown +

+ <% end %> +
+ <.icon name="hero-clock" class="h-8 w-8 text-gray-400 dark:text-gray-500" />
- <.icon name="hero-clock" class="h-8 w-8 text-gray-400 dark:text-gray-500 flex-shrink-0" />
- -
-
-
-

Uptime

- <%= if @agent_token.metadata["uptime_seconds"] do %> -

- {format_uptime(@agent_token.metadata["uptime_seconds"])} -

- <% else %> -

- Unknown -

- <% end %> -
- <.icon name="hero-clock" class="h-8 w-8 text-gray-400 dark:text-gray-500" /> -
-
-
- - <%= if @agent_token.metadata && map_size(@agent_token.metadata) > 0 do %> + <%= if @agent_token.metadata && map_size(@agent_token.metadata) > 0 do %> +
+

+ Agent Metadata +

+
+ <%= if @agent_token.metadata["hostname"] do %> +
+
Hostname
+
+ {@agent_token.metadata["hostname"]} +
+
+ <% end %> + <%= if @agent_token.metadata["uptime_seconds"] do %> +
+
Uptime
+
+ {format_uptime(@agent_token.metadata["uptime_seconds"])} +
+
+ <% end %> +
+
+ <% end %> + +

- Agent Metadata + Timestamps

- <%= if @agent_token.metadata["hostname"] do %> +
+
Created
+
+ <.timestamp datetime={@agent_token.inserted_at} timezone={@timezone} /> +
+
+ +
+
Last Updated
+
+ <.timestamp datetime={@agent_token.updated_at} timezone={@timezone} /> +
+
+ +
+
+ Last Seen (Heartbeat) +
+
+ <.timestamp datetime={@agent_token.last_seen_at} timezone={@timezone} /> +
+
+ + <%= if @agent_token.last_ip do %>
-
Hostname
+
Last IP Address
- {@agent_token.metadata["hostname"]} -
-
- <% end %> - <%= if @agent_token.metadata["uptime_seconds"] do %> -
-
Uptime
-
- {format_uptime(@agent_token.metadata["uptime_seconds"])} + {@agent_token.last_ip}
<% end %>
- <% end %> - - -
-

- Timestamps -

-
-
-
Created
-
- <.timestamp datetime={@agent_token.inserted_at} timezone={@timezone} /> -
-
- -
-
Last Updated
-
- <.timestamp datetime={@agent_token.updated_at} timezone={@timezone} /> -
-
- -
-
- Last Seen (Heartbeat) -
-
- <.timestamp datetime={@agent_token.last_seen_at} timezone={@timezone} /> -
-
- - <%= if @agent_token.last_ip do %> -
-
Last IP Address
-
- {@agent_token.last_ip} -
-
- <% end %> -
-
- + -
-
-

- Polling Targets -

-

- Device that this agent is responsible for polling -

-
- - <%= if @polling_targets == [] do %> -
- <.icon name="hero-server" class="mx-auto h-12 w-12 text-gray-400 dark:text-gray-500" /> -

- No devices assigned +
+
+

+ Polling Targets

-

- This agent is not currently polling any device. +

+ Device that this agent is responsible for polling

- <% else %> -
- <%= for device <- @polling_targets do %> - <.link - navigate={~p"/devices/#{device.id}"} - class="block hover:bg-gray-50 dark:hover:bg-gray-800 transition-colors" - > -
-
-
-
-

- {device.name} -

- <% {source, source_label} = assignment_source(device, @agent_token.id) %> - - {source_label} - -
-
- - <.icon name="hero-globe-alt" class="h-4 w-4" /> - {device.ip_address} - - <%= if device.site do %> - - <.icon name="hero-building-office" class="h-4 w-4" /> - {device.site.name} + + <%= if @polling_targets == [] do %> +
+ <.icon name="hero-server" class="mx-auto h-12 w-12 text-gray-400 dark:text-gray-500" /> +

+ No devices assigned +

+

+ This agent is not currently polling any device. +

+
+ <% else %> +
+ <%= for device <- @polling_targets do %> + <.link + navigate={~p"/devices/#{device.id}"} + class="block hover:bg-gray-50 dark:hover:bg-gray-800 transition-colors" + > +
+
+
+
+

+ {device.name} +

+ <% {source, source_label} = assignment_source(device, @agent_token.id) %> + + {source_label} - <% end %> +
+
+ + <.icon name="hero-globe-alt" class="h-4 w-4" /> + {device.ip_address} + + <%= if device.site do %> + + <.icon name="hero-building-office" class="h-4 w-4" /> + {device.site.name} + + <% end %> +
+ <.icon + name="hero-chevron-right" + class="h-5 w-5 text-gray-400 dark:text-gray-500" + />
- <.icon name="hero-chevron-right" class="h-5 w-5 text-gray-400 dark:text-gray-500" />
-
- - <% end %> -
- <% end %> + + <% end %> +
+ <% end %> +
+ diff --git a/priv/repo/migrations/20260127131415_add_allow_remote_debug_to_agent_tokens.exs b/priv/repo/migrations/20260127131415_add_allow_remote_debug_to_agent_tokens.exs new file mode 100644 index 00000000..1ca7021b --- /dev/null +++ b/priv/repo/migrations/20260127131415_add_allow_remote_debug_to_agent_tokens.exs @@ -0,0 +1,12 @@ +defmodule Towerops.Repo.Migrations.AddAllowRemoteDebugToAgentTokens do + use Ecto.Migration + + def change do + alter table(:agent_tokens) do + add :allow_remote_debug, :boolean, default: false, null: false + end + + # Optional: index for filtering agents with debug enabled + create index(:agent_tokens, [:allow_remote_debug]) + end +end diff --git a/priv/static/changelog.txt b/priv/static/changelog.txt index c6521a2c..e966d297 100644 --- a/priv/static/changelog.txt +++ b/priv/static/changelog.txt @@ -5,6 +5,7 @@ Devices Working 2025-01-27 * Infrastructure improvements * Bug fix: poller agent assignment on devices +* Feature: improve Ubiquiti LTU device handling 2025-01-26 * Cloud poller improvements