From 72f69b13ff6635b51255458185ae6dfecd3b31c6 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Tue, 27 Jan 2026 14:34:32 -0600
Subject: [PATCH] agent debug improvements

---
 lib/towerops/workers/discovery_worker.ex | 72 +++++++++++++++++++-----
 1 file changed, 57 insertions(+), 15 deletions(-)

diff --git a/lib/towerops/workers/discovery_worker.ex b/lib/towerops/workers/discovery_worker.ex
index 3ea0ebfc..48f891f5 100644
--- a/lib/towerops/workers/discovery_worker.ex
+++ b/lib/towerops/workers/discovery_worker.ex
@@ -38,20 +38,32 @@ defmodule Towerops.Workers.DiscoveryWorker do
         :discard
 
       device ->
+        Logger.info(
+          "Rediscovery triggered for device: #{device.name} (#{device.ip_address})",
+          device_id: device_id,
+          device_name: device.name,
+          device_ip: device.ip_address,
+          snmp_enabled: device.snmp_enabled,
+          snmp_port: device.snmp_port || 161
+        )
+
         case get_assigned_agent(device) do
           {agent_token_id, source} when not is_nil(agent_token_id) ->
             # Try to get the agent token, falling back if it doesn't exist
             try do
               agent = Agents.get_agent_token!(agent_token_id)
-              poller_type = if agent.cloud_poller, do: "cloud poller", else: "user's poller"
+              poller_type = if agent.cloud_poller, do: "cloud poller", else: "user's agent"
 
               Logger.info(
-                "Starting SNMP discovery for device #{device_id} using #{poller_type} '#{agent.name}' (#{source} assignment)",
+                "Agent assignment found: using #{poller_type} '#{agent.name}' (assigned via #{source})",
                 device_id: device_id,
+                device_name: device.name,
                 agent_token_id: agent_token_id,
                 agent_name: agent.name,
                 cloud_poller: agent.cloud_poller,
-                source: source
+                assignment_source: source,
+                agent_enabled: agent.enabled,
+                agent_last_seen: agent.last_seen_at
               )
 
               attempt_agent_discovery(device, agent_token_id, [agent_token_id])
@@ -140,9 +152,12 @@ defmodule Towerops.Workers.DiscoveryWorker do
   defp attempt_cloud_poller_discovery(device, tried_agents) do
     case get_next_cloud_poller(tried_agents) do
       nil ->
-        Logger.info(
-          "No more cloud pollers available, falling back to direct SNMP from Phoenix cluster",
-          device_id: device.id
+        Logger.warning(
+          "No available cloud pollers found, falling back to direct SNMP from Phoenix cluster",
+          device_id: device.id,
+          device_name: device.name,
+          device_ip: device.ip_address,
+          tried_agents: tried_agents
         )
 
         perform_direct_discovery(device)
@@ -151,10 +166,14 @@ defmodule Towerops.Workers.DiscoveryWorker do
         agent = Agents.get_agent_token!(next_agent_id)
 
         Logger.info(
-          "Trying cloud poller '#{agent.name}' for device #{device.id}",
+          "No user agent assigned - trying global cloud poller '#{agent.name}'",
           device_id: device.id,
+          device_name: device.name,
+          device_ip: device.ip_address,
           cloud_poller_id: next_agent_id,
-          agent_name: agent.name
+          cloud_poller_name: agent.name,
+          agent_enabled: agent.enabled,
+          agent_last_seen: agent.last_seen_at
         )
 
         attempt_agent_discovery(device, next_agent_id, [next_agent_id | tried_agents])
@@ -182,12 +201,28 @@ defmodule Towerops.Workers.DiscoveryWorker do
   defp trigger_agent_discovery(device, agent_token_id) do
     # Broadcast discovery request to agent's channel
     # The agent will receive this via PubSub and execute discovery
+    Logger.info(
+      "Sending discovery request to remote agent via PubSub",
+      device_id: device.id,
+      device_name: device.name,
+      device_ip: device.ip_address,
+      agent_token_id: agent_token_id,
+      pubsub_channel: "agent:#{agent_token_id}:discovery"
+    )
+
     Phoenix.PubSub.broadcast(
       Towerops.PubSub,
       "agent:#{agent_token_id}:discovery",
       {:discovery_requested, device.id}
     )
 
+    Logger.info(
+      "Discovery request sent, waiting up to #{@agent_discovery_timeout_ms}ms for agent to respond",
+      device_id: device.id,
+      agent_token_id: agent_token_id,
+      timeout_ms: @agent_discovery_timeout_ms
+    )
+
     # Wait for agent to complete discovery (with timeout)
     # The agent will update last_discovery_at when done
     wait_for_agent_discovery(device, agent_token_id, @agent_discovery_timeout_ms)
@@ -248,25 +283,32 @@ defmodule Towerops.Workers.DiscoveryWorker do
   end
 
   defp perform_direct_discovery(device) do
-    Logger.info(
-      "Performing SNMP discovery from Phoenix cluster for device #{device.id}",
-      device_id: device.id
+    Logger.warning(
+      "FALLBACK: Performing direct SNMP from Phoenix cluster (no remote agents available)",
+      device_id: device.id,
+      device_name: device.name,
+      device_ip: device.ip_address,
+      note: "Phoenix cluster must be able to reach device on network to succeed"
     )
 
     case Snmp.discover_device(device) do
       {:ok, _snmp_device} ->
         Logger.info(
-          "SNMP discovery completed successfully from Phoenix cluster for device #{device.id}",
-          device_id: device.id
+          "Direct SNMP discovery completed successfully from Phoenix cluster",
+          device_id: device.id,
+          device_name: device.name
         )
 
         :ok
 
       {:error, reason} ->
         Logger.error(
-          "SNMP discovery failed from Phoenix cluster for device #{device.id}: #{inspect(reason)}",
+          "Direct SNMP discovery failed from Phoenix cluster: #{inspect(reason)}",
           device_id: device.id,
-          reason: reason
+          device_name: device.name,
+          device_ip: device.ip_address,
+          error: reason,
+          note: "Device may not be reachable from Phoenix cluster network"
         )
 
         {:error, reason}