feat: add automatic polling job cleanup on deployment

Added JobCleanupTask that runs on production startup to cancel all
existing polling jobs and reschedule them with the latest worker code.

This ensures:
- No stale jobs run with old code after deployment (e.g., missing SNMPv3 credentials)
- All devices get fresh polling jobs with current logic
- Production deployments are seamless without manual intervention

The task:
- Only runs in production environment (skipped in dev/test)
- Runs asynchronously after supervisor initialization
- Cancels all DevicePollerWorker and DeviceMonitorWorker jobs
- Reschedules polling for all SNMP-enabled devices
- Is idempotent and safe to run multiple times

Removed debug logging from Client and DevicePollerWorker as SNMPv3
implementation is now verified and working correctly.

lib/towerops/application.ex

@@ -106,7 +106,17 @@ defmodule Towerops.Application do
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Towerops.Supervisor]
-    Supervisor.start_link(children, opts)
+    result = Supervisor.start_link(children, opts)
+
+    # Run post-startup cleanup tasks (production only)
+    # This ensures all polling jobs use the latest worker code after deployment
+    Task.start(fn ->
+      # Wait for supervisor to fully initialize
+      Process.sleep(2000)
+      Towerops.Workers.JobCleanupTask.run()
+    end)
+
+    result
   end
 
   # Tell Phoenix to update the endpoint configuration

lib/towerops/snmp/client.ex

@@ -66,11 +66,6 @@ defmodule Towerops.Snmp.Client do
     target = build_target(opts)
     snmp_opts = build_snmp_opts(opts)
 
-    # Debug logging for SNMPv3
-    if Keyword.get(snmp_opts, :version) == :v3 do
-      Logger.info("SNMPv3 request to #{target}: #{inspect(snmp_opts, pretty: true)}")
-    end
-
     # Resolve MIB name to numeric OID if needed
     resolved_oid = resolve_mib_name(oid)
 

lib/towerops/workers/device_poller_worker.ex

@@ -1122,28 +1122,21 @@ defmodule Towerops.Workers.DevicePollerWorker do
     ]
 
     # Add version-specific credentials
-    opts =
-      if snmp_config.version == "3" do
-        v3_config = Devices.get_snmpv3_config(device)
+    if snmp_config.version == "3" do
+      v3_config = Devices.get_snmpv3_config(device)
 
-        base_opts ++
-          [
-            security_name: v3_config.username,
-            security_level: v3_config.security_level,
-            auth_protocol: v3_config.auth_protocol,
-            auth_password: v3_config.auth_password,
-            priv_protocol: v3_config.priv_protocol,
-            priv_password: v3_config.priv_password
-          ]
-      else
-        base_opts ++ [community: snmp_config.community]
-      end
-
-    Logger.info(
-      "DevicePollerWorker building client opts for #{device.name} (#{device.ip_address}): version=#{snmp_config.version}, has_security_name=#{Keyword.has_key?(opts, :security_name)}, has_community=#{Keyword.has_key?(opts, :community)}"
-    )
-
-    opts
+      base_opts ++
+        [
+          security_name: v3_config.username,
+          security_level: v3_config.security_level,
+          auth_protocol: v3_config.auth_protocol,
+          auth_password: v3_config.auth_password,
+          priv_protocol: v3_config.priv_protocol,
+          priv_password: v3_config.priv_password
+        ]
+    else
+      base_opts ++ [community: snmp_config.community]
+    end
   end
 
   defp get_poll_interval(device) do

lib/towerops/workers/job_cleanup_task.ex

@@ -0,0 +1,95 @@
+defmodule Towerops.Workers.JobCleanupTask do
+  @moduledoc """
+  One-time startup task to clean up and reschedule all device polling jobs.
+
+  This task runs on application startup to ensure all polling jobs use the
+  latest worker code. This is particularly important when deploying fixes
+  that change how jobs are executed (e.g., SNMPv3 credential handling).
+
+  The task is idempotent and safe to run multiple times.
+  """
+
+  import Ecto.Query
+
+  alias Towerops.Devices
+  alias Towerops.Repo
+  alias Towerops.Workers.DeviceMonitorWorker
+  alias Towerops.Workers.DevicePollerWorker
+
+  require Logger
+
+  @doc """
+  Cancels all existing polling jobs and reschedules them with current code.
+  Only runs in production environment.
+  """
+  def run do
+    if Application.get_env(:towerops, :env) == :prod do
+      Logger.info("JobCleanupTask: Starting cleanup of polling jobs...")
+      cleanup_and_reschedule()
+      Logger.info("JobCleanupTask: Cleanup complete")
+    else
+      Logger.debug("JobCleanupTask: Skipping cleanup in #{Application.get_env(:towerops, :env)} environment")
+    end
+  end
+
+  defp cleanup_and_reschedule do
+    # Cancel all existing polling jobs
+    cancel_all_polling_jobs()
+
+    # Wait a moment for cancellations to process
+    Process.sleep(1000)
+
+    # Reschedule polling for all enabled devices
+    reschedule_all_devices()
+  end
+
+  defp cancel_all_polling_jobs do
+    {:ok, poller_count} =
+      Oban.cancel_all_jobs(
+        from(j in Oban.Job,
+          where: j.worker == "Towerops.Workers.DevicePollerWorker",
+          where: j.state in ["available", "scheduled", "executing", "retryable"]
+        )
+      )
+
+    {:ok, monitor_count} =
+      Oban.cancel_all_jobs(
+        from(j in Oban.Job,
+          where: j.worker == "Towerops.Workers.DeviceMonitorWorker",
+          where: j.state in ["available", "scheduled", "executing", "retryable"]
+        )
+      )
+
+    Logger.info("JobCleanupTask: Cancelled #{poller_count} polling jobs and #{monitor_count} monitor jobs")
+  end
+
+  defp reschedule_all_devices do
+    # Get all devices with SNMP enabled
+    devices =
+      Repo.all(
+        from(d in Towerops.Devices.Device,
+          where: d.snmp_enabled == true,
+          select: d.id
+        )
+      )
+
+    Logger.info("JobCleanupTask: Rescheduling polling for #{length(devices)} devices")
+
+    # Reschedule each device
+    Enum.each(devices, fn device_id ->
+      device = Devices.get_device!(device_id)
+
+      # Start polling job
+      case DevicePollerWorker.start_polling(device_id) do
+        {:ok, _} -> :ok
+        {:error, reason} -> Logger.warning("Failed to start polling for device #{device.name}: #{inspect(reason)}")
+      end
+
+      # Start monitor job
+      case DeviceMonitorWorker.start_monitoring(device_id) do
+        {:ok, _} -> :ok
+        {:error, reason} -> Logger.warning("Failed to start monitoring for device #{device.name}: #{inspect(reason)}")
+      end
+    end)
+  end
+end