From ae64ec257605db9885ce9d05fc896cb91a79d35c Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 5 Feb 2026 08:28:27 -0600 Subject: [PATCH] feat: add automatic polling job cleanup on deployment Added JobCleanupTask that runs on production startup to cancel all existing polling jobs and reschedule them with the latest worker code. This ensures: - No stale jobs run with old code after deployment (e.g., missing SNMPv3 credentials) - All devices get fresh polling jobs with current logic - Production deployments are seamless without manual intervention The task: - Only runs in production environment (skipped in dev/test) - Runs asynchronously after supervisor initialization - Cancels all DevicePollerWorker and DeviceMonitorWorker jobs - Reschedules polling for all SNMP-enabled devices - Is idempotent and safe to run multiple times Removed debug logging from Client and DevicePollerWorker as SNMPv3 implementation is now verified and working correctly. --- lib/towerops/application.ex | 12 ++- lib/towerops/snmp/client.ex | 5 -- lib/towerops/workers/device_poller_worker.ex | 35 +++----- lib/towerops/workers/job_cleanup_task.ex | 95 ++++++++++++++++++++ 4 files changed, 120 insertions(+), 27 deletions(-) create mode 100644 lib/towerops/workers/job_cleanup_task.ex diff --git a/lib/towerops/application.ex b/lib/towerops/application.ex index aab77585..b1d64a08 100644 --- a/lib/towerops/application.ex +++ b/lib/towerops/application.ex @@ -106,7 +106,17 @@ defmodule Towerops.Application do # See https://hexdocs.pm/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Towerops.Supervisor] - Supervisor.start_link(children, opts) + result = Supervisor.start_link(children, opts) + + # Run post-startup cleanup tasks (production only) + # This ensures all polling jobs use the latest worker code after deployment + Task.start(fn -> + # Wait for supervisor to fully initialize + Process.sleep(2000) + Towerops.Workers.JobCleanupTask.run() + end) + + result end # Tell Phoenix to update the endpoint configuration diff --git a/lib/towerops/snmp/client.ex b/lib/towerops/snmp/client.ex index 1e0e8ad3..5765eee9 100644 --- a/lib/towerops/snmp/client.ex +++ b/lib/towerops/snmp/client.ex @@ -66,11 +66,6 @@ defmodule Towerops.Snmp.Client do target = build_target(opts) snmp_opts = build_snmp_opts(opts) - # Debug logging for SNMPv3 - if Keyword.get(snmp_opts, :version) == :v3 do - Logger.info("SNMPv3 request to #{target}: #{inspect(snmp_opts, pretty: true)}") - end - # Resolve MIB name to numeric OID if needed resolved_oid = resolve_mib_name(oid) diff --git a/lib/towerops/workers/device_poller_worker.ex b/lib/towerops/workers/device_poller_worker.ex index b99f561a..13693eb7 100644 --- a/lib/towerops/workers/device_poller_worker.ex +++ b/lib/towerops/workers/device_poller_worker.ex @@ -1122,28 +1122,21 @@ defmodule Towerops.Workers.DevicePollerWorker do ] # Add version-specific credentials - opts = - if snmp_config.version == "3" do - v3_config = Devices.get_snmpv3_config(device) + if snmp_config.version == "3" do + v3_config = Devices.get_snmpv3_config(device) - base_opts ++ - [ - security_name: v3_config.username, - security_level: v3_config.security_level, - auth_protocol: v3_config.auth_protocol, - auth_password: v3_config.auth_password, - priv_protocol: v3_config.priv_protocol, - priv_password: v3_config.priv_password - ] - else - base_opts ++ [community: snmp_config.community] - end - - Logger.info( - "DevicePollerWorker building client opts for #{device.name} (#{device.ip_address}): version=#{snmp_config.version}, has_security_name=#{Keyword.has_key?(opts, :security_name)}, has_community=#{Keyword.has_key?(opts, :community)}" - ) - - opts + base_opts ++ + [ + security_name: v3_config.username, + security_level: v3_config.security_level, + auth_protocol: v3_config.auth_protocol, + auth_password: v3_config.auth_password, + priv_protocol: v3_config.priv_protocol, + priv_password: v3_config.priv_password + ] + else + base_opts ++ [community: snmp_config.community] + end end defp get_poll_interval(device) do diff --git a/lib/towerops/workers/job_cleanup_task.ex b/lib/towerops/workers/job_cleanup_task.ex new file mode 100644 index 00000000..eace5e48 --- /dev/null +++ b/lib/towerops/workers/job_cleanup_task.ex @@ -0,0 +1,95 @@ +defmodule Towerops.Workers.JobCleanupTask do + @moduledoc """ + One-time startup task to clean up and reschedule all device polling jobs. + + This task runs on application startup to ensure all polling jobs use the + latest worker code. This is particularly important when deploying fixes + that change how jobs are executed (e.g., SNMPv3 credential handling). + + The task is idempotent and safe to run multiple times. + """ + + import Ecto.Query + + alias Towerops.Devices + alias Towerops.Repo + alias Towerops.Workers.DeviceMonitorWorker + alias Towerops.Workers.DevicePollerWorker + + require Logger + + @doc """ + Cancels all existing polling jobs and reschedules them with current code. + Only runs in production environment. + """ + def run do + if Application.get_env(:towerops, :env) == :prod do + Logger.info("JobCleanupTask: Starting cleanup of polling jobs...") + cleanup_and_reschedule() + Logger.info("JobCleanupTask: Cleanup complete") + else + Logger.debug("JobCleanupTask: Skipping cleanup in #{Application.get_env(:towerops, :env)} environment") + end + end + + defp cleanup_and_reschedule do + # Cancel all existing polling jobs + cancel_all_polling_jobs() + + # Wait a moment for cancellations to process + Process.sleep(1000) + + # Reschedule polling for all enabled devices + reschedule_all_devices() + end + + defp cancel_all_polling_jobs do + {:ok, poller_count} = + Oban.cancel_all_jobs( + from(j in Oban.Job, + where: j.worker == "Towerops.Workers.DevicePollerWorker", + where: j.state in ["available", "scheduled", "executing", "retryable"] + ) + ) + + {:ok, monitor_count} = + Oban.cancel_all_jobs( + from(j in Oban.Job, + where: j.worker == "Towerops.Workers.DeviceMonitorWorker", + where: j.state in ["available", "scheduled", "executing", "retryable"] + ) + ) + + Logger.info("JobCleanupTask: Cancelled #{poller_count} polling jobs and #{monitor_count} monitor jobs") + end + + defp reschedule_all_devices do + # Get all devices with SNMP enabled + devices = + Repo.all( + from(d in Towerops.Devices.Device, + where: d.snmp_enabled == true, + select: d.id + ) + ) + + Logger.info("JobCleanupTask: Rescheduling polling for #{length(devices)} devices") + + # Reschedule each device + Enum.each(devices, fn device_id -> + device = Devices.get_device!(device_id) + + # Start polling job + case DevicePollerWorker.start_polling(device_id) do + {:ok, _} -> :ok + {:error, reason} -> Logger.warning("Failed to start polling for device #{device.name}: #{inspect(reason)}") + end + + # Start monitor job + case DeviceMonitorWorker.start_monitoring(device_id) do + {:ok, _} -> :ok + {:error, reason} -> Logger.warning("Failed to start monitoring for device #{device.name}: #{inspect(reason)}") + end + end) + end +end