Graham McIntire
d85b42c3ee
Production code (lib/): all Process.sleep calls replaced with receive after timeout - retry backoff, exponential backoff, batch delays, poll intervals, and settle waits. Test code: poll_until helpers across 4 agent channel test files now use receive after instead of Process.sleep. Various standalone Process.sleep(N) calls replaced with :sys.get_state sync barriers or receive after. deferred_discovery test simulation sleeps reduced 500ms -> 30ms.
114 lines
3.4 KiB
Elixir
114 lines
3.4 KiB
Elixir
defmodule Towerops.Workers.JobCleanupTask do
|
|
@moduledoc """
|
|
One-time startup task to clean up and reschedule all device polling jobs.
|
|
|
|
This task runs on application startup to ensure all polling jobs use the
|
|
latest worker code. This is particularly important when deploying fixes
|
|
that change how jobs are executed (e.g., SNMPv3 credential handling).
|
|
|
|
The task is idempotent and safe to run multiple times.
|
|
"""
|
|
|
|
import Ecto.Query
|
|
|
|
alias Towerops.Devices
|
|
alias Towerops.Repo
|
|
alias Towerops.Snmp.Client
|
|
alias Towerops.Workers.DeviceMonitorWorker
|
|
alias Towerops.Workers.DevicePollerWorker
|
|
|
|
require Logger
|
|
|
|
@doc """
|
|
Cancels all existing polling jobs and reschedules them with current code.
|
|
Only runs in production environment.
|
|
"""
|
|
def run do
|
|
if Application.get_env(:towerops, :env) == :prod do
|
|
Logger.info("JobCleanupTask: Starting cleanup of polling jobs...")
|
|
cleanup_and_reschedule()
|
|
Logger.info("JobCleanupTask: Cleanup complete")
|
|
else
|
|
Logger.debug("JobCleanupTask: Skipping cleanup in #{Application.get_env(:towerops, :env)} environment")
|
|
end
|
|
end
|
|
|
|
defp cleanup_and_reschedule do
|
|
# Cancel all existing polling jobs
|
|
cancel_all_polling_jobs()
|
|
|
|
# Wait a moment for cancellations to process
|
|
settle_ms = Application.get_env(:towerops, :job_cleanup_settle_ms, 1000)
|
|
|
|
receive do
|
|
after
|
|
settle_ms -> :ok
|
|
end
|
|
|
|
# Reschedule polling for all enabled devices
|
|
reschedule_all_devices()
|
|
end
|
|
|
|
defp cancel_all_polling_jobs do
|
|
# Exclude "executing" so in-flight polls/monitors complete naturally;
|
|
# cancelling them mid-run discards work and can leave partial state.
|
|
cancellable_states = ["available", "scheduled", "retryable"]
|
|
|
|
{:ok, poller_count} =
|
|
Oban.cancel_all_jobs(
|
|
from(j in Oban.Job,
|
|
where: j.worker == "Towerops.Workers.DevicePollerWorker",
|
|
where: j.state in ^cancellable_states
|
|
)
|
|
)
|
|
|
|
{:ok, monitor_count} =
|
|
Oban.cancel_all_jobs(
|
|
from(j in Oban.Job,
|
|
where: j.worker == "Towerops.Workers.DeviceMonitorWorker",
|
|
where: j.state in ^cancellable_states
|
|
)
|
|
)
|
|
|
|
Logger.info("JobCleanupTask: Cancelled #{poller_count} polling jobs and #{monitor_count} monitor jobs")
|
|
end
|
|
|
|
defp reschedule_all_devices do
|
|
if Client.phoenix_snmp_disabled() do
|
|
Logger.info("JobCleanupTask: Phoenix SNMP polling is disabled, skipping job rescheduling")
|
|
:ok
|
|
else
|
|
do_reschedule_all_devices()
|
|
end
|
|
end
|
|
|
|
defp do_reschedule_all_devices do
|
|
# Get all devices with SNMP enabled
|
|
devices =
|
|
Repo.all(
|
|
from(d in Towerops.Devices.Device,
|
|
where: d.snmp_enabled == true,
|
|
select: d.id
|
|
)
|
|
)
|
|
|
|
Logger.info("JobCleanupTask: Rescheduling polling for #{length(devices)} devices")
|
|
|
|
# Reschedule each device
|
|
Enum.each(devices, fn device_id ->
|
|
device = Devices.get_device!(device_id)
|
|
|
|
# Start polling job
|
|
case DevicePollerWorker.start_polling(device_id) do
|
|
{:ok, _} -> :ok
|
|
{:error, reason} -> Logger.warning("Failed to start polling for device #{device.name}: #{inspect(reason)}")
|
|
end
|
|
|
|
# Start monitor job
|
|
case DeviceMonitorWorker.start_monitoring(device_id) do
|
|
{:ok, _} -> :ok
|
|
{:error, reason} -> Logger.warning("Failed to start monitoring for device #{device.name}: #{inspect(reason)}")
|
|
end
|
|
end)
|
|
end
|
|
end
|