257 lines
8.7 KiB
Elixir
257 lines
8.7 KiB
Elixir
defmodule Towerops.Workers.StaleAgentWorkerTest do
|
|
use Towerops.DataCase, async: false
|
|
use Oban.Testing, repo: Towerops.Repo
|
|
|
|
import Towerops.AccountsFixtures
|
|
import Towerops.OrganizationsFixtures
|
|
|
|
alias Towerops.Agents
|
|
alias Towerops.Agents.AgentToken
|
|
alias Towerops.Workers.StaleAgentWorker
|
|
|
|
describe "perform/1" do
|
|
setup do
|
|
user = user_fixture()
|
|
organization = organization_fixture(user.id)
|
|
{:ok, organization: organization}
|
|
end
|
|
|
|
test "returns :ok when no stale agents exist" do
|
|
assert :ok = perform_job(StaleAgentWorker, %{})
|
|
end
|
|
|
|
test "returns :ok when all agents are fresh", %{organization: org} do
|
|
{:ok, agent, _token} = Agents.create_agent_token(org.id, "Fresh Agent")
|
|
|
|
# Update heartbeat to now
|
|
Agents.update_agent_token_heartbeat(agent.id, "192.168.1.1", %{})
|
|
|
|
assert :ok = perform_job(StaleAgentWorker, %{})
|
|
end
|
|
|
|
test "processes stale agents and broadcasts PubSub event", %{organization: org} do
|
|
{:ok, agent, _token} = Agents.create_agent_token(org.id, "Stale Agent")
|
|
|
|
# Set last_seen_at to 15 minutes ago
|
|
stale_time = DateTime.add(DateTime.utc_now(), -15, :minute)
|
|
|
|
Towerops.Repo.update_all(
|
|
from(a in AgentToken, where: a.id == ^agent.id),
|
|
set: [last_seen_at: stale_time]
|
|
)
|
|
|
|
# Subscribe to PubSub to verify broadcast
|
|
Phoenix.PubSub.subscribe(Towerops.PubSub, "agents:health")
|
|
|
|
assert :ok = perform_job(StaleAgentWorker, %{})
|
|
|
|
# Verify PubSub broadcast was sent
|
|
assert_received {:agents_stale, stale_agents}
|
|
assert length(stale_agents) == 1
|
|
assert hd(stale_agents).id == agent.id
|
|
end
|
|
|
|
test "broadcasts newly stale agents (10-15 minutes)", %{organization: org} do
|
|
{:ok, agent, _token} = Agents.create_agent_token(org.id, "Newly Stale")
|
|
|
|
# Set last_seen_at to 12 minutes ago (within newly stale window)
|
|
stale_time = DateTime.add(DateTime.utc_now(), -12, :minute)
|
|
|
|
Towerops.Repo.update_all(
|
|
from(a in AgentToken, where: a.id == ^agent.id),
|
|
set: [last_seen_at: stale_time]
|
|
)
|
|
|
|
Phoenix.PubSub.subscribe(Towerops.PubSub, "agents:health")
|
|
|
|
assert :ok = perform_job(StaleAgentWorker, %{})
|
|
|
|
assert_received {:agents_stale, stale_agents}
|
|
assert length(stale_agents) == 1
|
|
assert hd(stale_agents).id == agent.id
|
|
assert hd(stale_agents).organization_id == org.id
|
|
end
|
|
|
|
test "broadcasts long-term stale agents (>15 minutes)", %{organization: org} do
|
|
{:ok, agent, _token} = Agents.create_agent_token(org.id, "Long-term Stale")
|
|
|
|
# Set last_seen_at to 30 minutes ago
|
|
stale_time = DateTime.add(DateTime.utc_now(), -30, :minute)
|
|
|
|
Towerops.Repo.update_all(
|
|
from(a in AgentToken, where: a.id == ^agent.id),
|
|
set: [last_seen_at: stale_time]
|
|
)
|
|
|
|
Phoenix.PubSub.subscribe(Towerops.PubSub, "agents:health")
|
|
|
|
assert :ok = perform_job(StaleAgentWorker, %{})
|
|
|
|
assert_received {:agents_stale, stale_agents}
|
|
assert length(stale_agents) == 1
|
|
assert hd(stale_agents).id == agent.id
|
|
end
|
|
|
|
test "handles mix of newly stale and long-term stale agents", %{organization: org} do
|
|
{:ok, newly_stale, _} = Agents.create_agent_token(org.id, "Newly Stale")
|
|
{:ok, long_term_stale, _} = Agents.create_agent_token(org.id, "Long-term Stale")
|
|
|
|
# Newly stale: 12 minutes ago
|
|
newly_stale_time = DateTime.add(DateTime.utc_now(), -12, :minute)
|
|
# Long-term stale: 20 minutes ago
|
|
long_term_time = DateTime.add(DateTime.utc_now(), -20, :minute)
|
|
|
|
Towerops.Repo.update_all(
|
|
from(a in AgentToken, where: a.id == ^newly_stale.id),
|
|
set: [last_seen_at: newly_stale_time]
|
|
)
|
|
|
|
Towerops.Repo.update_all(
|
|
from(a in AgentToken, where: a.id == ^long_term_stale.id),
|
|
set: [last_seen_at: long_term_time]
|
|
)
|
|
|
|
Phoenix.PubSub.subscribe(Towerops.PubSub, "agents:health")
|
|
|
|
assert :ok = perform_job(StaleAgentWorker, %{})
|
|
|
|
# Verify both agents are included in PubSub broadcast
|
|
assert_received {:agents_stale, stale_agents}
|
|
assert length(stale_agents) == 2
|
|
|
|
stale_ids = Enum.map(stale_agents, & &1.id)
|
|
assert newly_stale.id in stale_ids
|
|
assert long_term_stale.id in stale_ids
|
|
end
|
|
|
|
test "does not broadcast when no stale agents exist" do
|
|
Phoenix.PubSub.subscribe(Towerops.PubSub, "agents:health")
|
|
|
|
assert :ok = perform_job(StaleAgentWorker, %{})
|
|
|
|
refute_received {:agents_stale, _}
|
|
end
|
|
|
|
test "preloads organization association for stale agents", %{organization: org} do
|
|
{:ok, agent, _token} = Agents.create_agent_token(org.id, "Stale Agent")
|
|
|
|
stale_time = DateTime.add(DateTime.utc_now(), -15, :minute)
|
|
|
|
Towerops.Repo.update_all(
|
|
from(a in AgentToken, where: a.id == ^agent.id),
|
|
set: [last_seen_at: stale_time]
|
|
)
|
|
|
|
Phoenix.PubSub.subscribe(Towerops.PubSub, "agents:health")
|
|
|
|
assert :ok = perform_job(StaleAgentWorker, %{})
|
|
|
|
assert_received {:agents_stale, [stale_agent]}
|
|
# Verify organization is preloaded
|
|
assert %Ecto.Association.NotLoaded{} != stale_agent.organization
|
|
assert stale_agent.organization.id == org.id
|
|
end
|
|
end
|
|
|
|
describe "find_stale_agents/0" do
|
|
setup do
|
|
user = user_fixture()
|
|
organization = organization_fixture(user.id)
|
|
{:ok, organization: organization}
|
|
end
|
|
|
|
test "returns empty list when no agents exist" do
|
|
assert StaleAgentWorker.find_stale_agents() == []
|
|
end
|
|
|
|
test "returns empty list when all agents are fresh", %{organization: org} do
|
|
{:ok, agent, _token} = Agents.create_agent_token(org.id, "Fresh Agent")
|
|
|
|
# Update heartbeat to now
|
|
Agents.update_agent_token_heartbeat(agent.id, "192.168.1.1", %{})
|
|
|
|
assert StaleAgentWorker.find_stale_agents() == []
|
|
end
|
|
|
|
test "returns agents that haven't been seen in over 10 minutes", %{organization: org} do
|
|
{:ok, agent, _token} = Agents.create_agent_token(org.id, "Stale Agent")
|
|
|
|
# Set last_seen_at to 15 minutes ago
|
|
stale_time = DateTime.add(DateTime.utc_now(), -15, :minute)
|
|
|
|
Towerops.Repo.update_all(
|
|
from(a in AgentToken, where: a.id == ^agent.id),
|
|
set: [last_seen_at: stale_time]
|
|
)
|
|
|
|
stale_agents = StaleAgentWorker.find_stale_agents()
|
|
assert length(stale_agents) == 1
|
|
assert hd(stale_agents).id == agent.id
|
|
end
|
|
|
|
test "ignores agents that have never checked in", %{organization: org} do
|
|
{:ok, _agent, _token} = Agents.create_agent_token(org.id, "New Agent")
|
|
|
|
# last_seen_at is nil by default
|
|
assert StaleAgentWorker.find_stale_agents() == []
|
|
end
|
|
|
|
test "ignores disabled agents", %{organization: org} do
|
|
{:ok, agent, _token} = Agents.create_agent_token(org.id, "Disabled Agent")
|
|
|
|
# Set last_seen_at to 15 minutes ago and disable
|
|
stale_time = DateTime.add(DateTime.utc_now(), -15, :minute)
|
|
|
|
Towerops.Repo.update_all(
|
|
from(a in AgentToken, where: a.id == ^agent.id),
|
|
set: [last_seen_at: stale_time, enabled: false]
|
|
)
|
|
|
|
assert StaleAgentWorker.find_stale_agents() == []
|
|
end
|
|
|
|
test "returns multiple stale agents", %{organization: org} do
|
|
{:ok, agent1, _token} = Agents.create_agent_token(org.id, "Stale Agent 1")
|
|
{:ok, agent2, _token} = Agents.create_agent_token(org.id, "Stale Agent 2")
|
|
|
|
# Set both to 20 minutes ago
|
|
stale_time = DateTime.add(DateTime.utc_now(), -20, :minute)
|
|
|
|
Towerops.Repo.update_all(
|
|
from(a in AgentToken, where: a.id in [^agent1.id, ^agent2.id]),
|
|
set: [last_seen_at: stale_time]
|
|
)
|
|
|
|
stale_agents = StaleAgentWorker.find_stale_agents()
|
|
assert length(stale_agents) == 2
|
|
|
|
stale_ids = Enum.map(stale_agents, & &1.id)
|
|
assert agent1.id in stale_ids
|
|
assert agent2.id in stale_ids
|
|
end
|
|
|
|
test "correctly identifies agents at the 10-minute boundary", %{organization: org} do
|
|
{:ok, agent_just_stale, _} = Agents.create_agent_token(org.id, "Just Stale")
|
|
{:ok, agent_just_fresh, _} = Agents.create_agent_token(org.id, "Just Fresh")
|
|
|
|
# 11 minutes ago - should be stale
|
|
stale_time = DateTime.add(DateTime.utc_now(), -11, :minute)
|
|
# 9 minutes ago - should be fresh
|
|
fresh_time = DateTime.add(DateTime.utc_now(), -9, :minute)
|
|
|
|
Towerops.Repo.update_all(
|
|
from(a in AgentToken, where: a.id == ^agent_just_stale.id),
|
|
set: [last_seen_at: stale_time]
|
|
)
|
|
|
|
Towerops.Repo.update_all(
|
|
from(a in AgentToken, where: a.id == ^agent_just_fresh.id),
|
|
set: [last_seen_at: fresh_time]
|
|
)
|
|
|
|
stale_agents = StaleAgentWorker.find_stale_agents()
|
|
assert length(stale_agents) == 1
|
|
assert hd(stale_agents).id == agent_just_stale.id
|
|
end
|
|
end
|
|
end
|