Add CI-triggered mass agent update webhook

POST /api/v1/webhooks/agent-release triggers all connected agents
to self-update via their existing WebSocket channels. Authenticated
with a shared secret (AGENT_WEBHOOK_SECRET env var).

- Add ReleaseChecker.invalidate_cache/0 for fresh GitHub fetch
- Add Agents.list_updatable_agents/0 (enabled + seen < 10min)
- Add Agents.broadcast_mass_update/0 orchestration function
- Add WebhookAuth plug with timing-safe secret comparison
- Add AgentReleaseWebhookController with :webhook pipeline
- Configure AGENT_WEBHOOK_SECRET in dev/test/runtime configs

config/dev.exs

@@ -161,6 +161,9 @@ config :towerops, ToweropsWeb.Endpoint,
     ]
   ]
 
+# Webhook secret for CI-triggered agent updates
+config :towerops, :agent_webhook_secret, "dev-webhook-secret"
+
 # Set environment identifier for runtime checks
 config :towerops, :env, :dev
 

config/runtime.exs

@@ -288,6 +288,8 @@ if config_env() == :prod do
     ],
     secret_key_base: secret_key_base
 
+  # Webhook secret for CI-triggered mass agent updates
+  config :towerops, :agent_webhook_secret, System.get_env("AGENT_WEBHOOK_SECRET")
   config :towerops, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY")
   # Set environment identifier for runtime checks
   config :towerops, :env, :prod

config/test.exs

@@ -64,6 +64,9 @@ config :towerops, ToweropsWeb.Endpoint,
   secret_key_base: "XDSSqVUtRXUjEfzxefIAaPIzBfonpNMOrCRyP1a0kjPzzyOpVNSRmMBVae/bwTqj",
   server: false
 
+# Webhook secret for CI-triggered agent updates
+config :towerops, :agent_webhook_secret, "test-webhook-secret"
+
 # Set environment identifier for runtime checks
 config :towerops, :env, :test
 

lib/towerops/agents.ex

@@ -10,9 +10,12 @@ defmodule Towerops.Agents do
 
   alias Towerops.Agents.AgentAssignment
   alias Towerops.Agents.AgentToken
+  alias Towerops.Agents.ReleaseChecker
   alias Towerops.Devices.Device
   alias Towerops.Repo
 
+  require Logger
+
   ## Token management
 
   @doc """
@@ -289,6 +292,64 @@ defmodule Towerops.Agents do
     )
   end
 
+  @doc """
+  Returns all agent tokens eligible for mass update.
+
+  An agent is updatable if it is enabled and was last seen within the past 10 minutes.
+  Includes both organization agents and cloud pollers.
+  """
+  @spec list_updatable_agents() :: [AgentToken.t()]
+  def list_updatable_agents do
+    cutoff = DateTime.add(DateTime.utc_now(), -10, :minute)
+
+    AgentToken
+    |> where([t], t.enabled == true and not is_nil(t.last_seen_at) and t.last_seen_at > ^cutoff)
+    |> Repo.all()
+  end
+
+  @doc """
+  Broadcasts an update command to all connected agents.
+
+  1. Invalidates the release cache
+  2. Fetches the latest release from GitHub
+  3. Finds all updatable agents
+  4. Sends update command to each agent matching their architecture
+
+  Returns `{:ok, %{notified: N, skipped: N, version: String.t()}}` on success.
+  """
+  @spec broadcast_mass_update() :: {:ok, map()} | {:error, term()}
+  def broadcast_mass_update do
+    ReleaseChecker.invalidate_cache()
+
+    case ReleaseChecker.get_latest_release() do
+      {:ok, %{version: version, assets: assets}} ->
+        agents = list_updatable_agents()
+        {notified, skipped} = notify_agents(agents, assets)
+
+        {:ok, %{notified: notified, skipped: skipped, version: version}}
+
+      {:error, reason} ->
+        Logger.error("Failed to fetch latest release for mass update: #{inspect(reason)}")
+        {:error, reason}
+    end
+  end
+
+  defp notify_agents(agents, assets) do
+    Enum.reduce(agents, {0, 0}, fn agent, {notified, skipped} ->
+      arch = get_in(agent.metadata, ["arch"]) || "x86_64"
+
+      case ReleaseChecker.asset_for_arch(assets, arch) do
+        {:ok, asset} ->
+          update_agent(agent.id, asset.url, asset.checksum)
+          {notified + 1, skipped}
+
+        {:error, :no_matching_asset} ->
+          Logger.warning("No matching asset for agent #{agent.id} (arch: #{arch})")
+          {notified, skipped + 1}
+      end
+    end)
+  end
+
   @doc """
   Toggles remote debugging for an agent.
 

lib/towerops/agents/release_checker.ex

@@ -15,6 +15,18 @@ defmodule Towerops.Agents.ReleaseChecker do
     "aarch64" => "arm64"
   }
 
+  @doc """
+  Clears the cached release info so the next call to `get_latest_release/0`
+  fetches fresh data from GitHub.
+  """
+  @spec invalidate_cache() :: :ok
+  def invalidate_cache do
+    :persistent_term.erase({__MODULE__, :release})
+    :ok
+  rescue
+    ArgumentError -> :ok
+  end
+
   @doc """
   Returns the latest release info from GitHub.
 

lib/towerops_web/controllers/api/v1/agent_release_webhook_controller.ex

@@ -0,0 +1,29 @@
+defmodule ToweropsWeb.Api.V1.AgentReleaseWebhookController do
+  @moduledoc """
+  Webhook endpoint for CI to trigger mass agent updates.
+
+  Called by CI after publishing a new GitHub Release. Broadcasts an update
+  command to all connected agents via their existing WebSocket channels.
+  """
+
+  use ToweropsWeb, :controller
+
+  alias Towerops.Agents
+
+  def create(conn, _params) do
+    case Agents.broadcast_mass_update() do
+      {:ok, result} ->
+        json(conn, %{
+          status: "ok",
+          notified: result.notified,
+          skipped: result.skipped,
+          version: result.version
+        })
+
+      {:error, reason} ->
+        conn
+        |> put_status(:bad_gateway)
+        |> json(%{status: "error", error: inspect(reason)})
+    end
+  end
+end

lib/towerops_web/plugs/webhook_auth.ex

@@ -0,0 +1,43 @@
+defmodule ToweropsWeb.Plugs.WebhookAuth do
+  @moduledoc """
+  Plug for authenticating webhook requests using a shared secret.
+
+  Expects the Authorization header to contain a Bearer token matching
+  the configured `agent_webhook_secret`.
+
+  Returns 401 Unauthorized if:
+  - No Authorization header is present
+  - Token does not match the configured secret
+  """
+
+  import Phoenix.Controller, only: [json: 2]
+  import Plug.Conn
+
+  def init(opts), do: opts
+
+  def call(conn, _opts) do
+    case get_req_header(conn, "authorization") do
+      ["Bearer " <> token] ->
+        verify_secret(conn, token)
+
+      _ ->
+        conn
+        |> put_status(:unauthorized)
+        |> json(%{error: "Missing or invalid Authorization header"})
+        |> halt()
+    end
+  end
+
+  defp verify_secret(conn, token) do
+    secret = Application.get_env(:towerops, :agent_webhook_secret)
+
+    if Plug.Crypto.secure_compare(token, secret) do
+      conn
+    else
+      conn
+      |> put_status(:unauthorized)
+      |> json(%{error: "Invalid webhook secret"})
+      |> halt()
+    end
+  end
+end

lib/towerops_web/router.ex

@@ -124,6 +124,18 @@ defmodule ToweropsWeb.Router do
     resources "/devices", DevicesController, except: [:new, :edit]
   end
 
+  # Webhook routes (shared secret authentication)
+  pipeline :webhook do
+    plug :accepts, ["json"]
+    plug ToweropsWeb.Plugs.WebhookAuth
+  end
+
+  scope "/api/v1/webhooks", V1 do
+    pipe_through :webhook
+
+    post "/agent-release", AgentReleaseWebhookController, :create
+  end
+
   # Admin API routes (requires superuser API token)
   scope "/admin/api", V1 do
     pipe_through :api_v1

test/towerops/agents/release_checker_test.exs

@@ -50,4 +50,30 @@ defmodule Towerops.Agents.ReleaseCheckerTest do
       assert {:error, :no_matching_asset} = ReleaseChecker.asset_for_arch(assets, "riscv64")
     end
   end
+
+  describe "invalidate_cache/0" do
+    test "returns :ok when no cache exists" do
+      # Ensure cache is clear
+      try do
+        :persistent_term.erase({ReleaseChecker, :release})
+      rescue
+        ArgumentError -> :ok
+      end
+
+      assert :ok = ReleaseChecker.invalidate_cache()
+    end
+
+    test "clears existing cache" do
+      # Seed the cache
+      :persistent_term.put(
+        {ReleaseChecker, :release},
+        {%{version: "1.0.0", assets: []}, System.monotonic_time(:millisecond)}
+      )
+
+      assert :ok = ReleaseChecker.invalidate_cache()
+
+      # Verify cache is gone
+      assert :persistent_term.get({ReleaseChecker, :release}, nil) == nil
+    end
+  end
 end

test/towerops/agents_test.exs

@@ -1814,6 +1814,57 @@ defmodule Towerops.AgentsTest do
     end
   end
 
+  describe "list_updatable_agents/0" do
+    test "returns agents with recent last_seen_at", %{organization: org} do
+      {:ok, agent, _token} = Agents.create_agent_token(org.id, "Active Agent")
+
+      # Set last_seen_at to now
+      Agents.update_agent_token_heartbeat(agent.id, "192.168.1.1", %{"version" => "0.1.0"})
+
+      agents = Agents.list_updatable_agents()
+      assert Enum.any?(agents, &(&1.id == agent.id))
+    end
+
+    test "excludes agents not seen in over 10 minutes", %{organization: org} do
+      {:ok, agent, _token} = Agents.create_agent_token(org.id, "Stale Agent")
+
+      # Set last_seen_at to 15 minutes ago
+      stale_time = DateTime.add(DateTime.utc_now(), -15, :minute)
+
+      Repo.update_all(
+        from(t in AgentToken, where: t.id == ^agent.id),
+        set: [last_seen_at: stale_time]
+      )
+
+      agents = Agents.list_updatable_agents()
+      refute Enum.any?(agents, &(&1.id == agent.id))
+    end
+
+    test "excludes disabled agents", %{organization: org} do
+      {:ok, agent, _token} = Agents.create_agent_token(org.id, "Disabled Agent")
+      Agents.update_agent_token_heartbeat(agent.id, "192.168.1.1", %{"version" => "0.1.0"})
+      {:ok, _} = Agents.revoke_agent_token(agent.id)
+
+      agents = Agents.list_updatable_agents()
+      refute Enum.any?(agents, &(&1.id == agent.id))
+    end
+
+    test "includes cloud pollers", %{} do
+      {:ok, cloud_poller, _token} = Agents.create_cloud_poller("Cloud Poller")
+      Agents.update_agent_token_heartbeat(cloud_poller.id, "10.0.0.1", %{"version" => "0.1.0"})
+
+      agents = Agents.list_updatable_agents()
+      assert Enum.any?(agents, &(&1.id == cloud_poller.id))
+    end
+
+    test "excludes agents with nil last_seen_at", %{organization: org} do
+      {:ok, agent, _token} = Agents.create_agent_token(org.id, "Never Seen Agent")
+
+      agents = Agents.list_updatable_agents()
+      refute Enum.any?(agents, &(&1.id == agent.id))
+    end
+  end
+
   describe "restart_agent/1" do
     test "broadcasts restart_requested on lifecycle topic", %{organization: org} do
       {:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Test Agent")
@@ -1838,4 +1889,11 @@ defmodule Towerops.AgentsTest do
       assert_receive {:update_requested, ^url, ^checksum}
     end
   end
+
+  describe "broadcast_mass_update/0" do
+    test "returns error when GitHub API is unavailable" do
+      # In test environment, GitHub returns 404 for the private repo
+      assert {:error, _reason} = Agents.broadcast_mass_update()
+    end
+  end
 end

test/towerops_web/controllers/api/v1/agent_release_webhook_controller_test.exs

@@ -0,0 +1,44 @@
+defmodule ToweropsWeb.Api.V1.AgentReleaseWebhookControllerTest do
+  use ToweropsWeb.ConnCase, async: true
+
+  setup do
+    conn =
+      build_conn()
+      |> put_req_header("authorization", "Bearer test-webhook-secret")
+      |> put_req_header("accept", "application/json")
+
+    %{conn: conn}
+  end
+
+  describe "create/2 authentication" do
+    test "returns 401 without authorization" do
+      conn =
+        build_conn()
+        |> put_req_header("accept", "application/json")
+        |> post(~p"/api/v1/webhooks/agent-release")
+
+      assert json_response(conn, 401)["error"] == "Missing or invalid Authorization header"
+    end
+
+    test "returns 401 with wrong secret" do
+      conn =
+        build_conn()
+        |> put_req_header("authorization", "Bearer wrong-secret")
+        |> put_req_header("accept", "application/json")
+        |> post(~p"/api/v1/webhooks/agent-release")
+
+      assert json_response(conn, 401)["error"] == "Invalid webhook secret"
+    end
+  end
+
+  describe "create/2 response" do
+    test "returns 502 when GitHub API is unavailable", %{conn: conn} do
+      # In test environment, GitHub API returns 404 (private repo, no auth token).
+      # The controller should surface this as a 502.
+      conn = post(conn, ~p"/api/v1/webhooks/agent-release")
+
+      response = json_response(conn, 502)
+      assert response["status"] == "error"
+    end
+  end
+end

test/towerops_web/plugs/webhook_auth_test.exs

@@ -0,0 +1,67 @@
+defmodule ToweropsWeb.Plugs.WebhookAuthTest do
+  use ToweropsWeb.ConnCase, async: true
+
+  alias ToweropsWeb.Plugs.WebhookAuth
+
+  describe "call/2 with valid authorization" do
+    test "passes through with valid Bearer token", %{conn: conn} do
+      conn =
+        conn
+        |> put_req_header("authorization", "Bearer test-webhook-secret")
+        |> WebhookAuth.call([])
+
+      refute conn.halted
+    end
+  end
+
+  describe "call/2 with missing authorization" do
+    test "returns 401 when Authorization header is missing", %{conn: conn} do
+      conn = WebhookAuth.call(conn, [])
+
+      assert conn.halted
+      assert conn.status == 401
+      assert json_response(conn, 401) == %{"error" => "Missing or invalid Authorization header"}
+    end
+
+    test "returns 401 when Authorization header is empty", %{conn: conn} do
+      conn =
+        conn
+        |> put_req_header("authorization", "")
+        |> WebhookAuth.call([])
+
+      assert conn.halted
+      assert conn.status == 401
+      assert json_response(conn, 401) == %{"error" => "Missing or invalid Authorization header"}
+    end
+  end
+
+  describe "call/2 with invalid authorization" do
+    test "returns 401 when token does not match", %{conn: conn} do
+      conn =
+        conn
+        |> put_req_header("authorization", "Bearer wrong-secret")
+        |> WebhookAuth.call([])
+
+      assert conn.halted
+      assert conn.status == 401
+      assert json_response(conn, 401) == %{"error" => "Invalid webhook secret"}
+    end
+
+    test "returns 401 when using Basic auth instead of Bearer", %{conn: conn} do
+      conn =
+        conn
+        |> put_req_header("authorization", "Basic dGVzdDp0ZXN0")
+        |> WebhookAuth.call([])
+
+      assert conn.halted
+      assert conn.status == 401
+      assert json_response(conn, 401) == %{"error" => "Missing or invalid Authorization header"}
+    end
+  end
+
+  describe "init/1" do
+    test "returns options unchanged" do
+      assert WebhookAuth.init([]) == []
+    end
+  end
+end