From c7df6a85694098ee009975d04555b4aec7f71415 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 10 Feb 2026 13:40:32 -0600 Subject: [PATCH] Add CI-triggered mass agent update webhook POST /api/v1/webhooks/agent-release triggers all connected agents to self-update via their existing WebSocket channels. Authenticated with a shared secret (AGENT_WEBHOOK_SECRET env var). - Add ReleaseChecker.invalidate_cache/0 for fresh GitHub fetch - Add Agents.list_updatable_agents/0 (enabled + seen < 10min) - Add Agents.broadcast_mass_update/0 orchestration function - Add WebhookAuth plug with timing-safe secret comparison - Add AgentReleaseWebhookController with :webhook pipeline - Configure AGENT_WEBHOOK_SECRET in dev/test/runtime configs --- config/dev.exs | 3 + config/runtime.exs | 2 + config/test.exs | 3 + lib/towerops/agents.ex | 61 +++++++++++++++++ lib/towerops/agents/release_checker.ex | 12 ++++ .../v1/agent_release_webhook_controller.ex | 29 ++++++++ lib/towerops_web/plugs/webhook_auth.ex | 43 ++++++++++++ lib/towerops_web/router.ex | 12 ++++ test/towerops/agents/release_checker_test.exs | 26 +++++++ test/towerops/agents_test.exs | 58 ++++++++++++++++ .../agent_release_webhook_controller_test.exs | 44 ++++++++++++ test/towerops_web/plugs/webhook_auth_test.exs | 67 +++++++++++++++++++ 12 files changed, 360 insertions(+) create mode 100644 lib/towerops_web/controllers/api/v1/agent_release_webhook_controller.ex create mode 100644 lib/towerops_web/plugs/webhook_auth.ex create mode 100644 test/towerops_web/controllers/api/v1/agent_release_webhook_controller_test.exs create mode 100644 test/towerops_web/plugs/webhook_auth_test.exs diff --git a/config/dev.exs b/config/dev.exs index a68bcd99..17f0f845 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -161,6 +161,9 @@ config :towerops, ToweropsWeb.Endpoint, ] ] +# Webhook secret for CI-triggered agent updates +config :towerops, :agent_webhook_secret, "dev-webhook-secret" + # Set environment identifier for runtime checks config :towerops, :env, :dev diff --git a/config/runtime.exs b/config/runtime.exs index c69a1bcb..5492b60c 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -288,6 +288,8 @@ if config_env() == :prod do ], secret_key_base: secret_key_base + # Webhook secret for CI-triggered mass agent updates + config :towerops, :agent_webhook_secret, System.get_env("AGENT_WEBHOOK_SECRET") config :towerops, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY") # Set environment identifier for runtime checks config :towerops, :env, :prod diff --git a/config/test.exs b/config/test.exs index f20c21e5..5a92a6a3 100644 --- a/config/test.exs +++ b/config/test.exs @@ -64,6 +64,9 @@ config :towerops, ToweropsWeb.Endpoint, secret_key_base: "XDSSqVUtRXUjEfzxefIAaPIzBfonpNMOrCRyP1a0kjPzzyOpVNSRmMBVae/bwTqj", server: false +# Webhook secret for CI-triggered agent updates +config :towerops, :agent_webhook_secret, "test-webhook-secret" + # Set environment identifier for runtime checks config :towerops, :env, :test diff --git a/lib/towerops/agents.ex b/lib/towerops/agents.ex index 098827c1..8e2828e8 100644 --- a/lib/towerops/agents.ex +++ b/lib/towerops/agents.ex @@ -10,9 +10,12 @@ defmodule Towerops.Agents do alias Towerops.Agents.AgentAssignment alias Towerops.Agents.AgentToken + alias Towerops.Agents.ReleaseChecker alias Towerops.Devices.Device alias Towerops.Repo + require Logger + ## Token management @doc """ @@ -289,6 +292,64 @@ defmodule Towerops.Agents do ) end + @doc """ + Returns all agent tokens eligible for mass update. + + An agent is updatable if it is enabled and was last seen within the past 10 minutes. + Includes both organization agents and cloud pollers. + """ + @spec list_updatable_agents() :: [AgentToken.t()] + def list_updatable_agents do + cutoff = DateTime.add(DateTime.utc_now(), -10, :minute) + + AgentToken + |> where([t], t.enabled == true and not is_nil(t.last_seen_at) and t.last_seen_at > ^cutoff) + |> Repo.all() + end + + @doc """ + Broadcasts an update command to all connected agents. + + 1. Invalidates the release cache + 2. Fetches the latest release from GitHub + 3. Finds all updatable agents + 4. Sends update command to each agent matching their architecture + + Returns `{:ok, %{notified: N, skipped: N, version: String.t()}}` on success. + """ + @spec broadcast_mass_update() :: {:ok, map()} | {:error, term()} + def broadcast_mass_update do + ReleaseChecker.invalidate_cache() + + case ReleaseChecker.get_latest_release() do + {:ok, %{version: version, assets: assets}} -> + agents = list_updatable_agents() + {notified, skipped} = notify_agents(agents, assets) + + {:ok, %{notified: notified, skipped: skipped, version: version}} + + {:error, reason} -> + Logger.error("Failed to fetch latest release for mass update: #{inspect(reason)}") + {:error, reason} + end + end + + defp notify_agents(agents, assets) do + Enum.reduce(agents, {0, 0}, fn agent, {notified, skipped} -> + arch = get_in(agent.metadata, ["arch"]) || "x86_64" + + case ReleaseChecker.asset_for_arch(assets, arch) do + {:ok, asset} -> + update_agent(agent.id, asset.url, asset.checksum) + {notified + 1, skipped} + + {:error, :no_matching_asset} -> + Logger.warning("No matching asset for agent #{agent.id} (arch: #{arch})") + {notified, skipped + 1} + end + end) + end + @doc """ Toggles remote debugging for an agent. diff --git a/lib/towerops/agents/release_checker.ex b/lib/towerops/agents/release_checker.ex index 505f5ba5..3e438283 100644 --- a/lib/towerops/agents/release_checker.ex +++ b/lib/towerops/agents/release_checker.ex @@ -15,6 +15,18 @@ defmodule Towerops.Agents.ReleaseChecker do "aarch64" => "arm64" } + @doc """ + Clears the cached release info so the next call to `get_latest_release/0` + fetches fresh data from GitHub. + """ + @spec invalidate_cache() :: :ok + def invalidate_cache do + :persistent_term.erase({__MODULE__, :release}) + :ok + rescue + ArgumentError -> :ok + end + @doc """ Returns the latest release info from GitHub. diff --git a/lib/towerops_web/controllers/api/v1/agent_release_webhook_controller.ex b/lib/towerops_web/controllers/api/v1/agent_release_webhook_controller.ex new file mode 100644 index 00000000..04203001 --- /dev/null +++ b/lib/towerops_web/controllers/api/v1/agent_release_webhook_controller.ex @@ -0,0 +1,29 @@ +defmodule ToweropsWeb.Api.V1.AgentReleaseWebhookController do + @moduledoc """ + Webhook endpoint for CI to trigger mass agent updates. + + Called by CI after publishing a new GitHub Release. Broadcasts an update + command to all connected agents via their existing WebSocket channels. + """ + + use ToweropsWeb, :controller + + alias Towerops.Agents + + def create(conn, _params) do + case Agents.broadcast_mass_update() do + {:ok, result} -> + json(conn, %{ + status: "ok", + notified: result.notified, + skipped: result.skipped, + version: result.version + }) + + {:error, reason} -> + conn + |> put_status(:bad_gateway) + |> json(%{status: "error", error: inspect(reason)}) + end + end +end diff --git a/lib/towerops_web/plugs/webhook_auth.ex b/lib/towerops_web/plugs/webhook_auth.ex new file mode 100644 index 00000000..74ba242a --- /dev/null +++ b/lib/towerops_web/plugs/webhook_auth.ex @@ -0,0 +1,43 @@ +defmodule ToweropsWeb.Plugs.WebhookAuth do + @moduledoc """ + Plug for authenticating webhook requests using a shared secret. + + Expects the Authorization header to contain a Bearer token matching + the configured `agent_webhook_secret`. + + Returns 401 Unauthorized if: + - No Authorization header is present + - Token does not match the configured secret + """ + + import Phoenix.Controller, only: [json: 2] + import Plug.Conn + + def init(opts), do: opts + + def call(conn, _opts) do + case get_req_header(conn, "authorization") do + ["Bearer " <> token] -> + verify_secret(conn, token) + + _ -> + conn + |> put_status(:unauthorized) + |> json(%{error: "Missing or invalid Authorization header"}) + |> halt() + end + end + + defp verify_secret(conn, token) do + secret = Application.get_env(:towerops, :agent_webhook_secret) + + if Plug.Crypto.secure_compare(token, secret) do + conn + else + conn + |> put_status(:unauthorized) + |> json(%{error: "Invalid webhook secret"}) + |> halt() + end + end +end diff --git a/lib/towerops_web/router.ex b/lib/towerops_web/router.ex index 913310d7..cf9411fd 100644 --- a/lib/towerops_web/router.ex +++ b/lib/towerops_web/router.ex @@ -124,6 +124,18 @@ defmodule ToweropsWeb.Router do resources "/devices", DevicesController, except: [:new, :edit] end + # Webhook routes (shared secret authentication) + pipeline :webhook do + plug :accepts, ["json"] + plug ToweropsWeb.Plugs.WebhookAuth + end + + scope "/api/v1/webhooks", V1 do + pipe_through :webhook + + post "/agent-release", AgentReleaseWebhookController, :create + end + # Admin API routes (requires superuser API token) scope "/admin/api", V1 do pipe_through :api_v1 diff --git a/test/towerops/agents/release_checker_test.exs b/test/towerops/agents/release_checker_test.exs index 637b4602..51090717 100644 --- a/test/towerops/agents/release_checker_test.exs +++ b/test/towerops/agents/release_checker_test.exs @@ -50,4 +50,30 @@ defmodule Towerops.Agents.ReleaseCheckerTest do assert {:error, :no_matching_asset} = ReleaseChecker.asset_for_arch(assets, "riscv64") end end + + describe "invalidate_cache/0" do + test "returns :ok when no cache exists" do + # Ensure cache is clear + try do + :persistent_term.erase({ReleaseChecker, :release}) + rescue + ArgumentError -> :ok + end + + assert :ok = ReleaseChecker.invalidate_cache() + end + + test "clears existing cache" do + # Seed the cache + :persistent_term.put( + {ReleaseChecker, :release}, + {%{version: "1.0.0", assets: []}, System.monotonic_time(:millisecond)} + ) + + assert :ok = ReleaseChecker.invalidate_cache() + + # Verify cache is gone + assert :persistent_term.get({ReleaseChecker, :release}, nil) == nil + end + end end diff --git a/test/towerops/agents_test.exs b/test/towerops/agents_test.exs index f48334ea..c57b5277 100644 --- a/test/towerops/agents_test.exs +++ b/test/towerops/agents_test.exs @@ -1814,6 +1814,57 @@ defmodule Towerops.AgentsTest do end end + describe "list_updatable_agents/0" do + test "returns agents with recent last_seen_at", %{organization: org} do + {:ok, agent, _token} = Agents.create_agent_token(org.id, "Active Agent") + + # Set last_seen_at to now + Agents.update_agent_token_heartbeat(agent.id, "192.168.1.1", %{"version" => "0.1.0"}) + + agents = Agents.list_updatable_agents() + assert Enum.any?(agents, &(&1.id == agent.id)) + end + + test "excludes agents not seen in over 10 minutes", %{organization: org} do + {:ok, agent, _token} = Agents.create_agent_token(org.id, "Stale Agent") + + # Set last_seen_at to 15 minutes ago + stale_time = DateTime.add(DateTime.utc_now(), -15, :minute) + + Repo.update_all( + from(t in AgentToken, where: t.id == ^agent.id), + set: [last_seen_at: stale_time] + ) + + agents = Agents.list_updatable_agents() + refute Enum.any?(agents, &(&1.id == agent.id)) + end + + test "excludes disabled agents", %{organization: org} do + {:ok, agent, _token} = Agents.create_agent_token(org.id, "Disabled Agent") + Agents.update_agent_token_heartbeat(agent.id, "192.168.1.1", %{"version" => "0.1.0"}) + {:ok, _} = Agents.revoke_agent_token(agent.id) + + agents = Agents.list_updatable_agents() + refute Enum.any?(agents, &(&1.id == agent.id)) + end + + test "includes cloud pollers", %{} do + {:ok, cloud_poller, _token} = Agents.create_cloud_poller("Cloud Poller") + Agents.update_agent_token_heartbeat(cloud_poller.id, "10.0.0.1", %{"version" => "0.1.0"}) + + agents = Agents.list_updatable_agents() + assert Enum.any?(agents, &(&1.id == cloud_poller.id)) + end + + test "excludes agents with nil last_seen_at", %{organization: org} do + {:ok, agent, _token} = Agents.create_agent_token(org.id, "Never Seen Agent") + + agents = Agents.list_updatable_agents() + refute Enum.any?(agents, &(&1.id == agent.id)) + end + end + describe "restart_agent/1" do test "broadcasts restart_requested on lifecycle topic", %{organization: org} do {:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Test Agent") @@ -1838,4 +1889,11 @@ defmodule Towerops.AgentsTest do assert_receive {:update_requested, ^url, ^checksum} end end + + describe "broadcast_mass_update/0" do + test "returns error when GitHub API is unavailable" do + # In test environment, GitHub returns 404 for the private repo + assert {:error, _reason} = Agents.broadcast_mass_update() + end + end end diff --git a/test/towerops_web/controllers/api/v1/agent_release_webhook_controller_test.exs b/test/towerops_web/controllers/api/v1/agent_release_webhook_controller_test.exs new file mode 100644 index 00000000..3e35a39e --- /dev/null +++ b/test/towerops_web/controllers/api/v1/agent_release_webhook_controller_test.exs @@ -0,0 +1,44 @@ +defmodule ToweropsWeb.Api.V1.AgentReleaseWebhookControllerTest do + use ToweropsWeb.ConnCase, async: true + + setup do + conn = + build_conn() + |> put_req_header("authorization", "Bearer test-webhook-secret") + |> put_req_header("accept", "application/json") + + %{conn: conn} + end + + describe "create/2 authentication" do + test "returns 401 without authorization" do + conn = + build_conn() + |> put_req_header("accept", "application/json") + |> post(~p"/api/v1/webhooks/agent-release") + + assert json_response(conn, 401)["error"] == "Missing or invalid Authorization header" + end + + test "returns 401 with wrong secret" do + conn = + build_conn() + |> put_req_header("authorization", "Bearer wrong-secret") + |> put_req_header("accept", "application/json") + |> post(~p"/api/v1/webhooks/agent-release") + + assert json_response(conn, 401)["error"] == "Invalid webhook secret" + end + end + + describe "create/2 response" do + test "returns 502 when GitHub API is unavailable", %{conn: conn} do + # In test environment, GitHub API returns 404 (private repo, no auth token). + # The controller should surface this as a 502. + conn = post(conn, ~p"/api/v1/webhooks/agent-release") + + response = json_response(conn, 502) + assert response["status"] == "error" + end + end +end diff --git a/test/towerops_web/plugs/webhook_auth_test.exs b/test/towerops_web/plugs/webhook_auth_test.exs new file mode 100644 index 00000000..09f3a639 --- /dev/null +++ b/test/towerops_web/plugs/webhook_auth_test.exs @@ -0,0 +1,67 @@ +defmodule ToweropsWeb.Plugs.WebhookAuthTest do + use ToweropsWeb.ConnCase, async: true + + alias ToweropsWeb.Plugs.WebhookAuth + + describe "call/2 with valid authorization" do + test "passes through with valid Bearer token", %{conn: conn} do + conn = + conn + |> put_req_header("authorization", "Bearer test-webhook-secret") + |> WebhookAuth.call([]) + + refute conn.halted + end + end + + describe "call/2 with missing authorization" do + test "returns 401 when Authorization header is missing", %{conn: conn} do + conn = WebhookAuth.call(conn, []) + + assert conn.halted + assert conn.status == 401 + assert json_response(conn, 401) == %{"error" => "Missing or invalid Authorization header"} + end + + test "returns 401 when Authorization header is empty", %{conn: conn} do + conn = + conn + |> put_req_header("authorization", "") + |> WebhookAuth.call([]) + + assert conn.halted + assert conn.status == 401 + assert json_response(conn, 401) == %{"error" => "Missing or invalid Authorization header"} + end + end + + describe "call/2 with invalid authorization" do + test "returns 401 when token does not match", %{conn: conn} do + conn = + conn + |> put_req_header("authorization", "Bearer wrong-secret") + |> WebhookAuth.call([]) + + assert conn.halted + assert conn.status == 401 + assert json_response(conn, 401) == %{"error" => "Invalid webhook secret"} + end + + test "returns 401 when using Basic auth instead of Bearer", %{conn: conn} do + conn = + conn + |> put_req_header("authorization", "Basic dGVzdDp0ZXN0") + |> WebhookAuth.call([]) + + assert conn.halted + assert conn.status == 401 + assert json_response(conn, 401) == %{"error" => "Missing or invalid Authorization header"} + end + end + + describe "init/1" do + test "returns options unchanged" do + assert WebhookAuth.init([]) == [] + end + end +end