Add agent restart and self-update commands
Add server-initiated restart and self-update capabilities for remote agents. Restart broadcasts on the lifecycle PubSub topic causing the agent to exit cleanly for Docker to restart. Self-update fetches the latest release from GitHub, finds the matching binary for the agent's architecture, and sends the download URL with SHA256 checksum. - Add restart_agent/1 and update_agent/3 to Agents context - Handle :restart_requested and :update_requested in AgentChannel - Add superuser-only Restart and Update buttons on agent show page - Add ReleaseChecker module for GitHub Releases API with 5-min cache - Add arch field to AgentHeartbeat protobuf for architecture detection - Store arch in agent metadata from heartbeat
This commit is contained in:
parent
c066399e18
commit
fbce65070a
9 changed files with 371 additions and 1 deletions
|
|
@ -258,6 +258,37 @@ defmodule Towerops.Agents do
|
|||
result
|
||||
end
|
||||
|
||||
@doc """
|
||||
Requests an agent to restart by broadcasting on its lifecycle topic.
|
||||
|
||||
The agent channel picks this up and pushes a "restart" event to the agent,
|
||||
which exits cleanly so Docker can restart it.
|
||||
"""
|
||||
@spec restart_agent(binary()) :: :ok | {:error, term()}
|
||||
def restart_agent(agent_token_id) do
|
||||
Phoenix.PubSub.broadcast(
|
||||
Towerops.PubSub,
|
||||
"agent:#{agent_token_id}:lifecycle",
|
||||
:restart_requested
|
||||
)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Requests an agent to self-update by broadcasting the download URL and checksum
|
||||
on its lifecycle topic.
|
||||
|
||||
The agent channel picks this up and pushes an "update" event to the agent,
|
||||
which downloads the binary, verifies the checksum, and replaces itself.
|
||||
"""
|
||||
@spec update_agent(binary(), String.t(), String.t()) :: :ok | {:error, term()}
|
||||
def update_agent(agent_token_id, url, checksum) do
|
||||
Phoenix.PubSub.broadcast(
|
||||
Towerops.PubSub,
|
||||
"agent:#{agent_token_id}:lifecycle",
|
||||
{:update_requested, url, checksum}
|
||||
)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Toggles remote debugging for an agent.
|
||||
|
||||
|
|
|
|||
120
lib/towerops/agents/release_checker.ex
Normal file
120
lib/towerops/agents/release_checker.ex
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
defmodule Towerops.Agents.ReleaseChecker do
|
||||
@moduledoc """
|
||||
Fetches latest release info from GitHub Releases API for the towerops-agent repo.
|
||||
|
||||
Caches results for 5 minutes to avoid excessive API calls.
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
@github_repo "towerops-app/towerops-agent"
|
||||
@cache_ttl_ms to_timeout(minute: 5)
|
||||
|
||||
@arch_map %{
|
||||
"x86_64" => "amd64",
|
||||
"aarch64" => "arm64"
|
||||
}
|
||||
|
||||
@doc """
|
||||
Returns the latest release info from GitHub.
|
||||
|
||||
Returns `{:ok, %{version: String.t(), assets: [asset]}}` where each asset
|
||||
has `name`, `url`, and `checksum` fields.
|
||||
|
||||
Results are cached for 5 minutes.
|
||||
"""
|
||||
@spec get_latest_release() :: {:ok, map()} | {:error, term()}
|
||||
def get_latest_release do
|
||||
case get_cached() do
|
||||
{:ok, release} ->
|
||||
{:ok, release}
|
||||
|
||||
:miss ->
|
||||
fetch_and_cache()
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Finds the asset matching the given architecture.
|
||||
|
||||
Maps Rust architecture strings (x86_64, aarch64) to release asset names (amd64, arm64).
|
||||
"""
|
||||
@spec asset_for_arch([map()], String.t()) :: {:ok, map()} | {:error, :no_matching_asset}
|
||||
def asset_for_arch(assets, arch) do
|
||||
target = Map.get(@arch_map, arch, arch)
|
||||
|
||||
case Enum.find(assets, fn asset -> String.contains?(asset.name, target) end) do
|
||||
nil -> {:error, :no_matching_asset}
|
||||
asset -> {:ok, asset}
|
||||
end
|
||||
end
|
||||
|
||||
defp get_cached do
|
||||
case :persistent_term.get({__MODULE__, :release}, nil) do
|
||||
{release, cached_at} ->
|
||||
if System.monotonic_time(:millisecond) - cached_at < @cache_ttl_ms do
|
||||
{:ok, release}
|
||||
else
|
||||
:miss
|
||||
end
|
||||
|
||||
nil ->
|
||||
:miss
|
||||
end
|
||||
end
|
||||
|
||||
defp fetch_and_cache do
|
||||
url = "https://api.github.com/repos/#{@github_repo}/releases/latest"
|
||||
|
||||
case Req.get(url, headers: [{"accept", "application/vnd.github+json"}]) do
|
||||
{:ok, %Req.Response{status: 200, body: body}} ->
|
||||
release = parse_release(body)
|
||||
:persistent_term.put({__MODULE__, :release}, {release, System.monotonic_time(:millisecond)})
|
||||
{:ok, release}
|
||||
|
||||
{:ok, %Req.Response{status: status}} ->
|
||||
Logger.warning("GitHub API returned status #{status} for latest release")
|
||||
{:error, {:github_api, status}}
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.warning("Failed to fetch latest release from GitHub: #{inspect(reason)}")
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_release(body) do
|
||||
version = String.trim_leading(body["tag_name"], "v")
|
||||
|
||||
assets =
|
||||
(body["assets"] || [])
|
||||
|> Enum.filter(fn asset ->
|
||||
name = asset["name"]
|
||||
String.starts_with?(name, "towerops-agent-linux-") and not String.ends_with?(name, ".sha256")
|
||||
end)
|
||||
|> Enum.map(fn asset ->
|
||||
checksum = fetch_checksum(body["assets"], asset["name"])
|
||||
|
||||
%{
|
||||
name: asset["name"],
|
||||
url: asset["browser_download_url"],
|
||||
checksum: checksum
|
||||
}
|
||||
end)
|
||||
|
||||
%{version: version, assets: assets}
|
||||
end
|
||||
|
||||
defp fetch_checksum(all_assets, binary_name) do
|
||||
sha_asset = Enum.find(all_assets, fn a -> a["name"] == "#{binary_name}.sha256" end)
|
||||
|
||||
if sha_asset do
|
||||
case Req.get(sha_asset["browser_download_url"]) do
|
||||
{:ok, %Req.Response{status: 200, body: body}} ->
|
||||
body |> String.trim() |> String.split(" ") |> List.first()
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -358,6 +358,7 @@ defmodule Towerops.Agent.AgentHeartbeat do
|
|||
field :hostname, 2, type: :string
|
||||
field :uptime_seconds, 3, type: :uint64, json_name: "uptimeSeconds"
|
||||
field :ip_address, 4, type: :string, json_name: "ipAddress"
|
||||
field :arch, 5, type: :string
|
||||
end
|
||||
|
||||
defmodule Towerops.Agent.AgentError do
|
||||
|
|
|
|||
|
|
@ -170,6 +170,27 @@ defmodule ToweropsWeb.AgentChannel do
|
|||
{:stop, :token_disabled, socket}
|
||||
end
|
||||
|
||||
# Handle restart request — push restart event to agent, then stop channel
|
||||
def handle_info(:restart_requested, socket) do
|
||||
Logger.info("Restart requested, notifying agent",
|
||||
agent_token_id: socket.assigns.agent_token_id
|
||||
)
|
||||
|
||||
push(socket, "restart", %{})
|
||||
{:stop, :restart_requested, socket}
|
||||
end
|
||||
|
||||
# Handle update request — push update event with download URL and checksum to agent
|
||||
def handle_info({:update_requested, url, checksum}, socket) do
|
||||
Logger.info("Update requested, notifying agent",
|
||||
agent_token_id: socket.assigns.agent_token_id,
|
||||
url: url
|
||||
)
|
||||
|
||||
push(socket, "update", %{url: url, checksum: checksum})
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
||||
# Handle PubSub broadcast when device assignments change.
|
||||
# Debounces rapid changes — cancels any pending refresh and schedules a new one in 500ms.
|
||||
def handle_info({:assignments_changed, _event}, socket) do
|
||||
|
|
@ -359,7 +380,8 @@ defmodule ToweropsWeb.AgentChannel do
|
|||
{:ok, heartbeat} <- Validator.validate_heartbeat(binary) do
|
||||
metadata = %{
|
||||
"version" => heartbeat.version,
|
||||
"uptime_seconds" => heartbeat.uptime_seconds
|
||||
"uptime_seconds" => heartbeat.uptime_seconds,
|
||||
"arch" => heartbeat.arch
|
||||
}
|
||||
|
||||
_ =
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ defmodule ToweropsWeb.AgentLive.Show do
|
|||
import ToweropsWeb.AgentLive.Helpers
|
||||
|
||||
alias Towerops.Agents
|
||||
alias Towerops.Agents.ReleaseChecker
|
||||
|
||||
@impl true
|
||||
def mount(%{"id" => id}, _session, socket) do
|
||||
|
|
@ -47,6 +48,41 @@ defmodule ToweropsWeb.AgentLive.Show do
|
|||
{:noreply, assign(socket, :active_tab, tab)}
|
||||
end
|
||||
|
||||
def handle_event("restart_agent", _params, socket) do
|
||||
if socket.assigns.current_scope.user.is_superuser do
|
||||
Agents.restart_agent(socket.assigns.agent_token.id)
|
||||
|
||||
{:noreply, put_flash(socket, :info, "Restart command sent to agent")}
|
||||
else
|
||||
{:noreply, socket}
|
||||
end
|
||||
end
|
||||
|
||||
def handle_event("update_agent", _params, socket) do
|
||||
if socket.assigns.current_scope.user.is_superuser do
|
||||
agent_token = socket.assigns.agent_token
|
||||
arch = get_in(agent_token.metadata, ["arch"]) || "x86_64"
|
||||
|
||||
{:noreply, send_update_command(socket, agent_token, arch)}
|
||||
else
|
||||
{:noreply, socket}
|
||||
end
|
||||
end
|
||||
|
||||
defp send_update_command(socket, agent_token, arch) do
|
||||
with {:ok, release} <- ReleaseChecker.get_latest_release(),
|
||||
{:ok, asset} <- ReleaseChecker.asset_for_arch(release.assets, arch) do
|
||||
Agents.update_agent(agent_token.id, asset.url, asset.checksum)
|
||||
put_flash(socket, :info, "Update command sent to agent (v#{release.version})")
|
||||
else
|
||||
{:error, :no_matching_asset} ->
|
||||
put_flash(socket, :error, "No binary available for architecture: #{arch}")
|
||||
|
||||
{:error, _reason} ->
|
||||
put_flash(socket, :error, "Failed to fetch latest release from GitHub")
|
||||
end
|
||||
end
|
||||
|
||||
defp assignment_source(device, agent_token_id) do
|
||||
cond do
|
||||
directly_assigned?(device, agent_token_id) ->
|
||||
|
|
|
|||
|
|
@ -7,6 +7,22 @@
|
|||
{@page_title}
|
||||
<:subtitle>Agent Details and Health</:subtitle>
|
||||
<:actions>
|
||||
<button
|
||||
:if={@current_scope.user.is_superuser}
|
||||
phx-click="update_agent"
|
||||
data-confirm="Are you sure you want to update this agent? It will download and replace the binary."
|
||||
class="inline-flex items-center gap-2 rounded-lg px-3.5 py-2.5 text-sm font-semibold shadow-sm focus:outline-none focus:ring-2 focus:ring-offset-2 bg-green-600 text-white hover:bg-green-700 focus:ring-green-500 dark:bg-green-500 dark:hover:bg-green-600"
|
||||
>
|
||||
<.icon name="hero-arrow-up-tray" class="h-5 w-5" /> Update
|
||||
</button>
|
||||
<button
|
||||
:if={@current_scope.user.is_superuser}
|
||||
phx-click="restart_agent"
|
||||
data-confirm="Are you sure you want to restart this agent? It will disconnect and restart its container."
|
||||
class="inline-flex items-center gap-2 rounded-lg px-3.5 py-2.5 text-sm font-semibold shadow-sm focus:outline-none focus:ring-2 focus:ring-offset-2 bg-amber-600 text-white hover:bg-amber-700 focus:ring-amber-500 dark:bg-amber-500 dark:hover:bg-amber-600"
|
||||
>
|
||||
<.icon name="hero-arrow-path" class="h-5 w-5" /> Restart
|
||||
</button>
|
||||
<.link
|
||||
navigate={~p"/agents/#{@agent_token.id}/edit"}
|
||||
class="inline-flex items-center gap-2 rounded-lg px-3.5 py-2.5 text-sm font-semibold shadow-sm focus:outline-none focus:ring-2 focus:ring-offset-2 disabled:opacity-50 disabled:cursor-not-allowed bg-blue-600 text-white hover:bg-blue-700 focus:ring-blue-500 dark:bg-blue-500 dark:hover:bg-blue-600"
|
||||
|
|
|
|||
53
test/towerops/agents/release_checker_test.exs
Normal file
53
test/towerops/agents/release_checker_test.exs
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
defmodule Towerops.Agents.ReleaseCheckerTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Towerops.Agents.ReleaseChecker
|
||||
|
||||
describe "get_latest_release/0" do
|
||||
@tag :integration
|
||||
test "returns release info with version and assets" do
|
||||
{:ok, release} = ReleaseChecker.get_latest_release()
|
||||
|
||||
assert is_binary(release.version)
|
||||
assert is_list(release.assets)
|
||||
|
||||
assert Enum.any?(release.assets, fn asset ->
|
||||
String.contains?(asset.name, "amd64")
|
||||
end)
|
||||
|
||||
assert Enum.any?(release.assets, fn asset ->
|
||||
String.contains?(asset.name, "arm64")
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
||||
describe "asset_for_arch/2" do
|
||||
test "returns matching asset for x86_64 arch" do
|
||||
assets = [
|
||||
%{name: "towerops-agent-linux-amd64", url: "https://example.com/amd64", checksum: "abc"},
|
||||
%{name: "towerops-agent-linux-arm64", url: "https://example.com/arm64", checksum: "def"}
|
||||
]
|
||||
|
||||
assert {:ok, asset} = ReleaseChecker.asset_for_arch(assets, "x86_64")
|
||||
assert asset.name == "towerops-agent-linux-amd64"
|
||||
end
|
||||
|
||||
test "returns matching asset for aarch64 arch" do
|
||||
assets = [
|
||||
%{name: "towerops-agent-linux-amd64", url: "https://example.com/amd64", checksum: "abc"},
|
||||
%{name: "towerops-agent-linux-arm64", url: "https://example.com/arm64", checksum: "def"}
|
||||
]
|
||||
|
||||
assert {:ok, asset} = ReleaseChecker.asset_for_arch(assets, "aarch64")
|
||||
assert asset.name == "towerops-agent-linux-arm64"
|
||||
end
|
||||
|
||||
test "returns error for unknown arch" do
|
||||
assets = [
|
||||
%{name: "towerops-agent-linux-amd64", url: "https://example.com/amd64", checksum: "abc"}
|
||||
]
|
||||
|
||||
assert {:error, :no_matching_asset} = ReleaseChecker.asset_for_arch(assets, "riscv64")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -1813,4 +1813,29 @@ defmodule Towerops.AgentsTest do
|
|||
assert Agents.should_phoenix_poll_device?(device) == true
|
||||
end
|
||||
end
|
||||
|
||||
describe "restart_agent/1" do
|
||||
test "broadcasts restart_requested on lifecycle topic", %{organization: org} do
|
||||
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Test Agent")
|
||||
Phoenix.PubSub.subscribe(Towerops.PubSub, "agent:#{agent_token.id}:lifecycle")
|
||||
|
||||
assert :ok = Agents.restart_agent(agent_token.id)
|
||||
|
||||
assert_receive :restart_requested
|
||||
end
|
||||
end
|
||||
|
||||
describe "update_agent/3" do
|
||||
test "broadcasts update_requested with url and checksum on lifecycle topic", %{organization: org} do
|
||||
{:ok, agent_token, _token} = Agents.create_agent_token(org.id, "Test Agent")
|
||||
Phoenix.PubSub.subscribe(Towerops.PubSub, "agent:#{agent_token.id}:lifecycle")
|
||||
|
||||
url = "https://github.com/towerops-app/towerops-agent/releases/download/v0.2.0/towerops-agent-linux-amd64"
|
||||
checksum = "abc123def456"
|
||||
|
||||
assert :ok = Agents.update_agent(agent_token.id, url, checksum)
|
||||
|
||||
assert_receive {:update_requested, ^url, ^checksum}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -524,6 +524,72 @@ defmodule ToweropsWeb.AgentLiveTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "Show - Restart Agent" do
|
||||
test "superuser sees restart button and can restart agent", %{conn: conn, organization: organization, user: user} do
|
||||
Towerops.Repo.update!(Ecto.Changeset.change(user, %{is_superuser: true}))
|
||||
{:ok, agent_token, _token} = Agents.create_agent_token(organization.id, "Test Agent")
|
||||
|
||||
{:ok, view, html} = live(conn, ~p"/agents/#{agent_token.id}")
|
||||
|
||||
assert html =~ "Restart"
|
||||
|
||||
# Subscribe to lifecycle topic to verify broadcast
|
||||
Phoenix.PubSub.subscribe(Towerops.PubSub, "agent:#{agent_token.id}:lifecycle")
|
||||
|
||||
view
|
||||
|> element("button", "Restart")
|
||||
|> render_click()
|
||||
|
||||
assert_receive :restart_requested
|
||||
assert render(view) =~ "Restart command sent"
|
||||
end
|
||||
|
||||
test "regular user does not see restart button", %{conn: conn, organization: organization} do
|
||||
{:ok, agent_token, _token} = Agents.create_agent_token(organization.id, "Test Agent")
|
||||
|
||||
{:ok, _view, html} = live(conn, ~p"/agents/#{agent_token.id}")
|
||||
|
||||
refute html =~ "Restart"
|
||||
end
|
||||
end
|
||||
|
||||
describe "Show - Update Agent" do
|
||||
test "superuser sees update button", %{conn: conn, organization: organization, user: user} do
|
||||
Towerops.Repo.update!(Ecto.Changeset.change(user, %{is_superuser: true}))
|
||||
{:ok, agent_token, _token} = Agents.create_agent_token(organization.id, "Test Agent")
|
||||
|
||||
{:ok, view, _html} = live(conn, ~p"/agents/#{agent_token.id}")
|
||||
|
||||
assert has_element?(view, "button", "Update")
|
||||
end
|
||||
|
||||
test "superuser clicking update shows error when no releases available", %{
|
||||
conn: conn,
|
||||
organization: organization,
|
||||
user: user
|
||||
} do
|
||||
Towerops.Repo.update!(Ecto.Changeset.change(user, %{is_superuser: true}))
|
||||
{:ok, agent_token, _token} = Agents.create_agent_token(organization.id, "Test Agent")
|
||||
|
||||
{:ok, view, _html} = live(conn, ~p"/agents/#{agent_token.id}")
|
||||
|
||||
html =
|
||||
view
|
||||
|> element("button", "Update")
|
||||
|> render_click()
|
||||
|
||||
assert html =~ "Failed to fetch latest release"
|
||||
end
|
||||
|
||||
test "regular user does not see update button", %{conn: conn, organization: organization} do
|
||||
{:ok, agent_token, _token} = Agents.create_agent_token(organization.id, "Test Agent")
|
||||
|
||||
{:ok, view, _html} = live(conn, ~p"/agents/#{agent_token.id}")
|
||||
|
||||
refute has_element?(view, "button", "Update")
|
||||
end
|
||||
end
|
||||
|
||||
describe "Show - Regular User Cross-Org Access" do
|
||||
test "regular user cannot view agent from another organization", %{conn: conn, organization: organization, user: user} do
|
||||
{:ok, other_org} = Towerops.Organizations.create_organization(%{name: "Other Org"}, user.id, bypass_limits: true)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue