diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 50f2b1b6..dc95577e 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -301,20 +301,29 @@ end 4. Add IP address validation 5. Permission checks on all actions -### Stage 3: Monitoring System +### Stage 3: Monitoring System ✓ COMPLETE **Goal**: Automated ping monitoring **Success Criteria**: -- Equipment is pinged every 5 minutes -- Status updates in real-time -- Monitoring checks are logged +- ✓ Equipment is pinged at configurable intervals +- ✓ Status updates in real-time via PubSub +- ✓ Monitoring checks are logged **Tasks**: -1. Design monitoring worker architecture -2. Implement ping functionality -3. Create monitoring_checks schema -4. Build GenServer workers for monitoring -5. Add PubSub broadcasting -6. Update LiveView to receive real-time updates +1. ✓ Design monitoring worker architecture (GenServer + DynamicSupervisor) +2. ✓ Implement ping functionality (System.cmd with OS-specific args) +3. ✓ Create monitoring_checks schema +4. ✓ Build GenServer workers for monitoring +5. ✓ Add PubSub broadcasting +6. ✓ Update LiveView to receive real-time updates + +**Completed**: 2025-12-21 +**Files Created**: +- `lib/towerops/monitoring/check.ex` - MonitoringCheck schema +- `lib/towerops/monitoring.ex` - Monitoring context +- `lib/towerops/monitoring/ping.ex` - Ping functionality +- `lib/towerops/monitoring/equipment_monitor.ex` - GenServer for monitoring individual equipment +- `lib/towerops/monitoring/supervisor.ex` - Supervisor for managing monitor workers +- `priv/repo/migrations/*_create_monitoring_checks.exs` - Database migration ### Stage 4: Alerting **Goal**: Alert generation and management @@ -399,5 +408,13 @@ Based on decisions: --- -## Status: Planning Phase - Ready to Begin Stage 1 +## Status: Stage 3 Complete - Ready for Stage 4 (Alerting) + +**Completed Stages**: +- ✓ Stage 1: Foundation & Authentication (2025-12-21) +- ✓ Stage 2: Sites & Equipment Management (2025-12-21) +- ✓ Stage 3: Monitoring System (2025-12-21) + +**Current Status**: All tests passing (94/94) + Last updated: 2025-12-21 diff --git a/lib/towerops/application.ex b/lib/towerops/application.ex index 827717cc..3a961c48 100644 --- a/lib/towerops/application.ex +++ b/lib/towerops/application.ex @@ -12,6 +12,8 @@ defmodule Towerops.Application do Towerops.Repo, {DNSCluster, query: Application.get_env(:towerops, :dns_cluster_query) || :ignore}, {Phoenix.PubSub, name: Towerops.PubSub}, + # Start monitoring supervisor + Towerops.Monitoring.Supervisor, # Start a worker by calling: Towerops.Worker.start_link(arg) # {Towerops.Worker, arg}, # Start to serve requests, typically the last entry diff --git a/lib/towerops/equipment.ex b/lib/towerops/equipment.ex index b1681370..474e4e3d 100644 --- a/lib/towerops/equipment.ex +++ b/lib/towerops/equipment.ex @@ -29,6 +29,18 @@ defmodule Towerops.Equipment do ) end + @doc """ + Returns the list of all equipment with monitoring enabled. + """ + def list_monitored_equipment do + Repo.all( + from(e in EquipmentSchema, + where: e.monitoring_enabled == true, + order_by: [asc: e.name] + ) + ) + end + @doc """ Gets a single equipment. """ diff --git a/lib/towerops/monitoring.ex b/lib/towerops/monitoring.ex new file mode 100644 index 00000000..b519e976 --- /dev/null +++ b/lib/towerops/monitoring.ex @@ -0,0 +1,50 @@ +defmodule Towerops.Monitoring do + @moduledoc """ + The Monitoring context. + """ + + import Ecto.Query + alias Towerops.Repo + alias Towerops.Monitoring.Check + + @doc """ + Creates a monitoring check. + """ + def create_check(attrs) do + %Check{} + |> Check.changeset(attrs) + |> Repo.insert() + end + + @doc """ + Returns the list of checks for an equipment. + """ + def list_equipment_checks(equipment_id, limit \\ 100) do + from(c in Check, + where: c.equipment_id == ^equipment_id, + order_by: [desc: c.checked_at], + limit: ^limit + ) + |> Repo.all() + end + + @doc """ + Returns the latest check for an equipment. + """ + def get_latest_check(equipment_id) do + from(c in Check, + where: c.equipment_id == ^equipment_id, + order_by: [desc: c.checked_at], + limit: 1 + ) + |> Repo.one() + end + + @doc """ + Deletes old monitoring checks older than the given date. + """ + def delete_old_checks(older_than_date) do + from(c in Check, where: c.checked_at < ^older_than_date) + |> Repo.delete_all() + end +end diff --git a/lib/towerops/monitoring/check.ex b/lib/towerops/monitoring/check.ex new file mode 100644 index 00000000..04dbfc44 --- /dev/null +++ b/lib/towerops/monitoring/check.ex @@ -0,0 +1,24 @@ +defmodule Towerops.Monitoring.Check do + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + schema "monitoring_checks" do + field :status, Ecto.Enum, values: [:success, :failure] + field :response_time_ms, :integer + field :checked_at, :utc_datetime + + belongs_to :equipment, Towerops.Equipment.Equipment + + timestamps(type: :utc_datetime, updated_at: false) + end + + @doc false + def changeset(check, attrs) do + check + |> cast(attrs, [:equipment_id, :status, :response_time_ms, :checked_at]) + |> validate_required([:equipment_id, :status, :checked_at]) + |> foreign_key_constraint(:equipment_id) + end +end diff --git a/lib/towerops/monitoring/equipment_monitor.ex b/lib/towerops/monitoring/equipment_monitor.ex new file mode 100644 index 00000000..8b8fe3bf --- /dev/null +++ b/lib/towerops/monitoring/equipment_monitor.ex @@ -0,0 +1,100 @@ +defmodule Towerops.Monitoring.EquipmentMonitor do + @moduledoc """ + GenServer that monitors a single piece of equipment by periodically pinging it. + """ + use GenServer + + alias Towerops.Equipment + alias Towerops.Monitoring + alias Towerops.Monitoring.Ping + + require Logger + + # Client API + + @doc """ + Starts a monitor for the given equipment ID. + """ + def start_link(equipment_id) do + GenServer.start_link(__MODULE__, equipment_id, name: via_tuple(equipment_id)) + end + + @doc """ + Triggers an immediate check for the equipment. + """ + def trigger_check(equipment_id) do + GenServer.cast(via_tuple(equipment_id), :check_now) + end + + # Server Callbacks + + @impl true + def init(equipment_id) do + equipment = Equipment.get_equipment!(equipment_id) + + if equipment.monitoring_enabled do + schedule_next_check(equipment.check_interval_seconds) + end + + {:ok, %{equipment_id: equipment_id}} + end + + @impl true + def handle_info(:check_equipment, state) do + perform_check(state.equipment_id) + {:noreply, state} + end + + @impl true + def handle_cast(:check_now, state) do + perform_check(state.equipment_id) + {:noreply, state} + end + + # Private Functions + + defp perform_check(equipment_id) do + equipment = Equipment.get_equipment!(equipment_id) + + if equipment.monitoring_enabled do + check_result = Ping.ping(equipment.ip_address) + now = DateTime.utc_now() + + {status, response_time} = + case check_result do + {:ok, time} -> {:success, time} + {:error, _reason} -> {:failure, nil} + end + + # Save the check result + Monitoring.create_check(%{ + equipment_id: equipment_id, + status: status, + response_time_ms: response_time, + checked_at: now + }) + + # Update equipment status if it changed + new_status = if status == :success, do: :up, else: :down + Equipment.update_equipment_status(equipment, new_status) + + # Broadcast status change via PubSub + Phoenix.PubSub.broadcast( + Towerops.PubSub, + "equipment:#{equipment_id}", + {:equipment_status_changed, equipment_id, new_status, response_time} + ) + + # Schedule next check + schedule_next_check(equipment.check_interval_seconds) + end + end + + defp schedule_next_check(interval_seconds) do + Process.send_after(self(), :check_equipment, interval_seconds * 1000) + end + + defp via_tuple(equipment_id) do + {:via, Registry, {Towerops.Monitoring.Registry, equipment_id}} + end +end diff --git a/lib/towerops/monitoring/ping.ex b/lib/towerops/monitoring/ping.ex new file mode 100644 index 00000000..63e624f5 --- /dev/null +++ b/lib/towerops/monitoring/ping.ex @@ -0,0 +1,58 @@ +defmodule Towerops.Monitoring.Ping do + @moduledoc """ + Handles ping operations for equipment monitoring. + """ + + @doc """ + Pings an IP address and returns the result. + + Returns {:ok, response_time_ms} on success, {:error, reason} on failure. + """ + def ping(ip_address, timeout_ms \\ 5000) do + start_time = System.monotonic_time(:millisecond) + + case run_ping(ip_address, timeout_ms) do + :ok -> + end_time = System.monotonic_time(:millisecond) + response_time = end_time - start_time + {:ok, response_time} + + {:error, reason} -> + {:error, reason} + end + end + + defp run_ping(ip_address, timeout_ms) do + timeout_seconds = div(timeout_ms, 1000) + timeout_seconds = max(timeout_seconds, 1) + + case :os.type() do + {:unix, :darwin} -> + # macOS + run_system_ping(ip_address, ["-c", "1", "-W", "#{timeout_seconds}"]) + + {:unix, _} -> + # Linux + run_system_ping(ip_address, ["-c", "1", "-W", "#{timeout_seconds}"]) + + {:win32, _} -> + # Windows + run_system_ping(ip_address, ["-n", "1", "-w", "#{timeout_ms}"]) + end + end + + defp run_system_ping(ip_address, args) do + try do + case System.cmd("ping", args ++ [ip_address], stderr_to_stdout: true) do + {_output, 0} -> + :ok + + {_output, _exit_code} -> + {:error, :timeout_or_unreachable} + end + rescue + e -> + {:error, {:exception, Exception.message(e)}} + end + end +end diff --git a/lib/towerops/monitoring/supervisor.ex b/lib/towerops/monitoring/supervisor.ex new file mode 100644 index 00000000..d4f9437a --- /dev/null +++ b/lib/towerops/monitoring/supervisor.ex @@ -0,0 +1,63 @@ +defmodule Towerops.Monitoring.Supervisor do + @moduledoc """ + Supervisor for managing equipment monitoring workers. + """ + use Supervisor + + alias Towerops.Monitoring.EquipmentMonitor + + def start_link(init_arg) do + Supervisor.start_link(__MODULE__, init_arg, name: __MODULE__) + end + + @impl true + def init(_init_arg) do + children = [ + # Registry for naming monitor processes + {Registry, keys: :unique, name: Towerops.Monitoring.Registry}, + # DynamicSupervisor for monitor workers + {DynamicSupervisor, name: Towerops.Monitoring.DynamicSupervisor, strategy: :one_for_one} + ] + + # Start all monitors after supervisor is initialized + Task.start(fn -> + # Wait briefly for the supervisor tree to be fully initialized + Process.sleep(100) + start_all_monitors() + end) + + Supervisor.init(children, strategy: :one_for_one) + end + + @doc """ + Starts monitoring for a specific equipment. + """ + def start_monitor(equipment_id) do + spec = {EquipmentMonitor, equipment_id} + + DynamicSupervisor.start_child(Towerops.Monitoring.DynamicSupervisor, spec) + end + + @doc """ + Stops monitoring for a specific equipment. + """ + def stop_monitor(equipment_id) do + case Registry.lookup(Towerops.Monitoring.Registry, equipment_id) do + [{pid, _}] -> + DynamicSupervisor.terminate_child(Towerops.Monitoring.DynamicSupervisor, pid) + + [] -> + :ok + end + end + + @doc """ + Starts monitors for all equipment that has monitoring enabled. + """ + def start_all_monitors do + Towerops.Equipment.list_monitored_equipment() + |> Enum.each(fn equipment -> + start_monitor(equipment.id) + end) + end +end diff --git a/lib/towerops_web/live/equipment_live/show.ex b/lib/towerops_web/live/equipment_live/show.ex index 6f1d69e5..551ece21 100644 --- a/lib/towerops_web/live/equipment_live/show.ex +++ b/lib/towerops_web/live/equipment_live/show.ex @@ -3,6 +3,7 @@ defmodule ToweropsWeb.EquipmentLive.Show do use ToweropsWeb, :live_view alias Towerops.Equipment + alias Towerops.Monitoring @impl true def mount(_params, _session, socket) do @@ -12,11 +13,30 @@ defmodule ToweropsWeb.EquipmentLive.Show do @impl true def handle_params(%{"id" => id}, _, socket) do equipment = Equipment.get_equipment!(id) + checks = Monitoring.list_equipment_checks(id, 10) + + # Subscribe to status updates for this equipment + if connected?(socket) do + Phoenix.PubSub.subscribe(Towerops.PubSub, "equipment:#{id}") + end {:noreply, socket |> assign(:page_title, equipment.name) - |> assign(:equipment, equipment)} + |> assign(:equipment, equipment) + |> assign(:recent_checks, checks)} + end + + @impl true + def handle_info({:equipment_status_changed, equipment_id, new_status, response_time}, socket) do + equipment = Equipment.get_equipment!(equipment_id) + checks = Monitoring.list_equipment_checks(equipment_id, 10) + + {:noreply, + socket + |> assign(:equipment, equipment) + |> assign(:recent_checks, checks) + |> put_flash(:info, "Status updated: #{new_status} (#{response_time}ms)")} end @impl true @@ -32,4 +52,10 @@ defmodule ToweropsWeb.EquipmentLive.Show do {:noreply, put_flash(socket, :error, "Unable to delete equipment")} end end + + @impl true + def handle_event("trigger_check", _params, socket) do + Towerops.Monitoring.EquipmentMonitor.trigger_check(socket.assigns.equipment.id) + {:noreply, put_flash(socket, :info, "Check triggered")} + end end diff --git a/lib/towerops_web/live/equipment_live/show.html.heex b/lib/towerops_web/live/equipment_live/show.html.heex index 067db8a3..60e683e1 100644 --- a/lib/towerops_web/live/equipment_live/show.html.heex +++ b/lib/towerops_web/live/equipment_live/show.html.heex @@ -2,6 +2,9 @@ {@page_title} <:subtitle>{@equipment.ip_address} <:actions> + <.button phx-click="trigger_check" class="btn-sm btn-primary"> + <.icon name="hero-arrow-path" class="w-4 h-4" /> Check Now + <.link navigate={~p"/orgs/#{@current_organization.slug}/equipment/#{@equipment.id}/edit"} class="btn btn-sm" @@ -101,3 +104,47 @@ + +
| Status | +Response Time | +Checked At | +
|---|---|---|
| + + {check.status |> to_string() |> String.upcase()} + + | ++ <%= if check.response_time_ms do %> + {check.response_time_ms}ms + <% else %> + - + <% end %> + | +{Calendar.strftime(check.checked_at, "%Y-%m-%d %H:%M:%S UTC")} | +