diff --git a/AGENT_NEXT_STEPS.md b/AGENT_NEXT_STEPS.md deleted file mode 100644 index feac6272..00000000 --- a/AGENT_NEXT_STEPS.md +++ /dev/null @@ -1,209 +0,0 @@ -# Agent System - Next Steps - -This document outlines the remaining work to complete the remote SNMP polling agent system. - -## Current Status - -✅ **Complete**: -- Backend API with token authentication -- Agent management UI -- **Hierarchical agent assignment** (Equipment > Site > Organization) -- Organization/site/equipment-level configuration -- Rust agent architecture (compiles successfully) -- **SNMP client fully implemented** (GET and WALK operations) -- SQLite buffering -- Docker deployment files -- Protocol Buffers integration for all API endpoints -- 775 tests passing - -## Critical Path to Production - -### 1. Integration Testing (Priority: HIGH) ⬅️ **START HERE** - -**Prerequisites**: ✅ All backend code complete - -**Test Environment Setup**: -1. Deploy test SNMP device (or use simulator) -2. Deploy Phoenix backend -3. Deploy Rust agent via Docker -4. Create agent token via UI -5. Assign test equipment to agent - -**Test Cases**: -- [ ] Agent connects and authenticates -- [ ] Agent fetches configuration -- [ ] Agent polls sensors successfully -- [ ] Agent polls interfaces successfully -- [ ] Metrics appear in Phoenix database -- [ ] Threshold violations trigger events -- [ ] Agent survives API outage (test 24h buffering) -- [ ] Agent reconnects after network issues -- [ ] Heartbeat updates agent status - -**Duration**: 1-2 days - -### 2. Load & Performance Testing (Priority: MEDIUM) - -**Test Scenarios**: - -**Scenario 1: Single Agent, Many Devices** -- 100 devices -- 500 sensors -- 200 interfaces -- 60-second poll interval -- Monitor: Memory, CPU, database growth - -**Scenario 2: Many Agents** -- 10 agents -- 10 devices each -- Concurrent polling -- Monitor: API performance, database connections - -**Scenario 3: Long-Running Stability** -- 1 agent -- 50 devices -- Run for 7 days -- Monitor: Memory leaks, database size, errors - -**Duration**: 3-5 days - -### 3. Production Readiness (Priority: MEDIUM) - -**Docker Image**: -- [x] Build optimized image (11.8 MB) -- [ ] Publish to container registry -- [ ] Tag releases (v0.1.0, latest) -- [x] Multi-architecture support (amd64, arm64) - -**Documentation**: -- [x] User guide with deployment methods (USER_GUIDE.md) -- [x] Troubleshooting guide -- [x] Network/firewall requirements -- [x] Upgrade procedure -- [x] Disaster recovery - -**Monitoring**: -- [ ] Grafana dashboard for agent health -- [ ] Alerts for offline agents -- [ ] Metrics on agent performance -- [x] Database queries for agent statistics (lib/towerops/agents/stats.ex) - -**Duration**: 2-3 days - -## Optional Enhancements - -### 4. Advanced Features (Priority: LOW) - -**SNMPv3 Support** -- User authentication -- Privacy encryption -- Configuration UI changes - -**Agent-Side Filtering** -- Threshold checks in agent -- Reduce bandwidth for high-sensor-count equipment -- Configurable sampling rates - -**Equipment Polling Modes** -- Add `polling_mode` enum: `:server`, `:agent`, `:both` -- Allows gradual migration -- Enables A/B testing - -**Agent Health Metrics** -- CPU/memory usage -- Metrics queue depth -- Polling success rate -- Average latency per device - -## Quick Start Guide - -### For Development - -1. **Test Locally** -```bash -# Terminal 1: Start Phoenix -mix phx.server - -# Terminal 2: Start agent -cd towerops-agent -cargo run -- \ - --api-url http://localhost:4000 \ - --token -``` - -2. **Create Test Agent** -- Visit http://localhost:4000/orgs/:slug/agents -- Create agent, copy token -- Use token in step 1 - -### For Production Deployment - -1. **Build Docker Image** -```bash -cd towerops-agent -docker build -t towerops/agent:0.1.0 . -docker tag towerops/agent:0.1.0 towerops/agent:latest -``` - -2. **Push to Registry** -```bash -docker push towerops/agent:0.1.0 -docker push towerops/agent:latest -``` - -3. **Deploy to Customer** -```bash -# Provide customer with docker-compose.yml -# They run: -docker-compose up -d -``` - -## Estimated Timeline - -| Phase | Effort | Duration | -|-------|--------|----------| -| ✅ ~~SNMP Integration~~ | ~~4-8 hours~~ | ~~1 day~~ | -| ✅ ~~Hierarchical Agent Assignment~~ | ~~12 hours~~ | ~~1.5 days~~ | -| Integration Testing | 16 hours | 2 days | -| Load Testing | 24 hours | 3 days | -| Production Prep | 16 hours | 2 days | -| **Remaining** | **56 hours** | **7 days** | - -## Success Criteria - -The agent system is ready for production when: - -- [x] Agent authenticates with token successfully ✅ -- [x] Agent fetches configuration from API ✅ -- [x] Agent polls SNMP devices (sensors + interfaces) ✅ -- [x] Metrics appear in database within 60 seconds ✅ -- [ ] Threshold violations trigger events ⚠️ (requires integration testing) -- [x] Agent survives 24h API outage without data loss ✅ (SQLite buffering, needs verification) -- [x] UI shows agent status (online/offline) ✅ -- [x] Token revocation works immediately ✅ -- [ ] Agent uses <256 MB memory with 50 devices ⚠️ (needs load testing) -- [x] Docker image is <50 MB ✅ (11.8 MB) -- [ ] Load test: 100 devices, 500 sensors, 200 interfaces ⚠️ (manual testing required) -- [ ] Stability test: 7 days continuous operation ⚠️ (manual testing required) - -## Questions & Decisions Needed - -1. **SNMP Library Choice**: Stick with `snmp` v0.2 or switch? -2. **Release Strategy**: Beta testing with select customers first? -3. **Support Plan**: How will customers get help with agent deployment? -4. **Monitoring**: What metrics should we track about agent usage? -5. **Pricing**: Does remote agent feature affect pricing tiers? - -## Resources - -- **Implementation Doc**: `/Users/graham/dev/towerops/AGENT_IMPLEMENTATION.md` -- **Agent README**: `/Users/graham/dev/towerops/towerops-agent/README.md` -- **SNMP Crate Docs**: https://docs.rs/snmp/0.2.2/snmp/ -- **Original Plan**: `/Users/graham/.claude/plans/melodic-coalescing-truffle.md` - -## Contact - -For questions about this implementation: -- Review AGENT_IMPLEMENTATION.md for architecture details -- Check git history for context on specific changes -- All code follows project conventions in CLAUDE.md diff --git a/lib/towerops/agents/agent_token.ex b/lib/towerops/agents/agent_token.ex index f629b688..092dd5a2 100644 --- a/lib/towerops/agents/agent_token.ex +++ b/lib/towerops/agents/agent_token.ex @@ -10,6 +10,8 @@ defmodule Towerops.Agents.AgentToken do import Ecto.Changeset + alias Towerops.Organizations.Organization + # 48 bytes = 384 bits of cryptographically secure randomness @rand_size 48 @@ -23,11 +25,25 @@ defmodule Towerops.Agents.AgentToken do field :enabled, :boolean, default: true field :metadata, :map, default: %{} - belongs_to :organization, Towerops.Organizations.Organization + belongs_to :organization, Organization timestamps(type: :utc_datetime) end + @type t :: %__MODULE__{ + id: Ecto.UUID.t(), + token: String.t(), + name: String.t(), + last_seen_at: DateTime.t() | nil, + last_ip: String.t() | nil, + enabled: boolean(), + metadata: map(), + organization_id: Ecto.UUID.t(), + organization: Ecto.Association.NotLoaded.t() | Organization.t(), + inserted_at: DateTime.t(), + updated_at: DateTime.t() + } + @doc """ Builds a new agent token with cryptographically secure random bytes. diff --git a/lib/towerops/equipment/equipment.ex b/lib/towerops/equipment/equipment.ex index 85bd9ff0..e32896fd 100644 --- a/lib/towerops/equipment/equipment.ex +++ b/lib/towerops/equipment/equipment.ex @@ -4,6 +4,11 @@ defmodule Towerops.Equipment.Equipment do import Ecto.Changeset + alias Ecto.Association.NotLoaded + alias Towerops.Agents.AgentAssignment + alias Towerops.Sites.Site + alias Towerops.Snmp.Device + @primary_key {:id, :binary_id, autogenerate: true} @foreign_key_type :binary_id schema "equipment" do @@ -24,14 +29,38 @@ defmodule Towerops.Equipment.Equipment do field :last_discovery_at, :utc_datetime field :last_snmp_poll_at, :utc_datetime - belongs_to :site, Towerops.Sites.Site + belongs_to :site, Site - has_one :snmp_device, Towerops.Snmp.Device - has_many :agent_assignments, Towerops.Agents.AgentAssignment + has_one :snmp_device, Device + has_many :agent_assignments, AgentAssignment timestamps(type: :utc_datetime) end + @type t :: %__MODULE__{ + id: Ecto.UUID.t(), + name: String.t(), + ip_address: String.t(), + description: String.t() | nil, + status: :up | :down | :unknown, + last_checked_at: DateTime.t() | nil, + last_status_change_at: DateTime.t() | nil, + monitoring_enabled: boolean(), + check_interval_seconds: integer(), + snmp_enabled: boolean(), + snmp_version: String.t(), + snmp_community: String.t() | nil, + snmp_port: integer(), + last_discovery_at: DateTime.t() | nil, + last_snmp_poll_at: DateTime.t() | nil, + site_id: Ecto.UUID.t(), + site: NotLoaded.t() | Site.t(), + snmp_device: NotLoaded.t() | Device.t() | nil, + agent_assignments: NotLoaded.t() | [AgentAssignment.t()], + inserted_at: DateTime.t(), + updated_at: DateTime.t() + } + @doc false def changeset(equipment, attrs) do equipment diff --git a/lib/towerops/snmp/device.ex b/lib/towerops/snmp/device.ex index 03523bc0..0ac78f63 100644 --- a/lib/towerops/snmp/device.ex +++ b/lib/towerops/snmp/device.ex @@ -4,6 +4,11 @@ defmodule Towerops.Snmp.Device do import Ecto.Changeset + alias Ecto.Association.NotLoaded + alias Towerops.Equipment.Equipment + alias Towerops.Snmp.Interface + alias Towerops.Snmp.Sensor + @primary_key {:id, :binary_id, autogenerate: true} @foreign_key_type :binary_id schema "snmp_devices" do @@ -17,14 +22,33 @@ defmodule Towerops.Snmp.Device do field :model, :string field :firmware_version, :string - belongs_to :equipment, Towerops.Equipment.Equipment + belongs_to :equipment, Equipment - has_many :interfaces, Towerops.Snmp.Interface, foreign_key: :snmp_device_id - has_many :sensors, Towerops.Snmp.Sensor, foreign_key: :snmp_device_id + has_many :interfaces, Interface, foreign_key: :snmp_device_id + has_many :sensors, Sensor, foreign_key: :snmp_device_id timestamps(type: :utc_datetime) end + @type t :: %__MODULE__{ + id: Ecto.UUID.t(), + sys_descr: String.t() | nil, + sys_object_id: String.t() | nil, + sys_name: String.t() | nil, + sys_uptime: integer() | nil, + sys_contact: String.t() | nil, + sys_location: String.t() | nil, + manufacturer: String.t() | nil, + model: String.t() | nil, + firmware_version: String.t() | nil, + equipment_id: Ecto.UUID.t(), + equipment: NotLoaded.t() | Equipment.t(), + interfaces: NotLoaded.t() | [Interface.t()], + sensors: NotLoaded.t() | [Sensor.t()], + inserted_at: DateTime.t(), + updated_at: DateTime.t() + } + @doc false def changeset(device, attrs) do device diff --git a/lib/towerops/snmp/interface.ex b/lib/towerops/snmp/interface.ex index 1b772731..78d4ce32 100644 --- a/lib/towerops/snmp/interface.ex +++ b/lib/towerops/snmp/interface.ex @@ -4,6 +4,10 @@ defmodule Towerops.Snmp.Interface do import Ecto.Changeset + alias Ecto.Association.NotLoaded + alias Towerops.Snmp.Device + alias Towerops.Snmp.InterfaceStat + @primary_key {:id, :binary_id, autogenerate: true} @foreign_key_type :binary_id schema "snmp_interfaces" do @@ -18,13 +22,32 @@ defmodule Towerops.Snmp.Interface do field :if_oper_status, :string field :monitored, :boolean, default: true - belongs_to :snmp_device, Towerops.Snmp.Device + belongs_to :snmp_device, Device - has_many :stats, Towerops.Snmp.InterfaceStat, foreign_key: :interface_id + has_many :stats, InterfaceStat, foreign_key: :interface_id timestamps(type: :utc_datetime) end + @type t :: %__MODULE__{ + id: Ecto.UUID.t(), + if_index: integer(), + if_name: String.t() | nil, + if_descr: String.t() | nil, + if_alias: String.t() | nil, + if_type: integer() | nil, + if_speed: integer() | nil, + if_phys_address: String.t() | nil, + if_admin_status: String.t() | nil, + if_oper_status: String.t() | nil, + monitored: boolean(), + snmp_device_id: Ecto.UUID.t(), + snmp_device: NotLoaded.t() | Device.t(), + stats: NotLoaded.t() | [InterfaceStat.t()], + inserted_at: DateTime.t(), + updated_at: DateTime.t() + } + @doc false def changeset(interface, attrs) do interface diff --git a/lib/towerops/snmp/sensor.ex b/lib/towerops/snmp/sensor.ex index 4eceea53..31b86d69 100644 --- a/lib/towerops/snmp/sensor.ex +++ b/lib/towerops/snmp/sensor.ex @@ -4,6 +4,10 @@ defmodule Towerops.Snmp.Sensor do import Ecto.Changeset + alias Ecto.Association.NotLoaded + alias Towerops.Snmp.Device + alias Towerops.Snmp.SensorReading + @primary_key {:id, :binary_id, autogenerate: true} @foreign_key_type :binary_id schema "snmp_sensors" do @@ -18,13 +22,32 @@ defmodule Towerops.Snmp.Sensor do field :last_checked_at, :utc_datetime field :metadata, :map, default: %{} - belongs_to :snmp_device, Towerops.Snmp.Device + belongs_to :snmp_device, Device - has_many :readings, Towerops.Snmp.SensorReading, foreign_key: :sensor_id + has_many :readings, SensorReading, foreign_key: :sensor_id timestamps(type: :utc_datetime) end + @type t :: %__MODULE__{ + id: Ecto.UUID.t(), + sensor_type: String.t(), + sensor_index: String.t(), + sensor_oid: String.t(), + sensor_descr: String.t(), + sensor_unit: String.t(), + sensor_divisor: integer(), + monitored: boolean(), + last_value: float() | nil, + last_checked_at: DateTime.t() | nil, + metadata: map(), + snmp_device_id: Ecto.UUID.t(), + snmp_device: NotLoaded.t() | Device.t(), + readings: NotLoaded.t() | [SensorReading.t()], + inserted_at: DateTime.t(), + updated_at: DateTime.t() + } + @doc false def changeset(sensor, attrs) do sensor diff --git a/lib/towerops_web/channels/agent_channel.ex b/lib/towerops_web/channels/agent_channel.ex index 4436aab9..9b483f2b 100644 --- a/lib/towerops_web/channels/agent_channel.ex +++ b/lib/towerops_web/channels/agent_channel.ex @@ -45,7 +45,7 @@ defmodule ToweropsWeb.AgentChannel do |> assign(:organization_id, agent_token.organization_id) # Update last_seen_at on join - Agents.update_agent_token_heartbeat(agent_token.id, nil, %{}) + _ = Agents.update_agent_token_heartbeat(agent_token.id, nil, %{}) # Send initial job list send(self(), :send_jobs) @@ -77,7 +77,7 @@ defmodule ToweropsWeb.AgentChannel do def handle_in("result", %{"binary" => binary_b64}, socket) do binary = Base.decode64!(binary_b64) result = SnmpResult.decode(binary) - process_snmp_result(socket.assigns.organization_id, result) + _ = process_snmp_result(socket.assigns.organization_id, result) {:noreply, socket} end @@ -91,7 +91,13 @@ defmodule ToweropsWeb.AgentChannel do "uptime_seconds" => heartbeat.uptime_seconds } - Agents.update_agent_token_heartbeat(socket.assigns.agent_token_id, heartbeat.ip_address, metadata) + _ = + Agents.update_agent_token_heartbeat( + socket.assigns.agent_token_id, + heartbeat.ip_address, + metadata + ) + {:noreply, socket} end