defmodule SnmpKit.SnmpLib.Dashboard do @moduledoc """ Real-time monitoring dashboard and metrics aggregation for SNMP operations. This module provides a comprehensive monitoring and visualization system for production SNMP deployments. Based on patterns proven in large-scale monitoring systems managing thousands of network devices. ## Features - **Real-Time Metrics**: Live updates of performance and health metrics - **Historical Analytics**: Trend analysis and capacity planning data - **Alert Management**: Configurable thresholds and notification routing - **Performance Insights**: Detailed breakdown of operation performance - **Device Health**: Per-device status monitoring and diagnostics - **Resource Utilization**: Pool, memory, and system resource tracking ## Metrics Categories ### Performance Metrics - Request/response times (min, max, average, percentiles) - Throughput (operations per second) - Error rates and failure classifications - Connection pool utilization ### Health Metrics - Device availability and reachability - Circuit breaker states - Retry counts and backoff status - Resource exhaustion indicators ### System Metrics - Memory usage and garbage collection - Process counts and supervision tree health - Network socket utilization - Queue depths and processing delays ## Dashboard Views ### Overview Dashboard Global health and performance summary with key indicators. ### Device Dashboard Per-device detailed metrics and troubleshooting information. ### Pool Dashboard Connection pool health, utilization, and performance metrics. ### Alerts Dashboard Active alerts, acknowledgments, and escalation status. ## Usage Patterns # Start the dashboard server {:ok, _pid} = SnmpKit.SnmpLib.Dashboard.start_link(port: 4000) # Record custom metrics SnmpKit.SnmpLib.Dashboard.record_metric(:custom_operation, %{ duration: 150, device: "192.168.1.1", status: :success }) # Create custom alert SnmpKit.SnmpLib.Dashboard.create_alert(:high_error_rate, %{ device: "192.168.1.100", error_rate: 0.15, threshold: 0.10 }) # Export metrics for external systems prometheus_data = SnmpKit.SnmpLib.Dashboard.export_prometheus() ## Integration with External Systems - **Prometheus**: Native metrics export in Prometheus format - **Grafana**: Pre-built dashboards and alerting rules - **PagerDuty**: Alert escalation and incident management - **Slack/Teams**: Notification integration for team alerting """ use GenServer require Logger @metrics_table :snmp_lib_metrics @alerts_table :snmp_lib_alerts @timeseries_table :snmp_lib_timeseries @default_port 4000 # 5 seconds @default_update_interval 5_000 @default_retention_days 7 @type metric_name :: atom() @type metric_value :: number() @type metric_tags :: map() @type alert_level :: :info | :warning | :critical @type dashboard_opts :: [ port: pos_integer(), update_interval: pos_integer(), retention_days: pos_integer(), prometheus_enabled: boolean(), grafana_integration: boolean() ] defstruct [ :port, :update_interval, :retention_days, :web_server_pid, :prometheus_enabled, :grafana_integration, metrics_buffer: [], alerts_buffer: [], last_cleanup: nil ] ## Public API @doc """ Starts the dashboard server with monitoring and web interface. ## Options - `port`: Web dashboard port (default: 4000) - `update_interval`: Metrics update frequency in milliseconds (default: 5000) - `retention_days`: How long to keep historical data (default: 7) - `prometheus_enabled`: Enable Prometheus metrics endpoint (default: false) - `grafana_integration`: Enable Grafana dashboard integration (default: false) ## Examples # Start with defaults {:ok, _pid} = SnmpKit.SnmpLib.Dashboard.start_link() # Start with custom configuration {:ok, _pid} = SnmpKit.SnmpLib.Dashboard.start_link( port: 8080, prometheus_enabled: true, retention_days: 14 ) """ @spec start_link(dashboard_opts()) :: {:ok, pid()} | {:error, any()} def start_link(opts \\ []) do GenServer.start_link(__MODULE__, opts, name: __MODULE__) end @doc """ Records a metric data point for monitoring and visualization. ## Parameters - `metric_name`: Unique identifier for the metric type - `value`: Numeric value for the metric - `tags`: Optional metadata for filtering and grouping ## Examples # Record response time metric SnmpKit.SnmpLib.Dashboard.record_metric(:snmp_response_time, 125, %{ device: "192.168.1.1", operation: "get", community: "public" }) # Record error count SnmpKit.SnmpLib.Dashboard.record_metric(:snmp_errors, 1, %{ device: "192.168.1.1", error_type: "timeout" }) # Record pool utilization SnmpKit.SnmpLib.Dashboard.record_metric(:pool_utilization, 0.75, %{ pool_name: "main_pool" }) """ @spec record_metric(metric_name(), metric_value(), metric_tags()) :: :ok def record_metric(metric_name, value, tags \\ %{}) when is_number(value) do timestamp = System.system_time(:millisecond) metric = %{ name: metric_name, value: value, tags: tags, timestamp: timestamp } GenServer.cast(__MODULE__, {:record_metric, metric}) end @doc """ Creates an alert for monitoring and notification systems. ## Parameters - `alert_name`: Unique identifier for the alert type - `level`: Alert severity level (:info, :warning, :critical) - `details`: Alert metadata and context information ## Examples # Create device unreachable alert SnmpKit.SnmpLib.Dashboard.create_alert(:device_unreachable, :critical, %{ device: "192.168.1.1", last_seen: DateTime.utc_now(), consecutive_failures: 5 }) # Create performance degradation warning SnmpKit.SnmpLib.Dashboard.create_alert(:slow_response, :warning, %{ device: "192.168.1.1", avg_response_time: 5000, threshold: 2000 }) """ @spec create_alert(atom(), alert_level(), map()) :: :ok def create_alert(alert_name, level, details \\ %{}) do alert = %{ name: alert_name, level: level, details: details, timestamp: System.system_time(:millisecond), acknowledged: false } GenServer.cast(__MODULE__, {:create_alert, alert}) end @doc """ Acknowledges an alert to stop notifications. ## Examples :ok = SnmpKit.SnmpLib.Dashboard.acknowledge_alert(:device_unreachable, "192.168.1.1") """ @spec acknowledge_alert(atom(), any()) :: :ok def acknowledge_alert(alert_name, identifier) do GenServer.cast(__MODULE__, {:acknowledge_alert, alert_name, identifier}) end @doc """ Gets current performance metrics summary. ## Returns A map containing aggregated metrics: - `total_operations`: Total SNMP operations performed - `success_rate`: Percentage of successful operations - `avg_response_time`: Average response time in milliseconds - `active_devices`: Number of devices being monitored - `pool_utilization`: Connection pool usage percentage - `error_rates`: Breakdown of error types and frequencies ## Examples metrics = SnmpKit.SnmpLib.Dashboard.get_metrics_summary() IO.puts "Success rate: " <> Float.to_string(metrics.success_rate * 100) <> "%" """ @spec get_metrics_summary() :: map() def get_metrics_summary do GenServer.call(__MODULE__, :get_metrics_summary) end @doc """ Gets detailed metrics for a specific device. ## Examples device_metrics = SnmpKit.SnmpLib.Dashboard.get_device_metrics("192.168.1.1") IO.inspect device_metrics.response_times """ @spec get_device_metrics(binary()) :: map() def get_device_metrics(device_id) do GenServer.call(__MODULE__, {:get_device_metrics, device_id}) end @doc """ Gets all active alerts with optional filtering. ## Examples all_alerts = SnmpKit.SnmpLib.Dashboard.get_active_alerts() critical_alerts = SnmpKit.SnmpLib.Dashboard.get_active_alerts(level: :critical) """ @spec get_active_alerts(keyword()) :: [map()] def get_active_alerts(filters \\ []) do GenServer.call(__MODULE__, {:get_active_alerts, filters}) end @doc """ Exports metrics in Prometheus format for external monitoring. ## Examples prometheus_data = SnmpKit.SnmpLib.Dashboard.export_prometheus() File.write!("/tmp/snmp_metrics.prom", prometheus_data) """ @spec export_prometheus() :: binary() def export_prometheus do GenServer.call(__MODULE__, :export_prometheus) end @doc """ Gets historical time series data for a metric. ## Parameters - `metric_name`: Name of the metric to retrieve - `duration`: Time window in milliseconds (default: 1 hour) - `tags`: Optional tag filters ## Examples # Get last hour of response times timeseries = SnmpKit.SnmpLib.Dashboard.get_timeseries(:snmp_response_time) # Get last 24 hours for specific device device_data = SnmpKit.SnmpLib.Dashboard.get_timeseries( :snmp_response_time, 24 * 60 * 60 * 1000, %{device: "192.168.1.1"} ) """ @spec get_timeseries(metric_name(), pos_integer(), map()) :: [map()] def get_timeseries(metric_name, duration \\ 3_600_000, tags \\ %{}) do GenServer.call(__MODULE__, {:get_timeseries, metric_name, duration, tags}) end ## GenServer Implementation @impl GenServer def init(opts) do # Create ETS tables for metrics storage :ets.new(@metrics_table, [:named_table, :bag, :public]) :ets.new(@alerts_table, [:named_table, :bag, :public]) :ets.new(@timeseries_table, [:named_table, :ordered_set, :public]) state = %__MODULE__{ port: Keyword.get(opts, :port, @default_port), update_interval: Keyword.get(opts, :update_interval, @default_update_interval), retention_days: Keyword.get(opts, :retention_days, @default_retention_days), prometheus_enabled: Keyword.get(opts, :prometheus_enabled, false), grafana_integration: Keyword.get(opts, :grafana_integration, false), last_cleanup: System.system_time(:millisecond) } # Start web server if enabled web_server_pid = start_web_server(state) state = %{state | web_server_pid: web_server_pid} # Schedule periodic tasks schedule_update() schedule_cleanup() Logger.info("SnmpKit.SnmpLib.Dashboard started on port #{state.port}") {:ok, state} end @impl GenServer def handle_cast({:record_metric, metric}, state) do # Store in main metrics table :ets.insert(@metrics_table, {metric.name, metric}) # Store in timeseries table for historical analysis timeseries_key = {metric.name, metric.timestamp} :ets.insert(@timeseries_table, {timeseries_key, metric}) # Add to buffer for processing new_buffer = [metric | state.metrics_buffer] new_state = %{state | metrics_buffer: new_buffer} {:noreply, new_state} end @impl GenServer def handle_cast({:create_alert, alert}, state) do # Store alert :ets.insert(@alerts_table, {alert.name, alert}) # Add to buffer for notification processing new_buffer = [alert | state.alerts_buffer] new_state = %{state | alerts_buffer: new_buffer} # Log alert based on level case alert.level do :critical -> Logger.error("CRITICAL ALERT: #{alert.name} - #{inspect(alert.details)}") :warning -> Logger.warning("WARNING ALERT: #{alert.name} - #{inspect(alert.details)}") :info -> Logger.info("INFO ALERT: #{alert.name} - #{inspect(alert.details)}") end {:noreply, new_state} end @impl GenServer def handle_cast({:acknowledge_alert, alert_name, identifier}, state) do # Find and update matching alerts alerts = :ets.lookup(@alerts_table, alert_name) Enum.each(alerts, fn {name, alert} -> if match_alert_identifier(alert, identifier) do updated_alert = %{alert | acknowledged: true} :ets.delete_object(@alerts_table, {name, alert}) :ets.insert(@alerts_table, {name, updated_alert}) end end) {:noreply, state} end @impl GenServer def handle_call(:get_metrics_summary, _from, state) do summary = calculate_metrics_summary() {:reply, summary, state} end @impl GenServer def handle_call({:get_device_metrics, device_id}, _from, state) do metrics = calculate_device_metrics(device_id) {:reply, metrics, state} end @impl GenServer def handle_call({:get_active_alerts, filters}, _from, state) do alerts = get_filtered_alerts(filters) {:reply, alerts, state} end @impl GenServer def handle_call(:export_prometheus, _from, state) do prometheus_data = generate_prometheus_export() {:reply, prometheus_data, state} end @impl GenServer def handle_call({:get_timeseries, metric_name, duration, tags}, _from, state) do timeseries = extract_timeseries_data(metric_name, duration, tags) {:reply, timeseries, state} end @impl GenServer def handle_info(:update_metrics, state) do # Process buffered metrics process_metrics_buffer(state.metrics_buffer) # Process buffered alerts process_alerts_buffer(state.alerts_buffer) # Clear buffers new_state = %{state | metrics_buffer: [], alerts_buffer: []} schedule_update() {:noreply, new_state} end @impl GenServer def handle_info(:cleanup_old_data, state) do current_time = System.system_time(:millisecond) cleanup_threshold = current_time - state.retention_days * 24 * 60 * 60 * 1000 # Clean up old timeseries data cleanup_old_timeseries(cleanup_threshold) # Clean up acknowledged alerts older than 24 hours alert_cleanup_threshold = current_time - 24 * 60 * 60 * 1000 cleanup_old_alerts(alert_cleanup_threshold) schedule_cleanup() new_state = %{state | last_cleanup: current_time} {:noreply, new_state} end ## Private Implementation # Web server management defp start_web_server(state) do # In a real implementation, this would start a web server # For now, we'll just return a placeholder pid spawn(fn -> Process.sleep(1000) Logger.info("Dashboard web interface would be available at http://localhost:#{state.port}") end) end # Metrics calculation defp calculate_metrics_summary do current_time = System.system_time(:millisecond) one_hour_ago = current_time - 3_600_000 # Get recent metrics recent_metrics = :ets.select(@timeseries_table, [ {{{:_, :"$1"}, :_}, [{:>=, :"$1", one_hour_ago}], [:"$_"]} ]) # Calculate summary statistics total_operations = length(recent_metrics) success_count = count_successful_operations(recent_metrics) response_times = extract_response_times(recent_metrics) %{ total_operations: total_operations, success_rate: if(total_operations > 0, do: success_count / total_operations, else: 0.0), avg_response_time: if(response_times == [], do: 0.0, else: Enum.sum(response_times) / length(response_times) ), active_devices: count_active_devices(recent_metrics), pool_utilization: get_current_pool_utilization(), error_rates: calculate_error_rates(recent_metrics) } end defp calculate_device_metrics(device_id) do # Get metrics for specific device device_metrics = :ets.select(@metrics_table, [ {{:_, %{tags: %{device: device_id}}}, [], [:"$_"]} ]) %{ device_id: device_id, total_operations: length(device_metrics), response_times: extract_device_response_times(device_metrics), error_count: count_device_errors(device_metrics), last_seen: get_device_last_seen(device_metrics) } end # Alert management defp get_filtered_alerts(filters) do level_filter = Keyword.get(filters, :level) acknowledged_filter = Keyword.get(filters, :acknowledged, false) @alerts_table |> :ets.tab2list() |> Enum.map(fn {_name, alert} -> alert end) |> Enum.filter(fn alert -> (is_nil(level_filter) or alert.level == level_filter) and alert.acknowledged == acknowledged_filter end) |> Enum.sort_by(& &1.timestamp, :desc) end defp match_alert_identifier(alert, identifier) do # Match alert by device ID or other identifier case alert.details do %{device: ^identifier} -> true _ -> false end end # Prometheus export defp generate_prometheus_export do metrics_summary = calculate_metrics_summary() prometheus_lines = [ "# HELP snmp_lib_total_operations Total SNMP operations performed", "# TYPE snmp_lib_total_operations counter", "snmp_lib_total_operations #{metrics_summary.total_operations}", "", "# HELP snmp_lib_success_rate Success rate of SNMP operations", "# TYPE snmp_lib_success_rate gauge", "snmp_lib_success_rate #{metrics_summary.success_rate}", "", "# HELP snmp_lib_avg_response_time Average response time in milliseconds", "# TYPE snmp_lib_avg_response_time gauge", "snmp_lib_avg_response_time #{metrics_summary.avg_response_time}", "" ] Enum.join(prometheus_lines, "\n") end # Timeseries data extraction defp extract_timeseries_data(metric_name, duration, tags) do current_time = System.system_time(:millisecond) start_time = current_time - duration # Get timeseries data within time window @timeseries_table |> :ets.select([ {{{metric_name, :"$1"}, :"$2"}, [{:>=, :"$1", start_time}], [:"$2"]} ]) |> Enum.filter(fn metric -> tags_match(metric.tags, tags) end) |> Enum.sort_by(& &1.timestamp) end defp tags_match(metric_tags, filter_tags) do Enum.all?(filter_tags, fn {key, value} -> Map.get(metric_tags, key) == value end) end # Data processing helpers defp count_successful_operations(metrics) do Enum.count(metrics, fn {_key, metric} -> case metric.tags do %{status: :success} -> true _ -> false end end) end defp extract_response_times(metrics) do metrics |> Enum.filter(fn {_key, metric} -> metric.name == :snmp_response_time end) |> Enum.map(fn {_key, metric} -> metric.value end) end defp extract_device_response_times(metrics) do metrics |> Enum.filter(fn {_name, metric} -> metric.name == :snmp_response_time end) |> Enum.map(fn {_name, metric} -> metric.value end) end defp count_device_errors(metrics) do Enum.count(metrics, fn {_name, metric} -> case metric.tags do %{status: :error} -> true _ -> false end end) end defp get_device_last_seen(metrics) do case metrics do [] -> nil _ -> metrics |> Enum.map(fn {_name, metric} -> metric.timestamp end) |> Enum.max() end end defp count_active_devices(metrics) do metrics |> Enum.map(fn {_key, metric} -> Map.get(metric.tags, :device) end) |> Enum.filter(& &1) |> Enum.uniq() |> length() end defp get_current_pool_utilization do # This would get actual pool utilization from SnmpKit.SnmpLib.Pool # For now, return a placeholder 0.5 end defp calculate_error_rates(metrics) do error_metrics = Enum.filter(metrics, fn {_key, metric} -> Map.get(metric.tags, :status) == :error end) error_metrics |> Enum.group_by(fn {_key, metric} -> Map.get(metric.tags, :error_type, :unknown) end) |> Map.new(fn {error_type, errors} -> {error_type, length(errors)} end) end # Cleanup functions defp cleanup_old_timeseries(threshold) do old_keys = :ets.select(@timeseries_table, [ {{{:_, :"$1"}, :_}, [{:<, :"$1", threshold}], [:"$_"]} ]) Enum.each(old_keys, fn {key, _metric} -> :ets.delete(@timeseries_table, key) end) Logger.debug("Cleaned up #{length(old_keys)} old timeseries points") end defp cleanup_old_alerts(threshold) do old_alerts = :ets.select(@alerts_table, [ {{:_, %{acknowledged: true, timestamp: :"$1"}}, [{:<, :"$1", threshold}], [:"$_"]} ]) Enum.each(old_alerts, fn {name, alert} -> :ets.delete_object(@alerts_table, {name, alert}) end) Logger.debug("Cleaned up #{length(old_alerts)} old acknowledged alerts") end # Buffer processing defp process_metrics_buffer(buffer) do # This would perform aggregation, anomaly detection, etc. Logger.debug("Processed #{length(buffer)} metrics in buffer") end defp process_alerts_buffer(buffer) do # This would send notifications, update external systems, etc. critical_alerts = Enum.filter(buffer, &(&1.level == :critical)) if critical_alerts != [] do Logger.warning("#{length(critical_alerts)} critical alerts need attention") end end # Scheduling defp schedule_update do Process.send_after(self(), :update_metrics, @default_update_interval) end defp schedule_cleanup do # Clean up old data every hour Process.send_after(self(), :cleanup_old_data, 3_600_000) end end