towerops/lib/towerops/snmp/sensor_change_detector.ex
Graham McIntire abbe117f5e
add sensor value change events for all sensor types
extract sensor change detection from DevicePollerWorker into shared
SensorChangeDetector module. add sensor_value_changed event for non-%
sensors when value differs from last_value. agent channel now detects
changes and updates last_value, matching phoenix poller behavior.
2026-02-11 15:06:55 -06:00

291 lines
9.2 KiB
Elixir

defmodule Towerops.Snmp.SensorChangeDetector do
@moduledoc """
Detects sensor value changes and broadcasts events.
Shared between DevicePollerWorker and AgentChannel to ensure consistent
event detection across both Phoenix-polled and agent-polled devices.
"""
alias Towerops.Snmp.Sensor
require Logger
@doc """
Detects sensor changes and broadcasts events via PubSub.
For percentage sensors: emits spike/drop events for >=30% swings.
For all other sensors: emits sensor_value_changed when value differs from last_value.
For all sensors with thresholds: emits threshold warning/critical/normal events.
No-ops when `sensor.last_value` is nil (first reading).
"""
@spec detect_and_broadcast(Sensor.t(), number(), DateTime.t()) :: :ok
def detect_and_broadcast(%Sensor{last_value: nil}, _current_value, _timestamp), do: :ok
def detect_and_broadcast(%Sensor{} = sensor, current_value, timestamp) do
device_id = get_device_id_from_sensor(sensor)
if device_id do
thresholds = extract_thresholds(sensor.metadata)
events =
[]
|> maybe_add_threshold_event(sensor, current_value, timestamp, device_id, thresholds)
|> maybe_add_change_event(sensor, current_value, timestamp, device_id)
broadcast_events(events)
end
:ok
end
# Threshold detection
defp extract_thresholds(metadata) do
%{
warning_high: metadata["warning_high"],
critical_high: metadata["critical_high"],
warning_low: metadata["warning_low"],
critical_low: metadata["critical_low"]
}
end
defp maybe_add_threshold_event(events, sensor, current_value, timestamp, device_id, thresholds) do
threshold_event = check_threshold_violation(sensor, current_value, timestamp, device_id, thresholds)
if threshold_event, do: [threshold_event | events], else: events
end
defp check_threshold_violation(sensor, current_value, timestamp, device_id, thresholds) do
threshold_checks = [
&check_critical_high/5,
&check_critical_low/5,
&check_warning_high/5,
&check_warning_low/5
]
Enum.find_value(threshold_checks, fn check_fn ->
check_fn.(sensor, current_value, timestamp, device_id, thresholds)
end) || check_returned_to_normal(sensor, current_value, timestamp, device_id, thresholds)
end
defp check_critical_high(sensor, current_value, timestamp, device_id, thresholds) do
if thresholds.critical_high && current_value >= thresholds.critical_high do
build_threshold_event(
sensor,
current_value,
timestamp,
device_id,
"critical_high",
thresholds.critical_high,
"critical",
"critically high"
)
end
end
defp check_critical_low(sensor, current_value, timestamp, device_id, thresholds) do
if thresholds.critical_low && current_value <= thresholds.critical_low do
build_threshold_event(
sensor,
current_value,
timestamp,
device_id,
"critical_low",
thresholds.critical_low,
"critical",
"critically low"
)
end
end
defp check_warning_high(sensor, current_value, timestamp, device_id, thresholds) do
if thresholds.warning_high && current_value >= thresholds.warning_high do
build_threshold_event(
sensor,
current_value,
timestamp,
device_id,
"warning_high",
thresholds.warning_high,
"warning",
"high"
)
end
end
defp check_warning_low(sensor, current_value, timestamp, device_id, thresholds) do
if thresholds.warning_low && current_value <= thresholds.warning_low do
build_threshold_event(
sensor,
current_value,
timestamp,
device_id,
"warning_low",
thresholds.warning_low,
"warning",
"low"
)
end
end
defp build_threshold_event(
sensor,
current_value,
timestamp,
device_id,
threshold_type,
threshold_value,
severity,
description
) do
event_type = if severity == "critical", do: "sensor_threshold_critical", else: "sensor_threshold_warning"
build_sensor_event(
device_id,
sensor,
event_type,
severity,
"#{sensor.sensor_descr} is #{description}: #{format_sensor_value(current_value, sensor.sensor_unit)} (threshold: #{threshold_value}#{sensor.sensor_unit})",
%{
current_value: current_value,
previous_value: sensor.last_value,
threshold_type: threshold_type,
threshold_value: threshold_value
},
timestamp
)
end
defp check_returned_to_normal(sensor, current_value, timestamp, device_id, thresholds) do
was_over_threshold = value_was_over_threshold?(sensor.last_value, thresholds)
is_now_normal = value_is_normal?(current_value, thresholds)
if was_over_threshold && is_now_normal do
build_sensor_event(
device_id,
sensor,
"sensor_threshold_normal",
"info",
"#{sensor.sensor_descr} returned to normal: #{format_sensor_value(current_value, sensor.sensor_unit)}",
%{current_value: current_value, previous_value: sensor.last_value},
timestamp
)
end
end
defp value_was_over_threshold?(last_value, thresholds) do
(thresholds.warning_high && last_value >= thresholds.warning_high) ||
(thresholds.critical_high && last_value >= thresholds.critical_high) ||
(thresholds.warning_low && last_value <= thresholds.warning_low) ||
(thresholds.critical_low && last_value <= thresholds.critical_low)
end
defp value_is_normal?(current_value, thresholds) do
(thresholds.warning_high == nil || current_value < thresholds.warning_high) &&
(thresholds.critical_high == nil || current_value < thresholds.critical_high) &&
(thresholds.warning_low == nil || current_value > thresholds.warning_low) &&
(thresholds.critical_low == nil || current_value > thresholds.critical_low)
end
# Change detection
defp maybe_add_change_event(events, sensor, current_value, timestamp, device_id) do
if sensor.sensor_unit == "%" do
# Percentage sensors use spike/drop events for large swings
change_event = check_significant_change(sensor, current_value, timestamp, device_id)
if change_event, do: [change_event | events], else: events
else
# All other sensors emit value_changed when value differs
if current_value == sensor.last_value do
events
else
event =
build_sensor_event(
device_id,
sensor,
"sensor_value_changed",
"info",
"#{sensor.sensor_descr} changed from #{format_sensor_value(sensor.last_value, sensor.sensor_unit)} to #{format_sensor_value(current_value, sensor.sensor_unit)}",
%{current_value: current_value, previous_value: sensor.last_value},
timestamp
)
[event | events]
end
end
end
defp check_significant_change(sensor, current_value, timestamp, device_id) do
change_percent = abs(current_value - sensor.last_value)
cond do
change_percent >= 30 && current_value > sensor.last_value ->
build_sensor_event(
device_id,
sensor,
"sensor_value_spike",
"warning",
"#{sensor.sensor_descr} spiked from #{format_sensor_value(sensor.last_value, sensor.sensor_unit)} to #{format_sensor_value(current_value, sensor.sensor_unit)}",
%{current_value: current_value, previous_value: sensor.last_value, change: change_percent},
timestamp
)
change_percent >= 30 && current_value < sensor.last_value ->
build_sensor_event(
device_id,
sensor,
"sensor_value_drop",
"info",
"#{sensor.sensor_descr} dropped from #{format_sensor_value(sensor.last_value, sensor.sensor_unit)} to #{format_sensor_value(current_value, sensor.sensor_unit)}",
%{current_value: current_value, previous_value: sensor.last_value, change: change_percent},
timestamp
)
true ->
nil
end
end
# Event building and broadcasting
defp build_sensor_event(device_id, sensor, event_type, severity, message, metadata, timestamp) do
base_metadata = %{
sensor_id: sensor.id,
sensor_name: sensor.sensor_descr,
sensor_type: sensor.sensor_type,
sensor_unit: sensor.sensor_unit
}
%{
device_id: device_id,
event_type: event_type,
severity: severity,
message: message,
metadata: Map.merge(base_metadata, metadata),
occurred_at: timestamp
}
end
defp broadcast_events(events) do
Enum.each(events, fn event ->
_ = Phoenix.PubSub.broadcast(Towerops.PubSub, "device:#{event.device_id}", {:device_event, event})
_ = Phoenix.PubSub.broadcast(Towerops.PubSub, "device:events", {:device_event, event})
Logger.debug("Sensor event: #{event.message}")
end)
end
defp get_device_id_from_sensor(sensor) do
case Towerops.Repo.get(Towerops.Snmp.Device, sensor.snmp_device_id) do
nil -> nil
device -> device.device_id
end
end
defp format_sensor_value(value, unit) when is_number(value) do
"#{Float.round(value / 1.0, 1)}#{unit}"
end
defp format_sensor_value(_, _), do: "N/A"
end