towerops/lib/towerops/snmp/sensor_change_detector.ex
2026-03-29 11:03:20 -05:00

336 lines
11 KiB
Elixir

defmodule Towerops.Snmp.SensorChangeDetector do
@moduledoc """
Detects sensor value changes and broadcasts events.
Shared between DevicePollerWorker and AgentChannel to ensure consistent
event detection across both Phoenix-polled and agent-polled devices.
Pure decision logic (threshold checking, value formatting) is implemented
as helper functions. PubSub broadcasting, Repo queries, and event building stay here.
"""
alias Towerops.Snmp.Sensor
require Logger
@doc """
Detects sensor changes and broadcasts events via PubSub.
For percentage sensors: emits spike/drop events for >=30% swings.
For all other sensors: emits sensor_value_changed when value differs from last_value.
For all sensors with thresholds: emits threshold warning/critical/normal events.
No-ops when `sensor.last_value` is nil (first reading).
"""
@spec detect_and_broadcast(Sensor.t(), number(), DateTime.t()) :: :ok
def detect_and_broadcast(%Sensor{last_value: nil}, _current_value, _timestamp), do: :ok
def detect_and_broadcast(%Sensor{} = sensor, current_value, timestamp) do
device_id = get_device_id_from_sensor(sensor)
if device_id do
thresholds = extract_thresholds(sensor.metadata)
events =
[]
|> maybe_add_threshold_event(sensor, current_value, timestamp, device_id, thresholds)
|> maybe_add_change_event(sensor, current_value, timestamp, device_id)
broadcast_events(events)
end
:ok
end
# Threshold detection
defp extract_thresholds(metadata) do
%{
warning_high: metadata["warning_high"],
critical_high: metadata["critical_high"],
warning_low: metadata["warning_low"],
critical_low: metadata["critical_low"]
}
end
defp maybe_add_threshold_event(events, sensor, current_value, timestamp, device_id, thresholds) do
threshold_event = check_threshold_violation(sensor, current_value, timestamp, device_id, thresholds)
if threshold_event, do: [threshold_event | events], else: events
end
defp check_threshold_violation(sensor, current_value, timestamp, device_id, thresholds) do
threshold_checks = [
&check_critical_high/5,
&check_critical_low/5,
&check_warning_high/5,
&check_warning_low/5
]
Enum.find_value(threshold_checks, fn check_fn ->
check_fn.(sensor, current_value, timestamp, device_id, thresholds)
end) || check_returned_to_normal(sensor, current_value, timestamp, device_id, thresholds)
end
defp check_critical_high(sensor, current_value, timestamp, device_id, thresholds) do
if thresholds.critical_high && current_value >= thresholds.critical_high do
build_threshold_event(
sensor,
current_value,
timestamp,
device_id,
"critical_high",
thresholds.critical_high,
"critical",
"critically high"
)
end
end
defp check_critical_low(sensor, current_value, timestamp, device_id, thresholds) do
if thresholds.critical_low && current_value <= thresholds.critical_low do
build_threshold_event(
sensor,
current_value,
timestamp,
device_id,
"critical_low",
thresholds.critical_low,
"critical",
"critically low"
)
end
end
defp check_warning_high(sensor, current_value, timestamp, device_id, thresholds) do
if thresholds.warning_high && current_value >= thresholds.warning_high do
build_threshold_event(
sensor,
current_value,
timestamp,
device_id,
"warning_high",
thresholds.warning_high,
"warning",
"high"
)
end
end
defp check_warning_low(sensor, current_value, timestamp, device_id, thresholds) do
if thresholds.warning_low && current_value <= thresholds.warning_low do
build_threshold_event(
sensor,
current_value,
timestamp,
device_id,
"warning_low",
thresholds.warning_low,
"warning",
"low"
)
end
end
defp build_threshold_event(
sensor,
current_value,
timestamp,
device_id,
threshold_type,
threshold_value,
severity,
description
) do
event_type = if severity == "critical", do: "sensor_threshold_critical", else: "sensor_threshold_warning"
build_sensor_event(
device_id,
sensor,
event_type,
severity,
"#{sensor.sensor_descr} is #{description}: #{format_sensor_value(current_value, sensor.sensor_unit)} (threshold: #{threshold_value}#{sensor.sensor_unit})",
%{
current_value: current_value,
previous_value: sensor.last_value,
threshold_type: threshold_type,
threshold_value: threshold_value
},
timestamp
)
end
defp check_returned_to_normal(sensor, current_value, timestamp, device_id, thresholds) do
was_over_threshold = value_was_over_threshold?(sensor.last_value, thresholds)
is_now_normal = value_is_normal?(current_value, thresholds)
if was_over_threshold && is_now_normal do
build_sensor_event(
device_id,
sensor,
"sensor_threshold_normal",
"info",
"#{sensor.sensor_descr} returned to normal: #{format_sensor_value(current_value, sensor.sensor_unit)}",
%{current_value: current_value, previous_value: sensor.last_value},
timestamp
)
end
end
# Check whether the last value exceeded any configured threshold.
# Returns true if the value was at or above any high threshold,
# or at or below any low threshold.
defp value_was_over_threshold?(last_value, thresholds) do
check_at_or_above(last_value, thresholds.warning_high) or
check_at_or_above(last_value, thresholds.critical_high) or
check_at_or_below(last_value, thresholds.warning_low) or
check_at_or_below(last_value, thresholds.critical_low)
end
# Check whether the current value is within all configured thresholds.
# Returns true only if the value is strictly below all high thresholds
# and strictly above all low thresholds. Unconfigured thresholds (nil)
# are treated as not restricting.
defp value_is_normal?(current_value, thresholds) do
below?(current_value, thresholds.warning_high) and
below?(current_value, thresholds.critical_high) and
above?(current_value, thresholds.warning_low) and
above?(current_value, thresholds.critical_low)
end
defp check_at_or_above(value, threshold) when not is_nil(threshold), do: value >= threshold
defp check_at_or_above(_value, nil), do: false
defp check_at_or_below(value, threshold) when not is_nil(threshold), do: value <= threshold
defp check_at_or_below(_value, nil), do: false
defp below?(value, threshold) when not is_nil(threshold), do: value < threshold
defp below?(_value, nil), do: true
defp above?(value, threshold) when not is_nil(threshold), do: value > threshold
defp above?(_value, nil), do: true
# Change detection
defp maybe_add_change_event(events, sensor, current_value, timestamp, device_id) do
if sensor.sensor_unit == "%" do
# Percentage sensors use spike/drop events for large swings
change_event = check_significant_change(sensor, current_value, timestamp, device_id)
if change_event, do: [change_event | events], else: events
else
# All other sensors emit value_changed when value differs
if current_value == sensor.last_value do
events
else
event =
build_sensor_event(
device_id,
sensor,
"sensor_value_changed",
"info",
"#{sensor.sensor_descr} changed from #{format_sensor_value(sensor.last_value, sensor.sensor_unit)} to #{format_sensor_value(current_value, sensor.sensor_unit)}",
%{current_value: current_value, previous_value: sensor.last_value},
timestamp
)
[event | events]
end
end
end
defp check_significant_change(sensor, current_value, timestamp, device_id) do
change_percent = abs(current_value - sensor.last_value)
cond do
change_percent >= 30 && current_value > sensor.last_value ->
build_sensor_event(
device_id,
sensor,
"sensor_value_spike",
"warning",
"#{sensor.sensor_descr} spiked from #{format_sensor_value(sensor.last_value, sensor.sensor_unit)} to #{format_sensor_value(current_value, sensor.sensor_unit)}",
%{current_value: current_value, previous_value: sensor.last_value, change: change_percent},
timestamp
)
change_percent >= 30 && current_value < sensor.last_value ->
build_sensor_event(
device_id,
sensor,
"sensor_value_drop",
"info",
"#{sensor.sensor_descr} dropped from #{format_sensor_value(sensor.last_value, sensor.sensor_unit)} to #{format_sensor_value(current_value, sensor.sensor_unit)}",
%{current_value: current_value, previous_value: sensor.last_value, change: change_percent},
timestamp
)
true ->
nil
end
end
# Event building and broadcasting
defp build_sensor_event(device_id, sensor, event_type, severity, message, metadata, timestamp) do
base_metadata = %{
sensor_id: sensor.id,
sensor_name: sensor.sensor_descr,
sensor_type: sensor.sensor_type,
sensor_unit: sensor.sensor_unit
}
%{
device_id: device_id,
event_type: event_type,
severity: severity,
message: message,
metadata: Map.merge(base_metadata, metadata),
occurred_at: timestamp
}
end
defp broadcast_events(events) do
Enum.each(events, fn event ->
_ = Phoenix.PubSub.broadcast(Towerops.PubSub, "device:#{event.device_id}", {:device_event, event})
_ = Phoenix.PubSub.broadcast(Towerops.PubSub, "device:events", {:device_event, event})
case get_org_id_from_device(event.device_id) do
nil ->
:ok
org_id ->
event_with_org = Map.put(event, :org_id, org_id)
_ = Phoenix.PubSub.broadcast(Towerops.PubSub, "device_events:org:#{org_id}", {:device_event, event_with_org})
end
Logger.debug("Sensor event: #{event.message}")
end)
end
defp get_org_id_from_device(device_id) do
case Towerops.Repo.get(Towerops.Devices.Device, device_id) do
nil -> nil
device -> device.organization_id
end
end
defp get_device_id_from_sensor(sensor) do
case Towerops.Repo.get(Towerops.Snmp.Device, sensor.snmp_device_id) do
nil -> nil
device -> device.device_id
end
end
# Format a sensor value with its unit for display in event messages.
# When the unit is empty (count sensors), the value is truncated to an integer
# string (e.g. 1000.0 -> "1000"). Otherwise, the value is rounded to 1 decimal
# place and the unit is appended (e.g. 50.0, "Hz" -> "50.0Hz").
defp format_sensor_value(value, unit) when is_number(value) do
case unit || "" do
"" -> Integer.to_string(trunc(value))
unit_str -> "#{Float.round(value, 1)}#{unit_str}"
end
end
defp format_sensor_value(_, _), do: "N/A"
end