Add sensor value change tracking and threshold monitoring
- Add new event types: sensor_threshold_warning, sensor_threshold_critical, sensor_threshold_normal, sensor_value_spike, sensor_value_drop - Implement sensor change detection in PollerWorker - Track threshold violations (warning/critical high/low) - Detect value spikes/drops for percentage sensors (30% change) - Track return to normal from threshold violations - Store threshold config in sensor metadata - Update sensor last_value and last_checked_at on each poll - Broadcast sensor events via PubSub for real-time logging
This commit is contained in:
parent
7ea79afea1
commit
6c631cbe26
3 changed files with 253 additions and 5 deletions
|
|
@ -17,6 +17,11 @@ defmodule Towerops.Equipment.Event do
|
|||
| :interface_admin_status_change
|
||||
| :device_discovered
|
||||
| :device_rediscovered
|
||||
| :sensor_threshold_warning
|
||||
| :sensor_threshold_critical
|
||||
| :sensor_threshold_normal
|
||||
| :sensor_value_spike
|
||||
| :sensor_value_drop
|
||||
|
||||
@primary_key {:id, :binary_id, autogenerate: true}
|
||||
@foreign_key_type :binary_id
|
||||
|
|
@ -46,7 +51,12 @@ defmodule Towerops.Equipment.Event do
|
|||
"interface_mac_change",
|
||||
"interface_admin_status_change",
|
||||
"device_discovered",
|
||||
"device_rediscovered"
|
||||
"device_rediscovered",
|
||||
"sensor_threshold_warning",
|
||||
"sensor_threshold_critical",
|
||||
"sensor_threshold_normal",
|
||||
"sensor_value_spike",
|
||||
"sensor_value_drop"
|
||||
])
|
||||
|> validate_inclusion(:severity, ["info", "warning", "critical"])
|
||||
|> foreign_key_constraint(:equipment_id)
|
||||
|
|
|
|||
|
|
@ -171,6 +171,15 @@ defmodule Towerops.Snmp.PollerWorker do
|
|||
checked_at: timestamp
|
||||
})
|
||||
|
||||
# Check for sensor value changes and threshold violations
|
||||
detect_sensor_changes(sensor, value, timestamp)
|
||||
|
||||
# Update sensor's last value and timestamp
|
||||
Snmp.update_sensor(sensor, %{
|
||||
last_value: value,
|
||||
last_checked_at: timestamp
|
||||
})
|
||||
|
||||
{:error, :non_numeric} ->
|
||||
Logger.debug("Non-numeric SNMP value for sensor #{sensor.sensor_descr}")
|
||||
|
||||
|
|
@ -564,6 +573,235 @@ defmodule Towerops.Snmp.PollerWorker do
|
|||
end
|
||||
end
|
||||
|
||||
defp detect_sensor_changes(sensor, current_value, timestamp) do
|
||||
# Skip change detection if there's no previous value to compare against
|
||||
if sensor.last_value == nil do
|
||||
:ok
|
||||
else
|
||||
equipment_id = get_equipment_id_from_sensor(sensor)
|
||||
events = []
|
||||
|
||||
# Get threshold configuration from sensor metadata
|
||||
warning_high = sensor.metadata["warning_high"]
|
||||
critical_high = sensor.metadata["critical_high"]
|
||||
warning_low = sensor.metadata["warning_low"]
|
||||
critical_low = sensor.metadata["critical_low"]
|
||||
|
||||
# Check for threshold violations
|
||||
events =
|
||||
cond do
|
||||
critical_high && current_value >= critical_high ->
|
||||
# Critical high threshold exceeded
|
||||
event =
|
||||
build_sensor_event(
|
||||
equipment_id,
|
||||
sensor,
|
||||
"sensor_threshold_critical",
|
||||
"critical",
|
||||
"#{sensor.sensor_descr} is critically high: #{format_sensor_value(current_value, sensor.sensor_unit)} (threshold: #{critical_high}#{sensor.sensor_unit})",
|
||||
%{
|
||||
current_value: current_value,
|
||||
previous_value: sensor.last_value,
|
||||
threshold_type: "critical_high",
|
||||
threshold_value: critical_high
|
||||
},
|
||||
timestamp
|
||||
)
|
||||
|
||||
[event | events]
|
||||
|
||||
critical_low && current_value <= critical_low ->
|
||||
# Critical low threshold exceeded
|
||||
event =
|
||||
build_sensor_event(
|
||||
equipment_id,
|
||||
sensor,
|
||||
"sensor_threshold_critical",
|
||||
"critical",
|
||||
"#{sensor.sensor_descr} is critically low: #{format_sensor_value(current_value, sensor.sensor_unit)} (threshold: #{critical_low}#{sensor.sensor_unit})",
|
||||
%{
|
||||
current_value: current_value,
|
||||
previous_value: sensor.last_value,
|
||||
threshold_type: "critical_low",
|
||||
threshold_value: critical_low
|
||||
},
|
||||
timestamp
|
||||
)
|
||||
|
||||
[event | events]
|
||||
|
||||
warning_high && current_value >= warning_high ->
|
||||
# Warning high threshold exceeded
|
||||
event =
|
||||
build_sensor_event(
|
||||
equipment_id,
|
||||
sensor,
|
||||
"sensor_threshold_warning",
|
||||
"warning",
|
||||
"#{sensor.sensor_descr} is high: #{format_sensor_value(current_value, sensor.sensor_unit)} (threshold: #{warning_high}#{sensor.sensor_unit})",
|
||||
%{
|
||||
current_value: current_value,
|
||||
previous_value: sensor.last_value,
|
||||
threshold_type: "warning_high",
|
||||
threshold_value: warning_high
|
||||
},
|
||||
timestamp
|
||||
)
|
||||
|
||||
[event | events]
|
||||
|
||||
warning_low && current_value <= warning_low ->
|
||||
# Warning low threshold exceeded
|
||||
event =
|
||||
build_sensor_event(
|
||||
equipment_id,
|
||||
sensor,
|
||||
"sensor_threshold_warning",
|
||||
"warning",
|
||||
"#{sensor.sensor_descr} is low: #{format_sensor_value(current_value, sensor.sensor_unit)} (threshold: #{warning_low}#{sensor.sensor_unit})",
|
||||
%{
|
||||
current_value: current_value,
|
||||
previous_value: sensor.last_value,
|
||||
threshold_type: "warning_low",
|
||||
threshold_value: warning_low
|
||||
},
|
||||
timestamp
|
||||
)
|
||||
|
||||
[event | events]
|
||||
|
||||
true ->
|
||||
# Check if value returned to normal from a threshold violation
|
||||
was_over_threshold =
|
||||
(warning_high && sensor.last_value >= warning_high) ||
|
||||
(critical_high && sensor.last_value >= critical_high) ||
|
||||
(warning_low && sensor.last_value <= warning_low) ||
|
||||
(critical_low && sensor.last_value <= critical_low)
|
||||
|
||||
is_now_normal =
|
||||
(warning_high == nil || current_value < warning_high) &&
|
||||
(critical_high == nil || current_value < critical_high) &&
|
||||
(warning_low == nil || current_value > warning_low) &&
|
||||
(critical_low == nil || current_value > critical_low)
|
||||
|
||||
if was_over_threshold && is_now_normal do
|
||||
event =
|
||||
build_sensor_event(
|
||||
equipment_id,
|
||||
sensor,
|
||||
"sensor_threshold_normal",
|
||||
"info",
|
||||
"#{sensor.sensor_descr} returned to normal: #{format_sensor_value(current_value, sensor.sensor_unit)}",
|
||||
%{
|
||||
current_value: current_value,
|
||||
previous_value: sensor.last_value
|
||||
},
|
||||
timestamp
|
||||
)
|
||||
|
||||
[event | events]
|
||||
else
|
||||
events
|
||||
end
|
||||
end
|
||||
|
||||
# Check for significant value changes (spikes or drops)
|
||||
# Only for percentage-based sensors (CPU, memory, storage)
|
||||
events =
|
||||
if sensor.sensor_unit == "%" do
|
||||
change_percent = abs(current_value - sensor.last_value)
|
||||
|
||||
cond do
|
||||
# Spike: increase of 30% or more
|
||||
change_percent >= 30 && current_value > sensor.last_value ->
|
||||
event =
|
||||
build_sensor_event(
|
||||
equipment_id,
|
||||
sensor,
|
||||
"sensor_value_spike",
|
||||
"warning",
|
||||
"#{sensor.sensor_descr} spiked from #{format_sensor_value(sensor.last_value, sensor.sensor_unit)} to #{format_sensor_value(current_value, sensor.sensor_unit)}",
|
||||
%{
|
||||
current_value: current_value,
|
||||
previous_value: sensor.last_value,
|
||||
change: change_percent
|
||||
},
|
||||
timestamp
|
||||
)
|
||||
|
||||
[event | events]
|
||||
|
||||
# Drop: decrease of 30% or more
|
||||
change_percent >= 30 && current_value < sensor.last_value ->
|
||||
event =
|
||||
build_sensor_event(
|
||||
equipment_id,
|
||||
sensor,
|
||||
"sensor_value_drop",
|
||||
"info",
|
||||
"#{sensor.sensor_descr} dropped from #{format_sensor_value(sensor.last_value, sensor.sensor_unit)} to #{format_sensor_value(current_value, sensor.sensor_unit)}",
|
||||
%{
|
||||
current_value: current_value,
|
||||
previous_value: sensor.last_value,
|
||||
change: change_percent
|
||||
},
|
||||
timestamp
|
||||
)
|
||||
|
||||
[event | events]
|
||||
|
||||
true ->
|
||||
events
|
||||
end
|
||||
else
|
||||
events
|
||||
end
|
||||
|
||||
# Broadcast all events
|
||||
Enum.each(events, fn event ->
|
||||
Phoenix.PubSub.broadcast(
|
||||
Towerops.PubSub,
|
||||
"equipment:events",
|
||||
{:equipment_event, event}
|
||||
)
|
||||
|
||||
Logger.debug("Sensor event: #{event.message}")
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
||||
defp build_sensor_event(equipment_id, sensor, event_type, severity, message, metadata, timestamp) do
|
||||
base_metadata = %{
|
||||
sensor_id: sensor.id,
|
||||
sensor_name: sensor.sensor_descr,
|
||||
sensor_type: sensor.sensor_type,
|
||||
sensor_unit: sensor.sensor_unit
|
||||
}
|
||||
|
||||
%{
|
||||
equipment_id: equipment_id,
|
||||
event_type: event_type,
|
||||
severity: severity,
|
||||
message: message,
|
||||
metadata: Map.merge(base_metadata, metadata),
|
||||
occurred_at: timestamp
|
||||
}
|
||||
end
|
||||
|
||||
defp get_equipment_id_from_sensor(sensor) do
|
||||
# Get the device to find equipment_id
|
||||
case Towerops.Repo.get(Towerops.Snmp.Device, sensor.snmp_device_id) do
|
||||
nil -> nil
|
||||
device -> device.equipment_id
|
||||
end
|
||||
end
|
||||
|
||||
defp format_sensor_value(value, unit) when is_number(value) do
|
||||
"#{Float.round(value, 1)}#{unit}"
|
||||
end
|
||||
|
||||
defp format_sensor_value(_, _), do: "N/A"
|
||||
|
||||
defp via_tuple(equipment_id) do
|
||||
{:via, Registry, {PollerRegistry, equipment_id}}
|
||||
end
|
||||
|
|
|
|||
|
|
@ -189,7 +189,7 @@
|
|||
>
|
||||
<div class="flex items-center justify-between">
|
||||
<h3 class="text-sm font-semibold text-zinc-900 dark:text-zinc-100">
|
||||
Overall Traffic (24 Hours)
|
||||
Overall Traffic
|
||||
</h3>
|
||||
<.icon name="hero-arrow-right" class="h-4 w-4 text-zinc-400" />
|
||||
</div>
|
||||
|
|
@ -223,7 +223,7 @@
|
|||
>
|
||||
<div class="flex items-center justify-between">
|
||||
<h3 class="text-sm font-semibold text-zinc-900 dark:text-zinc-100">
|
||||
Processors (24 Hours)
|
||||
Processors
|
||||
</h3>
|
||||
<.icon name="hero-arrow-right" class="h-4 w-4 text-zinc-400" />
|
||||
</div>
|
||||
|
|
@ -251,7 +251,7 @@
|
|||
>
|
||||
<div class="flex items-center justify-between">
|
||||
<h3 class="text-sm font-semibold text-zinc-900 dark:text-zinc-100">
|
||||
Memory Usage (24 Hours)
|
||||
Memory Usage
|
||||
</h3>
|
||||
<.icon name="hero-arrow-right" class="h-4 w-4 text-zinc-400" />
|
||||
</div>
|
||||
|
|
@ -279,7 +279,7 @@
|
|||
>
|
||||
<div class="flex items-center justify-between">
|
||||
<h3 class="text-sm font-semibold text-zinc-900 dark:text-zinc-100">
|
||||
Storage Usage (24 Hours)
|
||||
Storage Usage
|
||||
</h3>
|
||||
<.icon name="hero-arrow-right" class="h-4 w-4 text-zinc-400" />
|
||||
</div>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue