From 5b1b9a08c752c632637230da7a36e7b21ed44b97 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 18 Jan 2026 11:46:56 -0600 Subject: [PATCH] improve profile import --- lib/towerops/device_profiles/importer.ex | 20 ++++++-- .../device_profiles/sensor_definition.ex | 7 +-- lib/towerops/ecto_types/json_any.ex | 39 +++++++++++++++ ...4155_change_sensor_oid_fields_to_jsonb.exs | 49 +++++++++++++++++++ ...612_change_ent_physical_index_to_jsonb.exs | 26 ++++++++++ 5 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 lib/towerops/ecto_types/json_any.ex create mode 100644 priv/repo/migrations/20260118174155_change_sensor_oid_fields_to_jsonb.exs create mode 100644 priv/repo/migrations/20260118174612_change_ent_physical_index_to_jsonb.exs diff --git a/lib/towerops/device_profiles/importer.ex b/lib/towerops/device_profiles/importer.ex index 0e21a3c8..d16f5cef 100644 --- a/lib/towerops/device_profiles/importer.ex +++ b/lib/towerops/device_profiles/importer.ex @@ -319,12 +319,15 @@ defmodule Towerops.DeviceProfiles.Importer do defp import_sensors_of_class(profile_id, sensor_class, sensors) do Enum.each(sensors, fn sensor -> # Skip sensors without a valid OID (required for SNMP polling) - if sensor["oid"] || sensor["num_oid"] do + # Check for both nil and empty string + has_valid_oid = present?(sensor["oid"]) || present?(sensor["num_oid"]) + + if has_valid_oid do attrs = %{ device_profile_id: profile_id, sensor_class: sensor_class, - oid: sensor["oid"], - num_oid: sensor["num_oid"], + oid: nilify_blank(sensor["oid"]), + num_oid: nilify_blank(sensor["num_oid"]), index: to_string(sensor["index"] || ""), descr: sensor["descr"], divisor: calculate_divisor(sensor["precision"]), @@ -406,4 +409,15 @@ defmodule Towerops.DeviceProfiles.Importer do |> Enum.reject(fn {_k, v} -> is_nil(v) end) |> Map.new() end + + # Check if a value is present (not nil and not empty string) + defp present?(nil), do: false + defp present?(""), do: false + defp present?(_), do: true + + # Convert OID values, preserving all data types since the field is now JSONB + # Only convert empty strings to nil + defp nilify_blank(nil), do: nil + defp nilify_blank(""), do: nil + defp nilify_blank(value), do: value end diff --git a/lib/towerops/device_profiles/sensor_definition.ex b/lib/towerops/device_profiles/sensor_definition.ex index c5f937ee..7ddb442e 100644 --- a/lib/towerops/device_profiles/sensor_definition.ex +++ b/lib/towerops/device_profiles/sensor_definition.ex @@ -11,14 +11,15 @@ defmodule Towerops.DeviceProfiles.SensorDefinition do alias Towerops.DeviceProfiles.DeviceProfile alias Towerops.DeviceProfiles.SensorState + alias Towerops.EctoTypes.JsonAny @primary_key {:id, :binary_id, autogenerate: true} @foreign_key_type :binary_id schema "profile_sensor_definitions" do field :sensor_class, :string - field :oid, :string - field :num_oid, :string + field :oid, JsonAny + field :num_oid, JsonAny field :index, :string field :descr, :string field :divisor, :integer, default: 1 @@ -33,7 +34,7 @@ defmodule Towerops.DeviceProfiles.SensorDefinition do field :skip_value_lt, :float field :skip_value_gt, :float field :user_func, :string - field :entPhysicalIndex, :string + field :entPhysicalIndex, JsonAny field :group, :string field :options, :map diff --git a/lib/towerops/ecto_types/json_any.ex b/lib/towerops/ecto_types/json_any.ex new file mode 100644 index 00000000..e043d18c --- /dev/null +++ b/lib/towerops/ecto_types/json_any.ex @@ -0,0 +1,39 @@ +defmodule Towerops.EctoTypes.JsonAny do + @moduledoc """ + Custom Ecto type for JSONB columns that can store any JSON-encodable value. + + Unlike the built-in :map type which only accepts maps, this type accepts: + - strings + - numbers (integer, float) + - booleans + - nil + - lists + - maps + + All values are stored as JSONB in the database. + """ + use Ecto.Type + + def type, do: :jsonb + + # Cast accepts any JSON-encodable value + def cast(nil), do: {:ok, nil} + def cast(value) when is_binary(value), do: {:ok, value} + def cast(value) when is_number(value), do: {:ok, value} + def cast(value) when is_boolean(value), do: {:ok, value} + def cast(value) when is_list(value), do: {:ok, value} + def cast(value) when is_map(value), do: {:ok, value} + def cast(_), do: :error + + # Load from database - return as-is + def load(value), do: {:ok, value} + + # Dump to database - return as-is, Postgres will handle JSON encoding + def dump(nil), do: {:ok, nil} + def dump(value) when is_binary(value), do: {:ok, value} + def dump(value) when is_number(value), do: {:ok, value} + def dump(value) when is_boolean(value), do: {:ok, value} + def dump(value) when is_list(value), do: {:ok, value} + def dump(value) when is_map(value), do: {:ok, value} + def dump(_), do: :error +end diff --git a/priv/repo/migrations/20260118174155_change_sensor_oid_fields_to_jsonb.exs b/priv/repo/migrations/20260118174155_change_sensor_oid_fields_to_jsonb.exs new file mode 100644 index 00000000..cec28a8a --- /dev/null +++ b/priv/repo/migrations/20260118174155_change_sensor_oid_fields_to_jsonb.exs @@ -0,0 +1,49 @@ +defmodule Towerops.Repo.Migrations.ChangeSensorOidFieldsToJsonb do + use Ecto.Migration + + def up do + # Convert oid column from VARCHAR to JSONB + # Existing string values will be stored as JSON strings + execute """ + ALTER TABLE profile_sensor_definitions + ALTER COLUMN oid TYPE jsonb + USING CASE + WHEN oid IS NULL THEN NULL + ELSE to_jsonb(oid) + END + """ + + # Convert num_oid column from VARCHAR to JSONB + execute """ + ALTER TABLE profile_sensor_definitions + ALTER COLUMN num_oid TYPE jsonb + USING CASE + WHEN num_oid IS NULL THEN NULL + ELSE to_jsonb(num_oid) + END + """ + end + + def down do + # Convert back to VARCHAR(255), extracting string values from JSONB + execute """ + ALTER TABLE profile_sensor_definitions + ALTER COLUMN oid TYPE varchar(255) + USING CASE + WHEN oid IS NULL THEN NULL + WHEN jsonb_typeof(oid) = 'string' THEN oid #>> '{}' + ELSE oid::text + END + """ + + execute """ + ALTER TABLE profile_sensor_definitions + ALTER COLUMN num_oid TYPE varchar(255) + USING CASE + WHEN num_oid IS NULL THEN NULL + WHEN jsonb_typeof(num_oid) = 'string' THEN num_oid #>> '{}' + ELSE num_oid::text + END + """ + end +end diff --git a/priv/repo/migrations/20260118174612_change_ent_physical_index_to_jsonb.exs b/priv/repo/migrations/20260118174612_change_ent_physical_index_to_jsonb.exs new file mode 100644 index 00000000..ad2239af --- /dev/null +++ b/priv/repo/migrations/20260118174612_change_ent_physical_index_to_jsonb.exs @@ -0,0 +1,26 @@ +defmodule Towerops.Repo.Migrations.ChangeEntPhysicalIndexToJsonb do + use Ecto.Migration + + def up do + execute """ + ALTER TABLE profile_sensor_definitions + ALTER COLUMN "entPhysicalIndex" TYPE jsonb + USING CASE + WHEN "entPhysicalIndex" IS NULL THEN NULL + ELSE to_jsonb("entPhysicalIndex") + END + """ + end + + def down do + execute """ + ALTER TABLE profile_sensor_definitions + ALTER COLUMN "entPhysicalIndex" TYPE varchar(255) + USING CASE + WHEN "entPhysicalIndex" IS NULL THEN NULL + WHEN jsonb_typeof("entPhysicalIndex") = 'string' THEN "entPhysicalIndex" #>> '{}' + ELSE "entPhysicalIndex"::text + END + """ + end +end