From d6417ff16b1163328f8679e7700994a1c9d6cf0d Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 4 Jul 2025 12:15:38 -0500 Subject: [PATCH] more live updates and chart version update --- lib/aprsme/encoding_utils.ex | 172 +++++++++++++++++- lib/aprsme/is/is.ex | 6 +- lib/aprsme/packet.ex | 15 +- lib/aprsme/packet_consumer.ex | 31 +--- lib/aprsme/packets.ex | 60 +----- lib/aprsme/postgres_notifier.ex | 25 +++ lib/aprsme_web/live/map_live/packet_utils.ex | 20 +- .../live/weather_live/callsign_view.ex | 61 +++++-- .../live/weather_live/callsign_view.html.heex | 2 +- 9 files changed, 255 insertions(+), 137 deletions(-) diff --git a/lib/aprsme/encoding_utils.ex b/lib/aprsme/encoding_utils.ex index 524f7cb..0800765 100644 --- a/lib/aprsme/encoding_utils.ex +++ b/lib/aprsme/encoding_utils.ex @@ -45,8 +45,171 @@ defmodule Aprsme.EncodingUtils do def sanitize_string(nil), do: nil def sanitize_string(other), do: other + @doc """ + Converts various types to float for consistent display. + + ## Examples + + iex> Aprsme.EncodingUtils.to_float(1) + 1.0 + iex> Aprsme.EncodingUtils.to_float(1.5) + 1.5 + iex> Aprsme.EncodingUtils.to_float("2.3") + 2.3 + iex> Aprsme.EncodingUtils.to_float("bad") + nil + iex> Aprsme.EncodingUtils.to_float(nil) + nil + """ + @spec to_float(any()) :: float() | nil + def to_float(value) when is_float(value), do: value + def to_float(value) when is_integer(value), do: value * 1.0 + def to_float(%Decimal{} = value), do: Decimal.to_float(value) + + def to_float(value) when is_binary(value) do + case Float.parse(value) do + {float, _} -> float + :error -> nil + end + end + + def to_float(_), do: nil + + @doc """ + Converts various types to Decimal for database storage. + + ## Examples + + iex> is_struct(Aprsme.EncodingUtils.to_decimal(1), Decimal) + true + iex> is_struct(Aprsme.EncodingUtils.to_decimal(1.5), Decimal) + true + iex> is_struct(Aprsme.EncodingUtils.to_decimal("2.3"), Decimal) + true + iex> Aprsme.EncodingUtils.to_decimal("bad") + nil + iex> Aprsme.EncodingUtils.to_decimal(nil) + nil + """ + @spec to_decimal(any()) :: Decimal.t() | nil + def to_decimal(%Decimal{} = d), do: d + def to_decimal(f) when is_float(f), do: Decimal.from_float(f) + def to_decimal(i) when is_integer(i), do: Decimal.new(i) + + def to_decimal(s) when is_binary(s) do + case Decimal.parse(s) do + {d, _} -> + d + + :error -> + case Float.parse(s) do + {f, _} -> Decimal.from_float(f) + :error -> nil + end + end + end + + def to_decimal(_), do: nil + + @doc """ + Sanitizes all string fields in packet data before database storage. + + ## Examples + + iex> Aprsme.EncodingUtils.sanitize_packet_strings(["abc", <<255>>]) + ["abc", "ΓΏ"] + iex> Aprsme.EncodingUtils.sanitize_packet_strings(%{"foo" => <<0, 65, 66, 67>>}) + %{"foo" => "\0ABC"} + iex> Aprsme.EncodingUtils.sanitize_packet_strings(nil) + nil + """ + @spec sanitize_packet_strings(any()) :: any() + def sanitize_packet_strings(%DateTime{} = dt), do: dt + def sanitize_packet_strings(%NaiveDateTime{} = ndt), do: ndt + def sanitize_packet_strings(%_struct{} = struct), do: struct |> Map.from_struct() |> sanitize_packet_strings() + def sanitize_packet_strings(list) when is_list(list), do: Enum.map(list, &sanitize_packet_strings/1) + + def sanitize_packet_strings(binary) when is_binary(binary) do + s = sanitize_string(binary) + if is_binary(s), do: s, else: "" + end + + def sanitize_packet_strings(other), do: other + + @doc """ + Normalizes data_type field to ensure it's always a string. + + ## Examples + + iex> Aprsme.EncodingUtils.normalize_data_type(%{data_type: :weather}) + %{data_type: "weather"} + iex> Aprsme.EncodingUtils.normalize_data_type(%{"data_type" => :foo}) + %{"data_type" => "foo"} + iex> Aprsme.EncodingUtils.normalize_data_type(%{data_type: "bar"}) + %{data_type: "bar"} + iex> Aprsme.EncodingUtils.normalize_data_type(%{"data_type" => "baz"}) + %{"data_type" => "baz"} + iex> Aprsme.EncodingUtils.normalize_data_type(%{foo: 1}) + %{foo: 1} + """ + @spec normalize_data_type(map()) :: map() + def normalize_data_type(%{data_type: data_type} = attrs) when is_atom(data_type) do + %{attrs | data_type: to_string(data_type)} + end + + def normalize_data_type(%{"data_type" => data_type} = attrs) when is_atom(data_type) do + %{attrs | "data_type" => to_string(data_type)} + end + + def normalize_data_type(attrs) when is_map(attrs) do + case {Map.has_key?(attrs, :data_type), Map.get(attrs, :data_type)} do + {true, data_type} when is_atom(data_type) -> + %{attrs | data_type: to_string(data_type)} + + _ -> + attrs + end + end + + def normalize_data_type(attrs), do: attrs + + @doc """ + List of weather-related fields used for packet classification. + + ## Examples + + iex> Aprsme.EncodingUtils.weather_fields() |> Enum.member?(:temperature) + true + iex> :foo in Aprsme.EncodingUtils.weather_fields() + false + """ + @spec weather_fields() :: [atom()] + def weather_fields do + [ + :temperature, + :humidity, + :wind_speed, + :wind_direction, + :wind_gust, + :pressure, + :rain_1h, + :rain_24h, + :rain_since_midnight, + :snow, + :luminosity + ] + end + @doc """ Sanitizes all string fields in an APRS packet to ensure safe JSON encoding. + + ## Examples + + iex> result = Aprsme.EncodingUtils.sanitize_packet(%{"information_field" => <<0, 65, 66, 67>>, "data_extended" => %{"comment" => <<0, 68, 69, 70>>}}) + iex> result["information_field"] == "\0ABC" + true + iex> result["data_extended"]["comment"] == "\0DEF" + true """ @spec sanitize_packet(struct() | map()) :: struct() | map() def sanitize_packet(%Aprsme.Packet{} = packet) do @@ -65,6 +228,13 @@ defmodule Aprsme.EncodingUtils do @doc """ Sanitizes string fields in the data_extended structure. + + ## Examples + + iex> Aprsme.EncodingUtils.sanitize_data_extended(%{"comment" => <<0, 65, 66, 67>>}) + %{"comment" => "ABC"} + iex> Aprsme.EncodingUtils.sanitize_data_extended(nil) + nil """ @spec sanitize_data_extended(nil | map() | MicE.t() | any()) :: nil | map() | MicE.t() | any() def sanitize_data_extended(nil), do: nil @@ -106,6 +276,7 @@ defmodule Aprsme.EncodingUtils do |> :binary.bin_to_list() |> Enum.map(&Integer.to_string(&1, 16)) |> Enum.map_join("", &String.pad_leading(&1, 2, "0")) + |> String.upcase() end @doc """ @@ -115,7 +286,6 @@ defmodule Aprsme.EncodingUtils do iex> Aprsme.EncodingUtils.encoding_info("Hello") %{valid_utf8: true, byte_count: 5, char_count: 5} - iex> Aprsme.EncodingUtils.encoding_info(<<72, 101, 211, 108, 111>>) %{valid_utf8: false, byte_count: 5, invalid_at: 2} """ diff --git a/lib/aprsme/is/is.ex b/lib/aprsme/is/is.ex index f6e12f9..01401de 100644 --- a/lib/aprsme/is/is.ex +++ b/lib/aprsme/is/is.ex @@ -457,11 +457,7 @@ defmodule Aprsme.Is do # Normalize data_type to ensure proper storage @spec normalize_data_type(map()) :: map() - defp normalize_data_type(%{data_type: data_type} = attrs) when is_atom(data_type) do - %{attrs | data_type: to_string(data_type)} - end - - defp normalize_data_type(attrs), do: attrs + defp normalize_data_type(attrs), do: Aprsme.EncodingUtils.normalize_data_type(attrs) @spec update_packet_stats(map(), integer()) :: map() defp update_packet_stats(stats, current_time) do diff --git a/lib/aprsme/packet.ex b/lib/aprsme/packet.ex index 38f36df..f3c5223 100644 --- a/lib/aprsme/packet.ex +++ b/lib/aprsme/packet.ex @@ -347,20 +347,7 @@ defmodule Aprsme.Packet do end defp put_weather_fields(map, data_extended) do - weather_fields = [ - :temperature, - :humidity, - :wind_speed, - :wind_direction, - :wind_gust, - :pressure, - :rain_1h, - :rain_24h, - :rain_since_midnight, - :snow - ] - - Enum.reduce(weather_fields, map, fn field, acc -> + Enum.reduce(Aprsme.EncodingUtils.weather_fields(), map, fn field, acc -> value = data_extended[field] || data_extended[to_string(field)] maybe_put(acc, field, value) end) diff --git a/lib/aprsme/packet_consumer.ex b/lib/aprsme/packet_consumer.ex index a78e803..9289440 100644 --- a/lib/aprsme/packet_consumer.ex +++ b/lib/aprsme/packet_consumer.ex @@ -325,36 +325,11 @@ defmodule Aprsme.PacketConsumer do end end - defp sanitize_packet_strings(%DateTime{} = dt), do: dt - defp sanitize_packet_strings(%NaiveDateTime{} = ndt), do: ndt - defp sanitize_packet_strings(%_struct{} = struct), do: struct |> Map.from_struct() |> sanitize_packet_strings() - defp sanitize_packet_strings(list) when is_list(list), do: Enum.map(list, &sanitize_packet_strings/1) + defp sanitize_packet_strings(value), do: Aprsme.EncodingUtils.sanitize_packet_strings(value) - defp sanitize_packet_strings(binary) when is_binary(binary) do - s = Aprsme.EncodingUtils.sanitize_string(binary) - if is_binary(s), do: s, else: "" - end + defp to_float(value), do: Aprsme.EncodingUtils.to_float(value) - defp sanitize_packet_strings(other), do: other - - defp to_float(value) when is_float(value), do: value - defp to_float(value) when is_integer(value), do: value * 1.0 - defp to_float(%Decimal{} = value), do: Decimal.to_float(value) - - defp to_float(value) when is_binary(value) do - case Float.parse(value) do - {float, _} -> float - :error -> nil - end - end - - defp to_float(_), do: nil - - defp normalize_data_type(%{data_type: data_type} = attrs) when is_atom(data_type) do - %{attrs | data_type: to_string(data_type)} - end - - defp normalize_data_type(attrs), do: attrs + defp normalize_data_type(attrs), do: Aprsme.EncodingUtils.normalize_data_type(attrs) defp struct_to_map(%{__struct__: struct_type} = struct) do converted_map = diff --git a/lib/aprsme/packets.ex b/lib/aprsme/packets.ex index b05ed5e..afaffae 100644 --- a/lib/aprsme/packets.ex +++ b/lib/aprsme/packets.ex @@ -79,13 +79,7 @@ defmodule Aprsme.Packets do end defp normalize_data_type(attrs) do - case attrs do - %{data_type: data_type} when is_atom(data_type) -> - Map.put(attrs, :data_type, to_string(data_type)) - - _ -> - attrs - end + Aprsme.EncodingUtils.normalize_data_type(attrs) end defp set_received_at(attrs) do @@ -566,12 +560,10 @@ defmodule Aprsme.Packets do end @doc """ - Clean packets older than a specific number of days. + Clean packets older than a specific number of days. - This function allows for more granular cleanup operations by specifying - the exact age threshold for packet deletion. - @doc \""" - Removes packets older than the specified number of days. + This function allows for more granular cleanup operations by specifying + the exact age threshold for packet deletion. ## Parameters - `days` - Number of days to keep (packets older than this will be deleted) @@ -590,51 +582,13 @@ defmodule Aprsme.Packets do end # Helper to convert various types to float - defp to_float(value) when is_float(value), do: value - defp to_float(value) when is_integer(value), do: value * 1.0 - defp to_float(%Decimal{} = value), do: Decimal.to_float(value) - - defp to_float(value) when is_binary(value) do - case Float.parse(value) do - {float, _} -> float - :error -> nil - end - end - - defp to_float(_), do: nil + defp to_float(value), do: Aprsme.EncodingUtils.to_float(value) # Helper to convert various types to Decimal - defp to_decimal(%Decimal{} = d), do: d - defp to_decimal(f) when is_float(f), do: Decimal.from_float(f) - defp to_decimal(i) when is_integer(i), do: Decimal.new(i) - - defp to_decimal(s) when is_binary(s) do - case Decimal.parse(s) do - {d, _} -> - d - - :error -> - case Float.parse(s) do - {f, _} -> Decimal.from_float(f) - :error -> nil - end - end - end - - defp to_decimal(_), do: nil + defp to_decimal(value), do: Aprsme.EncodingUtils.to_decimal(value) # Helper to sanitize all string fields in packet data before database storage - defp sanitize_packet_strings(%DateTime{} = dt), do: dt - defp sanitize_packet_strings(%NaiveDateTime{} = ndt), do: ndt - defp sanitize_packet_strings(%_struct{} = struct), do: struct |> Map.from_struct() |> sanitize_packet_strings() - defp sanitize_packet_strings(list) when is_list(list), do: Enum.map(list, &sanitize_packet_strings/1) - - defp sanitize_packet_strings(binary) when is_binary(binary) do - s = Aprsme.EncodingUtils.sanitize_string(binary) - if is_binary(s), do: s, else: "" - end - - defp sanitize_packet_strings(other), do: other + defp sanitize_packet_strings(value), do: Aprsme.EncodingUtils.sanitize_packet_strings(value) # Get packets from last hour only - used to initialize the map @spec get_last_hour_packets() :: [struct()] diff --git a/lib/aprsme/postgres_notifier.ex b/lib/aprsme/postgres_notifier.ex index cadaed2..d3a6692 100644 --- a/lib/aprsme/postgres_notifier.ex +++ b/lib/aprsme/postgres_notifier.ex @@ -31,8 +31,12 @@ defmodule Aprsme.PostgresNotifier do def handle_info({:notification, _conn, _pid, @packet_channel, payload}, state) do case Jason.decode(payload) do {:ok, packet} -> + # Broadcast to the general packet topic Phoenix.PubSub.broadcast(Aprsme.PubSub, @packet_topic, {:postgres_packet, packet}) + # Also broadcast to callsign-specific weather topic if it's a weather packet + broadcast_weather_packet_if_relevant(packet) + _ -> :noop end @@ -41,4 +45,25 @@ defmodule Aprsme.PostgresNotifier do end def handle_info(_msg, state), do: {:noreply, state} + + defp broadcast_weather_packet_if_relevant(packet) do + # Check if this is a weather packet using centralized weather fields + has_weather_data = + Enum.any?(Aprsme.EncodingUtils.weather_fields(), fn field -> + value = Map.get(packet, to_string(field)) + not is_nil(value) + end) + + if has_weather_data do + # Extract callsign from packet + callsign = Map.get(packet, "sender") || Map.get(packet, :sender) + + if is_binary(callsign) and callsign != "" do + # Normalize callsign and broadcast to weather-specific topic + normalized_callsign = String.upcase(String.trim(callsign)) + weather_topic = "weather:#{normalized_callsign}" + Phoenix.PubSub.broadcast(Aprsme.PubSub, weather_topic, {:weather_packet, packet}) + end + end + end end diff --git a/lib/aprsme_web/live/map_live/packet_utils.ex b/lib/aprsme_web/live/map_live/packet_utils.ex index fa608c3..b118c86 100644 --- a/lib/aprsme_web/live/map_live/packet_utils.ex +++ b/lib/aprsme_web/live/map_live/packet_utils.ex @@ -54,25 +54,7 @@ defmodule AprsmeWeb.MapLive.PacketUtils do """ @spec to_float(any()) :: float() def to_float(value) do - case value do - %Decimal{} = d -> - Decimal.to_float(d) - - n when is_float(n) -> - n - - n when is_integer(n) -> - n * 1.0 - - n when is_binary(n) -> - case Float.parse(n) do - {f, _} -> f - :error -> 0.0 - end - - _ -> - 0.0 - end + Aprsme.EncodingUtils.to_float(value) || 0.0 end @doc """ diff --git a/lib/aprsme_web/live/weather_live/callsign_view.ex b/lib/aprsme_web/live/weather_live/callsign_view.ex index c49d6bd..9b9b577 100644 --- a/lib/aprsme_web/live/weather_live/callsign_view.ex +++ b/lib/aprsme_web/live/weather_live/callsign_view.ex @@ -9,9 +9,10 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do def mount(%{"callsign" => callsign}, _session, socket) do normalized_callsign = String.upcase(String.trim(callsign)) - # Subscribe to Postgres notifications for live updates + # Subscribe to weather-specific topic for targeted updates if connected?(socket) do - Phoenix.PubSub.subscribe(Aprsme.PubSub, "postgres:aprsme_packets") + weather_topic = "weather:#{normalized_callsign}" + Phoenix.PubSub.subscribe(Aprsme.PubSub, weather_topic) end weather_packet = get_latest_weather_packet(normalized_callsign) @@ -54,6 +55,46 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do end @impl true + def handle_info({:weather_packet, _packet}, socket) do + # Update weather data when new weather packet arrives for this callsign + weather_packet = get_latest_weather_packet(socket.assigns.callsign) + {start_time, end_time} = default_time_range() + weather_history = get_weather_history(socket.assigns.callsign, start_time, end_time) + + weather_history_json = + weather_history + |> Enum.map(fn pkt -> + dew_point = + if is_number(pkt.temperature) and is_number(pkt.humidity) do + calc_dew_point(pkt.temperature, pkt.humidity) + end + + %{ + timestamp: pkt.received_at, + temperature: pkt.temperature, + dew_point: dew_point, + humidity: pkt.humidity, + pressure: pkt.pressure, + wind_direction: pkt.wind_direction, + wind_speed: pkt.wind_speed, + rain_1h: pkt.rain_1h, + rain_24h: pkt.rain_24h, + rain_since_midnight: pkt.rain_since_midnight, + luminosity: pkt.luminosity + } + end) + |> Jason.encode!() + + socket = + socket + |> assign(:weather_packet, weather_packet) + |> assign(:weather_history, weather_history) + |> assign(:weather_history_json, weather_history_json) + + {:noreply, socket} + end + + # Keep the old handler for backward compatibility def handle_info({:postgres_packet, packet}, socket) do # Only update if the packet is for our callsign and contains weather data if packet_matches_callsign?(packet, socket.assigns.callsign) and has_weather_data?(packet) do @@ -107,20 +148,8 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do defp has_weather_data?(packet) do # Check if packet contains any weather-related fields - weather_fields = [ - "temperature", - "humidity", - "pressure", - "wind_speed", - "wind_direction", - "rain_1h", - "rain_24h", - "rain_since_midnight", - "luminosity" - ] - - Enum.any?(weather_fields, fn field -> - value = Map.get(packet, field) || Map.get(packet, String.to_atom(field)) + Enum.any?(Aprsme.EncodingUtils.weather_fields(), fn field -> + value = Map.get(packet, field) || Map.get(packet, to_string(field)) not is_nil(value) end) end diff --git a/lib/aprsme_web/live/weather_live/callsign_view.html.heex b/lib/aprsme_web/live/weather_live/callsign_view.html.heex index 9f66fad..8e1081f 100644 --- a/lib/aprsme_web/live/weather_live/callsign_view.html.heex +++ b/lib/aprsme_web/live/weather_live/callsign_view.html.heex @@ -235,7 +235,7 @@ > <% end %> -