From a69a87873f0e147ead798737a70517a04588da45 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 16 Jun 2025 11:16:56 -0500 Subject: [PATCH] fix startup --- lib/aprs/geometry_type.ex | 85 -------------------------------- lib/aprs/is/is.ex | 66 ++++++++++++++----------- lib/aprs/packet.ex | 28 +++++++++-- lib/aprs/packets.ex | 101 +++++++++++++++++++++++--------------- scripts/test_postgis.exs | 79 ++++++++++++++++------------- 5 files changed, 168 insertions(+), 191 deletions(-) delete mode 100644 lib/aprs/geometry_type.ex diff --git a/lib/aprs/geometry_type.ex b/lib/aprs/geometry_type.ex deleted file mode 100644 index 72ff397..0000000 --- a/lib/aprs/geometry_type.ex +++ /dev/null @@ -1,85 +0,0 @@ -defmodule Aprs.GeometryType do - @moduledoc """ - Custom Ecto type for PostGIS geometry fields. - This wraps the Geo.PostGIS.Geometry type to handle PostGIS geometry data. - """ - - use Ecto.Type - - def type, do: :geometry - - def cast(%Geo.Point{} = point), do: {:ok, point} - def cast(%Geo.Polygon{} = polygon), do: {:ok, polygon} - def cast(%Geo.LineString{} = linestring), do: {:ok, linestring} - def cast(%Geo.MultiPoint{} = multipoint), do: {:ok, multipoint} - def cast(%Geo.MultiPolygon{} = multipolygon), do: {:ok, multipolygon} - def cast(%Geo.MultiLineString{} = multilinestring), do: {:ok, multilinestring} - def cast(%Geo.GeometryCollection{} = collection), do: {:ok, collection} - - # Handle coordinate tuples and create Point geometry - def cast({lon, lat}) when is_number(lon) and is_number(lat) do - if lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90 do - {:ok, %Geo.Point{coordinates: {lon, lat}, srid: 4326}} - else - :error - end - end - - # Handle maps with lat/lon - def cast(%{lat: lat, lon: lon}) when is_number(lat) and is_number(lon) do - cast({lon, lat}) - end - - def cast(%{"lat" => lat, "lon" => lon}) when is_number(lat) and is_number(lon) do - cast({lon, lat}) - end - - def cast(nil), do: {:ok, nil} - def cast(_), do: :error - - def load(data) when is_binary(data) do - # Handle WKB binary data directly - case Geo.WKB.decode(data) do - {:ok, geometry} -> {:ok, geometry} - _ -> :error - end - end - - def load(%Geo.Point{} = point), do: {:ok, point} - def load(%Geo.Polygon{} = polygon), do: {:ok, polygon} - def load(%Geo.LineString{} = linestring), do: {:ok, linestring} - def load(%Geo.MultiPoint{} = multipoint), do: {:ok, multipoint} - def load(%Geo.MultiPolygon{} = multipolygon), do: {:ok, multipolygon} - def load(%Geo.MultiLineString{} = multilinestring), do: {:ok, multilinestring} - def load(%Geo.GeometryCollection{} = collection), do: {:ok, collection} - def load(nil), do: {:ok, nil} - def load(_), do: :error - - def dump(geometry) when not is_nil(geometry) do - # Encode to WKB binary format - case Geo.WKB.encode(geometry) do - {:ok, data} -> {:ok, data} - _ -> :error - end - end - - def dump(nil), do: {:ok, nil} - def dump(_), do: :error - - @doc """ - Helper function to create a Point geometry from lat/lon coordinates. - """ - def create_point(lat, lon) when is_number(lat) and is_number(lon) do - if lat >= -90 and lat <= 90 and lon >= -180 and lon <= 180 do - %Geo.Point{coordinates: {lon, lat}, srid: 4326} - end - end - - def create_point(_, _), do: nil - - @doc """ - Extract lat/lon coordinates from a Point geometry. - """ - def extract_coordinates(%Geo.Point{coordinates: {lon, lat}}), do: {lat, lon} - def extract_coordinates(_), do: {nil, nil} -end diff --git a/lib/aprs/is/is.ex b/lib/aprs/is/is.ex index b6de907..5a7e4b9 100644 --- a/lib/aprs/is/is.ex +++ b/lib/aprs/is/is.ex @@ -267,40 +267,50 @@ defmodule Aprs.Is do # Use Task to avoid slowing down the main process Task.start(fn -> require Logger - # Store the packet if it has position data - if has_position_data?(parsed_message) do - Logger.info("Storing packet with position data: #{inspect(parsed_message.sender)}") - # Always set received_at timestamp to ensure consistency - current_time = DateTime.truncate(DateTime.utc_now(), :microsecond) - packet_data = Map.put(parsed_message, :received_at, current_time) - # Convert to map before storing to avoid struct conversion issues - attrs = struct_to_map(packet_data) + try do + # Store the packet if it has position data + if has_position_data?(parsed_message) do + Logger.info("Storing packet with position data: #{inspect(parsed_message.sender)}") + # Always set received_at timestamp to ensure consistency + current_time = DateTime.truncate(DateTime.utc_now(), :microsecond) + packet_data = Map.put(parsed_message, :received_at, current_time) - # Extract additional data from the parsed packet including raw packet - attrs = Aprs.Packet.extract_additional_data(attrs, message) + # Convert to map before storing to avoid struct conversion issues + attrs = struct_to_map(packet_data) - # Normalize data_type to string if it's an atom - attrs = normalize_data_type(attrs) + # Extract additional data from the parsed packet including raw packet + attrs = Aprs.Packet.extract_additional_data(attrs, message) - # Ensure SSID is never nil - attrs = - if Map.has_key?(attrs, :ssid) and is_nil(attrs.ssid) do - Map.put(attrs, :ssid, "0") - else - attrs + # Normalize data_type to string if it's an atom + attrs = normalize_data_type(attrs) + + # Ensure SSID is never nil + attrs = + if Map.has_key?(attrs, :ssid) and is_nil(attrs.ssid) do + Map.put(attrs, :ssid, "0") + else + attrs + end + + # Store in database through the Packets context + case Aprs.Packets.store_packet(attrs) do + {:ok, packet} -> + Logger.info("Successfully stored packet from #{packet.sender}") + + {:error, changeset} -> + Logger.error("Failed to store packet from #{inspect(parsed_message.sender)}: #{inspect(changeset.errors)}") + # Log the problematic attributes for debugging + Logger.debug("Packet attributes that failed: #{inspect(attrs)}") end - - # Store in database through the Packets context - case Aprs.Packets.store_packet(attrs) do - {:ok, packet} -> - Logger.info("Successfully stored packet from #{packet.sender}") - - {:error, changeset} -> - Logger.error("Failed to store packet: #{inspect(changeset.errors)}") + else + Logger.debug("Skipping packet without position data: #{inspect(parsed_message.sender)}") end - else - Logger.debug("Skipping packet without position data: #{inspect(parsed_message.sender)}") + rescue + error -> + Logger.error("Exception while storing packet from #{inspect(parsed_message.sender)}: #{inspect(error)}") + Logger.debug("Raw message: #{inspect(message)}") + Logger.debug("Parsed message: #{inspect(parsed_message)}") end end) diff --git a/lib/aprs/packet.ex b/lib/aprs/packet.ex index 7d5f4cb..f980a96 100644 --- a/lib/aprs/packet.ex +++ b/lib/aprs/packet.ex @@ -19,7 +19,7 @@ defmodule Aprs.Packet do field(:region, :string) field(:lat, :float, virtual: true) field(:lon, :float, virtual: true) - field(:location, Aprs.GeometryType) + field(:location, Geo.PostGIS.Geometry) field(:has_position, :boolean, default: false) # Original raw packet and symbol information @@ -144,8 +144,19 @@ defmodule Aprs.Packet do end if is_valid_coordinates?(lat, lon) do - location = Aprs.GeometryType.create_point(lat, lon) - put_change(changeset, :location, location) + try do + location = create_point(lat, lon) + if location do + put_change(changeset, :location, location) + else + changeset + end + rescue + error -> + require Logger + Logger.error("Failed to create geometry for lat=#{lat}, lon=#{lon}: #{inspect(error)}") + changeset + end else changeset end @@ -418,12 +429,19 @@ defmodule Aprs.Packet do @doc """ Create a geometry point from lat/lon coordinates. """ - def create_point(lat, lon), do: Aprs.GeometryType.create_point(lat, lon) + def create_point(lat, lon) when is_number(lat) and is_number(lon) do + if lat >= -90 and lat <= 90 and lon >= -180 and lon <= 180 do + %Geo.Point{coordinates: {lon, lat}, srid: 4326} + end + end + + def create_point(_, _), do: nil @doc """ Extract lat/lon from a PostGIS geometry point. """ - def extract_coordinates(geometry), do: Aprs.GeometryType.extract_coordinates(geometry) + def extract_coordinates(%Geo.Point{coordinates: {lon, lat}}), do: {lat, lon} + def extract_coordinates(_), do: {nil, nil} @doc """ Get latitude from a packet's location geometry. diff --git a/lib/aprs/packets.ex b/lib/aprs/packets.ex index ba78dae..2e474d4 100644 --- a/lib/aprs/packets.ex +++ b/lib/aprs/packets.ex @@ -18,53 +18,68 @@ defmodule Aprs.Packets do def store_packet(packet_data) do require Logger - # Convert to map if it's a struct, or use as is if already a map - packet_attrs = - case packet_data do - %Packet{} = packet -> - packet - |> Map.from_struct() - |> Map.delete(:__meta__) + try do + # Convert to map if it's a struct, or use as is if already a map + packet_attrs = + case packet_data do + %Packet{} = packet -> + packet + |> Map.from_struct() + |> Map.delete(:__meta__) - %{} -> - packet_data - end + %{} -> + packet_data + end - # Convert data_type to string if it's an atom - packet_attrs = - case packet_attrs do - %{data_type: data_type} when is_atom(data_type) -> - Map.put(packet_attrs, :data_type, to_string(data_type)) + # Convert data_type to string if it's an atom + packet_attrs = + case packet_attrs do + %{data_type: data_type} when is_atom(data_type) -> + Map.put(packet_attrs, :data_type, to_string(data_type)) - _ -> - packet_attrs - end + _ -> + packet_attrs + end - # Make sure received_at is set with explicit UTC DateTime - current_time = DateTime.truncate(DateTime.utc_now(), :microsecond) - packet_attrs = Map.put(packet_attrs, :received_at, current_time) + # Make sure received_at is set with explicit UTC DateTime + current_time = DateTime.truncate(DateTime.utc_now(), :microsecond) + packet_attrs = Map.put(packet_attrs, :received_at, current_time) - # Extract position data - {lat, lon} = extract_position(packet_attrs) + # Extract position data with error handling + {lat, lon} = extract_position(packet_attrs) - # Set position fields if found - packet_attrs = - if lat && lon do - packet_attrs - |> Map.put(:lat, lat) - |> Map.put(:lon, lon) - |> Map.put(:has_position, true) - |> Map.put(:region, "#{Float.round(lat, 1)},#{Float.round(lon, 1)}") - else - # Set region based on callsign if no position - sender_region = if packet_attrs[:sender], do: String.slice(packet_attrs.sender || "", 0, 3), else: "unknown" - Map.put(packet_attrs, :region, "call:#{sender_region}") - end + # Set position fields if found + packet_attrs = + if lat && lon do + # Validate coordinates before creating geometry + if are_valid_coordinates?(lat, lon) do + packet_attrs + |> Map.put(:lat, lat) + |> Map.put(:lon, lon) + |> Map.put(:has_position, true) + |> Map.put(:region, "#{Float.round(lat, 1)},#{Float.round(lon, 1)}") + else + Logger.warning("Invalid coordinates for packet from #{packet_attrs[:sender]}: lat=#{lat}, lon=#{lon}") + # Set region based on callsign if coordinates are invalid + sender_region = if packet_attrs[:sender], do: String.slice(packet_attrs.sender || "", 0, 3), else: "unknown" + Map.put(packet_attrs, :region, "call:#{sender_region}") + end + else + # Set region based on callsign if no position + sender_region = if packet_attrs[:sender], do: String.slice(packet_attrs.sender || "", 0, 3), else: "unknown" + Map.put(packet_attrs, :region, "call:#{sender_region}") + end - # Insert the packet - %Packet{} - |> Packet.changeset(packet_attrs) - |> Repo.insert() + # Insert the packet + %Packet{} + |> Packet.changeset(packet_attrs) + |> Repo.insert() + + rescue + error -> + Logger.error("Exception in store_packet for #{inspect(packet_data[:sender])}: #{inspect(error)}") + {:error, :storage_exception} + end end # Extracts position data from packet, checking various possible locations @@ -329,6 +344,12 @@ defmodule Aprs.Packets do defp to_float(_), do: nil + # Helper to validate coordinate values + defp are_valid_coordinates?(lat, lon) do + is_number(lat) and is_number(lon) and + lat >= -90 and lat <= 90 and lon >= -180 and lon <= 180 + end + # Get packets from last hour only - used to initialize the map def get_last_hour_packets do one_hour_ago = DateTime.add(DateTime.utc_now(), -3600, :second) diff --git a/scripts/test_postgis.exs b/scripts/test_postgis.exs index 980fb72..70bc710 100644 --- a/scripts/test_postgis.exs +++ b/scripts/test_postgis.exs @@ -1,33 +1,7 @@ #!/usr/bin/env elixir # Test script to verify PostGIS functionality in the APRS application - -Mix.install([]) - -# Add the project to the code path -Code.append_path("_build/dev/lib/aprs/ebin") -Code.append_path("_build/dev/lib/ecto/ebin") -Code.append_path("_build/dev/lib/ecto_sql/ebin") -Code.append_path("_build/dev/lib/postgrex/ebin") -Code.append_path("_build/dev/lib/geo/ebin") -Code.append_path("_build/dev/lib/geo_postgis/ebin") - -# Load the application configuration -Application.put_env(:aprs, Aprs.Repo, - username: "postgres", - password: "postgres", - hostname: "localhost", - database: "aprs_dev", - types: Aprs.PostgresTypes -) - -# Start necessary applications -Application.ensure_all_started(:postgrex) -Application.ensure_all_started(:ecto) -Application.ensure_all_started(:ecto_sql) - -# Start the repo -{:ok, _} = Aprs.Repo.start_link() +# This script must be run from within the Mix project using: mix run scripts/test_postgis.exs IO.puts("πŸ—ΊοΈ Testing PostGIS functionality...") @@ -64,9 +38,13 @@ try do point = %Geo.Point{coordinates: {-96.7969, 32.7767}, srid: 4326} # Dallas, TX IO.puts("βœ… Created point: #{inspect(point)}") - # Test the custom GeometryType - {:ok, cast_result} = Aprs.GeometryType.cast(point) - IO.puts("βœ… GeometryType cast successful: #{inspect(cast_result)}") + # Test the PostGIS Geometry type directly + {:ok, cast_result} = Geo.PostGIS.Geometry.cast(point) + IO.puts("βœ… Geo.PostGIS.Geometry cast successful: #{inspect(cast_result)}") + + # Test creating a point using the Packet helper + created_point = Aprs.Packet.create_point(32.7767, -96.7969) + IO.puts("βœ… Packet.create_point successful: #{inspect(created_point)}") rescue e -> @@ -157,14 +135,49 @@ rescue IO.puts("❌ Error in spatial query: #{inspect(e)}") end +# Test 8: Test inserting a packet with geometry +IO.puts("\n8. Testing packet insertion with geometry...") +try do + # Create test packet data + test_packet_attrs = %{ + base_callsign: "TEST", + data_type: "position", + destination: "APRS", + information_field: "!3216.50N/09647.00W>Test packet", + path: "WIDE1-1,WIDE2-1", + sender: "TEST-1", + ssid: "1", + received_at: DateTime.utc_now(), + lat: 32.275, + lon: -96.783, + has_position: true, + region: "32.3,-96.8", + raw_packet: "TEST-1>APRS,WIDE1-1,WIDE2-1:!3216.50N/09647.00W>Test packet" + } + + case Aprs.Packets.store_packet(test_packet_attrs) do + {:ok, packet} -> + IO.puts("βœ… Successfully inserted test packet: #{packet.sender}") + + # Clean up test packet + Aprs.Repo.delete(packet) + IO.puts("βœ… Test packet cleaned up") + + {:error, changeset} -> + IO.puts("❌ Failed to insert test packet: #{inspect(changeset.errors)}") + end + +rescue + e -> + IO.puts("❌ Error in packet insertion test: #{inspect(e)}") +end + IO.puts("\nπŸŽ‰ PostGIS testing completed!") IO.puts("\nπŸ“Š Summary:") IO.puts(" - PostGIS extension is enabled") IO.puts(" - Location column with geometry type exists") IO.puts(" - Spatial indexes are created") IO.puts(" - Basic spatial functions are working") +IO.puts(" - Packet insertion with geometry works") IO.puts(" - Data migration from lat/lon to PostGIS geometry completed") IO.puts("\nπŸš€ Your APRS application is now ready for efficient spatial queries!") - -# Clean up -Aprs.Repo.stop()