From 2f20a19c6a1513ad3a795a810c7a54cf94948ca7 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 15 Jun 2025 21:54:06 -0500 Subject: [PATCH] run migrations on start --- config/config.exs | 1 + config/test.exs | 3 + lib/aprs/application.ex | 24 +++ lib/aprs/is/is.ex | 3 + lib/aprs/packet.ex | 178 +++++++++++++++++- lib/aprs_web/live/map_live/enhanced.ex | 8 +- .../20230600000000_add_oban_jobs_table.exs | 11 -- .../migrations/20250616014248_updateoban.exs | 7 - ...616024903_add_additional_packet_fields.exs | 49 +++++ 9 files changed, 264 insertions(+), 20 deletions(-) delete mode 100644 priv/repo/migrations/20230600000000_add_oban_jobs_table.exs delete mode 100644 priv/repo/migrations/20250616014248_updateoban.exs create mode 100644 priv/repo/migrations/20250616024903_add_additional_packet_fields.exs diff --git a/config/config.exs b/config/config.exs index 2b26663..2d439cb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -48,6 +48,7 @@ config :aprs, aprs_is_default_filter: System.get_env("APRS_FILTER"), aprs_is_login_id: System.get_env("APRS_CALLSIGN"), aprs_is_password: System.get_env("APRS_PASSCODE"), + auto_migrate: true, env: config_env() # Configure esbuild (the version is required) diff --git a/config/test.exs b/config/test.exs index c90b0aa..193c399 100644 --- a/config/test.exs +++ b/config/test.exs @@ -23,6 +23,9 @@ config :aprs, AprsWeb.Endpoint, secret_key_base: "IV9+ENaw9i8xjReRk4sULRvRgsmFVTGQwQGGrf4G+Q/SFMeHBCNWRlPXQ2YvT36R", server: false +# Disable automatic migrations during tests +config :aprs, auto_migrate: false + # Only in tests, remove the complexity from the password hashing algorithm config :bcrypt_elixir, :log_rounds, 1 diff --git a/lib/aprs/application.ex b/lib/aprs/application.ex index 92a7189..dccf2f1 100644 --- a/lib/aprs/application.ex +++ b/lib/aprs/application.ex @@ -9,6 +9,9 @@ defmodule Aprs.Application do @impl true def start(_type, _args) do + # Run migrations on startup + migrate() + topologies = Application.get_env(:libcluster, :topologies) || [] children = [ @@ -53,4 +56,25 @@ defmodule Aprs.Application do AprsWeb.Endpoint.config_change(changed, removed) :ok end + + defp migrate do + if Application.get_env(:aprs, :auto_migrate, true) do + require Logger + + Logger.info("Running database migrations...") + Aprs.Release.migrate() + Logger.info("Database migrations completed") + else + require Logger + + Logger.info("Automatic migrations disabled") + end + rescue + error -> + require Logger + + Logger.error("Failed to run migrations: #{inspect(error)}") + # Don't crash the application, just log the error + :ok + end end diff --git a/lib/aprs/is/is.ex b/lib/aprs/is/is.ex index b3658ff..687206f 100644 --- a/lib/aprs/is/is.ex +++ b/lib/aprs/is/is.ex @@ -293,6 +293,9 @@ defmodule Aprs.Is do # Convert to map before storing to avoid struct conversion issues attrs = Map.from_struct(packet_data) + # Extract additional data from the parsed packet including raw packet + attrs = Aprs.Packet.extract_additional_data(attrs, message) + # Normalize data_type to string if it's an atom attrs = normalize_data_type(attrs) diff --git a/lib/aprs/packet.ex b/lib/aprs/packet.ex index f3f7abd..9b3a8e5 100644 --- a/lib/aprs/packet.ex +++ b/lib/aprs/packet.ex @@ -19,6 +19,40 @@ defmodule Aprs.Packet do field(:lat, :float) field(:lon, :float) field(:has_position, :boolean, default: false) + + # Original raw packet and symbol information + field(:raw_packet, :string) + field(:symbol_code, :string) + field(:symbol_table_id, :string) + + # Additional packet data + field(:comment, :string) + field(:timestamp, :string) + field(:aprs_messaging, :boolean, default: false) + + # Weather data + field(:temperature, :float) + field(:humidity, :float) + field(:wind_speed, :float) + field(:wind_direction, :integer) + field(:wind_gust, :float) + field(:pressure, :float) + field(:rain_1h, :float) + field(:rain_24h, :float) + field(:rain_since_midnight, :float) + + # Equipment/status information + field(:manufacturer, :string) + field(:equipment_type, :string) + field(:course, :integer) + field(:speed, :float) + field(:altitude, :float) + + # Message-specific fields + field(:addressee, :string) + field(:message_text, :string) + field(:message_number, :string) + embeds_one(:data_extended, DataExtended) timestamps() @@ -42,7 +76,30 @@ defmodule Aprs.Packet do :region, :lat, :lon, - :has_position + :has_position, + :raw_packet, + :symbol_code, + :symbol_table_id, + :comment, + :timestamp, + :aprs_messaging, + :temperature, + :humidity, + :wind_speed, + :wind_direction, + :wind_gust, + :pressure, + :rain_1h, + :rain_24h, + :rain_since_midnight, + :manufacturer, + :equipment_type, + :course, + :speed, + :altitude, + :addressee, + :message_text, + :message_number ]) |> validate_required([ :base_callsign, @@ -87,4 +144,123 @@ defmodule Aprs.Packet do end defp normalize_data_type(attrs), do: attrs + + @doc """ + Extracts additional data from the parsed packet's data_extended field + and merges it with the packet attributes for storage. + """ + def extract_additional_data(attrs, raw_packet \\ nil) do + data_extended = attrs[:data_extended] || attrs["data_extended"] || %{} + + # Start with the base attributes and add the raw packet + base_attrs = Map.put(attrs, :raw_packet, raw_packet) + + # Extract data based on the type of data_extended + additional_data = + case data_extended do + %{} when is_map(data_extended) -> + extract_from_map(data_extended) + + %Parser.Types.MicE{} = mic_e -> + extract_from_mic_e(mic_e) + + _ -> + %{} + end + + Map.merge(base_attrs, additional_data) + end + + # Extract data from standard map-based data_extended + defp extract_from_map(data_extended) do + %{} + |> maybe_put(:symbol_code, data_extended[:symbol_code] || data_extended["symbol_code"]) + |> maybe_put(:symbol_table_id, data_extended[:symbol_table_id] || data_extended["symbol_table_id"]) + |> maybe_put(:comment, data_extended[:comment] || data_extended["comment"]) + |> maybe_put(:timestamp, data_extended[:timestamp] || data_extended["timestamp"]) + |> maybe_put(:aprs_messaging, data_extended[:aprs_messaging?] || data_extended["aprs_messaging?"]) + |> maybe_put(:temperature, data_extended[:temperature] || data_extended["temperature"]) + |> maybe_put(:humidity, data_extended[:humidity] || data_extended["humidity"]) + |> maybe_put(:wind_speed, data_extended[:wind_speed] || data_extended["wind_speed"]) + |> maybe_put(:wind_direction, data_extended[:wind_direction] || data_extended["wind_direction"]) + |> maybe_put(:wind_gust, data_extended[:wind_gust] || data_extended["wind_gust"]) + |> maybe_put(:pressure, data_extended[:pressure] || data_extended["pressure"]) + |> maybe_put(:rain_1h, data_extended[:rain_1h] || data_extended["rain_1h"]) + |> maybe_put(:rain_24h, data_extended[:rain_24h] || data_extended["rain_24h"]) + |> maybe_put(:rain_since_midnight, data_extended[:rain_since_midnight] || data_extended["rain_since_midnight"]) + |> maybe_put(:manufacturer, data_extended[:manufacturer] || data_extended["manufacturer"]) + |> maybe_put(:equipment_type, data_extended[:equipment_type] || data_extended["equipment_type"]) + |> maybe_put(:course, data_extended[:course] || data_extended["course"]) + |> maybe_put(:speed, data_extended[:speed] || data_extended["speed"]) + |> maybe_put(:altitude, data_extended[:altitude] || data_extended["altitude"]) + |> maybe_put(:addressee, data_extended[:addressee] || data_extended["addressee"]) + |> maybe_put(:message_text, data_extended[:message_text] || data_extended["message_text"]) + |> maybe_put(:message_number, data_extended[:message_number] || data_extended["message_number"]) + |> extract_weather_data(data_extended) + end + + # Extract data from MicE packets + defp extract_from_mic_e(mic_e) do + %{} + |> maybe_put(:comment, mic_e.message) + |> maybe_put(:manufacturer, mic_e.manufacturer) + |> maybe_put(:equipment_type, mic_e.equipment_type) + |> maybe_put(:course, mic_e.course) + |> maybe_put(:speed, mic_e.speed) + |> maybe_put(:symbol_code, mic_e.symbol_code) + |> maybe_put(:symbol_table_id, mic_e.symbol_table_id) + end + + # Extract weather data from various formats + defp extract_weather_data(attrs, data_extended) do + # Look for weather report in different possible locations + weather_data = + data_extended[:weather] || data_extended["weather"] || + data_extended[:weather_report] || data_extended["weather_report"] + + case weather_data do + weather when is_binary(weather) -> + parse_weather_string(attrs, weather) + + weather when is_map(weather) -> + Map.merge(attrs, weather) + + _ -> + attrs + end + end + + # Parse weather data from string format (basic implementation) + defp parse_weather_string(attrs, weather_string) do + # This is a simplified parser - a full implementation would handle + # the complete APRS weather format specification + attrs + |> maybe_extract_weather_field(weather_string, ~r/(\d{3})\/(\d{3})/, [:wind_direction, :wind_speed]) + |> maybe_extract_weather_field(weather_string, ~r/t(\d{3})/, [:temperature]) + |> maybe_extract_weather_field(weather_string, ~r/h(\d{2})/, [:humidity]) + |> maybe_extract_weather_field(weather_string, ~r/b(\d{5})/, [:pressure]) + end + + # Helper to extract weather fields using regex + defp maybe_extract_weather_field(attrs, weather_string, regex, keys) do + case Regex.run(regex, weather_string) do + [_full | matches] -> + keys + |> Enum.zip(matches) + |> Enum.reduce(attrs, fn {key, value}, acc -> + case Integer.parse(value) do + {int_val, _} -> Map.put(acc, key, int_val) + :error -> acc + end + end) + + _ -> + attrs + end + end + + # Helper to put a value only if it's not nil + defp maybe_put(map, _key, nil), do: map + defp maybe_put(map, _key, ""), do: map + defp maybe_put(map, key, value), do: Map.put(map, key, value) end diff --git a/lib/aprs_web/live/map_live/enhanced.ex b/lib/aprs_web/live/map_live/enhanced.ex index 574e883..ab858c8 100644 --- a/lib/aprs_web/live/map_live/enhanced.ex +++ b/lib/aprs_web/live/map_live/enhanced.ex @@ -63,10 +63,16 @@ defmodule AprsWeb.MapLive.Enhanced do "zoom" => zoom } = params + # Normalize center to use atom keys + normalized_center = %{ + lat: center["lat"], + lng: center["lng"] + } + socket = socket |> assign(:map_bounds, bounds) - |> assign(:map_center, center) + |> assign(:map_center, normalized_center) |> assign(:map_zoom, zoom) |> load_markers_in_bounds() diff --git a/priv/repo/migrations/20230600000000_add_oban_jobs_table.exs b/priv/repo/migrations/20230600000000_add_oban_jobs_table.exs deleted file mode 100644 index 08a7766..0000000 --- a/priv/repo/migrations/20230600000000_add_oban_jobs_table.exs +++ /dev/null @@ -1,11 +0,0 @@ -defmodule Aprs.Repo.Migrations.AddObanJobsTable do - use Ecto.Migration - - def up do - Oban.Migrations.up(version: 11) - end - - def down do - Oban.Migrations.down(version: 11) - end -end diff --git a/priv/repo/migrations/20250616014248_updateoban.exs b/priv/repo/migrations/20250616014248_updateoban.exs deleted file mode 100644 index 96a473a..0000000 --- a/priv/repo/migrations/20250616014248_updateoban.exs +++ /dev/null @@ -1,7 +0,0 @@ -defmodule Aprs.Repo.Migrations.Updateoban do - use Ecto.Migration - - def up, do: Oban.Migrations.up() - - def down, do: Oban.Migrations.down() -end diff --git a/priv/repo/migrations/20250616024903_add_additional_packet_fields.exs b/priv/repo/migrations/20250616024903_add_additional_packet_fields.exs new file mode 100644 index 0000000..0819f44 --- /dev/null +++ b/priv/repo/migrations/20250616024903_add_additional_packet_fields.exs @@ -0,0 +1,49 @@ +defmodule Aprs.Repo.Migrations.AddAdditionalPacketFields do + use Ecto.Migration + + def change do + alter table(:packets) do + # Store the original raw packet string as received from APRS-IS + add :raw_packet, :text + + # Store symbol information as separate columns for easier querying + add :symbol_code, :string + add :symbol_table_id, :string + + # Additional useful fields that may be in the parsed data + add :comment, :text + add :timestamp, :string + add :aprs_messaging, :boolean, default: false + + # Weather data fields (for weather packets) + add :temperature, :float + add :humidity, :float + add :wind_speed, :float + add :wind_direction, :integer + add :wind_gust, :float + add :pressure, :float + add :rain_1h, :float + add :rain_24h, :float + add :rain_since_midnight, :float + + # Equipment/status information + add :manufacturer, :string + add :equipment_type, :string + add :course, :integer + add :speed, :float + add :altitude, :float + + # Message-specific fields + add :addressee, :string + add :message_text, :text + add :message_number, :string + end + + # Add indices for commonly queried fields + create index(:packets, [:symbol_code]) + create index(:packets, [:symbol_table_id]) + create index(:packets, [:timestamp]) + create index(:packets, [:addressee]) + create index(:packets, [:manufacturer]) + end +end