From 5bdb40b9f89ba14622d77787d6448598a32974a3 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 4 Jul 2025 12:49:35 -0500 Subject: [PATCH] indexes --- lib/aprsme/packets.ex | 29 +++++++++++++ lib/aprsme_web/live/map_live/index.ex | 42 +++++++++++++++---- .../live/weather_live/callsign_view.html.heex | 4 +- .../20250704174406_optimize_map_queries.exs | 36 ++++++++++++++++ 4 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 priv/repo/migrations/20250704174406_optimize_map_queries.exs diff --git a/lib/aprsme/packets.ex b/lib/aprsme/packets.ex index afaffae..249080e 100644 --- a/lib/aprsme/packets.ex +++ b/lib/aprsme/packets.ex @@ -404,6 +404,35 @@ defmodule Aprsme.Packets do get_packets_for_replay(opts_with_time) end + @doc """ + Gets recent packets optimized for initial map load. + This uses a more efficient query pattern for the most common use case. + """ + @spec get_recent_packets_optimized(map()) :: [struct()] + def get_recent_packets_optimized(opts \\ %{}) do + # Always limit to the last hour for initial load + one_hour_ago = DateTime.add(DateTime.utc_now(), -3600, :second) + limit = Map.get(opts, :limit, 500) + + # Use a more efficient query that leverages the partial indexes + base_query = + from(p in Packet, + where: p.has_position == true, + where: p.received_at >= ^one_hour_ago, + order_by: [desc: p.received_at], + limit: ^limit + ) + + query = + base_query + |> filter_by_region(opts) + |> filter_by_callsign(opts) + |> filter_by_map_bounds(opts) + |> select_with_virtual_coordinates() + + Repo.all(query) + end + @doc """ Gets weather packets for a specific callsign within a time range. This is optimized for weather queries by filtering at the database level. diff --git a/lib/aprsme_web/live/map_live/index.ex b/lib/aprsme_web/live/map_live/index.ex index 5354ac6..39c8d7b 100644 --- a/lib/aprsme_web/live/map_live/index.ex +++ b/lib/aprsme_web/live/map_live/index.ex @@ -374,7 +374,9 @@ defmodule AprsmeWeb.MapLive.Index do } socket = assign(socket, map_bounds: map_bounds) - socket = load_historical_packets_for_bounds(socket, map_bounds) + + # Use optimized query for initial load + socket = load_historical_packets_for_bounds_optimized(socket, map_bounds) {:noreply, socket} else {:noreply, socket} @@ -695,7 +697,7 @@ defmodule AprsmeWeb.MapLive.Index do

APRS.me

- + - +
@@ -751,7 +753,7 @@ defmodule AprsmeWeb.MapLive.Index do
- +
- +
- +
@@ -879,7 +881,7 @@ defmodule AprsmeWeb.MapLive.Index do
<.navigation variant={:vertical} class="text-sm" />
- +
@@ -1123,6 +1125,32 @@ defmodule AprsmeWeb.MapLive.Index do end end + @spec load_historical_packets_for_bounds_optimized(Socket.t(), map()) :: Socket.t() + defp load_historical_packets_for_bounds_optimized(socket, map_bounds) do + bounds = [ + map_bounds.west, + map_bounds.south, + map_bounds.east, + map_bounds.north + ] + + # Use the optimized query for initial load + packets_module = Application.get_env(:aprsme, :packets_module, Aprsme.Packets) + + historical_packets = + packets_module.get_recent_packets_optimized(%{ + bounds: bounds, + # Smaller limit for faster initial load + limit: 500 + }) + + if Enum.empty?(historical_packets) do + assign(socket, historical_loaded: true) + else + process_historical_packets(socket, historical_packets) + end + end + @spec within_bounds?(map() | struct(), map()) :: boolean() defp within_bounds?(packet, bounds) do {lat, lon, _data_extended} = MapHelpers.get_coordinates(packet) diff --git a/lib/aprsme_web/live/weather_live/callsign_view.html.heex b/lib/aprsme_web/live/weather_live/callsign_view.html.heex index 650b4d8..d07dcb9 100644 --- a/lib/aprsme_web/live/weather_live/callsign_view.html.heex +++ b/lib/aprsme_web/live/weather_live/callsign_view.html.heex @@ -244,7 +244,9 @@ <% end %> -
diff --git a/priv/repo/migrations/20250704174406_optimize_map_queries.exs b/priv/repo/migrations/20250704174406_optimize_map_queries.exs new file mode 100644 index 0000000..40c1870 --- /dev/null +++ b/priv/repo/migrations/20250704174406_optimize_map_queries.exs @@ -0,0 +1,36 @@ +defmodule Aprsme.Repo.Migrations.OptimizeMapQueries do + use Ecto.Migration + @disable_ddl_transaction true + + def up do + # Add compound index for the most common map query pattern + create_if_not_exists index(:packets, [:has_position, :received_at]) + create_if_not_exists index(:packets, [:sender, :received_at]) + create_if_not_exists index(:packets, [:sender, :has_position]) + create_if_not_exists index(:packets, [:data_type]) + create_if_not_exists index(:packets, [:data_type, :received_at]) + create_if_not_exists index(:packets, [:symbol_table_id, :symbol_code, :received_at]) + create_if_not_exists index(:packets, [:device_identifier]) + create_if_not_exists index(:packets, [:region, :has_position, :received_at]) + + # Add partial index for weather packets (this is valid) + execute """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS packets_weather_idx + ON packets (received_at DESC) + WHERE data_type = 'weather' + OR (symbol_table_id = '/' AND symbol_code = '_') + """ + end + + def down do + execute "DROP INDEX IF EXISTS packets_weather_idx" + drop index(:packets, [:region, :has_position, :received_at]) + drop index(:packets, [:device_identifier]) + drop index(:packets, [:symbol_table_id, :symbol_code, :received_at]) + drop index(:packets, [:data_type, :received_at]) + drop index(:packets, [:data_type]) + drop index(:packets, [:sender, :has_position]) + drop index(:packets, [:sender, :received_at]) + drop index(:packets, [:has_position, :received_at]) + end +end