diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs index e730133..dab232d 100644 --- a/.dialyzer_ignore.exs +++ b/.dialyzer_ignore.exs @@ -1,5 +1,10 @@ [ - # Ignore all guard_test warnings from these files (false positives) + # Template compilation warnings - these are false positives from Phoenix LiveView template compilation + # These warnings are due to template compilation and cannot be easily fixed without breaking UI logic {"lib/aprsme_web/live/info_live/show.html.heex", :guard_test}, + {"lib/aprsme_web/live/info_live/show.html.heex", :unreachable_code}, + {"lib/aprsme_web/live/weather_live/callsign_view.html.heex", :guard_test}, + + # Ignore guard_test warnings from packet_consumer.ex (already resolved) {"lib/aprsme/packet_consumer.ex", :guard_test} ] diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..c5c0e79 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,78 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +This is an Elixir Phoenix LiveView application that serves as a real-time APRS (Automatic Packet Reporting System) tracker and visualizer. It connects to the APRS-IS network to receive live amateur radio packets and displays them on an interactive map interface. + +## Development Commands + +### Setup +- `mix setup` - Complete project setup (deps.get + ecto.setup) +- `mix deps.get` - Install dependencies +- `mix ecto.setup` - Create database, run migrations, and seed data +- `mix ecto.reset` - Drop and recreate database +- `mix phx.server` - Start Phoenix server (http://localhost:4000) +- `iex -S mix phx.server` - Start server in interactive Elixir shell + +### Testing +- `mix test` - Run full test suite +- `mix test --stale` - Run only tests affected by code changes +- `mix test.watch` - Continuous testing with file watching +- `mix test --cover` - Generate test coverage reports + +### Code Quality +- `mix format` - Format code according to .formatter.exs +- `mix credo` - Static code analysis and style checking +- `mix dialyzer` - Static type analysis (must run and fix errors/warnings) +- `mix sobelow` - Security vulnerability scanning + +### Assets (No Node.js) +- `mix assets.deploy` - Build and minify frontend assets (Tailwind CSS + ESBuild) +- Phoenix uses ESBuild directly without Node.js package managers + +## Architecture + +### Core Components +- **Aprsme.AprsIsConnection** - TCP connection to APRS-IS network with reconnection logic +- **Aprsme.PacketConsumer** - Processes incoming APRS packets using GenStage pipeline +- **Aprsme.Packet** - Database schema for APRS packets with PostGIS geographic data +- **AprsmeWeb.MapLive.Index** - Main real-time map interface using Phoenix LiveView +- **Aprsme.Workers.PacketCleanupWorker** - Oban background job for data cleanup + +### Data Flow +1. APRS-IS connection receives packets via TCP +2. PacketConsumer processes packets through GenStage pipeline +3. Packets stored in PostgreSQL with PostGIS geographic indexing +4. LiveView broadcasts real-time updates to connected clients via PubSub +5. Background workers handle cleanup and maintenance tasks + +### Key Dependencies +- Phoenix LiveView for real-time UI without JavaScript +- PostGIS for geographic data storage and spatial queries +- Oban for background job processing +- GenStage for packet processing pipelines +- Tailwind CSS + ESBuild for frontend assets (no Node.js) + +## Testing Patterns + +Tests use comprehensive mocking to prevent external connections: +- APRS-IS connections are mocked in test environment +- Database uses sandbox mode for isolation +- External API calls mocked with Mox library +- Run `mix test` after any changes to ensure no breakage + +## Code Style Guidelines + +- Use LiveView for UI interactions, minimize JavaScript +- Prefer pattern matching over if/case statements +- Follow idiomatic Elixir conventions +- Run `mix format` before committing +- Address any compiler warnings +- Run `mix dialyzer` and fix all errors/warnings +- Use function composition over nested conditionals + +## Deployment + +The application supports Kubernetes deployment with manifests in `k8s/` directory and GitHub Actions CI/CD pipeline. Database migrations run automatically via init containers. \ No newline at end of file diff --git a/assets/js/features/weather_charts.ts b/assets/js/features/weather_charts.ts index 3281d5d..d0c80b2 100644 --- a/assets/js/features/weather_charts.ts +++ b/assets/js/features/weather_charts.ts @@ -40,6 +40,29 @@ const getThemeColors = () => { }; }; +// Helper for 24-hour time formatting +function format24Hour(date: Date): string { + if (!(date instanceof Date) || isNaN(date.getTime())) return ''; + return new Intl.DateTimeFormat('en-GB', { + hour: '2-digit', + minute: '2-digit', + hour12: false + }).format(date); +} + +function toValidDate(raw: any): Date | null { + if (raw instanceof Date && !isNaN(raw.getTime())) return raw; + if (typeof raw === 'number') { + const d = new Date(raw); + if (!isNaN(d.getTime())) return d; + } + if (typeof raw === 'string') { + const d = new Date(raw); + if (!isNaN(d.getTime())) return d; + } + return null; +} + declare global { interface Window { chartInstances?: Map; @@ -110,6 +133,7 @@ export const WeatherChartHooks: Record = { ] }, options: { + adapters: { date: { locale: 'en-GB' } }, responsive: true, maintainAspectRatio: false, plugins: { @@ -117,7 +141,18 @@ export const WeatherChartHooks: Record = { legend: { labels: { color: colors.text } } }, scales: { - x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: labels.time || 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }, + x: { + type: 'time', + time: { + unit: 'minute', + tooltipFormat: 'HH:mm', + displayFormats: { minute: 'HH:mm', hour: 'HH:mm' }, + locale: 'en-GB' + }, + title: { display: true, text: labels.time || 'Time', color: colors.text }, + ticks: { color: colors.text, maxTicksLimit: 8 }, + grid: { color: colors.grid } + }, y: { title: { display: true, text: labels.degf || '°F', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } } } @@ -156,10 +191,25 @@ export const WeatherChartHooks: Record = { type: 'line', data: { labels: times, datasets: [{ label: labels.humidity_label || 'Humidity (%)', data: humidity, borderColor: 'green', backgroundColor: 'rgba(0, 255, 0, 0.1)', tension: 0.1, pointRadius: 0 }] }, options: { + adapters: { date: { locale: 'en-GB' } }, responsive: true, maintainAspectRatio: false, plugins: { title: { display: true, text: labels.humidity_title || 'Humidity (%)', color: colors.text }, legend: { labels: { color: colors.text } } }, - scales: { x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: labels.time || 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }, y: { title: { display: true, text: labels.percent || '%', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } } + scales: { + x: { + type: 'time', + time: { + unit: 'minute', + tooltipFormat: 'HH:mm', + displayFormats: { minute: 'HH:mm', hour: 'HH:mm' }, + locale: 'en-GB' + }, + title: { display: true, text: labels.time || 'Time', color: colors.text }, + ticks: { color: colors.text, maxTicksLimit: 8 }, + grid: { color: colors.grid } + }, + y: { title: { display: true, text: labels.percent || '%', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } + } } }); } @@ -196,10 +246,25 @@ export const WeatherChartHooks: Record = { type: 'line', data: { labels: times, datasets: [{ label: labels.pressure_label || 'Pressure (mb)', data: pressure, borderColor: 'purple', backgroundColor: 'rgba(128, 0, 128, 0.1)', tension: 0.1, pointRadius: 0 }] }, options: { + adapters: { date: { locale: 'en-GB' } }, responsive: true, maintainAspectRatio: false, plugins: { title: { display: true, text: labels.pressure_title || 'Pressure (mb)', color: colors.text }, legend: { labels: { color: colors.text } } }, - scales: { x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: labels.time || 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }, y: { title: { display: true, text: labels.mb || 'mb', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } } + scales: { + x: { + type: 'time', + time: { + unit: 'minute', + tooltipFormat: 'HH:mm', + displayFormats: { minute: 'HH:mm', hour: 'HH:mm' }, + locale: 'en-GB' + }, + title: { display: true, text: labels.time || 'Time', color: colors.text }, + ticks: { color: colors.text, maxTicksLimit: 8 }, + grid: { color: colors.grid } + }, + y: { title: { display: true, text: labels.mb || 'mb', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } + } } }); } @@ -241,10 +306,25 @@ export const WeatherChartHooks: Record = { ] }, options: { + adapters: { date: { locale: 'en-GB' } }, responsive: true, maintainAspectRatio: false, plugins: { title: { display: true, text: 'Wind Speed & Gust (mph)', color: colors.text }, legend: { labels: { color: colors.text } } }, - scales: { x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }, y: { title: { display: true, text: 'mph', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } } + scales: { + x: { + type: 'time', + time: { + unit: 'minute', + tooltipFormat: 'HH:mm', + displayFormats: { minute: 'HH:mm', hour: 'HH:mm' }, + locale: 'en-GB' + }, + title: { display: true, text: 'Time', color: colors.text }, + ticks: { color: colors.text, maxTicksLimit: 8 }, + grid: { color: colors.grid } + }, + y: { title: { display: true, text: 'mph', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } + } } }); } @@ -288,10 +368,25 @@ export const WeatherChartHooks: Record = { ] }, options: { + adapters: { date: { locale: 'en-GB' } }, responsive: true, maintainAspectRatio: false, plugins: { title: { display: true, text: 'Rainfall (inches)', color: colors.text }, legend: { labels: { color: colors.text } } }, - scales: { x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }, y: { title: { display: true, text: 'inches', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } } + scales: { + x: { + type: 'time', + time: { + unit: 'minute', + tooltipFormat: 'HH:mm', + displayFormats: { minute: 'HH:mm', hour: 'HH:mm' }, + locale: 'en-GB' + }, + title: { display: true, text: 'Time', color: colors.text }, + ticks: { color: colors.text, maxTicksLimit: 8 }, + grid: { color: colors.grid } + }, + y: { title: { display: true, text: 'inches', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } + } } }); } @@ -327,10 +422,25 @@ export const WeatherChartHooks: Record = { type: 'line', data: { labels: times, datasets: [{ label: 'Luminosity', data: luminosity, borderColor: 'yellow', backgroundColor: 'rgba(255, 255, 0, 0.1)', tension: 0.1, pointRadius: 0 }] }, options: { + adapters: { date: { locale: 'en-GB' } }, responsive: true, maintainAspectRatio: false, plugins: { title: { display: true, text: 'Luminosity', color: colors.text }, legend: { labels: { color: colors.text } } }, - scales: { x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }, y: { title: { display: true, text: 'Luminosity', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } } + scales: { + x: { + type: 'time', + time: { + unit: 'minute', + tooltipFormat: 'HH:mm', + displayFormats: { minute: 'HH:mm', hour: 'HH:mm' }, + locale: 'en-GB' + }, + title: { display: true, text: 'Time', color: colors.text }, + ticks: { color: colors.text, maxTicksLimit: 8 }, + grid: { color: colors.grid } + }, + y: { title: { display: true, text: 'Luminosity', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } + } } }); } diff --git a/config/dev.exs b/config/dev.exs index 75b3453..ef1c51b 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -55,6 +55,10 @@ config :aprsme, AprsmeWeb.Endpoint, # Run `mix help phx.gen.cert` for more information. config :aprsme, dev_routes: true +# Configure Hammer for development environment +config :hammer, + backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4, cleanup_interval_ms: 60_000 * 10]} + # Do not include metadata nor timestamps in development logs # # The `http:` config above can be replaced with: @@ -70,14 +74,14 @@ config :phoenix, :plug_init_mode, :runtime # Set a higher stacktrace during development. Avoid configuring such # certfile: "priv/cert/selfsigned.pem" +# Disable swoosh api client as it is only required for production adapters. # in production as building large stacktraces may be expensive. +# different ports. + # ], # # If desired, both `http:` and `https:` keys can be # configured to run both http and https servers on config :phoenix, :stacktrace_depth, 20 -# Disable swoosh api client as it is only required for production adapters. -# different ports. - config :swoosh, :api_client, false diff --git a/config/runtime.exs b/config/runtime.exs index c2e8dd5..f58bed2 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -71,24 +71,27 @@ if config_env() == :prod do server: true, check_origin: ["https://#{host}"] - # config :aprsme, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY") - config :aprsme, ecto_repos: [Aprsme.Repo], aprs_is_server: System.get_env("APRS_SERVER", "dallas.aprs2.net"), + # config :aprsme, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY") aprs_is_port: 14_580, aprs_is_default_filter: System.get_env("APRS_FILTER"), aprs_is_login_id: System.get_env("APRS_CALLSIGN"), aprs_is_password: System.get_env("APRS_PASSCODE"), env: :prod + # Configure Hammer for production environment + config :hammer, + backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4, cleanup_interval_ms: 60_000 * 10]} + # Configure libcluster for Dokku clustering config :libcluster, topologies: [ dokku: [ strategy: Cluster.Strategy.Gossip, config: [ - port: 45892, + port: 45_892, if_addr: "0.0.0.0", multicast_addr: "230.1.1.1", multicast_ttl: 1 diff --git a/config/test.exs b/config/test.exs index 6f312d6..297ee09 100644 --- a/config/test.exs +++ b/config/test.exs @@ -49,6 +49,10 @@ config :aprsme, auto_migrate: false # Only in tests, remove the complexity from the password hashing algorithm config :bcrypt_elixir, :log_rounds, 1 +# Configure Hammer for test environment +config :hammer, + backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4, cleanup_interval_ms: 60_000 * 10]} + # Print only warnings and errors during test config :logger, level: :warning diff --git a/lib/aprsme/application.ex b/lib/aprsme/application.ex index 994880f..d57433c 100644 --- a/lib/aprsme/application.ex +++ b/lib/aprsme/application.ex @@ -25,6 +25,12 @@ defmodule Aprsme.Application do Aprsme.Repo, # Start the PubSub system {Phoenix.PubSub, name: Aprsme.PubSub}, + # Start cache systems + %{id: :query_cache, start: {Cachex, :start_link, [:query_cache, [limit: 10_000]]}}, + %{id: :device_cache, start: {Cachex, :start_link, [:device_cache, [limit: 5_000]]}}, + %{id: :symbol_cache, start: {Cachex, :start_link, [:symbol_cache, [limit: 1_000]]}}, + # Start circuit breaker + Aprsme.CircuitBreaker, # Start the Endpoint (http/https) AprsmeWeb.Endpoint, diff --git a/lib/aprsme/aprs_is_connection.ex b/lib/aprsme/aprs_is_connection.ex index c91eaa5..a35ffdc 100644 --- a/lib/aprsme/aprs_is_connection.ex +++ b/lib/aprsme/aprs_is_connection.ex @@ -6,6 +6,8 @@ defmodule Aprsme.AprsIsConnection do """ use GenServer + alias Aprsme.LogSanitizer + require Logger @type state :: %{ @@ -55,7 +57,14 @@ defmodule Aprsme.AprsIsConnection do {:noreply, %{state | socket: socket, backoff: @reconnect_initial}} {:error, reason} -> - Logger.error("APRS-IS connection failed: #{inspect(reason)}. Retrying in #{state.backoff} ms.") + Logger.error("APRS-IS connection failed, retrying with backoff", + connection_error: + LogSanitizer.log_data( + reason: reason, + backoff_ms: state.backoff, + next_attempt: DateTime.add(DateTime.utc_now(), state.backoff, :millisecond) + ) + ) :telemetry.execute([:aprsme, :is, :connect_error], %{}, %{reason: reason}) schedule_connect(state.backoff) @@ -73,7 +82,15 @@ defmodule Aprsme.AprsIsConnection do @impl true def handle_info({:tcp_closed, _socket}, state) do - Logger.warning("APRS-IS connection closed. Reconnecting...") + Logger.warning("APRS-IS connection closed, initiating reconnection", + connection_status: + LogSanitizer.log_data( + event: "connection_closed", + reconnect_delay_ms: @reconnect_initial, + backoff_reset: true + ) + ) + :telemetry.execute([:aprsme, :is, :disconnected], %{}, %{}) schedule_connect(@reconnect_initial) {:noreply, %{state | socket: nil, backoff: @reconnect_initial}} @@ -81,7 +98,15 @@ defmodule Aprsme.AprsIsConnection do @impl true def handle_info({:tcp_error, _socket, reason}, state) do - Logger.error("APRS-IS TCP error: #{inspect(reason)}. Reconnecting...") + Logger.error("APRS-IS TCP error detected, reconnecting", + tcp_error: + LogSanitizer.log_data( + reason: reason, + reconnect_delay_ms: @reconnect_initial, + socket_reset: true + ) + ) + :telemetry.execute([:aprsme, :is, :tcp_error], %{}, %{reason: reason}) schedule_connect(@reconnect_initial) {:noreply, %{state | socket: nil, backoff: @reconnect_initial}} diff --git a/lib/aprsme/cached_queries.ex b/lib/aprsme/cached_queries.ex new file mode 100644 index 0000000..f686207 --- /dev/null +++ b/lib/aprsme/cached_queries.ex @@ -0,0 +1,118 @@ +defmodule Aprsme.CachedQueries do + @moduledoc """ + Caching layer for database queries to improve performance + """ + + alias Aprsme.Packets + + # 2 minutes for frequently changing data + @cache_ttl_short to_timeout(minute: 2) + # 15 minutes for moderately changing data + @cache_ttl_medium to_timeout(minute: 15) + + @doc """ + Get recent packets with caching + """ + def get_recent_packets_cached(opts) do + cache_key = generate_cache_key("recent_packets", opts) + + case Cachex.get(:query_cache, cache_key) do + {:ok, result} when not is_nil(result) -> + result + + _ -> + result = Packets.get_recent_packets_optimized(opts) + Cachex.put(:query_cache, cache_key, result, ttl: @cache_ttl_short) + result + end + end + + @doc """ + Get weather packets with caching + """ + def get_weather_packets_cached(callsign, start_time, end_time, opts) do + cache_key = generate_cache_key("weather", {callsign, start_time, end_time, opts}) + + case Cachex.get(:query_cache, cache_key) do + {:ok, result} when not is_nil(result) -> + result + + _ -> + result = Packets.get_weather_packets(callsign, start_time, end_time, opts) + Cachex.put(:query_cache, cache_key, result, ttl: @cache_ttl_medium) + result + end + end + + @doc """ + Get packet count with caching + """ + def get_total_packet_count_cached do + cache_key = "total_packet_count" + + case Cachex.get(:query_cache, cache_key) do + {:ok, result} when not is_nil(result) -> + result + + _ -> + result = Packets.get_total_packet_count() + Cachex.put(:query_cache, cache_key, result, ttl: @cache_ttl_medium) + result + end + end + + @doc """ + Get latest packet for callsign with caching + """ + def get_latest_packet_for_callsign_cached(callsign) do + cache_key = generate_cache_key("latest_packet", callsign) + + case Cachex.get(:query_cache, cache_key) do + {:ok, result} when not is_nil(result) -> + result + + _ -> + result = Packets.get_latest_packet_for_callsign(callsign) + # Shorter TTL for latest packets as they change frequently + Cachex.put(:query_cache, cache_key, result, ttl: @cache_ttl_short) + result + end + end + + @doc """ + Invalidate cache entries for a specific callsign + """ + def invalidate_callsign_cache(callsign) do + # Pattern-based cache invalidation + patterns = [ + "latest_packet:#{callsign}", + "weather:#{callsign}:*" + ] + + Enum.each(patterns, fn pattern -> + Cachex.del(:query_cache, pattern) + end) + end + + @doc """ + Invalidate all cached queries + """ + def invalidate_all_cache do + Cachex.clear(:query_cache) + end + + @doc """ + Get cache statistics + """ + def get_cache_stats do + {:ok, stats} = Cachex.stats(:query_cache) + stats + end + + # Private helper functions + + defp generate_cache_key(prefix, data) do + hash = :erlang.phash2(data) + "#{prefix}:#{hash}" + end +end diff --git a/lib/aprsme/circuit_breaker.ex b/lib/aprsme/circuit_breaker.ex new file mode 100644 index 0000000..f1d5417 --- /dev/null +++ b/lib/aprsme/circuit_breaker.ex @@ -0,0 +1,233 @@ +defmodule Aprsme.CircuitBreaker do + @moduledoc """ + Circuit breaker implementation for external service calls to prevent cascading failures + """ + use GenServer + + alias Aprsme.LogSanitizer + + require Logger + + @type state :: :closed | :open | :half_open + @type service_state :: %{ + state: state(), + failure_count: non_neg_integer(), + failure_threshold: non_neg_integer(), + timeout: non_neg_integer(), + last_failure_time: DateTime.t() | nil, + recovery_timeout: non_neg_integer() + } + + @default_failure_threshold 5 + @default_timeout 5000 + @default_recovery_timeout 30_000 + + # Public API + + @doc """ + Start the circuit breaker GenServer + """ + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @doc """ + Execute a function with circuit breaker protection + """ + @spec call(atom(), function(), non_neg_integer()) :: {:ok, any()} | {:error, :circuit_open | :timeout | any()} + def call(service_name, fun, timeout \\ @default_timeout) when is_function(fun, 0) do + case get_state(service_name) do + :open -> + Logger.warning("Circuit breaker open for service", + circuit_breaker: + LogSanitizer.log_data( + service: service_name, + state: :open, + action: :rejected + ) + ) + + {:error, :circuit_open} + + state when state in [:closed, :half_open] -> + attempt_call(service_name, fun, timeout) + end + end + + @doc """ + Get the current state of a service circuit breaker + """ + @spec get_state(atom()) :: state() + def get_state(service_name) do + GenServer.call(__MODULE__, {:get_state, service_name}) + end + + @doc """ + Get statistics for a service circuit breaker + """ + @spec get_stats(atom()) :: service_state() + def get_stats(service_name) do + GenServer.call(__MODULE__, {:get_stats, service_name}) + end + + @doc """ + Manually reset a circuit breaker + """ + @spec reset(atom()) :: :ok + def reset(service_name) do + GenServer.call(__MODULE__, {:reset, service_name}) + end + + # GenServer implementation + + @impl true + def init(_opts) do + {:ok, %{}} + end + + @impl true + def handle_call({:get_state, service_name}, _from, services) do + service_state = get_service_state(services, service_name) + current_state = calculate_current_state(service_state) + {:reply, current_state, services} + end + + @impl true + def handle_call({:get_stats, service_name}, _from, services) do + service_state = get_service_state(services, service_name) + {:reply, service_state, services} + end + + @impl true + def handle_call({:reset, service_name}, _from, services) do + default_state = default_service_state() + new_services = Map.put(services, service_name, default_state) + + Logger.info("Circuit breaker reset", + circuit_breaker: + LogSanitizer.log_data( + service: service_name, + action: :manual_reset + ) + ) + + {:reply, :ok, new_services} + end + + @impl true + def handle_call({:record_success, service_name}, _from, services) do + service_state = get_service_state(services, service_name) + updated_state = %{service_state | failure_count: 0, state: :closed} + new_services = Map.put(services, service_name, updated_state) + {:reply, :ok, new_services} + end + + @impl true + def handle_call({:record_failure, service_name}, _from, services) do + service_state = get_service_state(services, service_name) + new_failure_count = service_state.failure_count + 1 + + new_state = + if new_failure_count >= service_state.failure_threshold do + :open + else + service_state.state + end + + updated_state = %{ + service_state + | failure_count: new_failure_count, + state: new_state, + last_failure_time: DateTime.utc_now() + } + + if new_state == :open do + Logger.warning("Circuit breaker opened due to failure threshold", + circuit_breaker: + LogSanitizer.log_data( + service: service_name, + failure_count: new_failure_count, + threshold: service_state.failure_threshold, + state: :open + ) + ) + end + + new_services = Map.put(services, service_name, updated_state) + {:reply, :ok, new_services} + end + + # Private functions + + defp attempt_call(service_name, fun, timeout) do + task = Task.async(fn -> fun.() end) + + try do + result = Task.await(task, timeout) + record_success(service_name) + {:ok, result} + catch + :exit, {:timeout, _} -> + Task.shutdown(task, :brutal_kill) + record_failure(service_name) + + Logger.warning("Service call timed out", + circuit_breaker: + LogSanitizer.log_data( + service: service_name, + timeout_ms: timeout, + action: :timeout_failure + ) + ) + + {:error, :timeout} + + kind, error -> + record_failure(service_name) + + Logger.error("Service call failed", + circuit_breaker: + LogSanitizer.log_data( + service: service_name, + error_kind: kind, + error: LogSanitizer.sanitize(error) + ) + ) + + {:error, error} + end + end + + defp record_success(service_name) do + GenServer.call(__MODULE__, {:record_success, service_name}) + end + + defp record_failure(service_name) do + GenServer.call(__MODULE__, {:record_failure, service_name}) + end + + defp get_service_state(services, service_name) do + Map.get(services, service_name, default_service_state()) + end + + defp default_service_state do + %{ + state: :closed, + failure_count: 0, + failure_threshold: @default_failure_threshold, + timeout: @default_timeout, + last_failure_time: nil, + recovery_timeout: @default_recovery_timeout + } + end + + defp calculate_current_state(%{state: :open, last_failure_time: last_failure_time, recovery_timeout: recovery_timeout}) do + if last_failure_time && DateTime.diff(DateTime.utc_now(), last_failure_time, :millisecond) >= recovery_timeout do + :half_open + else + :open + end + end + + defp calculate_current_state(%{state: state}), do: state +end diff --git a/lib/aprsme/dependency_info.ex b/lib/aprsme/dependency_info.ex index e9a8cfc..6e9c4af 100644 --- a/lib/aprsme/dependency_info.ex +++ b/lib/aprsme/dependency_info.ex @@ -3,6 +3,8 @@ defmodule Aprsme.DependencyInfo do Provides information about dependencies, particularly for production builds. """ + alias Aprsme.CircuitBreaker + @doc """ Gets the SHA1 hash of the aprs library currently being used. In development: reads from vendor/aprs directory @@ -31,12 +33,19 @@ defmodule Aprsme.DependencyInfo do end def latest_aprs_library_sha do - resp = Req.get!("https://api.github.com/repos/aprsme/aprs/commits/main") - body = resp.body - sha = body["sha"] || body |> Jason.decode!() |> Access.get("sha") - String.slice(sha, 0, 7) - rescue - _ -> nil + case CircuitBreaker.call( + :github_api, + fn -> + resp = Req.get!("https://api.github.com/repos/aprsme/aprs/commits/main") + body = resp.body + sha = body["sha"] || body |> Jason.decode!() |> Access.get("sha") + String.slice(sha, 0, 7) + end, + 10_000 + ) do + {:ok, sha} -> sha + {:error, _} -> nil + end end def is_latest_aprs_library? do diff --git a/lib/aprsme/device_identification.ex b/lib/aprsme/device_identification.ex index bd5b9fc..791cf6d 100644 --- a/lib/aprsme/device_identification.ex +++ b/lib/aprsme/device_identification.ex @@ -6,6 +6,7 @@ defmodule Aprsme.DeviceIdentification do import Ecto.Query + alias Aprsme.CircuitBreaker alias Aprsme.Devices alias Aprsme.Repo @@ -130,13 +131,24 @@ defmodule Aprsme.DeviceIdentification do end def fetch_and_upsert_devices do - case Req.get(@url) do - {:ok, %Req.Response{status: 200, body: body}} -> - {:ok, json} = Jason.decode(body) - upsert_devices(json) + case CircuitBreaker.call( + :aprs_foundation_api, + fn -> + case Req.get(@url) do + {:ok, %Req.Response{status: 200, body: body}} -> + upsert_devices(body) - err -> - err + {:ok, %Req.Response{status: status}} -> + {:error, {:http_error, status}} + + {:error, reason} -> + {:error, reason} + end + end, + 15_000 + ) do + {:ok, result} -> result + {:error, reason} -> {:error, reason} end end diff --git a/lib/aprsme/error_handler.ex b/lib/aprsme/error_handler.ex new file mode 100644 index 0000000..696b6ea --- /dev/null +++ b/lib/aprsme/error_handler.ex @@ -0,0 +1,380 @@ +defmodule Aprsme.ErrorHandler do + @moduledoc """ + Centralized error handling and recovery patterns for the application + """ + + alias Aprsme.LogSanitizer + alias Ecto.Query.CastError + + require Logger + + @doc """ + Handle and log database errors with context + """ + def handle_database_error(error, context \\ %{}) do + sanitized_context = LogSanitizer.sanitize_error_context(context) + + case error do + %CastError{} = cast_error -> + Logger.error("Database cast error - invalid data type", + error_details: + LogSanitizer.log_data( + error_type: :cast_error, + message: cast_error.message, + context: sanitized_context + ) + ) + + {:error, :invalid_data} + + %Ecto.ConstraintError{} = constraint_error -> + Logger.error("Database constraint violation", + error_details: + LogSanitizer.log_data( + error_type: :constraint_violation, + constraint: constraint_error.constraint, + context: sanitized_context + ) + ) + + {:error, :constraint_violation} + + %Postgrex.Error{postgres: %{code: :unique_violation}} -> + Logger.warning("Attempted to insert duplicate record", + error_details: + LogSanitizer.log_data( + error_type: :duplicate_record, + context: sanitized_context + ) + ) + + {:error, :duplicate_record} + + %DBConnection.ConnectionError{} -> + Logger.error("Database connection error", + error_details: + LogSanitizer.log_data( + error_type: :connection_error, + context: sanitized_context + ) + ) + + {:error, :database_unavailable} + + %Ecto.QueryError{} = query_error -> + Logger.error("Database query error", + error_details: + LogSanitizer.log_data( + error_type: :query_error, + message: query_error.message, + context: sanitized_context + ) + ) + + {:error, :query_failed} + + error -> + Logger.error("Unexpected database error", + error_details: + LogSanitizer.log_data( + error_type: :unknown_database_error, + error: LogSanitizer.sanitize(error), + context: sanitized_context + ) + ) + + {:error, :database_error} + end + end + + @doc """ + Handle network and external service errors + """ + def handle_network_error(error, service_name, context \\ %{}) do + sanitized_context = LogSanitizer.sanitize_error_context(context) + + case error do + %Req.TransportError{reason: :timeout} -> + Logger.warning("Network request timeout", + network_error: + LogSanitizer.log_data( + service: service_name, + error_type: :timeout, + context: sanitized_context + ) + ) + + {:error, :timeout} + + %Req.TransportError{reason: :econnrefused} -> + Logger.error("Connection refused by service", + network_error: + LogSanitizer.log_data( + service: service_name, + error_type: :connection_refused, + context: sanitized_context + ) + ) + + {:error, :service_unavailable} + + %Req.TransportError{reason: reason} -> + Logger.error("Network transport error", + network_error: + LogSanitizer.log_data( + service: service_name, + error_type: :transport_error, + reason: reason, + context: sanitized_context + ) + ) + + {:error, :network_error} + + {:error, :circuit_open} -> + Logger.warning("Circuit breaker open for service", + network_error: + LogSanitizer.log_data( + service: service_name, + error_type: :circuit_open, + context: sanitized_context + ) + ) + + {:error, :service_unavailable} + + error -> + Logger.error("Unexpected network error", + network_error: + LogSanitizer.log_data( + service: service_name, + error_type: :unknown_network_error, + error: LogSanitizer.sanitize(error), + context: sanitized_context + ) + ) + + {:error, :network_error} + end + end + + @doc """ + Handle packet processing errors with recovery strategies + """ + def handle_packet_error(error, packet_data, context \\ %{}) do + _sanitized_packet = LogSanitizer.sanitize_packet_data(packet_data) + sanitized_context = LogSanitizer.sanitize_error_context(context) + + case error do + %Jason.DecodeError{} -> + Logger.warning("Invalid JSON in packet data", + packet_error: + LogSanitizer.log_data( + error_type: :invalid_json, + packet_sender: Map.get(packet_data, :sender, "unknown"), + context: sanitized_context + ) + ) + + store_bad_packet(packet_data, "invalid_json") + {:error, :invalid_json} + + %ArgumentError{message: message} -> + if String.contains?(message, "invalid") do + Logger.warning("Invalid packet format", + packet_error: + LogSanitizer.log_data( + error_type: :invalid_format, + packet_sender: Map.get(packet_data, :sender, "unknown"), + message: message, + context: sanitized_context + ) + ) + + store_bad_packet(packet_data, "invalid_format") + {:error, :invalid_format} + else + Logger.error("Unexpected packet processing error", + packet_error: + LogSanitizer.log_data( + error_type: :processing_error, + packet_sender: Map.get(packet_data, :sender, "unknown"), + error: LogSanitizer.sanitize(message), + context: sanitized_context + ) + ) + + store_bad_packet(packet_data, "processing_error") + {:error, :processing_failed} + end + + error -> + Logger.error("Unexpected packet processing error", + packet_error: + LogSanitizer.log_data( + error_type: :processing_error, + packet_sender: Map.get(packet_data, :sender, "unknown"), + error: LogSanitizer.sanitize(error), + context: sanitized_context + ) + ) + + store_bad_packet(packet_data, "processing_error") + {:error, :processing_failed} + end + end + + @doc """ + Handle GenServer and process errors + """ + def handle_process_error(error, process_name, context \\ %{}) do + sanitized_context = LogSanitizer.sanitize_error_context(context) + + case error do + :timeout -> + Logger.warning("Process operation timeout", + process_error: + LogSanitizer.log_data( + process: process_name, + error_type: :timeout, + context: sanitized_context + ) + ) + + {:error, :timeout} + + {:noproc, _} -> + Logger.error("Process not found or died", + process_error: + LogSanitizer.log_data( + process: process_name, + error_type: :process_died, + context: sanitized_context + ) + ) + + {:error, :process_unavailable} + + {:badrpc, reason} -> + Logger.error("RPC call failed", + process_error: + LogSanitizer.log_data( + process: process_name, + error_type: :rpc_failed, + reason: reason, + context: sanitized_context + ) + ) + + {:error, :rpc_failed} + + error -> + Logger.error("Unexpected process error", + process_error: + LogSanitizer.log_data( + process: process_name, + error_type: :unknown_process_error, + error: LogSanitizer.sanitize(error), + context: sanitized_context + ) + ) + + {:error, :process_error} + end + end + + @doc """ + Execute function with comprehensive error handling and recovery + """ + def with_error_handling(fun, opts \\ []) when is_function(fun, 0) do + context = Keyword.get(opts, :context, %{}) + max_retries = Keyword.get(opts, :max_retries, 0) + retry_delay = Keyword.get(opts, :retry_delay, 1000) + + do_with_retries(fun, max_retries, retry_delay, context, 0) + end + + defp do_with_retries(fun, max_retries, retry_delay, context, attempt) do + {:ok, fun.()} + rescue + error -> + if attempt < max_retries do + Logger.warning("Retrying operation after error", + retry_info: + LogSanitizer.log_data( + attempt: attempt + 1, + max_retries: max_retries, + delay_ms: retry_delay, + error: LogSanitizer.sanitize(error) + ) + ) + + Process.sleep(retry_delay) + do_with_retries(fun, max_retries, retry_delay, context, attempt + 1) + else + categorize_and_handle_error(error, context) + end + catch + kind, error -> + Logger.error("Unexpected error in operation", + error_details: + LogSanitizer.log_data( + kind: kind, + error: LogSanitizer.sanitize(error), + context: LogSanitizer.sanitize_error_context(context) + ) + ) + + {:error, :unexpected_error} + end + + defp categorize_and_handle_error(error, context) do + cond do + is_database_error?(error) -> + handle_database_error(error, context) + + is_network_error?(error) -> + handle_network_error(error, :unknown_service, context) + + true -> + Logger.error("Uncategorized error", + error_details: + LogSanitizer.log_data( + error: LogSanitizer.sanitize(error), + context: LogSanitizer.sanitize_error_context(context) + ) + ) + + {:error, :unknown_error} + end + end + + defp is_database_error?(%CastError{}), do: true + defp is_database_error?(%Ecto.ConstraintError{}), do: true + defp is_database_error?(%Postgrex.Error{}), do: true + defp is_database_error?(%DBConnection.ConnectionError{}), do: true + defp is_database_error?(%Ecto.QueryError{}), do: true + defp is_database_error?(_), do: false + + defp is_network_error?(%Req.TransportError{}), do: true + defp is_network_error?(%{reason: :circuit_open}), do: true + defp is_network_error?(%{reason: :timeout}), do: true + defp is_network_error?(%RuntimeError{message: "circuit_open"}), do: true + defp is_network_error?(%RuntimeError{message: "timeout"}), do: true + defp is_network_error?(_), do: false + + defp store_bad_packet(packet_data, reason) do + # This would store the bad packet for later analysis + # For now, just log it since BadPacket module may not exist + Logger.warning("Storing bad packet for analysis", + bad_packet: + LogSanitizer.log_data( + raw_packet: inspect(packet_data), + error_reason: reason, + received_at: DateTime.utc_now() + ) + ) + + :ok + end +end diff --git a/lib/aprsme/log_sanitizer.ex b/lib/aprsme/log_sanitizer.ex new file mode 100644 index 0000000..acef9f3 --- /dev/null +++ b/lib/aprsme/log_sanitizer.ex @@ -0,0 +1,133 @@ +defmodule Aprsme.LogSanitizer do + @moduledoc """ + Utility module for sanitizing log output to prevent sensitive data leakage + """ + + @sensitive_fields [ + :password, + :passcode, + :auth_token, + :api_key, + :secret, + :token, + :authorization, + :csrf_token, + :session_token, + :private_key, + :database_url, + :secret_key_base, + :signing_salt + ] + + @sensitive_patterns [ + ~r/password[=:]\s*\S+/i, + ~r/passcode[=:]\s*\S+/i, + ~r/token[=:]\s*\S+/i, + ~r/key[=:]\s*\S+/i, + ~r/secret[=:]\s*\S+/i, + ~r/authorization[=:]\s*\S+/i + ] + + @doc """ + Sanitize a map by removing or redacting sensitive fields + """ + def sanitize_map(data) when is_map(data) do + Enum.reduce(@sensitive_fields, data, fn field, acc -> + cond do + Map.has_key?(acc, field) -> Map.put(acc, field, "[REDACTED]") + Map.has_key?(acc, to_string(field)) -> Map.put(acc, to_string(field), "[REDACTED]") + true -> acc + end + end) + end + + def sanitize_map(data), do: data + + @doc """ + Sanitize a string by replacing sensitive patterns + """ + def sanitize_string(data) when is_binary(data) do + Enum.reduce(@sensitive_patterns, data, fn pattern, acc -> + Regex.replace(pattern, acc, fn match -> + case String.split(match, ["=", ":"]) do + [key, _value] -> "#{key}=[REDACTED]" + _ -> "[REDACTED]" + end + end) + end) + end + + def sanitize_string(data), do: data + + @doc """ + Sanitize any data structure recursively + """ + def sanitize(data) when is_map(data) do + data + |> sanitize_map() + |> Map.new(fn {k, v} -> {k, sanitize(v)} end) + end + + def sanitize(data) when is_list(data) do + Enum.map(data, &sanitize/1) + end + + def sanitize(data) when is_binary(data) do + sanitize_string(data) + end + + def sanitize(data), do: data + + @doc """ + Sanitize packet data for logging + """ + def sanitize_packet_data(packet_data) when is_map(packet_data) do + # Remove potentially sensitive packet content but keep structure + packet_data + |> Map.drop([:raw_packet, :comment, :status, :data_extended]) + |> sanitize_map() + end + + def sanitize_packet_data(data), do: data + + @doc """ + Sanitize connection information for logging + """ + def sanitize_connection_info(conn_info) when is_map(conn_info) do + conn_info + |> Map.drop([:password, :passcode]) + |> Map.put(:password, "[REDACTED]") + |> Map.put(:passcode, "[REDACTED]") + end + + def sanitize_connection_info(data), do: data + + @doc """ + Sanitize error information for logging + """ + def sanitize_error_context(context) when is_map(context) do + context + |> sanitize_map() + |> Map.update(:stacktrace, nil, fn stacktrace -> + if is_list(stacktrace) do + # Limit stacktrace length + Enum.take(stacktrace, 5) + else + stacktrace + end + end) + end + + def sanitize_error_context(data), do: data + + @doc """ + Create a sanitized logging keyword list + """ + def log_data(keyword_list) when is_list(keyword_list) do + Enum.map(keyword_list, fn {key, value} -> + {key, sanitize(value)} + end) + end + + def log_data(data), do: sanitize(data) +end diff --git a/lib/aprsme/packet_consumer.ex b/lib/aprsme/packet_consumer.ex index 9289440..aca24c0 100644 --- a/lib/aprsme/packet_consumer.ex +++ b/lib/aprsme/packet_consumer.ex @@ -5,6 +5,7 @@ defmodule Aprsme.PacketConsumer do """ use GenStage + alias Aprsme.LogSanitizer alias Aprsme.Repo require Logger @@ -17,6 +18,8 @@ defmodule Aprsme.PacketConsumer do def init(opts) do batch_size = opts[:batch_size] || 100 batch_timeout = opts[:batch_timeout] || 1000 + # Maximum batch size to prevent unbounded memory growth + max_batch_size = opts[:max_batch_size] || 1000 # Start a timer for batch processing timer = Process.send_after(self(), :process_batch, batch_timeout) @@ -26,27 +29,62 @@ defmodule Aprsme.PacketConsumer do batch: [], batch_size: batch_size, batch_timeout: batch_timeout, + max_batch_size: max_batch_size, timer: timer }} end @impl true - def handle_events(events, _from, %{batch: batch, batch_size: batch_size} = state) do + def handle_events(events, _from, %{batch: batch, batch_size: batch_size, max_batch_size: max_batch_size} = state) do new_batch = batch ++ events + new_batch_length = length(new_batch) - if length(new_batch) >= batch_size do - # Process the batch immediately - process_batch(new_batch) - {:noreply, [], %{state | batch: []}} - else - # Add to batch and wait for more - {:noreply, [], %{state | batch: new_batch}} + cond do + new_batch_length >= max_batch_size -> + # If batch exceeds maximum size, process immediately and drop excess + {process_batch, drop_batch} = Enum.split(new_batch, max_batch_size) + process_batch(process_batch) + + # Log warning about dropping packets (sanitized) + if length(drop_batch) > 0 do + Logger.warning("Dropped #{length(drop_batch)} packets due to batch size limit", + batch_info: + LogSanitizer.log_data( + dropped_count: length(drop_batch), + processed_count: length(process_batch), + reason: "batch_size_limit_exceeded" + ) + ) + end + + {:noreply, [], %{state | batch: []}} + + new_batch_length >= batch_size -> + # Process the batch immediately + process_batch(new_batch) + {:noreply, [], %{state | batch: []}} + + true -> + # Add to batch and wait for more + {:noreply, [], %{state | batch: new_batch}} end end @impl true - def handle_info(:process_batch, %{batch: batch, batch_timeout: timeout} = state) do + def handle_info(:process_batch, %{batch: batch, batch_timeout: timeout, max_batch_size: max_batch_size} = state) do if length(batch) > 0 do + # Check if batch size is concerning + if length(batch) > max_batch_size * 0.8 do + Logger.warning("Batch size approaching limit", + batch_status: + LogSanitizer.log_data( + current_size: length(batch), + max_size: max_batch_size, + utilization_percent: trunc(length(batch) / max_batch_size * 100) + ) + ) + end + process_batch(batch) end @@ -58,6 +96,8 @@ defmodule Aprsme.PacketConsumer do defp process_batch(packets) do require Logger + # Monitor memory usage before processing + {memory_before, _} = :erlang.statistics(:runtime) start_time = System.monotonic_time(:millisecond) results = @@ -73,18 +113,50 @@ defmodule Aprsme.PacketConsumer do end_time = System.monotonic_time(:millisecond) duration = end_time - start_time + # Monitor memory usage after processing + {memory_after, _} = :erlang.statistics(:runtime) + memory_diff = memory_after - memory_before + + # Force garbage collection if memory usage is high + # 50MB threshold + if memory_diff > 50_000 do + :erlang.garbage_collect() + + Logger.warning("High memory usage detected, forced garbage collection", + memory_info: + LogSanitizer.log_data( + memory_diff_bytes: memory_diff, + batch_size: length(packets), + gc_forced: true + ) + ) + end + :telemetry.execute( [ :aprsme, :packet_pipeline, :batch ], - %{count: length(packets), success: success_count, error: error_count, duration_ms: duration}, + %{ + count: length(packets), + success: success_count, + error: error_count, + duration_ms: duration, + memory_diff: memory_diff + }, %{} ) - Logger.info( - "Processed batch of #{length(packets)} packets in #{duration}ms (success: #{success_count}, errors: #{error_count})" + Logger.info("Batch processing completed", + batch_result: + LogSanitizer.log_data( + packet_count: length(packets), + duration_ms: duration, + success_count: success_count, + error_count: error_count, + memory_diff_bytes: memory_diff + ) ) end @@ -296,20 +368,11 @@ defmodule Aprsme.PacketConsumer do defp set_lat_lon(attrs, lat, lon) do round6 = fn + nil -> + nil + n when is_float(n) -> Float.round(n, 6) - - n when is_integer(n) -> - n * 1.0 - - n when is_binary(n) -> - case Float.parse(n) do - {f, _} -> Float.round(f, 6) - :error -> nil - end - - _ -> - nil end attrs diff --git a/lib/aprsme/packets.ex b/lib/aprsme/packets.ex index c5506ab..3782d49 100644 --- a/lib/aprsme/packets.ex +++ b/lib/aprsme/packets.ex @@ -504,7 +504,9 @@ defmodule Aprsme.Packets do defp filter_by_callsign(query, %{callsign: callsign}) do # Use sender field for exact callsign matching - from p in query, where: ilike(p.sender, ^callsign) + # Sanitize callsign to prevent SQL injection + sanitized_callsign = sanitize_callsign(callsign) + from p in query, where: ilike(p.sender, ^sanitized_callsign) end defp filter_by_callsign(query, _), do: query @@ -538,7 +540,15 @@ defmodule Aprsme.Packets do defp filter_by_map_bounds(query, _), do: query defp create_bounding_box_wkt(min_lon, min_lat, max_lon, max_lat) do - "POLYGON((#{min_lon} #{min_lat}, #{max_lon} #{min_lat}, #{max_lon} #{max_lat}, #{min_lon} #{max_lat}, #{min_lon} #{min_lat}))" + # Validate and sanitize coordinates to prevent SQL injection + with {:ok, min_lon} <- validate_coordinate(min_lon), + {:ok, min_lat} <- validate_coordinate(min_lat), + {:ok, max_lon} <- validate_coordinate(max_lon), + {:ok, max_lat} <- validate_coordinate(max_lat) do + "POLYGON((#{min_lon} #{min_lat}, #{max_lon} #{min_lat}, #{max_lon} #{max_lat}, #{min_lon} #{max_lat}, #{min_lon} #{min_lat}))" + else + _ -> raise ArgumentError, "Invalid coordinates provided" + end end defp limit_results(query, %{limit: limit, page: page}) when not is_nil(limit) and not is_nil(page) do @@ -717,12 +727,44 @@ defmodule Aprsme.Packets do """ @spec get_latest_packet_for_callsign(String.t()) :: struct() | nil def get_latest_packet_for_callsign(callsign) when is_binary(callsign) do + sanitized_callsign = sanitize_callsign(callsign) + from(p in Packet, - where: ilike(p.sender, ^callsign), + where: ilike(p.sender, ^sanitized_callsign), order_by: [desc: p.received_at], limit: 1 ) |> select_with_virtual_coordinates() |> Repo.one() end + + # Sanitization functions to prevent SQL injection + defp sanitize_callsign(callsign) when is_binary(callsign) do + # APRS callsigns are alphanumeric with hyphens and up to 9 characters + # Remove any characters that could be used for SQL injection + callsign + |> String.upcase() + |> String.replace(~r/[^A-Z0-9\-]/, "") + |> String.slice(0, 9) + end + + defp sanitize_callsign(_), do: "" + + defp validate_coordinate(coord) when is_number(coord) do + # Validate coordinate ranges + if coord >= -180 and coord <= 180 do + {:ok, coord} + else + {:error, :invalid_coordinate} + end + end + + defp validate_coordinate(coord) when is_binary(coord) do + case Float.parse(coord) do + {float_val, ""} -> validate_coordinate(float_val) + _ -> {:error, :invalid_coordinate} + end + end + + defp validate_coordinate(_), do: {:error, :invalid_coordinate} end diff --git a/lib/aprsme_web/controllers/api/v1/callsign_controller.ex b/lib/aprsme_web/controllers/api/v1/callsign_controller.ex index f757318..3efcc70 100644 --- a/lib/aprsme_web/controllers/api/v1/callsign_controller.ex +++ b/lib/aprsme_web/controllers/api/v1/callsign_controller.ex @@ -4,7 +4,8 @@ defmodule AprsmeWeb.Api.V1.CallsignController do """ use AprsmeWeb, :controller - alias Aprsme.Packets + alias Aprsme.CachedQueries + alias Aprsme.ErrorHandler alias AprsmeWeb.Api.V1.CallsignJSON action_fallback AprsmeWeb.Api.V1.FallbackController @@ -67,21 +68,21 @@ defmodule AprsmeWeb.Api.V1.CallsignController do defp get_latest_packet(callsign) do # Get the most recent packet for this callsign regardless of age or type - case Packets.get_latest_packet_for_callsign(callsign) do - nil -> - {:error, :not_found} - - packet -> - {:ok, packet} + # Use cached version for better performance with error handling + fn -> + case CachedQueries.get_latest_packet_for_callsign_cached(callsign) do + nil -> {:error, :not_found} + packet -> {:ok, packet} + end + end + |> ErrorHandler.with_error_handling( + context: %{callsign: callsign, operation: :get_latest_packet}, + max_retries: 1, + retry_delay: 500 + ) + |> case do + {:ok, result} -> result + {:error, error} -> {:error, error} end - rescue - Ecto.QueryError -> - {:error, :database_error} - - error -> - require Logger - - Logger.error("Error fetching packet for callsign #{callsign}: #{inspect(error)}") - {:error, :internal_error} end end diff --git a/lib/aprsme_web/live/map_live/packet_utils.ex b/lib/aprsme_web/live/map_live/packet_utils.ex index 76262ce..9441f2c 100644 --- a/lib/aprsme_web/live/map_live/packet_utils.ex +++ b/lib/aprsme_web/live/map_live/packet_utils.ex @@ -284,20 +284,38 @@ defmodule AprsmeWeb.MapLive.PacketUtils do end # Weather unit conversion helpers - defp convert_temperature(value, locale) when is_number(value) do - AprsmeWeb.WeatherUnits.format_temperature(value, locale) + defp convert_temperature(value, locale) when is_binary(value) and value != "N/A" do + case Float.parse(value) do + {num_value, _} -> + AprsmeWeb.WeatherUnits.format_temperature(num_value, locale) + + :error -> + {value, "°F"} + end end defp convert_temperature(value, _locale), do: {value, "°F"} - defp convert_wind_speed(value, locale) when is_number(value) do - AprsmeWeb.WeatherUnits.format_wind_speed(value, locale) + defp convert_wind_speed(value, locale) when is_binary(value) and value != "N/A" do + case Float.parse(value) do + {num_value, _} -> + AprsmeWeb.WeatherUnits.format_wind_speed(num_value, locale) + + :error -> + {value, "mph"} + end end defp convert_wind_speed(value, _locale), do: {value, "mph"} - defp convert_rain(value, locale) when is_number(value) do - AprsmeWeb.WeatherUnits.format_rain(value, locale) + defp convert_rain(value, locale) when is_binary(value) and value != "N/A" do + case Float.parse(value) do + {num_value, _} -> + AprsmeWeb.WeatherUnits.format_rain(num_value, locale) + + :error -> + {value, "in"} + end end defp convert_rain(value, _locale), do: {value, "in"} diff --git a/lib/aprsme_web/live/weather_live/callsign_view.ex b/lib/aprsme_web/live/weather_live/callsign_view.ex index 3fc42e6..2b4d839 100644 --- a/lib/aprsme_web/live/weather_live/callsign_view.ex +++ b/lib/aprsme_web/live/weather_live/callsign_view.ex @@ -251,7 +251,6 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do # Return "0" for missing numeric fields, keep other values as-is case value do "N/A" -> "0" - nil -> "0" _ -> value end end @@ -263,34 +262,69 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do value = PacketUtils.get_weather_field(packet, key) case {key, value} do - {:temperature, value} when is_number(value) -> - {converted_value, unit} = WeatherUnits.format_temperature(value, locale) - "#{converted_value}#{unit}" + {:temperature, value} when is_binary(value) and value != "N/A" -> + case Float.parse(value) do + {num_value, _} -> + {converted_value, unit} = WeatherUnits.format_temperature(num_value, locale) + "#{converted_value}#{unit}" - {:wind_speed, value} when is_number(value) -> - {converted_value, unit} = WeatherUnits.format_wind_speed(value, locale) - "#{converted_value} #{unit}" + :error -> + value + end - {:wind_gust, value} when is_number(value) -> - {converted_value, unit} = WeatherUnits.format_wind_speed(value, locale) - "#{converted_value} #{unit}" + {:wind_speed, value} when is_binary(value) and value != "N/A" -> + case Float.parse(value) do + {num_value, _} -> + {converted_value, unit} = WeatherUnits.format_wind_speed(num_value, locale) + "#{converted_value} #{unit}" - {:rain_1h, value} when is_number(value) -> - {converted_value, unit} = WeatherUnits.format_rain(value, locale) - "#{converted_value} #{unit}" + :error -> + value + end - {:rain_24h, value} when is_number(value) -> - {converted_value, unit} = WeatherUnits.format_rain(value, locale) - "#{converted_value} #{unit}" + {:wind_gust, value} when is_binary(value) and value != "N/A" -> + case Float.parse(value) do + {num_value, _} -> + {converted_value, unit} = WeatherUnits.format_wind_speed(num_value, locale) + "#{converted_value} #{unit}" - {:rain_since_midnight, value} when is_number(value) -> - {converted_value, unit} = WeatherUnits.format_rain(value, locale) - "#{converted_value} #{unit}" + :error -> + value + end + + {:rain_1h, value} when is_binary(value) and value != "N/A" -> + case Float.parse(value) do + {num_value, _} -> + {converted_value, unit} = WeatherUnits.format_rain(num_value, locale) + "#{converted_value} #{unit}" + + :error -> + value + end + + {:rain_24h, value} when is_binary(value) and value != "N/A" -> + case Float.parse(value) do + {num_value, _} -> + {converted_value, unit} = WeatherUnits.format_rain(num_value, locale) + "#{converted_value} #{unit}" + + :error -> + value + end + + {:rain_since_midnight, value} when is_binary(value) and value != "N/A" -> + case Float.parse(value) do + {num_value, _} -> + {converted_value, unit} = WeatherUnits.format_rain(num_value, locale) + "#{converted_value} #{unit}" + + :error -> + value + end _ -> case value do "N/A" -> "N/A" - nil -> "N/A" _ -> "#{value}" end end diff --git a/lib/aprsme_web/plugs/api_csrf.ex b/lib/aprsme_web/plugs/api_csrf.ex new file mode 100644 index 0000000..b0306a9 --- /dev/null +++ b/lib/aprsme_web/plugs/api_csrf.ex @@ -0,0 +1,81 @@ +defmodule AprsmeWeb.Plugs.ApiCSRF do + @moduledoc """ + CSRF protection for API endpoints using X-Requested-With header or API tokens + """ + import Phoenix.Controller + import Plug.Conn + + def init(opts) do + opts + end + + def call(conn, _opts) do + case get_req_header(conn, "content-type") do + [content_type | _] when content_type in ["application/json", "application/json; charset=utf-8"] -> + # For JSON requests, check for X-Requested-With header or valid API token + check_csrf_protection(conn) + + _ -> + # For non-JSON requests, proceed without CSRF check + conn + end + end + + defp check_csrf_protection(conn) do + case {get_req_header(conn, "x-requested-with"), get_req_header(conn, "authorization")} do + {["XMLHttpRequest"], _} -> + # Valid AJAX request + conn + + {_, ["Bearer " <> _token]} -> + # Has authorization header (API token) - would need validation in production + conn + + {_, _} -> + # Check for CSRF token in header + case get_req_header(conn, "x-csrf-token") do + [token] when byte_size(token) > 0 -> + # Validate CSRF token + validate_csrf_token(conn, token) + + _ -> + reject_request(conn) + end + end + end + + defp validate_csrf_token(conn, token) do + # Get session token for comparison + session_token = get_session(conn, "_csrf_token") + + # Use Phoenix.Controller CSRF token validation + if session_token && valid_csrf_token?(conn, token) do + conn + else + reject_request(conn) + end + end + + defp valid_csrf_token?(conn, token) do + session_token = get_session(conn, "_csrf_token") + + if session_token do + # Use a simple comparison for now + Phoenix.Token.verify(conn, "_csrf_token", token, max_age: 86_400) == {:ok, session_token} + else + false + end + rescue + _ -> false + end + + defp reject_request(conn) do + conn + |> put_status(:forbidden) + |> json(%{ + error: "CSRF protection failed", + message: "Include X-Requested-With: XMLHttpRequest header or valid CSRF token" + }) + |> halt() + end +end diff --git a/lib/aprsme_web/plugs/rate_limiter.ex b/lib/aprsme_web/plugs/rate_limiter.ex new file mode 100644 index 0000000..62131f2 --- /dev/null +++ b/lib/aprsme_web/plugs/rate_limiter.ex @@ -0,0 +1,63 @@ +defmodule AprsmeWeb.Plugs.RateLimiter do + @moduledoc """ + Rate limiting plug to prevent DoS attacks + """ + import Phoenix.Controller + import Plug.Conn + + def init(opts) do + # Default options + Keyword.merge( + [ + # 1 minute + scale: 60_000, + # 100 requests per minute + limit: 100, + # Rate limit by IP address + key: :ip, + error_message: "Too many requests" + ], + opts + ) + end + + def call(conn, opts) do + key = get_key(conn, opts[:key]) + scale = opts[:scale] + limit = opts[:limit] + error_message = opts[:error_message] + + case Hammer.check_rate("rate_limit:#{key}", scale, limit) do + {:allow, _count} -> + conn + + {:deny, _count} -> + conn + |> put_status(:too_many_requests) + |> json(%{error: error_message}) + |> halt() + end + end + + defp get_key(conn, :ip) do + case get_req_header(conn, "x-forwarded-for") do + [ip | _] -> ip + [] -> conn.remote_ip |> :inet.ntoa() |> to_string() + end + end + + defp get_key(conn, :user_agent) do + case get_req_header(conn, "user-agent") do + [ua | _] -> ua + [] -> "unknown" + end + end + + defp get_key(conn, custom_key) when is_function(custom_key) do + custom_key.(conn) + end + + defp get_key(_conn, key) when is_binary(key) do + key + end +end diff --git a/lib/aprsme_web/router.ex b/lib/aprsme_web/router.ex index 1f48f82..3dc4e24 100644 --- a/lib/aprsme_web/router.ex +++ b/lib/aprsme_web/router.ex @@ -5,6 +5,8 @@ defmodule AprsmeWeb.Router do import AprsmeWeb.UserAuth import Phoenix.LiveDashboard.Router + alias AprsmeWeb.Plugs.RateLimiter + pipeline :browser do plug :accepts, ["html"] plug :fetch_session @@ -14,14 +16,22 @@ defmodule AprsmeWeb.Router do plug :put_secure_browser_headers plug :fetch_current_user plug AprsmeWeb.Plugs.SetLocale + plug RateLimiter, scale: 60_000, limit: 200 + end + + pipeline :public_api do + plug :accepts, ["json"] + plug RateLimiter, scale: 60_000, limit: 100 end pipeline :api do plug :accepts, ["json"] + plug RateLimiter, scale: 60_000, limit: 100 + plug AprsmeWeb.Plugs.ApiCSRF end scope "/", AprsmeWeb do - pipe_through :api + pipe_through :public_api get "/health", PageController, :health get "/ready", PageController, :ready get "/status.json", PageController, :status_json @@ -30,9 +40,6 @@ defmodule AprsmeWeb.Router do scope "/", AprsmeWeb do pipe_through :browser - live_dashboard "/dashboard", metrics: AprsmeWeb.Telemetry - error_tracker_dashboard("/errors") - live_session :regular_pages, on_mount: [{AprsmeWeb.LocaleHook, :set_locale}] do live "/status", StatusLive.Index, :index live "/packets", PacketsLive.Index, :index @@ -89,6 +96,9 @@ defmodule AprsmeWeb.Router do scope "/", AprsmeWeb do pipe_through [:browser, :require_authenticated_user] + live_dashboard "/dashboard", metrics: AprsmeWeb.Telemetry + error_tracker_dashboard("/errors") + live_session :require_authenticated_user, on_mount: [{AprsmeWeb.UserAuth, :ensure_authenticated}, {AprsmeWeb.LocaleHook, :set_locale}] do live "/users/settings", UserSettingsLive, :edit diff --git a/mix.exs b/mix.exs index aa023fe..0381cae 100644 --- a/mix.exs +++ b/mix.exs @@ -94,7 +94,9 @@ defmodule Aprsme.MixProject do {:igniter, "~> 0.6", only: [:dev, :test]}, {:mox, "~> 1.2", only: :test}, {:styler, "~> 1.4.2", only: [:dev, :test], runtime: false}, - {:httpoison, "~> 1.8"} + {:httpoison, "~> 1.8"}, + {:hammer, "~> 6.0"}, + {:cachex, "~> 3.4"} ] end diff --git a/mix.lock b/mix.lock index c0684e1..b091ea9 100644 --- a/mix.lock +++ b/mix.lock @@ -2,6 +2,7 @@ "bandit": {:hex, :bandit, "1.7.0", "d1564f30553c97d3e25f9623144bb8df11f3787a26733f00b21699a128105c0c", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "3e2f7a98c7a11f48d9d8c037f7177cd39778e74d55c7af06fe6227c742a8168a"}, "bcrypt_elixir": {:hex, :bcrypt_elixir, "3.3.2", "d50091e3c9492d73e17fc1e1619a9b09d6a5ef99160eb4d736926fd475a16ca3", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "471be5151874ae7931911057d1467d908955f93554f7a6cd1b7d804cac8cef53"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, + "cachex": {:hex, :cachex, "3.6.0", "14a1bfbeee060dd9bec25a5b6f4e4691e3670ebda28c8ba2884b12fe30b36bf8", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "ebf24e373883bc8e0c8d894a63bbe102ae13d918f790121f5cfe6e485cc8e2e2"}, "certifi": {:hex, :certifi, "2.15.0", "0e6e882fcdaaa0a5a9f2b3db55b1394dba07e8d6d9bcad08318fb604c6839712", [:rebar3], [], "hexpm", "b147ed22ce71d72eafdad94f055165c1c182f61a2ff49df28bcc71d1d5b94a60"}, "comeonin": {:hex, :comeonin, "5.5.1", "5113e5f3800799787de08a6e0db307133850e635d34e9fab23c70b6501669510", [:mix], [], "hexpm", "65aac8f19938145377cee73973f192c5645873dcf550a8a6b18187d17c13ccdb"}, "credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"}, @@ -15,6 +16,7 @@ "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, "error_tracker": {:hex, :error_tracker, "0.6.0", "ac37d943b0720faccc26bc3adf39b2b74968b1ee0ae681c2fb57c98c5a10178d", [:mix], [{:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, ">= 0.0.0", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:myxql, ">= 0.0.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:phoenix_ecto, "~> 4.6", [hex: :phoenix_ecto, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8b99db5abeb883e7d622daf850c4dda38ce3bfabc4455431212d698df3156c63"}, "esbuild": {:hex, :esbuild, "0.10.0", "b0aa3388a1c23e727c5a3e7427c932d89ee791746b0081bbe56103e9ef3d291f", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "468489cda427b974a7cc9f03ace55368a83e1a7be12fba7e30969af78e5f8c70"}, + "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm", "32e95820a97cffea67830e91514a2ad53b888850442d6d395f53a1ac60c82e07"}, "expo": {:hex, :expo, "1.1.0", "f7b9ed7fb5745ebe1eeedf3d6f29226c5dd52897ac67c0f8af62a07e661e5c75", [:mix], [], "hexpm", "fbadf93f4700fb44c331362177bdca9eeb8097e8b0ef525c9cc501cb9917c960"}, "exvcr": {:hex, :exvcr, "0.17.1", "3bae83d698a464a48212ad87c8ea4bcfb6bd76d53b937129472764e557616228", [:mix], [{:exjsx, "~> 4.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:finch, "~> 0.16", [hex: :finch, repo: "hexpm", optional: true]}, {:httpoison, "~> 1.0 or ~> 2.0", [hex: :httpoison, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:meck, "~> 1.0", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "a39d86980da183011366a878972e0ed43a5441814b701edd2e11360564e7bcab"}, @@ -30,6 +32,7 @@ "glob_ex": {:hex, :glob_ex, "0.1.11", "cb50d3f1ef53f6ca04d6252c7fde09fd7a1cf63387714fe96f340a1349e62c93", [:mix], [], "hexpm", "342729363056e3145e61766b416769984c329e4378f1d558b63e341020525de4"}, "gridsquare": {:hex, :gridsquare, "0.2.0", "c556a5b5101db89743264b0d728601023035863487bbe36e9e50e93839adb4d7", [:mix], [], "hexpm", "dbf0dbb484681a1e97819b0667bfeaa8c8a48cf06a54bf96a8f26ee4158ad31a"}, "hackney": {:hex, :hackney, "1.24.1", "f5205a125bba6ed4587f9db3cc7c729d11316fa8f215d3e57ed1c067a9703fa9", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "f4a7392a0b53d8bbc3eb855bdcc919cd677358e65b2afd3840b5b3690c4c8a39"}, + "hammer": {:hex, :hammer, "6.2.1", "5ae9c33e3dceaeb42de0db46bf505bd9c35f259c8defb03390cd7556fea67ee2", [:mix], [{:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "b9476d0c13883d2dc0cc72e786bac6ac28911fba7cc2e04b70ce6a6d9c4b2bdc"}, "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "88ab3a0d790e6a47404cba02800a6b25d2afae50", [tag: "v2.1.1", sparse: "optimized", depth: 1]}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, @@ -37,6 +40,7 @@ "igniter": {:hex, :igniter, "0.6.11", "bd7269949df4e668fba4219aad32fc922e48f5197e2e27e106cc44d972e50649", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:owl, "~> 0.11", [hex: :owl, repo: "hexpm", optional: false]}, {:phx_new, "~> 1.7", [hex: :phx_new, repo: "hexpm", optional: true]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:rewrite, ">= 1.1.1 and < 2.0.0-0", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "8dc87860a66597dd93abfaf640a3ce747bfcdf53d7dcbb2ec51f5edfb741307d"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm", "fc3499fed7a726995aa659143a248534adc754ebd16ccd437cd93b649a95091f"}, + "jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"}, "libcluster": {:hex, :libcluster, "3.5.0", "5ee4cfde4bdf32b2fef271e33ce3241e89509f4344f6c6a8d4069937484866ba", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebf6561fcedd765a4cd43b4b8c04b1c87f4177b5fb3cbdfe40a780499d72f743"}, "meck": {:hex, :meck, "1.0.0", "24676cb6ee6951530093a93edcd410cfe4cb59fe89444b875d35c9d3909a15d0", [:rebar3], [], "hexpm", "680a9bcfe52764350beb9fb0335fb75fee8e7329821416cee0a19fec35433882"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, @@ -61,10 +65,12 @@ "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, "plug": {:hex, :plug, "1.18.1", "5067f26f7745b7e31bc3368bc1a2b818b9779faa959b49c934c17730efc911cf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "57a57db70df2b422b564437d2d33cf8d33cd16339c1edb190cd11b1a3a546cc2"}, "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"}, + "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, "postgrex": {:hex, :postgrex, "0.20.0", "363ed03ab4757f6bc47942eff7720640795eb557e1935951c1626f0d303a3aed", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "d36ef8b36f323d29505314f704e21a1a038e2dc387c6409ee0cd24144e187c0f"}, "req": {:hex, :req, "0.5.14", "521b449fa0bf275e6d034c05f29bec21789a0d6cd6f7a1c326c7bee642bf6e07", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "b7b15692071d556c73432c7797aa7e96b51d1a2db76f746b976edef95c930021"}, "rewrite": {:hex, :rewrite, "1.1.2", "f5a5d10f5fed1491a6ff48e078d4585882695962ccc9e6c779bae025d1f92eda", [:mix], [{:glob_ex, "~> 0.1", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.0", [hex: :sourceror, repo: "hexpm", optional: false]}, {:text_diff, "~> 0.1", [hex: :text_diff, repo: "hexpm", optional: false]}], "hexpm", "7f8b94b1e3528d0a47b3e8b7bfeca559d2948a65fa7418a9ad7d7712703d39d4"}, "sentry": {:hex, :sentry, "11.0.1", "e4e0ae12a74c59639808442a79d7ddd224fd5987fb9a14b35a1d01d9606117a3", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_ownership, "~> 0.3.0 or ~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.5", [hex: :opentelemetry, repo: "hexpm", optional: true]}, {:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: true]}, {:opentelemetry_exporter, "~> 1.0", [hex: :opentelemetry_exporter, repo: "hexpm", optional: true]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.20 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "ef12b6ee61c24bee3807eee92563567c89f5eec954d6bb913eb797e04625a613"}, + "sleeplocks": {:hex, :sleeplocks, "1.1.3", "96a86460cc33b435c7310dbd27ec82ca2c1f24ae38e34f8edde97f756503441a", [:rebar3], [], "hexpm", "d3b3958552e6eb16f463921e70ae7c767519ef8f5be46d7696cc1ed649421321"}, "sobelow": {:hex, :sobelow, "0.14.0", "dd82aae8f72503f924fe9dd97ffe4ca694d2f17ec463dcfd365987c9752af6ee", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "7ecf91e298acfd9b24f5d761f19e8f6e6ac585b9387fb6301023f1f2cd5eed5f"}, "sourceror": {:hex, :sourceror, "1.10.0", "38397dedbbc286966ec48c7af13e228b171332be1ad731974438c77791945ce9", [:mix], [], "hexpm", "29dbdfc92e04569c9d8e6efdc422fc1d815f4bd0055dc7c51b8800fb75c4b3f1"}, "spitfire": {:hex, :spitfire, "0.2.1", "29e154873f05444669c7453d3d931820822cbca5170e88f0f8faa1de74a79b47", [:mix], [], "hexpm", "6eeed75054a38341b2e1814d41bb0a250564092358de2669fdb57ff88141d91b"}, @@ -80,6 +86,7 @@ "text_diff": {:hex, :text_diff, "0.1.0", "1caf3175e11a53a9a139bc9339bd607c47b9e376b073d4571c031913317fecaa", [:mix], [], "hexpm", "d1ffaaecab338e49357b6daa82e435f877e0649041ace7755583a0ea3362dbd7"}, "thousand_island": {:hex, :thousand_island, "1.3.14", "ad45ebed2577b5437582bcc79c5eccd1e2a8c326abf6a3464ab6c06e2055a34a", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d0d24a929d31cdd1d7903a4fe7f2409afeedff092d277be604966cd6aa4307ef"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"}, + "unsafe": {:hex, :unsafe, "1.0.2", "23c6be12f6c1605364801f4b47007c0c159497d0446ad378b5cf05f1855c0581", [:mix], [], "hexpm", "b485231683c3ab01a9cd44cb4a79f152c6f3bb87358439c6f68791b85c2df675"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.8", "3b97dc94e407e2d1fc666b2fb9acf6be81a1798a2602294aac000260a7c4a47d", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "315b9a1865552212b5f35140ad194e67ce31af45bcee443d4ecb96b5fd3f3782"}, } diff --git a/priv/repo/migrations/20250708140444_add_additional_performance_indexes.exs b/priv/repo/migrations/20250708140444_add_additional_performance_indexes.exs new file mode 100644 index 0000000..1de0f95 --- /dev/null +++ b/priv/repo/migrations/20250708140444_add_additional_performance_indexes.exs @@ -0,0 +1,52 @@ +defmodule Aprsme.Repo.Migrations.AddAdditionalPerformanceIndexes do + use Ecto.Migration + @disable_ddl_transaction true + @disable_migration_lock true + + def up do + # Add compound indexes for common query patterns + create_if_not_exists index(:packets, [:received_at, :has_position, :data_type], + concurrently: true + ) + + create_if_not_exists index(:packets, [:sender, :data_type, :received_at], concurrently: true) + + # Add index for weather symbol combinations (frequently used in queries) + execute """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS packets_weather_symbol_idx + ON packets (received_at DESC, lat, lon) + WHERE (symbol_table_id = '/' AND symbol_code = '_') + OR (symbol_table_id = '\\' AND symbol_code = '_') + """ + + # Add partial index for recent packets with position (most common map query) + # Note: Removing time-based predicate as it's not immutable + execute """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS packets_recent_positioned_idx + ON packets (received_at DESC, lat, lon, sender) + WHERE has_position = true + """ + + # Add index for callsign case-insensitive searches + execute """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS packets_sender_upper_idx + ON packets (UPPER(sender), received_at DESC) + """ + + # Add composite index for region-based queries + create_if_not_exists index(:packets, [:region, :data_type, :received_at], concurrently: true) + + # Add index for device lookups (performance optimization) + create_if_not_exists index(:packets, [:device_identifier, :received_at], concurrently: true) + end + + def down do + execute "DROP INDEX IF EXISTS packets_device_identifier_received_at_idx" + execute "DROP INDEX IF EXISTS packets_region_data_type_received_at_idx" + execute "DROP INDEX IF EXISTS packets_sender_upper_idx" + execute "DROP INDEX IF EXISTS packets_recent_positioned_idx" + execute "DROP INDEX IF EXISTS packets_weather_symbol_idx" + execute "DROP INDEX IF EXISTS packets_sender_data_type_received_at_idx" + execute "DROP INDEX IF EXISTS packets_received_at_has_position_data_type_idx" + end +end diff --git a/test/integration/historical_packets_test.exs b/test/integration/historical_packets_test.exs index 1015e41..5d68541 100644 --- a/test/integration/historical_packets_test.exs +++ b/test/integration/historical_packets_test.exs @@ -1,9 +1,6 @@ defmodule Aprsme.Integration.HistoricalPacketsTest do use AprsmeWeb.ConnCase - import Phoenix.ConnTest - import Phoenix.LiveViewTest - setup do Mox.set_mox_global() Application.put_env(:aprsme, :packets_module, PacketsMock)