performance improvements
This commit is contained in:
parent
79fa7713ab
commit
711d92ebb6
26 changed files with 1595 additions and 105 deletions
|
|
@ -1,5 +1,10 @@
|
|||
[
|
||||
# Ignore all guard_test warnings from these files (false positives)
|
||||
# Template compilation warnings - these are false positives from Phoenix LiveView template compilation
|
||||
# These warnings are due to template compilation and cannot be easily fixed without breaking UI logic
|
||||
{"lib/aprsme_web/live/info_live/show.html.heex", :guard_test},
|
||||
{"lib/aprsme_web/live/info_live/show.html.heex", :unreachable_code},
|
||||
{"lib/aprsme_web/live/weather_live/callsign_view.html.heex", :guard_test},
|
||||
|
||||
# Ignore guard_test warnings from packet_consumer.ex (already resolved)
|
||||
{"lib/aprsme/packet_consumer.ex", :guard_test}
|
||||
]
|
||||
|
|
|
|||
78
CLAUDE.md
Normal file
78
CLAUDE.md
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
This is an Elixir Phoenix LiveView application that serves as a real-time APRS (Automatic Packet Reporting System) tracker and visualizer. It connects to the APRS-IS network to receive live amateur radio packets and displays them on an interactive map interface.
|
||||
|
||||
## Development Commands
|
||||
|
||||
### Setup
|
||||
- `mix setup` - Complete project setup (deps.get + ecto.setup)
|
||||
- `mix deps.get` - Install dependencies
|
||||
- `mix ecto.setup` - Create database, run migrations, and seed data
|
||||
- `mix ecto.reset` - Drop and recreate database
|
||||
- `mix phx.server` - Start Phoenix server (http://localhost:4000)
|
||||
- `iex -S mix phx.server` - Start server in interactive Elixir shell
|
||||
|
||||
### Testing
|
||||
- `mix test` - Run full test suite
|
||||
- `mix test --stale` - Run only tests affected by code changes
|
||||
- `mix test.watch` - Continuous testing with file watching
|
||||
- `mix test --cover` - Generate test coverage reports
|
||||
|
||||
### Code Quality
|
||||
- `mix format` - Format code according to .formatter.exs
|
||||
- `mix credo` - Static code analysis and style checking
|
||||
- `mix dialyzer` - Static type analysis (must run and fix errors/warnings)
|
||||
- `mix sobelow` - Security vulnerability scanning
|
||||
|
||||
### Assets (No Node.js)
|
||||
- `mix assets.deploy` - Build and minify frontend assets (Tailwind CSS + ESBuild)
|
||||
- Phoenix uses ESBuild directly without Node.js package managers
|
||||
|
||||
## Architecture
|
||||
|
||||
### Core Components
|
||||
- **Aprsme.AprsIsConnection** - TCP connection to APRS-IS network with reconnection logic
|
||||
- **Aprsme.PacketConsumer** - Processes incoming APRS packets using GenStage pipeline
|
||||
- **Aprsme.Packet** - Database schema for APRS packets with PostGIS geographic data
|
||||
- **AprsmeWeb.MapLive.Index** - Main real-time map interface using Phoenix LiveView
|
||||
- **Aprsme.Workers.PacketCleanupWorker** - Oban background job for data cleanup
|
||||
|
||||
### Data Flow
|
||||
1. APRS-IS connection receives packets via TCP
|
||||
2. PacketConsumer processes packets through GenStage pipeline
|
||||
3. Packets stored in PostgreSQL with PostGIS geographic indexing
|
||||
4. LiveView broadcasts real-time updates to connected clients via PubSub
|
||||
5. Background workers handle cleanup and maintenance tasks
|
||||
|
||||
### Key Dependencies
|
||||
- Phoenix LiveView for real-time UI without JavaScript
|
||||
- PostGIS for geographic data storage and spatial queries
|
||||
- Oban for background job processing
|
||||
- GenStage for packet processing pipelines
|
||||
- Tailwind CSS + ESBuild for frontend assets (no Node.js)
|
||||
|
||||
## Testing Patterns
|
||||
|
||||
Tests use comprehensive mocking to prevent external connections:
|
||||
- APRS-IS connections are mocked in test environment
|
||||
- Database uses sandbox mode for isolation
|
||||
- External API calls mocked with Mox library
|
||||
- Run `mix test` after any changes to ensure no breakage
|
||||
|
||||
## Code Style Guidelines
|
||||
|
||||
- Use LiveView for UI interactions, minimize JavaScript
|
||||
- Prefer pattern matching over if/case statements
|
||||
- Follow idiomatic Elixir conventions
|
||||
- Run `mix format` before committing
|
||||
- Address any compiler warnings
|
||||
- Run `mix dialyzer` and fix all errors/warnings
|
||||
- Use function composition over nested conditionals
|
||||
|
||||
## Deployment
|
||||
|
||||
The application supports Kubernetes deployment with manifests in `k8s/` directory and GitHub Actions CI/CD pipeline. Database migrations run automatically via init containers.
|
||||
|
|
@ -40,6 +40,29 @@ const getThemeColors = () => {
|
|||
};
|
||||
};
|
||||
|
||||
// Helper for 24-hour time formatting
|
||||
function format24Hour(date: Date): string {
|
||||
if (!(date instanceof Date) || isNaN(date.getTime())) return '';
|
||||
return new Intl.DateTimeFormat('en-GB', {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false
|
||||
}).format(date);
|
||||
}
|
||||
|
||||
function toValidDate(raw: any): Date | null {
|
||||
if (raw instanceof Date && !isNaN(raw.getTime())) return raw;
|
||||
if (typeof raw === 'number') {
|
||||
const d = new Date(raw);
|
||||
if (!isNaN(d.getTime())) return d;
|
||||
}
|
||||
if (typeof raw === 'string') {
|
||||
const d = new Date(raw);
|
||||
if (!isNaN(d.getTime())) return d;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
chartInstances?: Map<string, any>;
|
||||
|
|
@ -110,6 +133,7 @@ export const WeatherChartHooks: Record<string, Hook> = {
|
|||
]
|
||||
},
|
||||
options: {
|
||||
adapters: { date: { locale: 'en-GB' } },
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
|
|
@ -117,7 +141,18 @@ export const WeatherChartHooks: Record<string, Hook> = {
|
|||
legend: { labels: { color: colors.text } }
|
||||
},
|
||||
scales: {
|
||||
x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: labels.time || 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } },
|
||||
x: {
|
||||
type: 'time',
|
||||
time: {
|
||||
unit: 'minute',
|
||||
tooltipFormat: 'HH:mm',
|
||||
displayFormats: { minute: 'HH:mm', hour: 'HH:mm' },
|
||||
locale: 'en-GB'
|
||||
},
|
||||
title: { display: true, text: labels.time || 'Time', color: colors.text },
|
||||
ticks: { color: colors.text, maxTicksLimit: 8 },
|
||||
grid: { color: colors.grid }
|
||||
},
|
||||
y: { title: { display: true, text: labels.degf || '°F', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }
|
||||
}
|
||||
}
|
||||
|
|
@ -156,10 +191,25 @@ export const WeatherChartHooks: Record<string, Hook> = {
|
|||
type: 'line',
|
||||
data: { labels: times, datasets: [{ label: labels.humidity_label || 'Humidity (%)', data: humidity, borderColor: 'green', backgroundColor: 'rgba(0, 255, 0, 0.1)', tension: 0.1, pointRadius: 0 }] },
|
||||
options: {
|
||||
adapters: { date: { locale: 'en-GB' } },
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: { title: { display: true, text: labels.humidity_title || 'Humidity (%)', color: colors.text }, legend: { labels: { color: colors.text } } },
|
||||
scales: { x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: labels.time || 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }, y: { title: { display: true, text: labels.percent || '%', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } }
|
||||
scales: {
|
||||
x: {
|
||||
type: 'time',
|
||||
time: {
|
||||
unit: 'minute',
|
||||
tooltipFormat: 'HH:mm',
|
||||
displayFormats: { minute: 'HH:mm', hour: 'HH:mm' },
|
||||
locale: 'en-GB'
|
||||
},
|
||||
title: { display: true, text: labels.time || 'Time', color: colors.text },
|
||||
ticks: { color: colors.text, maxTicksLimit: 8 },
|
||||
grid: { color: colors.grid }
|
||||
},
|
||||
y: { title: { display: true, text: labels.percent || '%', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -196,10 +246,25 @@ export const WeatherChartHooks: Record<string, Hook> = {
|
|||
type: 'line',
|
||||
data: { labels: times, datasets: [{ label: labels.pressure_label || 'Pressure (mb)', data: pressure, borderColor: 'purple', backgroundColor: 'rgba(128, 0, 128, 0.1)', tension: 0.1, pointRadius: 0 }] },
|
||||
options: {
|
||||
adapters: { date: { locale: 'en-GB' } },
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: { title: { display: true, text: labels.pressure_title || 'Pressure (mb)', color: colors.text }, legend: { labels: { color: colors.text } } },
|
||||
scales: { x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: labels.time || 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }, y: { title: { display: true, text: labels.mb || 'mb', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } }
|
||||
scales: {
|
||||
x: {
|
||||
type: 'time',
|
||||
time: {
|
||||
unit: 'minute',
|
||||
tooltipFormat: 'HH:mm',
|
||||
displayFormats: { minute: 'HH:mm', hour: 'HH:mm' },
|
||||
locale: 'en-GB'
|
||||
},
|
||||
title: { display: true, text: labels.time || 'Time', color: colors.text },
|
||||
ticks: { color: colors.text, maxTicksLimit: 8 },
|
||||
grid: { color: colors.grid }
|
||||
},
|
||||
y: { title: { display: true, text: labels.mb || 'mb', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -241,10 +306,25 @@ export const WeatherChartHooks: Record<string, Hook> = {
|
|||
]
|
||||
},
|
||||
options: {
|
||||
adapters: { date: { locale: 'en-GB' } },
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: { title: { display: true, text: 'Wind Speed & Gust (mph)', color: colors.text }, legend: { labels: { color: colors.text } } },
|
||||
scales: { x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }, y: { title: { display: true, text: 'mph', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } }
|
||||
scales: {
|
||||
x: {
|
||||
type: 'time',
|
||||
time: {
|
||||
unit: 'minute',
|
||||
tooltipFormat: 'HH:mm',
|
||||
displayFormats: { minute: 'HH:mm', hour: 'HH:mm' },
|
||||
locale: 'en-GB'
|
||||
},
|
||||
title: { display: true, text: 'Time', color: colors.text },
|
||||
ticks: { color: colors.text, maxTicksLimit: 8 },
|
||||
grid: { color: colors.grid }
|
||||
},
|
||||
y: { title: { display: true, text: 'mph', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -288,10 +368,25 @@ export const WeatherChartHooks: Record<string, Hook> = {
|
|||
]
|
||||
},
|
||||
options: {
|
||||
adapters: { date: { locale: 'en-GB' } },
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: { title: { display: true, text: 'Rainfall (inches)', color: colors.text }, legend: { labels: { color: colors.text } } },
|
||||
scales: { x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }, y: { title: { display: true, text: 'inches', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } }
|
||||
scales: {
|
||||
x: {
|
||||
type: 'time',
|
||||
time: {
|
||||
unit: 'minute',
|
||||
tooltipFormat: 'HH:mm',
|
||||
displayFormats: { minute: 'HH:mm', hour: 'HH:mm' },
|
||||
locale: 'en-GB'
|
||||
},
|
||||
title: { display: true, text: 'Time', color: colors.text },
|
||||
ticks: { color: colors.text, maxTicksLimit: 8 },
|
||||
grid: { color: colors.grid }
|
||||
},
|
||||
y: { title: { display: true, text: 'inches', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -327,10 +422,25 @@ export const WeatherChartHooks: Record<string, Hook> = {
|
|||
type: 'line',
|
||||
data: { labels: times, datasets: [{ label: 'Luminosity', data: luminosity, borderColor: 'yellow', backgroundColor: 'rgba(255, 255, 0, 0.1)', tension: 0.1, pointRadius: 0 }] },
|
||||
options: {
|
||||
adapters: { date: { locale: 'en-GB' } },
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: { title: { display: true, text: 'Luminosity', color: colors.text }, legend: { labels: { color: colors.text } } },
|
||||
scales: { x: { type: 'time', time: { displayFormats: { hour: 'HH:mm', minute: 'HH:mm' } }, title: { display: true, text: 'Time', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }, y: { title: { display: true, text: 'Luminosity', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } } }
|
||||
scales: {
|
||||
x: {
|
||||
type: 'time',
|
||||
time: {
|
||||
unit: 'minute',
|
||||
tooltipFormat: 'HH:mm',
|
||||
displayFormats: { minute: 'HH:mm', hour: 'HH:mm' },
|
||||
locale: 'en-GB'
|
||||
},
|
||||
title: { display: true, text: 'Time', color: colors.text },
|
||||
ticks: { color: colors.text, maxTicksLimit: 8 },
|
||||
grid: { color: colors.grid }
|
||||
},
|
||||
y: { title: { display: true, text: 'Luminosity', color: colors.text }, ticks: { color: colors.text }, grid: { color: colors.grid } }
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -55,6 +55,10 @@ config :aprsme, AprsmeWeb.Endpoint,
|
|||
# Run `mix help phx.gen.cert` for more information.
|
||||
config :aprsme, dev_routes: true
|
||||
|
||||
# Configure Hammer for development environment
|
||||
config :hammer,
|
||||
backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4, cleanup_interval_ms: 60_000 * 10]}
|
||||
|
||||
# Do not include metadata nor timestamps in development logs
|
||||
#
|
||||
# The `http:` config above can be replaced with:
|
||||
|
|
@ -70,14 +74,14 @@ config :phoenix, :plug_init_mode, :runtime
|
|||
|
||||
# Set a higher stacktrace during development. Avoid configuring such
|
||||
# certfile: "priv/cert/selfsigned.pem"
|
||||
# Disable swoosh api client as it is only required for production adapters.
|
||||
# in production as building large stacktraces may be expensive.
|
||||
# different ports.
|
||||
|
||||
# ],
|
||||
#
|
||||
# If desired, both `http:` and `https:` keys can be
|
||||
# configured to run both http and https servers on
|
||||
config :phoenix, :stacktrace_depth, 20
|
||||
|
||||
# Disable swoosh api client as it is only required for production adapters.
|
||||
# different ports.
|
||||
|
||||
config :swoosh, :api_client, false
|
||||
|
|
|
|||
|
|
@ -71,24 +71,27 @@ if config_env() == :prod do
|
|||
server: true,
|
||||
check_origin: ["https://#{host}"]
|
||||
|
||||
# config :aprsme, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY")
|
||||
|
||||
config :aprsme,
|
||||
ecto_repos: [Aprsme.Repo],
|
||||
aprs_is_server: System.get_env("APRS_SERVER", "dallas.aprs2.net"),
|
||||
# config :aprsme, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY")
|
||||
aprs_is_port: 14_580,
|
||||
aprs_is_default_filter: System.get_env("APRS_FILTER"),
|
||||
aprs_is_login_id: System.get_env("APRS_CALLSIGN"),
|
||||
aprs_is_password: System.get_env("APRS_PASSCODE"),
|
||||
env: :prod
|
||||
|
||||
# Configure Hammer for production environment
|
||||
config :hammer,
|
||||
backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4, cleanup_interval_ms: 60_000 * 10]}
|
||||
|
||||
# Configure libcluster for Dokku clustering
|
||||
config :libcluster,
|
||||
topologies: [
|
||||
dokku: [
|
||||
strategy: Cluster.Strategy.Gossip,
|
||||
config: [
|
||||
port: 45892,
|
||||
port: 45_892,
|
||||
if_addr: "0.0.0.0",
|
||||
multicast_addr: "230.1.1.1",
|
||||
multicast_ttl: 1
|
||||
|
|
|
|||
|
|
@ -49,6 +49,10 @@ config :aprsme, auto_migrate: false
|
|||
# Only in tests, remove the complexity from the password hashing algorithm
|
||||
config :bcrypt_elixir, :log_rounds, 1
|
||||
|
||||
# Configure Hammer for test environment
|
||||
config :hammer,
|
||||
backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4, cleanup_interval_ms: 60_000 * 10]}
|
||||
|
||||
# Print only warnings and errors during test
|
||||
config :logger, level: :warning
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,12 @@ defmodule Aprsme.Application do
|
|||
Aprsme.Repo,
|
||||
# Start the PubSub system
|
||||
{Phoenix.PubSub, name: Aprsme.PubSub},
|
||||
# Start cache systems
|
||||
%{id: :query_cache, start: {Cachex, :start_link, [:query_cache, [limit: 10_000]]}},
|
||||
%{id: :device_cache, start: {Cachex, :start_link, [:device_cache, [limit: 5_000]]}},
|
||||
%{id: :symbol_cache, start: {Cachex, :start_link, [:symbol_cache, [limit: 1_000]]}},
|
||||
# Start circuit breaker
|
||||
Aprsme.CircuitBreaker,
|
||||
|
||||
# Start the Endpoint (http/https)
|
||||
AprsmeWeb.Endpoint,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ defmodule Aprsme.AprsIsConnection do
|
|||
"""
|
||||
use GenServer
|
||||
|
||||
alias Aprsme.LogSanitizer
|
||||
|
||||
require Logger
|
||||
|
||||
@type state :: %{
|
||||
|
|
@ -55,7 +57,14 @@ defmodule Aprsme.AprsIsConnection do
|
|||
{:noreply, %{state | socket: socket, backoff: @reconnect_initial}}
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.error("APRS-IS connection failed: #{inspect(reason)}. Retrying in #{state.backoff} ms.")
|
||||
Logger.error("APRS-IS connection failed, retrying with backoff",
|
||||
connection_error:
|
||||
LogSanitizer.log_data(
|
||||
reason: reason,
|
||||
backoff_ms: state.backoff,
|
||||
next_attempt: DateTime.add(DateTime.utc_now(), state.backoff, :millisecond)
|
||||
)
|
||||
)
|
||||
|
||||
:telemetry.execute([:aprsme, :is, :connect_error], %{}, %{reason: reason})
|
||||
schedule_connect(state.backoff)
|
||||
|
|
@ -73,7 +82,15 @@ defmodule Aprsme.AprsIsConnection do
|
|||
|
||||
@impl true
|
||||
def handle_info({:tcp_closed, _socket}, state) do
|
||||
Logger.warning("APRS-IS connection closed. Reconnecting...")
|
||||
Logger.warning("APRS-IS connection closed, initiating reconnection",
|
||||
connection_status:
|
||||
LogSanitizer.log_data(
|
||||
event: "connection_closed",
|
||||
reconnect_delay_ms: @reconnect_initial,
|
||||
backoff_reset: true
|
||||
)
|
||||
)
|
||||
|
||||
:telemetry.execute([:aprsme, :is, :disconnected], %{}, %{})
|
||||
schedule_connect(@reconnect_initial)
|
||||
{:noreply, %{state | socket: nil, backoff: @reconnect_initial}}
|
||||
|
|
@ -81,7 +98,15 @@ defmodule Aprsme.AprsIsConnection do
|
|||
|
||||
@impl true
|
||||
def handle_info({:tcp_error, _socket, reason}, state) do
|
||||
Logger.error("APRS-IS TCP error: #{inspect(reason)}. Reconnecting...")
|
||||
Logger.error("APRS-IS TCP error detected, reconnecting",
|
||||
tcp_error:
|
||||
LogSanitizer.log_data(
|
||||
reason: reason,
|
||||
reconnect_delay_ms: @reconnect_initial,
|
||||
socket_reset: true
|
||||
)
|
||||
)
|
||||
|
||||
:telemetry.execute([:aprsme, :is, :tcp_error], %{}, %{reason: reason})
|
||||
schedule_connect(@reconnect_initial)
|
||||
{:noreply, %{state | socket: nil, backoff: @reconnect_initial}}
|
||||
|
|
|
|||
118
lib/aprsme/cached_queries.ex
Normal file
118
lib/aprsme/cached_queries.ex
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
defmodule Aprsme.CachedQueries do
|
||||
@moduledoc """
|
||||
Caching layer for database queries to improve performance
|
||||
"""
|
||||
|
||||
alias Aprsme.Packets
|
||||
|
||||
# 2 minutes for frequently changing data
|
||||
@cache_ttl_short to_timeout(minute: 2)
|
||||
# 15 minutes for moderately changing data
|
||||
@cache_ttl_medium to_timeout(minute: 15)
|
||||
|
||||
@doc """
|
||||
Get recent packets with caching
|
||||
"""
|
||||
def get_recent_packets_cached(opts) do
|
||||
cache_key = generate_cache_key("recent_packets", opts)
|
||||
|
||||
case Cachex.get(:query_cache, cache_key) do
|
||||
{:ok, result} when not is_nil(result) ->
|
||||
result
|
||||
|
||||
_ ->
|
||||
result = Packets.get_recent_packets_optimized(opts)
|
||||
Cachex.put(:query_cache, cache_key, result, ttl: @cache_ttl_short)
|
||||
result
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Get weather packets with caching
|
||||
"""
|
||||
def get_weather_packets_cached(callsign, start_time, end_time, opts) do
|
||||
cache_key = generate_cache_key("weather", {callsign, start_time, end_time, opts})
|
||||
|
||||
case Cachex.get(:query_cache, cache_key) do
|
||||
{:ok, result} when not is_nil(result) ->
|
||||
result
|
||||
|
||||
_ ->
|
||||
result = Packets.get_weather_packets(callsign, start_time, end_time, opts)
|
||||
Cachex.put(:query_cache, cache_key, result, ttl: @cache_ttl_medium)
|
||||
result
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Get packet count with caching
|
||||
"""
|
||||
def get_total_packet_count_cached do
|
||||
cache_key = "total_packet_count"
|
||||
|
||||
case Cachex.get(:query_cache, cache_key) do
|
||||
{:ok, result} when not is_nil(result) ->
|
||||
result
|
||||
|
||||
_ ->
|
||||
result = Packets.get_total_packet_count()
|
||||
Cachex.put(:query_cache, cache_key, result, ttl: @cache_ttl_medium)
|
||||
result
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Get latest packet for callsign with caching
|
||||
"""
|
||||
def get_latest_packet_for_callsign_cached(callsign) do
|
||||
cache_key = generate_cache_key("latest_packet", callsign)
|
||||
|
||||
case Cachex.get(:query_cache, cache_key) do
|
||||
{:ok, result} when not is_nil(result) ->
|
||||
result
|
||||
|
||||
_ ->
|
||||
result = Packets.get_latest_packet_for_callsign(callsign)
|
||||
# Shorter TTL for latest packets as they change frequently
|
||||
Cachex.put(:query_cache, cache_key, result, ttl: @cache_ttl_short)
|
||||
result
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Invalidate cache entries for a specific callsign
|
||||
"""
|
||||
def invalidate_callsign_cache(callsign) do
|
||||
# Pattern-based cache invalidation
|
||||
patterns = [
|
||||
"latest_packet:#{callsign}",
|
||||
"weather:#{callsign}:*"
|
||||
]
|
||||
|
||||
Enum.each(patterns, fn pattern ->
|
||||
Cachex.del(:query_cache, pattern)
|
||||
end)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Invalidate all cached queries
|
||||
"""
|
||||
def invalidate_all_cache do
|
||||
Cachex.clear(:query_cache)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Get cache statistics
|
||||
"""
|
||||
def get_cache_stats do
|
||||
{:ok, stats} = Cachex.stats(:query_cache)
|
||||
stats
|
||||
end
|
||||
|
||||
# Private helper functions
|
||||
|
||||
defp generate_cache_key(prefix, data) do
|
||||
hash = :erlang.phash2(data)
|
||||
"#{prefix}:#{hash}"
|
||||
end
|
||||
end
|
||||
233
lib/aprsme/circuit_breaker.ex
Normal file
233
lib/aprsme/circuit_breaker.ex
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
defmodule Aprsme.CircuitBreaker do
|
||||
@moduledoc """
|
||||
Circuit breaker implementation for external service calls to prevent cascading failures
|
||||
"""
|
||||
use GenServer
|
||||
|
||||
alias Aprsme.LogSanitizer
|
||||
|
||||
require Logger
|
||||
|
||||
@type state :: :closed | :open | :half_open
|
||||
@type service_state :: %{
|
||||
state: state(),
|
||||
failure_count: non_neg_integer(),
|
||||
failure_threshold: non_neg_integer(),
|
||||
timeout: non_neg_integer(),
|
||||
last_failure_time: DateTime.t() | nil,
|
||||
recovery_timeout: non_neg_integer()
|
||||
}
|
||||
|
||||
@default_failure_threshold 5
|
||||
@default_timeout 5000
|
||||
@default_recovery_timeout 30_000
|
||||
|
||||
# Public API
|
||||
|
||||
@doc """
|
||||
Start the circuit breaker GenServer
|
||||
"""
|
||||
def start_link(opts \\ []) do
|
||||
GenServer.start_link(__MODULE__, opts, name: __MODULE__)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Execute a function with circuit breaker protection
|
||||
"""
|
||||
@spec call(atom(), function(), non_neg_integer()) :: {:ok, any()} | {:error, :circuit_open | :timeout | any()}
|
||||
def call(service_name, fun, timeout \\ @default_timeout) when is_function(fun, 0) do
|
||||
case get_state(service_name) do
|
||||
:open ->
|
||||
Logger.warning("Circuit breaker open for service",
|
||||
circuit_breaker:
|
||||
LogSanitizer.log_data(
|
||||
service: service_name,
|
||||
state: :open,
|
||||
action: :rejected
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :circuit_open}
|
||||
|
||||
state when state in [:closed, :half_open] ->
|
||||
attempt_call(service_name, fun, timeout)
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Get the current state of a service circuit breaker
|
||||
"""
|
||||
@spec get_state(atom()) :: state()
|
||||
def get_state(service_name) do
|
||||
GenServer.call(__MODULE__, {:get_state, service_name})
|
||||
end
|
||||
|
||||
@doc """
|
||||
Get statistics for a service circuit breaker
|
||||
"""
|
||||
@spec get_stats(atom()) :: service_state()
|
||||
def get_stats(service_name) do
|
||||
GenServer.call(__MODULE__, {:get_stats, service_name})
|
||||
end
|
||||
|
||||
@doc """
|
||||
Manually reset a circuit breaker
|
||||
"""
|
||||
@spec reset(atom()) :: :ok
|
||||
def reset(service_name) do
|
||||
GenServer.call(__MODULE__, {:reset, service_name})
|
||||
end
|
||||
|
||||
# GenServer implementation
|
||||
|
||||
@impl true
|
||||
def init(_opts) do
|
||||
{:ok, %{}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call({:get_state, service_name}, _from, services) do
|
||||
service_state = get_service_state(services, service_name)
|
||||
current_state = calculate_current_state(service_state)
|
||||
{:reply, current_state, services}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call({:get_stats, service_name}, _from, services) do
|
||||
service_state = get_service_state(services, service_name)
|
||||
{:reply, service_state, services}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call({:reset, service_name}, _from, services) do
|
||||
default_state = default_service_state()
|
||||
new_services = Map.put(services, service_name, default_state)
|
||||
|
||||
Logger.info("Circuit breaker reset",
|
||||
circuit_breaker:
|
||||
LogSanitizer.log_data(
|
||||
service: service_name,
|
||||
action: :manual_reset
|
||||
)
|
||||
)
|
||||
|
||||
{:reply, :ok, new_services}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call({:record_success, service_name}, _from, services) do
|
||||
service_state = get_service_state(services, service_name)
|
||||
updated_state = %{service_state | failure_count: 0, state: :closed}
|
||||
new_services = Map.put(services, service_name, updated_state)
|
||||
{:reply, :ok, new_services}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call({:record_failure, service_name}, _from, services) do
|
||||
service_state = get_service_state(services, service_name)
|
||||
new_failure_count = service_state.failure_count + 1
|
||||
|
||||
new_state =
|
||||
if new_failure_count >= service_state.failure_threshold do
|
||||
:open
|
||||
else
|
||||
service_state.state
|
||||
end
|
||||
|
||||
updated_state = %{
|
||||
service_state
|
||||
| failure_count: new_failure_count,
|
||||
state: new_state,
|
||||
last_failure_time: DateTime.utc_now()
|
||||
}
|
||||
|
||||
if new_state == :open do
|
||||
Logger.warning("Circuit breaker opened due to failure threshold",
|
||||
circuit_breaker:
|
||||
LogSanitizer.log_data(
|
||||
service: service_name,
|
||||
failure_count: new_failure_count,
|
||||
threshold: service_state.failure_threshold,
|
||||
state: :open
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
new_services = Map.put(services, service_name, updated_state)
|
||||
{:reply, :ok, new_services}
|
||||
end
|
||||
|
||||
# Private functions
|
||||
|
||||
defp attempt_call(service_name, fun, timeout) do
|
||||
task = Task.async(fn -> fun.() end)
|
||||
|
||||
try do
|
||||
result = Task.await(task, timeout)
|
||||
record_success(service_name)
|
||||
{:ok, result}
|
||||
catch
|
||||
:exit, {:timeout, _} ->
|
||||
Task.shutdown(task, :brutal_kill)
|
||||
record_failure(service_name)
|
||||
|
||||
Logger.warning("Service call timed out",
|
||||
circuit_breaker:
|
||||
LogSanitizer.log_data(
|
||||
service: service_name,
|
||||
timeout_ms: timeout,
|
||||
action: :timeout_failure
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :timeout}
|
||||
|
||||
kind, error ->
|
||||
record_failure(service_name)
|
||||
|
||||
Logger.error("Service call failed",
|
||||
circuit_breaker:
|
||||
LogSanitizer.log_data(
|
||||
service: service_name,
|
||||
error_kind: kind,
|
||||
error: LogSanitizer.sanitize(error)
|
||||
)
|
||||
)
|
||||
|
||||
{:error, error}
|
||||
end
|
||||
end
|
||||
|
||||
defp record_success(service_name) do
|
||||
GenServer.call(__MODULE__, {:record_success, service_name})
|
||||
end
|
||||
|
||||
defp record_failure(service_name) do
|
||||
GenServer.call(__MODULE__, {:record_failure, service_name})
|
||||
end
|
||||
|
||||
defp get_service_state(services, service_name) do
|
||||
Map.get(services, service_name, default_service_state())
|
||||
end
|
||||
|
||||
defp default_service_state do
|
||||
%{
|
||||
state: :closed,
|
||||
failure_count: 0,
|
||||
failure_threshold: @default_failure_threshold,
|
||||
timeout: @default_timeout,
|
||||
last_failure_time: nil,
|
||||
recovery_timeout: @default_recovery_timeout
|
||||
}
|
||||
end
|
||||
|
||||
defp calculate_current_state(%{state: :open, last_failure_time: last_failure_time, recovery_timeout: recovery_timeout}) do
|
||||
if last_failure_time && DateTime.diff(DateTime.utc_now(), last_failure_time, :millisecond) >= recovery_timeout do
|
||||
:half_open
|
||||
else
|
||||
:open
|
||||
end
|
||||
end
|
||||
|
||||
defp calculate_current_state(%{state: state}), do: state
|
||||
end
|
||||
|
|
@ -3,6 +3,8 @@ defmodule Aprsme.DependencyInfo do
|
|||
Provides information about dependencies, particularly for production builds.
|
||||
"""
|
||||
|
||||
alias Aprsme.CircuitBreaker
|
||||
|
||||
@doc """
|
||||
Gets the SHA1 hash of the aprs library currently being used.
|
||||
In development: reads from vendor/aprs directory
|
||||
|
|
@ -31,12 +33,19 @@ defmodule Aprsme.DependencyInfo do
|
|||
end
|
||||
|
||||
def latest_aprs_library_sha do
|
||||
resp = Req.get!("https://api.github.com/repos/aprsme/aprs/commits/main")
|
||||
body = resp.body
|
||||
sha = body["sha"] || body |> Jason.decode!() |> Access.get("sha")
|
||||
String.slice(sha, 0, 7)
|
||||
rescue
|
||||
_ -> nil
|
||||
case CircuitBreaker.call(
|
||||
:github_api,
|
||||
fn ->
|
||||
resp = Req.get!("https://api.github.com/repos/aprsme/aprs/commits/main")
|
||||
body = resp.body
|
||||
sha = body["sha"] || body |> Jason.decode!() |> Access.get("sha")
|
||||
String.slice(sha, 0, 7)
|
||||
end,
|
||||
10_000
|
||||
) do
|
||||
{:ok, sha} -> sha
|
||||
{:error, _} -> nil
|
||||
end
|
||||
end
|
||||
|
||||
def is_latest_aprs_library? do
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ defmodule Aprsme.DeviceIdentification do
|
|||
|
||||
import Ecto.Query
|
||||
|
||||
alias Aprsme.CircuitBreaker
|
||||
alias Aprsme.Devices
|
||||
alias Aprsme.Repo
|
||||
|
||||
|
|
@ -130,13 +131,24 @@ defmodule Aprsme.DeviceIdentification do
|
|||
end
|
||||
|
||||
def fetch_and_upsert_devices do
|
||||
case Req.get(@url) do
|
||||
{:ok, %Req.Response{status: 200, body: body}} ->
|
||||
{:ok, json} = Jason.decode(body)
|
||||
upsert_devices(json)
|
||||
case CircuitBreaker.call(
|
||||
:aprs_foundation_api,
|
||||
fn ->
|
||||
case Req.get(@url) do
|
||||
{:ok, %Req.Response{status: 200, body: body}} ->
|
||||
upsert_devices(body)
|
||||
|
||||
err ->
|
||||
err
|
||||
{:ok, %Req.Response{status: status}} ->
|
||||
{:error, {:http_error, status}}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
end,
|
||||
15_000
|
||||
) do
|
||||
{:ok, result} -> result
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
380
lib/aprsme/error_handler.ex
Normal file
380
lib/aprsme/error_handler.ex
Normal file
|
|
@ -0,0 +1,380 @@
|
|||
defmodule Aprsme.ErrorHandler do
|
||||
@moduledoc """
|
||||
Centralized error handling and recovery patterns for the application
|
||||
"""
|
||||
|
||||
alias Aprsme.LogSanitizer
|
||||
alias Ecto.Query.CastError
|
||||
|
||||
require Logger
|
||||
|
||||
@doc """
|
||||
Handle and log database errors with context
|
||||
"""
|
||||
def handle_database_error(error, context \\ %{}) do
|
||||
sanitized_context = LogSanitizer.sanitize_error_context(context)
|
||||
|
||||
case error do
|
||||
%CastError{} = cast_error ->
|
||||
Logger.error("Database cast error - invalid data type",
|
||||
error_details:
|
||||
LogSanitizer.log_data(
|
||||
error_type: :cast_error,
|
||||
message: cast_error.message,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :invalid_data}
|
||||
|
||||
%Ecto.ConstraintError{} = constraint_error ->
|
||||
Logger.error("Database constraint violation",
|
||||
error_details:
|
||||
LogSanitizer.log_data(
|
||||
error_type: :constraint_violation,
|
||||
constraint: constraint_error.constraint,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :constraint_violation}
|
||||
|
||||
%Postgrex.Error{postgres: %{code: :unique_violation}} ->
|
||||
Logger.warning("Attempted to insert duplicate record",
|
||||
error_details:
|
||||
LogSanitizer.log_data(
|
||||
error_type: :duplicate_record,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :duplicate_record}
|
||||
|
||||
%DBConnection.ConnectionError{} ->
|
||||
Logger.error("Database connection error",
|
||||
error_details:
|
||||
LogSanitizer.log_data(
|
||||
error_type: :connection_error,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :database_unavailable}
|
||||
|
||||
%Ecto.QueryError{} = query_error ->
|
||||
Logger.error("Database query error",
|
||||
error_details:
|
||||
LogSanitizer.log_data(
|
||||
error_type: :query_error,
|
||||
message: query_error.message,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :query_failed}
|
||||
|
||||
error ->
|
||||
Logger.error("Unexpected database error",
|
||||
error_details:
|
||||
LogSanitizer.log_data(
|
||||
error_type: :unknown_database_error,
|
||||
error: LogSanitizer.sanitize(error),
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :database_error}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Handle network and external service errors
|
||||
"""
|
||||
def handle_network_error(error, service_name, context \\ %{}) do
|
||||
sanitized_context = LogSanitizer.sanitize_error_context(context)
|
||||
|
||||
case error do
|
||||
%Req.TransportError{reason: :timeout} ->
|
||||
Logger.warning("Network request timeout",
|
||||
network_error:
|
||||
LogSanitizer.log_data(
|
||||
service: service_name,
|
||||
error_type: :timeout,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :timeout}
|
||||
|
||||
%Req.TransportError{reason: :econnrefused} ->
|
||||
Logger.error("Connection refused by service",
|
||||
network_error:
|
||||
LogSanitizer.log_data(
|
||||
service: service_name,
|
||||
error_type: :connection_refused,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :service_unavailable}
|
||||
|
||||
%Req.TransportError{reason: reason} ->
|
||||
Logger.error("Network transport error",
|
||||
network_error:
|
||||
LogSanitizer.log_data(
|
||||
service: service_name,
|
||||
error_type: :transport_error,
|
||||
reason: reason,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :network_error}
|
||||
|
||||
{:error, :circuit_open} ->
|
||||
Logger.warning("Circuit breaker open for service",
|
||||
network_error:
|
||||
LogSanitizer.log_data(
|
||||
service: service_name,
|
||||
error_type: :circuit_open,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :service_unavailable}
|
||||
|
||||
error ->
|
||||
Logger.error("Unexpected network error",
|
||||
network_error:
|
||||
LogSanitizer.log_data(
|
||||
service: service_name,
|
||||
error_type: :unknown_network_error,
|
||||
error: LogSanitizer.sanitize(error),
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :network_error}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Handle packet processing errors with recovery strategies
|
||||
"""
|
||||
def handle_packet_error(error, packet_data, context \\ %{}) do
|
||||
_sanitized_packet = LogSanitizer.sanitize_packet_data(packet_data)
|
||||
sanitized_context = LogSanitizer.sanitize_error_context(context)
|
||||
|
||||
case error do
|
||||
%Jason.DecodeError{} ->
|
||||
Logger.warning("Invalid JSON in packet data",
|
||||
packet_error:
|
||||
LogSanitizer.log_data(
|
||||
error_type: :invalid_json,
|
||||
packet_sender: Map.get(packet_data, :sender, "unknown"),
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
store_bad_packet(packet_data, "invalid_json")
|
||||
{:error, :invalid_json}
|
||||
|
||||
%ArgumentError{message: message} ->
|
||||
if String.contains?(message, "invalid") do
|
||||
Logger.warning("Invalid packet format",
|
||||
packet_error:
|
||||
LogSanitizer.log_data(
|
||||
error_type: :invalid_format,
|
||||
packet_sender: Map.get(packet_data, :sender, "unknown"),
|
||||
message: message,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
store_bad_packet(packet_data, "invalid_format")
|
||||
{:error, :invalid_format}
|
||||
else
|
||||
Logger.error("Unexpected packet processing error",
|
||||
packet_error:
|
||||
LogSanitizer.log_data(
|
||||
error_type: :processing_error,
|
||||
packet_sender: Map.get(packet_data, :sender, "unknown"),
|
||||
error: LogSanitizer.sanitize(message),
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
store_bad_packet(packet_data, "processing_error")
|
||||
{:error, :processing_failed}
|
||||
end
|
||||
|
||||
error ->
|
||||
Logger.error("Unexpected packet processing error",
|
||||
packet_error:
|
||||
LogSanitizer.log_data(
|
||||
error_type: :processing_error,
|
||||
packet_sender: Map.get(packet_data, :sender, "unknown"),
|
||||
error: LogSanitizer.sanitize(error),
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
store_bad_packet(packet_data, "processing_error")
|
||||
{:error, :processing_failed}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Handle GenServer and process errors
|
||||
"""
|
||||
def handle_process_error(error, process_name, context \\ %{}) do
|
||||
sanitized_context = LogSanitizer.sanitize_error_context(context)
|
||||
|
||||
case error do
|
||||
:timeout ->
|
||||
Logger.warning("Process operation timeout",
|
||||
process_error:
|
||||
LogSanitizer.log_data(
|
||||
process: process_name,
|
||||
error_type: :timeout,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :timeout}
|
||||
|
||||
{:noproc, _} ->
|
||||
Logger.error("Process not found or died",
|
||||
process_error:
|
||||
LogSanitizer.log_data(
|
||||
process: process_name,
|
||||
error_type: :process_died,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :process_unavailable}
|
||||
|
||||
{:badrpc, reason} ->
|
||||
Logger.error("RPC call failed",
|
||||
process_error:
|
||||
LogSanitizer.log_data(
|
||||
process: process_name,
|
||||
error_type: :rpc_failed,
|
||||
reason: reason,
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :rpc_failed}
|
||||
|
||||
error ->
|
||||
Logger.error("Unexpected process error",
|
||||
process_error:
|
||||
LogSanitizer.log_data(
|
||||
process: process_name,
|
||||
error_type: :unknown_process_error,
|
||||
error: LogSanitizer.sanitize(error),
|
||||
context: sanitized_context
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :process_error}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Execute function with comprehensive error handling and recovery
|
||||
"""
|
||||
def with_error_handling(fun, opts \\ []) when is_function(fun, 0) do
|
||||
context = Keyword.get(opts, :context, %{})
|
||||
max_retries = Keyword.get(opts, :max_retries, 0)
|
||||
retry_delay = Keyword.get(opts, :retry_delay, 1000)
|
||||
|
||||
do_with_retries(fun, max_retries, retry_delay, context, 0)
|
||||
end
|
||||
|
||||
defp do_with_retries(fun, max_retries, retry_delay, context, attempt) do
|
||||
{:ok, fun.()}
|
||||
rescue
|
||||
error ->
|
||||
if attempt < max_retries do
|
||||
Logger.warning("Retrying operation after error",
|
||||
retry_info:
|
||||
LogSanitizer.log_data(
|
||||
attempt: attempt + 1,
|
||||
max_retries: max_retries,
|
||||
delay_ms: retry_delay,
|
||||
error: LogSanitizer.sanitize(error)
|
||||
)
|
||||
)
|
||||
|
||||
Process.sleep(retry_delay)
|
||||
do_with_retries(fun, max_retries, retry_delay, context, attempt + 1)
|
||||
else
|
||||
categorize_and_handle_error(error, context)
|
||||
end
|
||||
catch
|
||||
kind, error ->
|
||||
Logger.error("Unexpected error in operation",
|
||||
error_details:
|
||||
LogSanitizer.log_data(
|
||||
kind: kind,
|
||||
error: LogSanitizer.sanitize(error),
|
||||
context: LogSanitizer.sanitize_error_context(context)
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :unexpected_error}
|
||||
end
|
||||
|
||||
defp categorize_and_handle_error(error, context) do
|
||||
cond do
|
||||
is_database_error?(error) ->
|
||||
handle_database_error(error, context)
|
||||
|
||||
is_network_error?(error) ->
|
||||
handle_network_error(error, :unknown_service, context)
|
||||
|
||||
true ->
|
||||
Logger.error("Uncategorized error",
|
||||
error_details:
|
||||
LogSanitizer.log_data(
|
||||
error: LogSanitizer.sanitize(error),
|
||||
context: LogSanitizer.sanitize_error_context(context)
|
||||
)
|
||||
)
|
||||
|
||||
{:error, :unknown_error}
|
||||
end
|
||||
end
|
||||
|
||||
defp is_database_error?(%CastError{}), do: true
|
||||
defp is_database_error?(%Ecto.ConstraintError{}), do: true
|
||||
defp is_database_error?(%Postgrex.Error{}), do: true
|
||||
defp is_database_error?(%DBConnection.ConnectionError{}), do: true
|
||||
defp is_database_error?(%Ecto.QueryError{}), do: true
|
||||
defp is_database_error?(_), do: false
|
||||
|
||||
defp is_network_error?(%Req.TransportError{}), do: true
|
||||
defp is_network_error?(%{reason: :circuit_open}), do: true
|
||||
defp is_network_error?(%{reason: :timeout}), do: true
|
||||
defp is_network_error?(%RuntimeError{message: "circuit_open"}), do: true
|
||||
defp is_network_error?(%RuntimeError{message: "timeout"}), do: true
|
||||
defp is_network_error?(_), do: false
|
||||
|
||||
defp store_bad_packet(packet_data, reason) do
|
||||
# This would store the bad packet for later analysis
|
||||
# For now, just log it since BadPacket module may not exist
|
||||
Logger.warning("Storing bad packet for analysis",
|
||||
bad_packet:
|
||||
LogSanitizer.log_data(
|
||||
raw_packet: inspect(packet_data),
|
||||
error_reason: reason,
|
||||
received_at: DateTime.utc_now()
|
||||
)
|
||||
)
|
||||
|
||||
:ok
|
||||
end
|
||||
end
|
||||
133
lib/aprsme/log_sanitizer.ex
Normal file
133
lib/aprsme/log_sanitizer.ex
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
defmodule Aprsme.LogSanitizer do
|
||||
@moduledoc """
|
||||
Utility module for sanitizing log output to prevent sensitive data leakage
|
||||
"""
|
||||
|
||||
@sensitive_fields [
|
||||
:password,
|
||||
:passcode,
|
||||
:auth_token,
|
||||
:api_key,
|
||||
:secret,
|
||||
:token,
|
||||
:authorization,
|
||||
:csrf_token,
|
||||
:session_token,
|
||||
:private_key,
|
||||
:database_url,
|
||||
:secret_key_base,
|
||||
:signing_salt
|
||||
]
|
||||
|
||||
@sensitive_patterns [
|
||||
~r/password[=:]\s*\S+/i,
|
||||
~r/passcode[=:]\s*\S+/i,
|
||||
~r/token[=:]\s*\S+/i,
|
||||
~r/key[=:]\s*\S+/i,
|
||||
~r/secret[=:]\s*\S+/i,
|
||||
~r/authorization[=:]\s*\S+/i
|
||||
]
|
||||
|
||||
@doc """
|
||||
Sanitize a map by removing or redacting sensitive fields
|
||||
"""
|
||||
def sanitize_map(data) when is_map(data) do
|
||||
Enum.reduce(@sensitive_fields, data, fn field, acc ->
|
||||
cond do
|
||||
Map.has_key?(acc, field) -> Map.put(acc, field, "[REDACTED]")
|
||||
Map.has_key?(acc, to_string(field)) -> Map.put(acc, to_string(field), "[REDACTED]")
|
||||
true -> acc
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
def sanitize_map(data), do: data
|
||||
|
||||
@doc """
|
||||
Sanitize a string by replacing sensitive patterns
|
||||
"""
|
||||
def sanitize_string(data) when is_binary(data) do
|
||||
Enum.reduce(@sensitive_patterns, data, fn pattern, acc ->
|
||||
Regex.replace(pattern, acc, fn match ->
|
||||
case String.split(match, ["=", ":"]) do
|
||||
[key, _value] -> "#{key}=[REDACTED]"
|
||||
_ -> "[REDACTED]"
|
||||
end
|
||||
end)
|
||||
end)
|
||||
end
|
||||
|
||||
def sanitize_string(data), do: data
|
||||
|
||||
@doc """
|
||||
Sanitize any data structure recursively
|
||||
"""
|
||||
def sanitize(data) when is_map(data) do
|
||||
data
|
||||
|> sanitize_map()
|
||||
|> Map.new(fn {k, v} -> {k, sanitize(v)} end)
|
||||
end
|
||||
|
||||
def sanitize(data) when is_list(data) do
|
||||
Enum.map(data, &sanitize/1)
|
||||
end
|
||||
|
||||
def sanitize(data) when is_binary(data) do
|
||||
sanitize_string(data)
|
||||
end
|
||||
|
||||
def sanitize(data), do: data
|
||||
|
||||
@doc """
|
||||
Sanitize packet data for logging
|
||||
"""
|
||||
def sanitize_packet_data(packet_data) when is_map(packet_data) do
|
||||
# Remove potentially sensitive packet content but keep structure
|
||||
packet_data
|
||||
|> Map.drop([:raw_packet, :comment, :status, :data_extended])
|
||||
|> sanitize_map()
|
||||
end
|
||||
|
||||
def sanitize_packet_data(data), do: data
|
||||
|
||||
@doc """
|
||||
Sanitize connection information for logging
|
||||
"""
|
||||
def sanitize_connection_info(conn_info) when is_map(conn_info) do
|
||||
conn_info
|
||||
|> Map.drop([:password, :passcode])
|
||||
|> Map.put(:password, "[REDACTED]")
|
||||
|> Map.put(:passcode, "[REDACTED]")
|
||||
end
|
||||
|
||||
def sanitize_connection_info(data), do: data
|
||||
|
||||
@doc """
|
||||
Sanitize error information for logging
|
||||
"""
|
||||
def sanitize_error_context(context) when is_map(context) do
|
||||
context
|
||||
|> sanitize_map()
|
||||
|> Map.update(:stacktrace, nil, fn stacktrace ->
|
||||
if is_list(stacktrace) do
|
||||
# Limit stacktrace length
|
||||
Enum.take(stacktrace, 5)
|
||||
else
|
||||
stacktrace
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
def sanitize_error_context(data), do: data
|
||||
|
||||
@doc """
|
||||
Create a sanitized logging keyword list
|
||||
"""
|
||||
def log_data(keyword_list) when is_list(keyword_list) do
|
||||
Enum.map(keyword_list, fn {key, value} ->
|
||||
{key, sanitize(value)}
|
||||
end)
|
||||
end
|
||||
|
||||
def log_data(data), do: sanitize(data)
|
||||
end
|
||||
|
|
@ -5,6 +5,7 @@ defmodule Aprsme.PacketConsumer do
|
|||
"""
|
||||
use GenStage
|
||||
|
||||
alias Aprsme.LogSanitizer
|
||||
alias Aprsme.Repo
|
||||
|
||||
require Logger
|
||||
|
|
@ -17,6 +18,8 @@ defmodule Aprsme.PacketConsumer do
|
|||
def init(opts) do
|
||||
batch_size = opts[:batch_size] || 100
|
||||
batch_timeout = opts[:batch_timeout] || 1000
|
||||
# Maximum batch size to prevent unbounded memory growth
|
||||
max_batch_size = opts[:max_batch_size] || 1000
|
||||
|
||||
# Start a timer for batch processing
|
||||
timer = Process.send_after(self(), :process_batch, batch_timeout)
|
||||
|
|
@ -26,27 +29,62 @@ defmodule Aprsme.PacketConsumer do
|
|||
batch: [],
|
||||
batch_size: batch_size,
|
||||
batch_timeout: batch_timeout,
|
||||
max_batch_size: max_batch_size,
|
||||
timer: timer
|
||||
}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_events(events, _from, %{batch: batch, batch_size: batch_size} = state) do
|
||||
def handle_events(events, _from, %{batch: batch, batch_size: batch_size, max_batch_size: max_batch_size} = state) do
|
||||
new_batch = batch ++ events
|
||||
new_batch_length = length(new_batch)
|
||||
|
||||
if length(new_batch) >= batch_size do
|
||||
# Process the batch immediately
|
||||
process_batch(new_batch)
|
||||
{:noreply, [], %{state | batch: []}}
|
||||
else
|
||||
# Add to batch and wait for more
|
||||
{:noreply, [], %{state | batch: new_batch}}
|
||||
cond do
|
||||
new_batch_length >= max_batch_size ->
|
||||
# If batch exceeds maximum size, process immediately and drop excess
|
||||
{process_batch, drop_batch} = Enum.split(new_batch, max_batch_size)
|
||||
process_batch(process_batch)
|
||||
|
||||
# Log warning about dropping packets (sanitized)
|
||||
if length(drop_batch) > 0 do
|
||||
Logger.warning("Dropped #{length(drop_batch)} packets due to batch size limit",
|
||||
batch_info:
|
||||
LogSanitizer.log_data(
|
||||
dropped_count: length(drop_batch),
|
||||
processed_count: length(process_batch),
|
||||
reason: "batch_size_limit_exceeded"
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
{:noreply, [], %{state | batch: []}}
|
||||
|
||||
new_batch_length >= batch_size ->
|
||||
# Process the batch immediately
|
||||
process_batch(new_batch)
|
||||
{:noreply, [], %{state | batch: []}}
|
||||
|
||||
true ->
|
||||
# Add to batch and wait for more
|
||||
{:noreply, [], %{state | batch: new_batch}}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:process_batch, %{batch: batch, batch_timeout: timeout} = state) do
|
||||
def handle_info(:process_batch, %{batch: batch, batch_timeout: timeout, max_batch_size: max_batch_size} = state) do
|
||||
if length(batch) > 0 do
|
||||
# Check if batch size is concerning
|
||||
if length(batch) > max_batch_size * 0.8 do
|
||||
Logger.warning("Batch size approaching limit",
|
||||
batch_status:
|
||||
LogSanitizer.log_data(
|
||||
current_size: length(batch),
|
||||
max_size: max_batch_size,
|
||||
utilization_percent: trunc(length(batch) / max_batch_size * 100)
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
process_batch(batch)
|
||||
end
|
||||
|
||||
|
|
@ -58,6 +96,8 @@ defmodule Aprsme.PacketConsumer do
|
|||
defp process_batch(packets) do
|
||||
require Logger
|
||||
|
||||
# Monitor memory usage before processing
|
||||
{memory_before, _} = :erlang.statistics(:runtime)
|
||||
start_time = System.monotonic_time(:millisecond)
|
||||
|
||||
results =
|
||||
|
|
@ -73,18 +113,50 @@ defmodule Aprsme.PacketConsumer do
|
|||
end_time = System.monotonic_time(:millisecond)
|
||||
duration = end_time - start_time
|
||||
|
||||
# Monitor memory usage after processing
|
||||
{memory_after, _} = :erlang.statistics(:runtime)
|
||||
memory_diff = memory_after - memory_before
|
||||
|
||||
# Force garbage collection if memory usage is high
|
||||
# 50MB threshold
|
||||
if memory_diff > 50_000 do
|
||||
:erlang.garbage_collect()
|
||||
|
||||
Logger.warning("High memory usage detected, forced garbage collection",
|
||||
memory_info:
|
||||
LogSanitizer.log_data(
|
||||
memory_diff_bytes: memory_diff,
|
||||
batch_size: length(packets),
|
||||
gc_forced: true
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
:telemetry.execute(
|
||||
[
|
||||
:aprsme,
|
||||
:packet_pipeline,
|
||||
:batch
|
||||
],
|
||||
%{count: length(packets), success: success_count, error: error_count, duration_ms: duration},
|
||||
%{
|
||||
count: length(packets),
|
||||
success: success_count,
|
||||
error: error_count,
|
||||
duration_ms: duration,
|
||||
memory_diff: memory_diff
|
||||
},
|
||||
%{}
|
||||
)
|
||||
|
||||
Logger.info(
|
||||
"Processed batch of #{length(packets)} packets in #{duration}ms (success: #{success_count}, errors: #{error_count})"
|
||||
Logger.info("Batch processing completed",
|
||||
batch_result:
|
||||
LogSanitizer.log_data(
|
||||
packet_count: length(packets),
|
||||
duration_ms: duration,
|
||||
success_count: success_count,
|
||||
error_count: error_count,
|
||||
memory_diff_bytes: memory_diff
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
|
|
@ -296,20 +368,11 @@ defmodule Aprsme.PacketConsumer do
|
|||
|
||||
defp set_lat_lon(attrs, lat, lon) do
|
||||
round6 = fn
|
||||
nil ->
|
||||
nil
|
||||
|
||||
n when is_float(n) ->
|
||||
Float.round(n, 6)
|
||||
|
||||
n when is_integer(n) ->
|
||||
n * 1.0
|
||||
|
||||
n when is_binary(n) ->
|
||||
case Float.parse(n) do
|
||||
{f, _} -> Float.round(f, 6)
|
||||
:error -> nil
|
||||
end
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
|
||||
attrs
|
||||
|
|
|
|||
|
|
@ -504,7 +504,9 @@ defmodule Aprsme.Packets do
|
|||
|
||||
defp filter_by_callsign(query, %{callsign: callsign}) do
|
||||
# Use sender field for exact callsign matching
|
||||
from p in query, where: ilike(p.sender, ^callsign)
|
||||
# Sanitize callsign to prevent SQL injection
|
||||
sanitized_callsign = sanitize_callsign(callsign)
|
||||
from p in query, where: ilike(p.sender, ^sanitized_callsign)
|
||||
end
|
||||
|
||||
defp filter_by_callsign(query, _), do: query
|
||||
|
|
@ -538,7 +540,15 @@ defmodule Aprsme.Packets do
|
|||
defp filter_by_map_bounds(query, _), do: query
|
||||
|
||||
defp create_bounding_box_wkt(min_lon, min_lat, max_lon, max_lat) do
|
||||
"POLYGON((#{min_lon} #{min_lat}, #{max_lon} #{min_lat}, #{max_lon} #{max_lat}, #{min_lon} #{max_lat}, #{min_lon} #{min_lat}))"
|
||||
# Validate and sanitize coordinates to prevent SQL injection
|
||||
with {:ok, min_lon} <- validate_coordinate(min_lon),
|
||||
{:ok, min_lat} <- validate_coordinate(min_lat),
|
||||
{:ok, max_lon} <- validate_coordinate(max_lon),
|
||||
{:ok, max_lat} <- validate_coordinate(max_lat) do
|
||||
"POLYGON((#{min_lon} #{min_lat}, #{max_lon} #{min_lat}, #{max_lon} #{max_lat}, #{min_lon} #{max_lat}, #{min_lon} #{min_lat}))"
|
||||
else
|
||||
_ -> raise ArgumentError, "Invalid coordinates provided"
|
||||
end
|
||||
end
|
||||
|
||||
defp limit_results(query, %{limit: limit, page: page}) when not is_nil(limit) and not is_nil(page) do
|
||||
|
|
@ -717,12 +727,44 @@ defmodule Aprsme.Packets do
|
|||
"""
|
||||
@spec get_latest_packet_for_callsign(String.t()) :: struct() | nil
|
||||
def get_latest_packet_for_callsign(callsign) when is_binary(callsign) do
|
||||
sanitized_callsign = sanitize_callsign(callsign)
|
||||
|
||||
from(p in Packet,
|
||||
where: ilike(p.sender, ^callsign),
|
||||
where: ilike(p.sender, ^sanitized_callsign),
|
||||
order_by: [desc: p.received_at],
|
||||
limit: 1
|
||||
)
|
||||
|> select_with_virtual_coordinates()
|
||||
|> Repo.one()
|
||||
end
|
||||
|
||||
# Sanitization functions to prevent SQL injection
|
||||
defp sanitize_callsign(callsign) when is_binary(callsign) do
|
||||
# APRS callsigns are alphanumeric with hyphens and up to 9 characters
|
||||
# Remove any characters that could be used for SQL injection
|
||||
callsign
|
||||
|> String.upcase()
|
||||
|> String.replace(~r/[^A-Z0-9\-]/, "")
|
||||
|> String.slice(0, 9)
|
||||
end
|
||||
|
||||
defp sanitize_callsign(_), do: ""
|
||||
|
||||
defp validate_coordinate(coord) when is_number(coord) do
|
||||
# Validate coordinate ranges
|
||||
if coord >= -180 and coord <= 180 do
|
||||
{:ok, coord}
|
||||
else
|
||||
{:error, :invalid_coordinate}
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_coordinate(coord) when is_binary(coord) do
|
||||
case Float.parse(coord) do
|
||||
{float_val, ""} -> validate_coordinate(float_val)
|
||||
_ -> {:error, :invalid_coordinate}
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_coordinate(_), do: {:error, :invalid_coordinate}
|
||||
end
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ defmodule AprsmeWeb.Api.V1.CallsignController do
|
|||
"""
|
||||
use AprsmeWeb, :controller
|
||||
|
||||
alias Aprsme.Packets
|
||||
alias Aprsme.CachedQueries
|
||||
alias Aprsme.ErrorHandler
|
||||
alias AprsmeWeb.Api.V1.CallsignJSON
|
||||
|
||||
action_fallback AprsmeWeb.Api.V1.FallbackController
|
||||
|
|
@ -67,21 +68,21 @@ defmodule AprsmeWeb.Api.V1.CallsignController do
|
|||
|
||||
defp get_latest_packet(callsign) do
|
||||
# Get the most recent packet for this callsign regardless of age or type
|
||||
case Packets.get_latest_packet_for_callsign(callsign) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
packet ->
|
||||
{:ok, packet}
|
||||
# Use cached version for better performance with error handling
|
||||
fn ->
|
||||
case CachedQueries.get_latest_packet_for_callsign_cached(callsign) do
|
||||
nil -> {:error, :not_found}
|
||||
packet -> {:ok, packet}
|
||||
end
|
||||
end
|
||||
|> ErrorHandler.with_error_handling(
|
||||
context: %{callsign: callsign, operation: :get_latest_packet},
|
||||
max_retries: 1,
|
||||
retry_delay: 500
|
||||
)
|
||||
|> case do
|
||||
{:ok, result} -> result
|
||||
{:error, error} -> {:error, error}
|
||||
end
|
||||
rescue
|
||||
Ecto.QueryError ->
|
||||
{:error, :database_error}
|
||||
|
||||
error ->
|
||||
require Logger
|
||||
|
||||
Logger.error("Error fetching packet for callsign #{callsign}: #{inspect(error)}")
|
||||
{:error, :internal_error}
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -284,20 +284,38 @@ defmodule AprsmeWeb.MapLive.PacketUtils do
|
|||
end
|
||||
|
||||
# Weather unit conversion helpers
|
||||
defp convert_temperature(value, locale) when is_number(value) do
|
||||
AprsmeWeb.WeatherUnits.format_temperature(value, locale)
|
||||
defp convert_temperature(value, locale) when is_binary(value) and value != "N/A" do
|
||||
case Float.parse(value) do
|
||||
{num_value, _} ->
|
||||
AprsmeWeb.WeatherUnits.format_temperature(num_value, locale)
|
||||
|
||||
:error ->
|
||||
{value, "°F"}
|
||||
end
|
||||
end
|
||||
|
||||
defp convert_temperature(value, _locale), do: {value, "°F"}
|
||||
|
||||
defp convert_wind_speed(value, locale) when is_number(value) do
|
||||
AprsmeWeb.WeatherUnits.format_wind_speed(value, locale)
|
||||
defp convert_wind_speed(value, locale) when is_binary(value) and value != "N/A" do
|
||||
case Float.parse(value) do
|
||||
{num_value, _} ->
|
||||
AprsmeWeb.WeatherUnits.format_wind_speed(num_value, locale)
|
||||
|
||||
:error ->
|
||||
{value, "mph"}
|
||||
end
|
||||
end
|
||||
|
||||
defp convert_wind_speed(value, _locale), do: {value, "mph"}
|
||||
|
||||
defp convert_rain(value, locale) when is_number(value) do
|
||||
AprsmeWeb.WeatherUnits.format_rain(value, locale)
|
||||
defp convert_rain(value, locale) when is_binary(value) and value != "N/A" do
|
||||
case Float.parse(value) do
|
||||
{num_value, _} ->
|
||||
AprsmeWeb.WeatherUnits.format_rain(num_value, locale)
|
||||
|
||||
:error ->
|
||||
{value, "in"}
|
||||
end
|
||||
end
|
||||
|
||||
defp convert_rain(value, _locale), do: {value, "in"}
|
||||
|
|
|
|||
|
|
@ -251,7 +251,6 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do
|
|||
# Return "0" for missing numeric fields, keep other values as-is
|
||||
case value do
|
||||
"N/A" -> "0"
|
||||
nil -> "0"
|
||||
_ -> value
|
||||
end
|
||||
end
|
||||
|
|
@ -263,34 +262,69 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do
|
|||
value = PacketUtils.get_weather_field(packet, key)
|
||||
|
||||
case {key, value} do
|
||||
{:temperature, value} when is_number(value) ->
|
||||
{converted_value, unit} = WeatherUnits.format_temperature(value, locale)
|
||||
"#{converted_value}#{unit}"
|
||||
{:temperature, value} when is_binary(value) and value != "N/A" ->
|
||||
case Float.parse(value) do
|
||||
{num_value, _} ->
|
||||
{converted_value, unit} = WeatherUnits.format_temperature(num_value, locale)
|
||||
"#{converted_value}#{unit}"
|
||||
|
||||
{:wind_speed, value} when is_number(value) ->
|
||||
{converted_value, unit} = WeatherUnits.format_wind_speed(value, locale)
|
||||
"#{converted_value} #{unit}"
|
||||
:error ->
|
||||
value
|
||||
end
|
||||
|
||||
{:wind_gust, value} when is_number(value) ->
|
||||
{converted_value, unit} = WeatherUnits.format_wind_speed(value, locale)
|
||||
"#{converted_value} #{unit}"
|
||||
{:wind_speed, value} when is_binary(value) and value != "N/A" ->
|
||||
case Float.parse(value) do
|
||||
{num_value, _} ->
|
||||
{converted_value, unit} = WeatherUnits.format_wind_speed(num_value, locale)
|
||||
"#{converted_value} #{unit}"
|
||||
|
||||
{:rain_1h, value} when is_number(value) ->
|
||||
{converted_value, unit} = WeatherUnits.format_rain(value, locale)
|
||||
"#{converted_value} #{unit}"
|
||||
:error ->
|
||||
value
|
||||
end
|
||||
|
||||
{:rain_24h, value} when is_number(value) ->
|
||||
{converted_value, unit} = WeatherUnits.format_rain(value, locale)
|
||||
"#{converted_value} #{unit}"
|
||||
{:wind_gust, value} when is_binary(value) and value != "N/A" ->
|
||||
case Float.parse(value) do
|
||||
{num_value, _} ->
|
||||
{converted_value, unit} = WeatherUnits.format_wind_speed(num_value, locale)
|
||||
"#{converted_value} #{unit}"
|
||||
|
||||
{:rain_since_midnight, value} when is_number(value) ->
|
||||
{converted_value, unit} = WeatherUnits.format_rain(value, locale)
|
||||
"#{converted_value} #{unit}"
|
||||
:error ->
|
||||
value
|
||||
end
|
||||
|
||||
{:rain_1h, value} when is_binary(value) and value != "N/A" ->
|
||||
case Float.parse(value) do
|
||||
{num_value, _} ->
|
||||
{converted_value, unit} = WeatherUnits.format_rain(num_value, locale)
|
||||
"#{converted_value} #{unit}"
|
||||
|
||||
:error ->
|
||||
value
|
||||
end
|
||||
|
||||
{:rain_24h, value} when is_binary(value) and value != "N/A" ->
|
||||
case Float.parse(value) do
|
||||
{num_value, _} ->
|
||||
{converted_value, unit} = WeatherUnits.format_rain(num_value, locale)
|
||||
"#{converted_value} #{unit}"
|
||||
|
||||
:error ->
|
||||
value
|
||||
end
|
||||
|
||||
{:rain_since_midnight, value} when is_binary(value) and value != "N/A" ->
|
||||
case Float.parse(value) do
|
||||
{num_value, _} ->
|
||||
{converted_value, unit} = WeatherUnits.format_rain(num_value, locale)
|
||||
"#{converted_value} #{unit}"
|
||||
|
||||
:error ->
|
||||
value
|
||||
end
|
||||
|
||||
_ ->
|
||||
case value do
|
||||
"N/A" -> "N/A"
|
||||
nil -> "N/A"
|
||||
_ -> "#{value}"
|
||||
end
|
||||
end
|
||||
|
|
|
|||
81
lib/aprsme_web/plugs/api_csrf.ex
Normal file
81
lib/aprsme_web/plugs/api_csrf.ex
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
defmodule AprsmeWeb.Plugs.ApiCSRF do
|
||||
@moduledoc """
|
||||
CSRF protection for API endpoints using X-Requested-With header or API tokens
|
||||
"""
|
||||
import Phoenix.Controller
|
||||
import Plug.Conn
|
||||
|
||||
def init(opts) do
|
||||
opts
|
||||
end
|
||||
|
||||
def call(conn, _opts) do
|
||||
case get_req_header(conn, "content-type") do
|
||||
[content_type | _] when content_type in ["application/json", "application/json; charset=utf-8"] ->
|
||||
# For JSON requests, check for X-Requested-With header or valid API token
|
||||
check_csrf_protection(conn)
|
||||
|
||||
_ ->
|
||||
# For non-JSON requests, proceed without CSRF check
|
||||
conn
|
||||
end
|
||||
end
|
||||
|
||||
defp check_csrf_protection(conn) do
|
||||
case {get_req_header(conn, "x-requested-with"), get_req_header(conn, "authorization")} do
|
||||
{["XMLHttpRequest"], _} ->
|
||||
# Valid AJAX request
|
||||
conn
|
||||
|
||||
{_, ["Bearer " <> _token]} ->
|
||||
# Has authorization header (API token) - would need validation in production
|
||||
conn
|
||||
|
||||
{_, _} ->
|
||||
# Check for CSRF token in header
|
||||
case get_req_header(conn, "x-csrf-token") do
|
||||
[token] when byte_size(token) > 0 ->
|
||||
# Validate CSRF token
|
||||
validate_csrf_token(conn, token)
|
||||
|
||||
_ ->
|
||||
reject_request(conn)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_csrf_token(conn, token) do
|
||||
# Get session token for comparison
|
||||
session_token = get_session(conn, "_csrf_token")
|
||||
|
||||
# Use Phoenix.Controller CSRF token validation
|
||||
if session_token && valid_csrf_token?(conn, token) do
|
||||
conn
|
||||
else
|
||||
reject_request(conn)
|
||||
end
|
||||
end
|
||||
|
||||
defp valid_csrf_token?(conn, token) do
|
||||
session_token = get_session(conn, "_csrf_token")
|
||||
|
||||
if session_token do
|
||||
# Use a simple comparison for now
|
||||
Phoenix.Token.verify(conn, "_csrf_token", token, max_age: 86_400) == {:ok, session_token}
|
||||
else
|
||||
false
|
||||
end
|
||||
rescue
|
||||
_ -> false
|
||||
end
|
||||
|
||||
defp reject_request(conn) do
|
||||
conn
|
||||
|> put_status(:forbidden)
|
||||
|> json(%{
|
||||
error: "CSRF protection failed",
|
||||
message: "Include X-Requested-With: XMLHttpRequest header or valid CSRF token"
|
||||
})
|
||||
|> halt()
|
||||
end
|
||||
end
|
||||
63
lib/aprsme_web/plugs/rate_limiter.ex
Normal file
63
lib/aprsme_web/plugs/rate_limiter.ex
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
defmodule AprsmeWeb.Plugs.RateLimiter do
|
||||
@moduledoc """
|
||||
Rate limiting plug to prevent DoS attacks
|
||||
"""
|
||||
import Phoenix.Controller
|
||||
import Plug.Conn
|
||||
|
||||
def init(opts) do
|
||||
# Default options
|
||||
Keyword.merge(
|
||||
[
|
||||
# 1 minute
|
||||
scale: 60_000,
|
||||
# 100 requests per minute
|
||||
limit: 100,
|
||||
# Rate limit by IP address
|
||||
key: :ip,
|
||||
error_message: "Too many requests"
|
||||
],
|
||||
opts
|
||||
)
|
||||
end
|
||||
|
||||
def call(conn, opts) do
|
||||
key = get_key(conn, opts[:key])
|
||||
scale = opts[:scale]
|
||||
limit = opts[:limit]
|
||||
error_message = opts[:error_message]
|
||||
|
||||
case Hammer.check_rate("rate_limit:#{key}", scale, limit) do
|
||||
{:allow, _count} ->
|
||||
conn
|
||||
|
||||
{:deny, _count} ->
|
||||
conn
|
||||
|> put_status(:too_many_requests)
|
||||
|> json(%{error: error_message})
|
||||
|> halt()
|
||||
end
|
||||
end
|
||||
|
||||
defp get_key(conn, :ip) do
|
||||
case get_req_header(conn, "x-forwarded-for") do
|
||||
[ip | _] -> ip
|
||||
[] -> conn.remote_ip |> :inet.ntoa() |> to_string()
|
||||
end
|
||||
end
|
||||
|
||||
defp get_key(conn, :user_agent) do
|
||||
case get_req_header(conn, "user-agent") do
|
||||
[ua | _] -> ua
|
||||
[] -> "unknown"
|
||||
end
|
||||
end
|
||||
|
||||
defp get_key(conn, custom_key) when is_function(custom_key) do
|
||||
custom_key.(conn)
|
||||
end
|
||||
|
||||
defp get_key(_conn, key) when is_binary(key) do
|
||||
key
|
||||
end
|
||||
end
|
||||
|
|
@ -5,6 +5,8 @@ defmodule AprsmeWeb.Router do
|
|||
import AprsmeWeb.UserAuth
|
||||
import Phoenix.LiveDashboard.Router
|
||||
|
||||
alias AprsmeWeb.Plugs.RateLimiter
|
||||
|
||||
pipeline :browser do
|
||||
plug :accepts, ["html"]
|
||||
plug :fetch_session
|
||||
|
|
@ -14,14 +16,22 @@ defmodule AprsmeWeb.Router do
|
|||
plug :put_secure_browser_headers
|
||||
plug :fetch_current_user
|
||||
plug AprsmeWeb.Plugs.SetLocale
|
||||
plug RateLimiter, scale: 60_000, limit: 200
|
||||
end
|
||||
|
||||
pipeline :public_api do
|
||||
plug :accepts, ["json"]
|
||||
plug RateLimiter, scale: 60_000, limit: 100
|
||||
end
|
||||
|
||||
pipeline :api do
|
||||
plug :accepts, ["json"]
|
||||
plug RateLimiter, scale: 60_000, limit: 100
|
||||
plug AprsmeWeb.Plugs.ApiCSRF
|
||||
end
|
||||
|
||||
scope "/", AprsmeWeb do
|
||||
pipe_through :api
|
||||
pipe_through :public_api
|
||||
get "/health", PageController, :health
|
||||
get "/ready", PageController, :ready
|
||||
get "/status.json", PageController, :status_json
|
||||
|
|
@ -30,9 +40,6 @@ defmodule AprsmeWeb.Router do
|
|||
scope "/", AprsmeWeb do
|
||||
pipe_through :browser
|
||||
|
||||
live_dashboard "/dashboard", metrics: AprsmeWeb.Telemetry
|
||||
error_tracker_dashboard("/errors")
|
||||
|
||||
live_session :regular_pages, on_mount: [{AprsmeWeb.LocaleHook, :set_locale}] do
|
||||
live "/status", StatusLive.Index, :index
|
||||
live "/packets", PacketsLive.Index, :index
|
||||
|
|
@ -89,6 +96,9 @@ defmodule AprsmeWeb.Router do
|
|||
scope "/", AprsmeWeb do
|
||||
pipe_through [:browser, :require_authenticated_user]
|
||||
|
||||
live_dashboard "/dashboard", metrics: AprsmeWeb.Telemetry
|
||||
error_tracker_dashboard("/errors")
|
||||
|
||||
live_session :require_authenticated_user,
|
||||
on_mount: [{AprsmeWeb.UserAuth, :ensure_authenticated}, {AprsmeWeb.LocaleHook, :set_locale}] do
|
||||
live "/users/settings", UserSettingsLive, :edit
|
||||
|
|
|
|||
4
mix.exs
4
mix.exs
|
|
@ -94,7 +94,9 @@ defmodule Aprsme.MixProject do
|
|||
{:igniter, "~> 0.6", only: [:dev, :test]},
|
||||
{:mox, "~> 1.2", only: :test},
|
||||
{:styler, "~> 1.4.2", only: [:dev, :test], runtime: false},
|
||||
{:httpoison, "~> 1.8"}
|
||||
{:httpoison, "~> 1.8"},
|
||||
{:hammer, "~> 6.0"},
|
||||
{:cachex, "~> 3.4"}
|
||||
]
|
||||
end
|
||||
|
||||
|
|
|
|||
7
mix.lock
7
mix.lock
|
|
@ -2,6 +2,7 @@
|
|||
"bandit": {:hex, :bandit, "1.7.0", "d1564f30553c97d3e25f9623144bb8df11f3787a26733f00b21699a128105c0c", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "3e2f7a98c7a11f48d9d8c037f7177cd39778e74d55c7af06fe6227c742a8168a"},
|
||||
"bcrypt_elixir": {:hex, :bcrypt_elixir, "3.3.2", "d50091e3c9492d73e17fc1e1619a9b09d6a5ef99160eb4d736926fd475a16ca3", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "471be5151874ae7931911057d1467d908955f93554f7a6cd1b7d804cac8cef53"},
|
||||
"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
|
||||
"cachex": {:hex, :cachex, "3.6.0", "14a1bfbeee060dd9bec25a5b6f4e4691e3670ebda28c8ba2884b12fe30b36bf8", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "ebf24e373883bc8e0c8d894a63bbe102ae13d918f790121f5cfe6e485cc8e2e2"},
|
||||
"certifi": {:hex, :certifi, "2.15.0", "0e6e882fcdaaa0a5a9f2b3db55b1394dba07e8d6d9bcad08318fb604c6839712", [:rebar3], [], "hexpm", "b147ed22ce71d72eafdad94f055165c1c182f61a2ff49df28bcc71d1d5b94a60"},
|
||||
"comeonin": {:hex, :comeonin, "5.5.1", "5113e5f3800799787de08a6e0db307133850e635d34e9fab23c70b6501669510", [:mix], [], "hexpm", "65aac8f19938145377cee73973f192c5645873dcf550a8a6b18187d17c13ccdb"},
|
||||
"credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"},
|
||||
|
|
@ -15,6 +16,7 @@
|
|||
"erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"},
|
||||
"error_tracker": {:hex, :error_tracker, "0.6.0", "ac37d943b0720faccc26bc3adf39b2b74968b1ee0ae681c2fb57c98c5a10178d", [:mix], [{:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, ">= 0.0.0", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:myxql, ">= 0.0.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:phoenix_ecto, "~> 4.6", [hex: :phoenix_ecto, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8b99db5abeb883e7d622daf850c4dda38ce3bfabc4455431212d698df3156c63"},
|
||||
"esbuild": {:hex, :esbuild, "0.10.0", "b0aa3388a1c23e727c5a3e7427c932d89ee791746b0081bbe56103e9ef3d291f", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "468489cda427b974a7cc9f03ace55368a83e1a7be12fba7e30969af78e5f8c70"},
|
||||
"eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"},
|
||||
"exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm", "32e95820a97cffea67830e91514a2ad53b888850442d6d395f53a1ac60c82e07"},
|
||||
"expo": {:hex, :expo, "1.1.0", "f7b9ed7fb5745ebe1eeedf3d6f29226c5dd52897ac67c0f8af62a07e661e5c75", [:mix], [], "hexpm", "fbadf93f4700fb44c331362177bdca9eeb8097e8b0ef525c9cc501cb9917c960"},
|
||||
"exvcr": {:hex, :exvcr, "0.17.1", "3bae83d698a464a48212ad87c8ea4bcfb6bd76d53b937129472764e557616228", [:mix], [{:exjsx, "~> 4.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:finch, "~> 0.16", [hex: :finch, repo: "hexpm", optional: true]}, {:httpoison, "~> 1.0 or ~> 2.0", [hex: :httpoison, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:meck, "~> 1.0", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "a39d86980da183011366a878972e0ed43a5441814b701edd2e11360564e7bcab"},
|
||||
|
|
@ -30,6 +32,7 @@
|
|||
"glob_ex": {:hex, :glob_ex, "0.1.11", "cb50d3f1ef53f6ca04d6252c7fde09fd7a1cf63387714fe96f340a1349e62c93", [:mix], [], "hexpm", "342729363056e3145e61766b416769984c329e4378f1d558b63e341020525de4"},
|
||||
"gridsquare": {:hex, :gridsquare, "0.2.0", "c556a5b5101db89743264b0d728601023035863487bbe36e9e50e93839adb4d7", [:mix], [], "hexpm", "dbf0dbb484681a1e97819b0667bfeaa8c8a48cf06a54bf96a8f26ee4158ad31a"},
|
||||
"hackney": {:hex, :hackney, "1.24.1", "f5205a125bba6ed4587f9db3cc7c729d11316fa8f215d3e57ed1c067a9703fa9", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "f4a7392a0b53d8bbc3eb855bdcc919cd677358e65b2afd3840b5b3690c4c8a39"},
|
||||
"hammer": {:hex, :hammer, "6.2.1", "5ae9c33e3dceaeb42de0db46bf505bd9c35f259c8defb03390cd7556fea67ee2", [:mix], [{:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "b9476d0c13883d2dc0cc72e786bac6ac28911fba7cc2e04b70ce6a6d9c4b2bdc"},
|
||||
"heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "88ab3a0d790e6a47404cba02800a6b25d2afae50", [tag: "v2.1.1", sparse: "optimized", depth: 1]},
|
||||
"hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"},
|
||||
"httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"},
|
||||
|
|
@ -37,6 +40,7 @@
|
|||
"igniter": {:hex, :igniter, "0.6.11", "bd7269949df4e668fba4219aad32fc922e48f5197e2e27e106cc44d972e50649", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:owl, "~> 0.11", [hex: :owl, repo: "hexpm", optional: false]}, {:phx_new, "~> 1.7", [hex: :phx_new, repo: "hexpm", optional: true]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:rewrite, ">= 1.1.1 and < 2.0.0-0", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "8dc87860a66597dd93abfaf640a3ce747bfcdf53d7dcbb2ec51f5edfb741307d"},
|
||||
"jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
|
||||
"jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm", "fc3499fed7a726995aa659143a248534adc754ebd16ccd437cd93b649a95091f"},
|
||||
"jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"},
|
||||
"libcluster": {:hex, :libcluster, "3.5.0", "5ee4cfde4bdf32b2fef271e33ce3241e89509f4344f6c6a8d4069937484866ba", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebf6561fcedd765a4cd43b4b8c04b1c87f4177b5fb3cbdfe40a780499d72f743"},
|
||||
"meck": {:hex, :meck, "1.0.0", "24676cb6ee6951530093a93edcd410cfe4cb59fe89444b875d35c9d3909a15d0", [:rebar3], [], "hexpm", "680a9bcfe52764350beb9fb0335fb75fee8e7329821416cee0a19fec35433882"},
|
||||
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
|
||||
|
|
@ -61,10 +65,12 @@
|
|||
"phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"},
|
||||
"plug": {:hex, :plug, "1.18.1", "5067f26f7745b7e31bc3368bc1a2b818b9779faa959b49c934c17730efc911cf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "57a57db70df2b422b564437d2d33cf8d33cd16339c1edb190cd11b1a3a546cc2"},
|
||||
"plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"},
|
||||
"poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},
|
||||
"postgrex": {:hex, :postgrex, "0.20.0", "363ed03ab4757f6bc47942eff7720640795eb557e1935951c1626f0d303a3aed", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "d36ef8b36f323d29505314f704e21a1a038e2dc387c6409ee0cd24144e187c0f"},
|
||||
"req": {:hex, :req, "0.5.14", "521b449fa0bf275e6d034c05f29bec21789a0d6cd6f7a1c326c7bee642bf6e07", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "b7b15692071d556c73432c7797aa7e96b51d1a2db76f746b976edef95c930021"},
|
||||
"rewrite": {:hex, :rewrite, "1.1.2", "f5a5d10f5fed1491a6ff48e078d4585882695962ccc9e6c779bae025d1f92eda", [:mix], [{:glob_ex, "~> 0.1", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.0", [hex: :sourceror, repo: "hexpm", optional: false]}, {:text_diff, "~> 0.1", [hex: :text_diff, repo: "hexpm", optional: false]}], "hexpm", "7f8b94b1e3528d0a47b3e8b7bfeca559d2948a65fa7418a9ad7d7712703d39d4"},
|
||||
"sentry": {:hex, :sentry, "11.0.1", "e4e0ae12a74c59639808442a79d7ddd224fd5987fb9a14b35a1d01d9606117a3", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_ownership, "~> 0.3.0 or ~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.5", [hex: :opentelemetry, repo: "hexpm", optional: true]}, {:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: true]}, {:opentelemetry_exporter, "~> 1.0", [hex: :opentelemetry_exporter, repo: "hexpm", optional: true]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.20 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "ef12b6ee61c24bee3807eee92563567c89f5eec954d6bb913eb797e04625a613"},
|
||||
"sleeplocks": {:hex, :sleeplocks, "1.1.3", "96a86460cc33b435c7310dbd27ec82ca2c1f24ae38e34f8edde97f756503441a", [:rebar3], [], "hexpm", "d3b3958552e6eb16f463921e70ae7c767519ef8f5be46d7696cc1ed649421321"},
|
||||
"sobelow": {:hex, :sobelow, "0.14.0", "dd82aae8f72503f924fe9dd97ffe4ca694d2f17ec463dcfd365987c9752af6ee", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "7ecf91e298acfd9b24f5d761f19e8f6e6ac585b9387fb6301023f1f2cd5eed5f"},
|
||||
"sourceror": {:hex, :sourceror, "1.10.0", "38397dedbbc286966ec48c7af13e228b171332be1ad731974438c77791945ce9", [:mix], [], "hexpm", "29dbdfc92e04569c9d8e6efdc422fc1d815f4bd0055dc7c51b8800fb75c4b3f1"},
|
||||
"spitfire": {:hex, :spitfire, "0.2.1", "29e154873f05444669c7453d3d931820822cbca5170e88f0f8faa1de74a79b47", [:mix], [], "hexpm", "6eeed75054a38341b2e1814d41bb0a250564092358de2669fdb57ff88141d91b"},
|
||||
|
|
@ -80,6 +86,7 @@
|
|||
"text_diff": {:hex, :text_diff, "0.1.0", "1caf3175e11a53a9a139bc9339bd607c47b9e376b073d4571c031913317fecaa", [:mix], [], "hexpm", "d1ffaaecab338e49357b6daa82e435f877e0649041ace7755583a0ea3362dbd7"},
|
||||
"thousand_island": {:hex, :thousand_island, "1.3.14", "ad45ebed2577b5437582bcc79c5eccd1e2a8c326abf6a3464ab6c06e2055a34a", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d0d24a929d31cdd1d7903a4fe7f2409afeedff092d277be604966cd6aa4307ef"},
|
||||
"unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"},
|
||||
"unsafe": {:hex, :unsafe, "1.0.2", "23c6be12f6c1605364801f4b47007c0c159497d0446ad378b5cf05f1855c0581", [:mix], [], "hexpm", "b485231683c3ab01a9cd44cb4a79f152c6f3bb87358439c6f68791b85c2df675"},
|
||||
"websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},
|
||||
"websock_adapter": {:hex, :websock_adapter, "0.5.8", "3b97dc94e407e2d1fc666b2fb9acf6be81a1798a2602294aac000260a7c4a47d", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "315b9a1865552212b5f35140ad194e67ce31af45bcee443d4ecb96b5fd3f3782"},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
defmodule Aprsme.Repo.Migrations.AddAdditionalPerformanceIndexes do
|
||||
use Ecto.Migration
|
||||
@disable_ddl_transaction true
|
||||
@disable_migration_lock true
|
||||
|
||||
def up do
|
||||
# Add compound indexes for common query patterns
|
||||
create_if_not_exists index(:packets, [:received_at, :has_position, :data_type],
|
||||
concurrently: true
|
||||
)
|
||||
|
||||
create_if_not_exists index(:packets, [:sender, :data_type, :received_at], concurrently: true)
|
||||
|
||||
# Add index for weather symbol combinations (frequently used in queries)
|
||||
execute """
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS packets_weather_symbol_idx
|
||||
ON packets (received_at DESC, lat, lon)
|
||||
WHERE (symbol_table_id = '/' AND symbol_code = '_')
|
||||
OR (symbol_table_id = '\\' AND symbol_code = '_')
|
||||
"""
|
||||
|
||||
# Add partial index for recent packets with position (most common map query)
|
||||
# Note: Removing time-based predicate as it's not immutable
|
||||
execute """
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS packets_recent_positioned_idx
|
||||
ON packets (received_at DESC, lat, lon, sender)
|
||||
WHERE has_position = true
|
||||
"""
|
||||
|
||||
# Add index for callsign case-insensitive searches
|
||||
execute """
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS packets_sender_upper_idx
|
||||
ON packets (UPPER(sender), received_at DESC)
|
||||
"""
|
||||
|
||||
# Add composite index for region-based queries
|
||||
create_if_not_exists index(:packets, [:region, :data_type, :received_at], concurrently: true)
|
||||
|
||||
# Add index for device lookups (performance optimization)
|
||||
create_if_not_exists index(:packets, [:device_identifier, :received_at], concurrently: true)
|
||||
end
|
||||
|
||||
def down do
|
||||
execute "DROP INDEX IF EXISTS packets_device_identifier_received_at_idx"
|
||||
execute "DROP INDEX IF EXISTS packets_region_data_type_received_at_idx"
|
||||
execute "DROP INDEX IF EXISTS packets_sender_upper_idx"
|
||||
execute "DROP INDEX IF EXISTS packets_recent_positioned_idx"
|
||||
execute "DROP INDEX IF EXISTS packets_weather_symbol_idx"
|
||||
execute "DROP INDEX IF EXISTS packets_sender_data_type_received_at_idx"
|
||||
execute "DROP INDEX IF EXISTS packets_received_at_has_position_data_type_idx"
|
||||
end
|
||||
end
|
||||
|
|
@ -1,9 +1,6 @@
|
|||
defmodule Aprsme.Integration.HistoricalPacketsTest do
|
||||
use AprsmeWeb.ConnCase
|
||||
|
||||
import Phoenix.ConnTest
|
||||
import Phoenix.LiveViewTest
|
||||
|
||||
setup do
|
||||
Mox.set_mox_global()
|
||||
Application.put_env(:aprsme, :packets_module, PacketsMock)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue