This commit is contained in:
Graham McIntire 2025-07-27 11:16:16 -05:00
parent 9270bc1aa5
commit 1596e95eba
No known key found for this signature in database
12 changed files with 182 additions and 166 deletions

View file

@ -31,24 +31,11 @@ config :aprsme, AprsmeWeb.Gettext,
locales: ~w(en es de fr), locales: ~w(en es de fr),
default_locale: "en" default_locale: "en"
# Configure Exq for background jobs
# Redis connection settings are configured in runtime.exs
config :exq,
name: Exq,
concurrency: :infinite,
queues: [
{"default", 10},
{"maintenance", 2}
],
scheduler_enable: true,
scheduler_poll_timeout: 200,
poll_timeout: 50,
redis_timeout: 5000
# Configure periodic cleanup job # Configure periodic cleanup job
config :aprsme, :cleanup_scheduler, config :aprsme, :cleanup_scheduler,
enabled: true, enabled: true,
interval: 6 * 60 * 60 * 1000 # 6 hours in milliseconds # 6 hours in milliseconds
interval: 6 * 60 * 60 * 1000
config :aprsme, config :aprsme,
ecto_repos: [Aprsme.Repo] ecto_repos: [Aprsme.Repo]
@ -128,6 +115,20 @@ config :esbuild,
cd: Path.expand("../assets", __DIR__) cd: Path.expand("../assets", __DIR__)
] ]
# Configure Exq for background jobs
# Redis connection settings are configured in runtime.exs
config :exq,
name: Exq,
concurrency: :infinite,
queues: [
{"default", 10},
{"maintenance", 2}
],
scheduler_enable: true,
scheduler_poll_timeout: 200,
poll_timeout: 50,
redis_timeout: 5000
config :gettext, :plural_forms, GettextPseudolocalize.Plural config :gettext, :plural_forms, GettextPseudolocalize.Plural
# Configures Elixir's Logger # Configures Elixir's Logger

View file

@ -49,6 +49,55 @@ if config_env() == :prod do
# Configure clustering based on environment # Configure clustering based on environment
cluster_enabled = System.get_env("CLUSTER_ENABLED", "false") == "true" cluster_enabled = System.get_env("CLUSTER_ENABLED", "false") == "true"
# Configure Exq with Redis connection from environment
redis_url = System.get_env("REDIS_URL", "redis://localhost:6379")
redis_uri = URI.parse(redis_url)
redis_host = redis_uri.host || "localhost"
redis_port = redis_uri.port || 6379
redis_password =
case redis_uri.userinfo do
nil ->
""
userinfo ->
case String.split(userinfo, ":") do
[_, password] -> password
_ -> ""
end
end
# Extract database from path if present
redis_database =
try do
case redis_uri.path do
nil ->
0
"/" ->
0
path ->
path |> String.trim_leading("/") |> String.to_integer()
end
rescue
_ -> 0
end
exq_config = [
host: redis_host,
port: redis_port,
database: redis_database
]
# Only add password if it's not empty
exq_config =
if redis_password != "" and redis_password != nil do
Keyword.put(exq_config, :password, redis_password)
else
exq_config
end
# ## Configuring the mailer # ## Configuring the mailer
# #
# Configure Resend for email delivery # Configure Resend for email delivery
@ -119,48 +168,6 @@ if config_env() == :prod do
config :aprsme, config :aprsme,
default_from_email: "w5isp@aprs.me" default_from_email: "w5isp@aprs.me"
# Configure Exq with Redis connection from environment
redis_url = System.get_env("REDIS_URL", "redis://localhost:6379")
redis_uri = URI.parse(redis_url)
redis_host = redis_uri.host || "localhost"
redis_port = redis_uri.port || 6379
redis_password = case redis_uri.userinfo do
nil -> ""
userinfo ->
case String.split(userinfo, ":") do
[_, password] -> password
_ -> ""
end
end
# Extract database from path if present
redis_database = try do
case redis_uri.path do
nil -> 0
"/" -> 0
path ->
path |> String.trim_leading("/") |> String.to_integer()
end
rescue
_ -> 0
end
exq_config = [
host: redis_host,
port: redis_port,
database: redis_database
]
# Only add password if it's not empty
exq_config = if redis_password != "" and redis_password != nil do
Keyword.put(exq_config, :password, redis_password)
else
exq_config
end
config :exq, exq_config
config :aprsme, config :aprsme,
ecto_repos: [Aprsme.Repo], ecto_repos: [Aprsme.Repo],
aprs_is_server: System.get_env("APRS_SERVER", "dallas.aprs2.net"), aprs_is_server: System.get_env("APRS_SERVER", "dallas.aprs2.net"),
@ -171,6 +178,8 @@ if config_env() == :prod do
aprs_is_password: System.get_env("APRS_PASSCODE"), aprs_is_password: System.get_env("APRS_PASSCODE"),
env: :prod env: :prod
config :exq, exq_config
# Configure Hammer for production environment # Configure Hammer for production environment
config :hammer, config :hammer,
backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4, cleanup_interval_ms: 60_000 * 10]} backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4, cleanup_interval_ms: 60_000 * 10]}

View file

@ -27,9 +27,6 @@ config :aprsme, AprsmeWeb.Endpoint,
secret_key_base: "IV9+ENaw9i8xjReRk4sULRvRgsmFVTGQwQGGrf4G+Q/SFMeHBCNWRlPXQ2YvT36R", secret_key_base: "IV9+ENaw9i8xjReRk4sULRvRgsmFVTGQwQGGrf4G+Q/SFMeHBCNWRlPXQ2YvT36R",
server: false server: false
# Disable Exq during tests to prevent background job execution
config :exq, mode: :inline
# Disable cleanup scheduler in test environment # Disable cleanup scheduler in test environment
config :aprsme, :cleanup_scheduler, enabled: false config :aprsme, :cleanup_scheduler, enabled: false
@ -54,6 +51,9 @@ config :aprsme, auto_migrate: false
# Only in tests, remove the complexity from the password hashing algorithm # Only in tests, remove the complexity from the password hashing algorithm
config :bcrypt_elixir, :log_rounds, 1 config :bcrypt_elixir, :log_rounds, 1
# Disable Exq during tests to prevent background job execution
config :exq, mode: :inline
# Configure ExVCR # Configure ExVCR
config :exvcr, config :exvcr,
vcr_cassette_library_dir: "test/fixtures/vcr_cassettes", vcr_cassette_library_dir: "test/fixtures/vcr_cassettes",

View file

@ -99,6 +99,7 @@ defmodule Aprsme.Application do
do_migrate(true) do_migrate(true)
else else
require Logger require Logger
if cluster_enabled do if cluster_enabled do
Logger.info("Skipping auto-migration in cluster mode") Logger.info("Skipping auto-migration in cluster mode")
else else
@ -176,7 +177,6 @@ defmodule Aprsme.Application do
Logger.info("Database migrations completed") Logger.info("Database migrations completed")
end end
defp pubsub_config do defp pubsub_config do
cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false) cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false)
redis_url = System.get_env("REDIS_URL") redis_url = System.get_env("REDIS_URL")

View file

@ -7,6 +7,7 @@ defmodule Aprsme.CleanupScheduler do
""" """
use GenServer use GenServer
require Logger require Logger
def start_link(opts) do def start_link(opts) do
@ -17,7 +18,8 @@ defmodule Aprsme.CleanupScheduler do
def init(_opts) do def init(_opts) do
config = Application.get_env(:aprsme, :cleanup_scheduler, []) config = Application.get_env(:aprsme, :cleanup_scheduler, [])
enabled = Keyword.get(config, :enabled, true) enabled = Keyword.get(config, :enabled, true)
interval = Keyword.get(config, :interval, 6 * 60 * 60 * 1000) # 6 hours # 6 hours
interval = Keyword.get(config, :interval, 6 * 60 * 60 * 1000)
if enabled do if enabled do
Logger.info("Starting packet cleanup scheduler with #{interval}ms interval") Logger.info("Starting packet cleanup scheduler with #{interval}ms interval")
@ -37,6 +39,7 @@ defmodule Aprsme.CleanupScheduler do
case Exq.enqueue(Exq, "maintenance", Aprsme.Workers.PacketCleanupWorker, []) do case Exq.enqueue(Exq, "maintenance", Aprsme.Workers.PacketCleanupWorker, []) do
{:ok, _job_id} -> {:ok, _job_id} ->
Logger.info("Packet cleanup job scheduled successfully") Logger.info("Packet cleanup job scheduled successfully")
{:error, reason} -> {:error, reason} ->
Logger.error("Failed to schedule packet cleanup job: #{inspect(reason)}") Logger.error("Failed to schedule packet cleanup job: #{inspect(reason)}")
end end

View file

@ -24,13 +24,16 @@ defmodule Aprsme.Encoding do
""" """
@spec to_float_safe(binary()) :: {:ok, float()} | nil @spec to_float_safe(binary()) :: {:ok, float()} | nil
def to_float_safe(value) when is_binary(value) do def to_float_safe(value) when is_binary(value) do
sanitized = value sanitized =
|> String.trim() value
|> String.slice(0, 30) # Reasonable max length for a number |> String.trim()
# Reasonable max length for a number
|> String.slice(0, 30)
case Float.parse(sanitized) do case Float.parse(sanitized) do
{f, _} when f > -9.0e15 and f < 9.0e15 -> {f, _} when f > -9.0e15 and f < 9.0e15 ->
{:ok, f} {:ok, f}
_ -> _ ->
nil nil
end end
@ -56,48 +59,48 @@ defmodule Aprsme.Encoding do
@spec has_weather_data(any(), any(), any(), any()) :: boolean() @spec has_weather_data(any(), any(), any(), any()) :: boolean()
def has_weather_data(temperature, humidity, wind_speed, pressure) do def has_weather_data(temperature, humidity, wind_speed, pressure) do
not is_nil(temperature) or not is_nil(temperature) or
not is_nil(humidity) or not is_nil(humidity) or
not is_nil(wind_speed) or not is_nil(wind_speed) or
not is_nil(pressure) not is_nil(pressure)
end end
@doc """ @doc """
Get encoding information about a binary Get encoding information about a binary
""" """
@spec encoding_info(binary()) :: %{ @spec encoding_info(binary()) :: %{
valid_utf8: boolean(), valid_utf8: boolean(),
byte_count: non_neg_integer(), byte_count: non_neg_integer(),
char_count: non_neg_integer() | nil, char_count: non_neg_integer() | nil,
invalid_at: non_neg_integer() | nil invalid_at: non_neg_integer() | nil
} }
def encoding_info(input) when is_binary(input) do def encoding_info(input) when is_binary(input) do
byte_count = byte_size(input) byte_count = byte_size(input)
case String.valid?(input) do if String.valid?(input) do
true -> %{
%{ valid_utf8: true,
valid_utf8: true, byte_count: byte_count,
byte_count: byte_count, char_count: String.length(input),
char_count: String.length(input), invalid_at: nil
invalid_at: nil }
} else
false -> invalid_pos = find_invalid_byte_position(input)
invalid_pos = find_invalid_byte_position(input)
%{ %{
valid_utf8: false, valid_utf8: false,
byte_count: byte_count, byte_count: byte_count,
char_count: nil, char_count: nil,
invalid_at: invalid_pos invalid_at: invalid_pos
} }
end end
end end
@spec encoding_info(any()) :: %{ @spec encoding_info(any()) :: %{
valid_utf8: boolean(), valid_utf8: boolean(),
byte_count: non_neg_integer(), byte_count: non_neg_integer(),
char_count: nil, char_count: nil,
invalid_at: nil invalid_at: nil
} }
def encoding_info(_) do def encoding_info(_) do
%{ %{
valid_utf8: false, valid_utf8: false,
@ -121,14 +124,12 @@ defmodule Aprsme.Encoding do
@spec latin1_to_utf8(binary()) :: binary() @spec latin1_to_utf8(binary()) :: binary()
defp latin1_to_utf8(input) do defp latin1_to_utf8(input) do
try do input
input |> :binary.bin_to_list()
|> :binary.bin_to_list() |> Enum.map(&latin1_char_to_utf8/1)
|> Enum.map(&latin1_char_to_utf8/1) |> IO.iodata_to_binary()
|> IO.iodata_to_binary() rescue
rescue _ -> ""
_ -> ""
end
end end
@spec latin1_char_to_utf8(0..255) :: binary() @spec latin1_char_to_utf8(0..255) :: binary()
@ -166,6 +167,7 @@ defmodule Aprsme.Encoding do
c when c >= 128 and c <= 159 -> false c when c >= 128 and c <= 159 -> false
_ -> true _ -> true
end end
_ -> _ ->
# Multi-codepoint grapheme, keep it # Multi-codepoint grapheme, keep it
true true
@ -186,11 +188,9 @@ defmodule Aprsme.Encoding do
@spec valid_utf8_continuation?(binary(), non_neg_integer()) :: boolean() @spec valid_utf8_continuation?(binary(), non_neg_integer()) :: boolean()
defp valid_utf8_continuation?(binary, pos) do defp valid_utf8_continuation?(binary, pos) do
try do <<_::binary-size(pos), _char::utf8, _::binary>> = binary
<<_::binary-size(pos), _char::utf8, _::binary>> = binary true
true rescue
rescue _ -> false
_ -> false
end
end end
end end

View file

@ -49,14 +49,12 @@ defmodule Aprsme.EncodingUtils do
""" """
@spec to_float(any()) :: float() | nil @spec to_float(any()) :: float() | nil
def to_float(value) when is_float(value) do def to_float(value) when is_float(value) do
if finite_float?(value), do: value, else: nil if finite_float?(value), do: value
end end
def to_float(value) when is_integer(value) do def to_float(value) when is_integer(value) do
if value >= -9.0e15 and value <= 9.0e15 do if value >= -9.0e15 and value <= 9.0e15 do
value * 1.0 value * 1.0
else
nil
end end
end end
@ -72,7 +70,7 @@ defmodule Aprsme.EncodingUtils do
def to_float(%Decimal{} = value) do def to_float(%Decimal{} = value) do
float = Decimal.to_float(value) float = Decimal.to_float(value)
if finite_float?(float), do: float, else: nil if finite_float?(float), do: float
end end
def to_float(_), do: nil def to_float(_), do: nil

View file

@ -23,14 +23,16 @@ defmodule Aprsme.Release do
cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false) cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false)
# Configure longer timeouts for migration operations # Configure longer timeouts for migration operations
migration_timeout = String.to_integer(System.get_env("MIGRATION_TIMEOUT", "7200000")) # 2 hours default # 2 hours default
migration_timeout = String.to_integer(System.get_env("MIGRATION_TIMEOUT", "7200000"))
if cluster_enabled do if cluster_enabled do
Logger.info("Running migrations with distributed lock...") Logger.info("Running migrations with distributed lock...")
# Ensure repo is started for advisory lock with extended timeout # Ensure repo is started for advisory lock with extended timeout
repo_config = Aprsme.Repo.config() repo_config =
|> Keyword.put(:timeout, migration_timeout) Aprsme.Repo.config()
|> Keyword.put(:pool_timeout, 60_000) |> Keyword.put(:timeout, migration_timeout)
|> Keyword.put(:pool_timeout, 60_000)
{:ok, _} = Aprsme.Repo.start_link(repo_config) {:ok, _} = Aprsme.Repo.start_link(repo_config)
@ -95,15 +97,19 @@ defmodule Aprsme.Release do
Logger.info("Running migrations with timeout: #{timeout}ms") Logger.info("Running migrations with timeout: #{timeout}ms")
# Run with extended timeout configuration # Run with extended timeout configuration
_repo_config = Aprsme.Repo.config() _repo_config =
|> Keyword.put(:timeout, timeout) Aprsme.Repo.config()
|> Keyword.put(:pool_timeout, 60_000) |> Keyword.put(:timeout, timeout)
|> Keyword.put(:pool_timeout, 60_000)
{:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, fn repo -> {:ok, _, _} =
# Set session-level timeout for this connection Ecto.Migrator.with_repo(
Ecto.Adapters.SQL.query!(repo, "SET statement_timeout = '#{div(timeout, 1000)}s'") Aprsme.Repo,
Ecto.Migrator.run(repo, :up, all: true) fn repo ->
end, [timeout: timeout]) # Set session-level timeout for this connection
Ecto.Adapters.SQL.query!(repo, "SET statement_timeout = '#{div(timeout, 1000)}s'")
Ecto.Migrator.run(repo, :up, all: true)
end, timeout: timeout)
end end
defp read_deployment_timestamp do defp read_deployment_timestamp do

View file

@ -52,7 +52,6 @@ defmodule Aprsme.MixProject do
defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(:test), do: ["lib", "test/support"]
defp elixirc_paths(_), do: ["lib"] defp elixirc_paths(_), do: ["lib"]
# Specifies your project dependencies. # Specifies your project dependencies.
# #
# Type `mix help deps` for examples and options. # Type `mix help deps` for examples and options.
@ -107,7 +106,7 @@ defmodule Aprsme.MixProject do
{:hammer, "~> 7.0"}, {:hammer, "~> 7.0"},
{:cachex, "~> 4.1"}, {:cachex, "~> 4.1"},
{:gettext_pseudolocalize, "~> 0.1"}, {:gettext_pseudolocalize, "~> 0.1"},
{:sentry, "~> 11.0"}, {:sentry, "~> 11.0"}
# Gleam dependencies # Gleam dependencies
] ]
end end