From 13f447f1b0ceb949116789f686abdc251d07b414 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sat, 17 Jan 2026 16:36:49 -0600 Subject: [PATCH] feat: add Exq background job processor - Add exq ~> 0.23 dependency with redix and elixir_uuid - Configure Exq in application supervisor with 4 queues (default, discovery, polling, maintenance) - Set Exq as included_application to prevent auto-start - Only start Exq in dev/prod environments, skip in test - Use :env config to determine runtime environment - Configure 10 concurrent workers with exq namespace - Connect to Redis/Valkey sidecar via existing :redis config --- config/dev.exs | 3 ++ config/runtime.exs | 2 ++ lib/towerops/application.ex | 58 +++++++++++++++++++++++++++---------- mix.exs | 4 ++- mix.lock | 3 ++ 5 files changed, 54 insertions(+), 16 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index f75ac8d1..77f8d4ef 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -92,6 +92,9 @@ config :towerops, ToweropsWeb.Endpoint, ] ] +# Set environment identifier for runtime checks +config :towerops, :env, :dev + # Configure Redis/Valkey for development config :towerops, :redis, host: "localhost", diff --git a/config/runtime.exs b/config/runtime.exs index 00045bd7..1f32b9ac 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -148,6 +148,8 @@ if config_env() == :prod do secret_key_base: secret_key_base config :towerops, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY") + # Set environment identifier for runtime checks + config :towerops, :env, :prod # Set default sender for all emails config :towerops, :mailer_from, {"Towerops", "hi@towerops.net"} diff --git a/lib/towerops/application.ex b/lib/towerops/application.ex index 82848da4..36bbc4f2 100644 --- a/lib/towerops/application.ex +++ b/lib/towerops/application.ex @@ -20,21 +20,25 @@ defmodule Towerops.Application do topologies = Application.get_env(:libcluster, :topologies, []) - children = [ - {Cluster.Supervisor, [topologies, [name: Towerops.ClusterSupervisor]]}, - ToweropsWeb.Telemetry, - Towerops.Repo, - {DNSCluster, query: Application.get_env(:towerops, :dns_cluster_query) || :ignore}, - {Phoenix.PubSub, name: Towerops.PubSub}, - # Start event logger (subscribes to PubSub) - Towerops.Devices.EventLogger, - # Start monitoring supervisor - Towerops.Monitoring.Supervisor, - # Start a worker by calling: Towerops.Worker.start_link(arg) - # {Towerops.Worker, arg}, - # Start to serve requests, typically the last entry - ToweropsWeb.Endpoint - ] + children = + [ + {Cluster.Supervisor, [topologies, [name: Towerops.ClusterSupervisor]]}, + ToweropsWeb.Telemetry, + Towerops.Repo, + {DNSCluster, query: Application.get_env(:towerops, :dns_cluster_query) || :ignore}, + {Phoenix.PubSub, name: Towerops.PubSub}, + # Start event logger (subscribes to PubSub) + Towerops.Devices.EventLogger, + # Start monitoring supervisor + Towerops.Monitoring.Supervisor + # Start a worker by calling: Towerops.Worker.start_link(arg) + # {Towerops.Worker, arg}, + ] ++ + exq_workers() ++ + [ + # Start to serve requests, typically the last entry + ToweropsWeb.Endpoint + ] # See https://hexdocs.pm/elixir/Supervisor.html # for other strategies and supported options @@ -50,6 +54,30 @@ defmodule Towerops.Application do :ok end + # Configure Exq workers - don't start in test environment + defp exq_workers do + if Application.get_env(:towerops, :env) == :test do + [] + else + [ + {Exq, exq_config()} + ] + end + end + + defp exq_config do + redis_config = Application.get_env(:towerops, :redis, []) + + [ + name: Exq, + host: Keyword.get(redis_config, :host, "localhost"), + port: Keyword.get(redis_config, :port, 6379), + namespace: "exq", + concurrency: 10, + queues: ["default", "discovery", "polling", "maintenance"] + ] + end + @doc """ Returns the deployment timestamp. diff --git a/mix.exs b/mix.exs index a81c65ae..ad785ee4 100644 --- a/mix.exs +++ b/mix.exs @@ -22,7 +22,8 @@ defmodule Towerops.MixProject do def application do [ mod: {Towerops.Application, []}, - extra_applications: [:logger, :runtime_tools, :os_mon] + extra_applications: [:logger, :runtime_tools, :os_mon], + included_applications: [:exq] ] end @@ -70,6 +71,7 @@ defmodule Towerops.MixProject do {:libcluster, "~> 3.4"}, {:bandit, "~> 1.5"}, {:ecto_psql_extras, "~> 0.6"}, + {:exq, "~> 0.19"}, {:mox, "~> 1.0", only: :test}, {:stream_data, "~> 1.1", only: :test}, {:styler, "~> 1.10", only: [:dev, :test], runtime: false}, diff --git a/mix.lock b/mix.lock index 37d6436f..abc8aa05 100644 --- a/mix.lock +++ b/mix.lock @@ -14,9 +14,11 @@ "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.8.8", "aa02529c97f69aed5722899f5dc6360128735a92dd169f23c5d50b1f7fdede08", [:mix], [{:ecto_sql, "~> 3.7", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, "> 0.16.0", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1 or ~> 4.0", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "04c63d92b141723ad6fed2e60a4b461ca00b3594d16df47bbc48f1f4534f2c49"}, "ecto_sql": {:hex, :ecto_sql, "3.13.4", "b6e9d07557ddba62508a9ce4a484989a5bb5e9a048ae0e695f6d93f095c25d60", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2b38cf0749ca4d1c5a8bcbff79bbe15446861ca12a61f9fba604486cb6b62a14"}, "elixir_make": {:hex, :elixir_make, "0.9.0", "6484b3cd8c0cee58f09f05ecaf1a140a8c97670671a6a0e7ab4dc326c3109726", [:mix], [], "hexpm", "db23d4fd8b757462ad02f8aa73431a426fe6671c80b200d9710caf3d1dd0ffdb"}, + "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"}, "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, "esbuild": {:hex, :esbuild, "0.10.0", "b0aa3388a1c23e727c5a3e7427c932d89ee791746b0081bbe56103e9ef3d291f", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "468489cda427b974a7cc9f03ace55368a83e1a7be12fba7e30969af78e5f8c70"}, "expo": {:hex, :expo, "1.1.1", "4202e1d2ca6e2b3b63e02f69cfe0a404f77702b041d02b58597c00992b601db5", [:mix], [], "hexpm", "5fb308b9cb359ae200b7e23d37c76978673aa1b06e2b3075d814ce12c5811640"}, + "exq": {:hex, :exq, "0.23.0", "143079d49cec13673849704e9407521047f4016c72117d0067405115a65247da", [:mix], [{:elixir_uuid, ">= 1.2.0", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0 and < 7.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:redix, ">= 0.9.0", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "38bc3216051d8278aa32b147f1eca6a74dd84283b7785834c7a3609c00f5114a"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, "finch": {:hex, :finch, "0.20.0", "5330aefb6b010f424dcbbc4615d914e9e3deae40095e73ab0c1bb0968933cadf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2658131a74d051aabfcba936093c903b8e89da9a1b63e430bee62045fa9b2ee2"}, "fine": {:hex, :fine, "0.1.4", "b19a89c1476c7c57afb5f9314aed5960b5bc95d5277de4cb5ee8e1d1616ce379", [:mix], [], "hexpm", "be3324cc454a42d80951cf6023b9954e9ff27c6daa255483b3e8d608670303f5"}, @@ -49,6 +51,7 @@ "postgrex": {:hex, :postgrex, "0.22.0", "fb027b58b6eab1f6de5396a2abcdaaeb168f9ed4eccbb594e6ac393b02078cbd", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a68c4261e299597909e03e6f8ff5a13876f5caadaddd0d23af0d0a61afcc5d84"}, "protobuf": {:hex, :protobuf, "0.16.0", "d1878725105d49162977cf3408ccc3eac4f3532e26e5a9e250f2c624175d10f6", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "f0d0d3edd8768130f24cc2cfc41320637d32c80110e80d13f160fa699102c828"}, "ranch": {:hex, :ranch, "2.2.0", "25528f82bc8d7c6152c57666ca99ec716510fe0925cb188172f41ce93117b1b0", [:make, :rebar3], [], "hexpm", "fa0b99a1780c80218a4197a59ea8d3bdae32fbff7e88527d7d8a4787eff4f8e7"}, + "redix": {:hex, :redix, "1.5.3", "4eaae29c75e3285c0ff9957046b7c209aa7f72a023a17f0a9ea51c2a50ab5b0f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7b06fb5246373af41f5826b03334dfa3f636347d4d5d98b4d455b699d425ae7e"}, "req": {:hex, :req, "0.5.17", "0096ddd5b0ed6f576a03dde4b158a0c727215b15d2795e59e0916c6971066ede", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0b8bc6ffdfebbc07968e59d3ff96d52f2202d0536f10fef4dc11dc02a2a43e39"}, "snmpkit": {:hex, :snmpkit, "1.3.19", "b09c38cea619a0a67d4fb0f3bfa3b9b749d42c400c2e7bd9446fef82f0d728a7", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}, {:yaml_elixir, "~> 2.9", [hex: :yaml_elixir, repo: "hexpm", optional: true]}], "hexpm", "b05c1f3911204c4d781234187a6f1350c369fcffe2058a85b49735b1259eb973"}, "sobelow": {:hex, :sobelow, "0.14.1", "2f81e8632f15574cba2402bcddff5497b413c01e6f094bc0ab94e83c2f74db81", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8fac9a2bd90fdc4b15d6fca6e1608efb7f7c600fa75800813b794ee9364c87f2"},