diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 1a4cdef8..9da1d270 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,17 @@ +2026-04-04 +perf: database performance and maintenance migrations + - Added GiST index on geoip_blocks using int8range for IP range containment queries + (geoip.ex now uses @> operator; expected to reduce 530ms seq scan to fast index scan) + - Enabled btree_gist extension required for GiST index on int8range expression + - Dropped unused checks indexes: checks_source_type_index, checks_enabled_index, + checks_device_type_source_unique_index (all 0 scans since 2026-03-08) + - Tuned oban_jobs per-table autovacuum: scale_factor=0.01, threshold=100, + analyze_scale_factor=0.005, cost_delay=0 (table has 1.6B inserts/deletes) + Files: priv/repo/migrations/20260404000001_add_geoip_gist_index.exs, + priv/repo/migrations/20260404000002_drop_unused_check_indexes.exs, + priv/repo/migrations/20260404000003_tune_oban_jobs_autovacuum.exs, + lib/towerops/geoip.ex + 2026-03-27 test: simplify validator tests to match actual validation - Removed tests expecting validation not present in current implementation diff --git a/lib/towerops/geoip.ex b/lib/towerops/geoip.ex index b8871313..3986b233 100644 --- a/lib/towerops/geoip.ex +++ b/lib/towerops/geoip.ex @@ -64,13 +64,14 @@ defmodule Towerops.GeoIP do end defp find_location(ip_int) do - # Query for IP block that contains this IP address - # Uses index on (start_ip_int, end_ip_int) for fast range lookup + # Query for IP block that contains this IP address. + # Uses GiST index on int8range(start_ip_int, end_ip_int, '[]') via the @> + # (contains) operator. The double-inequality form does not use the GiST index. query = from b in Block, join: l in Location, on: b.geoname_id == l.geoname_id, - where: b.start_ip_int <= ^ip_int and b.end_ip_int >= ^ip_int, + where: fragment("int8range(?, ?, '[]') @> ?::bigint", b.start_ip_int, b.end_ip_int, ^ip_int), select: %{ country_code: l.country_code, country_name: l.country_name, diff --git a/mix.lock b/mix.lock index 7f4d48e9..5e426da7 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,5 @@ %{ - "absinthe": {:hex, :absinthe, "1.9.1", "19fe8614d5cdabefaf127ee224cb89eceea48314de4d709737451b43b5bdedd5", [:mix], [{:dataloader, "~> 1.0.0 or ~> 2.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2.1 or ~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d93e1aa61d68b974f48d5660104cb911ae045ee3a5d69954d251f91f3dbe2077"}, + "absinthe": {:hex, :absinthe, "1.10.0", "58e4923c2a96bf12cba9aad9298f36f624533039f2e5305badf0d60904eee0a0", [:mix], [{:dataloader, "~> 1.0.0 or ~> 2.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2.1 or ~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6810d21f13a2b4783d1b7f468c63a5174d769a3035f188e3fc93f440128a46af"}, "absinthe_phoenix": {:hex, :absinthe_phoenix, "2.0.4", "f36999412fbd6a2339abb5b7e24a4cc9492bbc7909d5806deeef83b06f55c508", [:mix], [{:absinthe, "~> 1.5", [hex: :absinthe, repo: "hexpm", optional: false]}, {:absinthe_plug, "~> 1.5", [hex: :absinthe_plug, repo: "hexpm", optional: false]}, {:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.5", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.13 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}], "hexpm", "66617ee63b725256ca16264364148b10b19e2ecb177488cd6353584f2e6c1cf3"}, "absinthe_plug": {:hex, :absinthe_plug, "1.5.9", "4f66fd46aecf969b349dd94853e6132db6d832ae6a4b951312b6926ad4ee7ca3", [:mix], [{:absinthe, "~> 1.7", [hex: :absinthe, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "dcdc84334b0e9e2cd439bd2653678a822623f212c71088edf0a4a7d03f1fa225"}, "argon2_elixir": {:hex, :argon2_elixir, "4.1.3", "4f28318286f89453364d7fbb53e03d4563fd7ed2438a60237eba5e426e97785f", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "7c295b8d8e0eaf6f43641698f962526cdf87c6feb7d14bd21e599271b510608c"}, @@ -43,13 +43,13 @@ "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "inet_cidr": {:hex, :inet_cidr, "1.0.9", "e0ef72a2942529da78c8e4147d53f2ef5f6f5293335c3637b0fdf83c012cc816", [:mix], [], "hexpm", "172da15ff7cf635b1feaf14f5818be28c811b37cc5fb7c5f7c01058c1c1066cc"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, - "lazy_html": {:hex, :lazy_html, "0.1.10", "ffe42a0b4e70859cf21a33e12a251e0c76c1dff76391609bd56702a0ef5bc429", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9.0", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "50f67e5faa09d45a99c1ddf3fac004f051997877dc8974c5797bb5ccd8e27058"}, + "lazy_html": {:hex, :lazy_html, "0.1.11", "136c8e9cd616b4f4e9c1562daa683880891120b759606dc4c3b6b18058ba5d79", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9.0", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "3b1be592929c31eca1a21673d25696e5c14cddfe922d9d1a3e3b48be4163883b"}, "libcluster": {:hex, :libcluster, "3.5.0", "5ee4cfde4bdf32b2fef271e33ce3241e89509f4344f6c6a8d4069937484866ba", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebf6561fcedd765a4cd43b4b8c04b1c87f4177b5fb3cbdfe40a780499d72f743"}, "logger_backends": {:hex, :logger_backends, "1.0.0", "09c4fad6202e08cb0fbd37f328282f16539aca380f512523ce9472b28edc6bdf", [:mix], [], "hexpm", "1faceb3e7ec3ef66a8f5746c5afd020e63996df6fd4eb8cdb789e5665ae6c9ce"}, "logger_file_backend": {:hex, :logger_file_backend, "0.0.14", "774bb661f1c3fed51b624d2859180c01e386eb1273dc22de4f4a155ef749a602", [:mix], [], "hexpm", "071354a18196468f3904ef09413af20971d55164267427f6257b52cfba03f9e6"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, - "mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"}, + "mimerl": {:hex, :mimerl, "1.5.0", "f35aca6f23242339b3666e0ac0702379e362b469d0aea167f6cc713547e777ed", [:rebar3], [], "hexpm", "db648ce065bae14ea84ca8b5dd123f42f49417cef693541110bf6f9e9be9ecc4"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "mix_audit": {:hex, :mix_audit, "2.1.5", "c0f77cee6b4ef9d97e37772359a187a166c7a1e0e08b50edf5bf6959dfe5a016", [:make, :mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.11", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "87f9298e21da32f697af535475860dc1d3617a010e0b418d2ec6142bc8b42d69"}, "mix_test_watch": {:hex, :mix_test_watch, "1.4.0", "d88bcc4fbe3198871266e9d2f00cd8ae350938efbb11d3fa1da091586345adbb", [:mix], [{:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "2b4693e17c8ead2ef56d4f48a0329891e8c2d0d73752c0f09272a2b17dc38d1b"}, @@ -82,7 +82,7 @@ "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, "styler": {:hex, :styler, "1.11.0", "35010d970689a23c2bcc8e97bd8bf7d20e3561d60c49be84654df5c37d051a9c", [:mix], [], "hexpm", "70f36165d0cf238a32b7a456fdef6a9c72e77e657d7ac4a0ace33aeba3f2b8c0"}, "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, - "swoosh": {:hex, :swoosh, "1.24.0", "4df9645aeeef925a2eb10f7a588a6a09ddd6d370c5dfbd3e821b699c574bdf57", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:idna, "~> 6.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6ddd84550800468d0e2c15a8aaff924a64c014ed6cff90318077efd1672b8b3b"}, + "swoosh": {:hex, :swoosh, "1.25.0", "d60dcba6d1ce538b1994f8712a3d55bc9519ffba4654cc4665a75683881d11dd", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:idna, "~> 6.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c59db3d838b595b95954a3d0a13782e56881cecfe7ba7b793b1a1a6775273a6e"}, "table_rex": {:hex, :table_rex, "4.1.0", "fbaa8b1ce154c9772012bf445bfb86b587430fb96f3b12022d3f35ee4a68c918", [:mix], [], "hexpm", "95932701df195d43bc2d1c6531178fc8338aa8f38c80f098504d529c43bc2601"}, "tailwind": {:hex, :tailwind, "0.4.1", "e7bcc222fe96a1e55f948e76d13dd84a1a7653fb051d2a167135db3b4b08d3e9", [:mix], [], "hexpm", "6249d4f9819052911120dbdbe9e532e6bd64ea23476056adb7f730aa25c220d1"}, "telemetry": {:hex, :telemetry, "1.4.1", "ab6de178e2b29b58e8256b92b382ea3f590a47152ca3651ea857a6cae05ac423", [:rebar3], [], "hexpm", "2172e05a27531d3d31dd9782841065c50dd5c3c7699d95266b2edd54c2dafa1c"}, diff --git a/priv/repo/migrations/20260404000001_add_geoip_gist_index.exs b/priv/repo/migrations/20260404000001_add_geoip_gist_index.exs new file mode 100644 index 00000000..6d147f78 --- /dev/null +++ b/priv/repo/migrations/20260404000001_add_geoip_gist_index.exs @@ -0,0 +1,27 @@ +defmodule Towerops.Repo.Migrations.AddGeoipGistIndex do + use Ecto.Migration + + @disable_ddl_transaction true + @disable_migration_lock true + + def up do + # btree_gist is required to create a GiST index on int8range expressions + execute "CREATE EXTENSION IF NOT EXISTS btree_gist" + + # GiST index for IP range containment queries using the @> operator. + # Turns the full sequential scan (530ms, 22K calls) into a fast index scan. + # The companion query change in geoip.ex uses: + # WHERE int8range(start_ip_int, end_ip_int, '[]') @> $1::bigint + execute """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS geoip_blocks_ip_range_gist + ON geoip_blocks USING gist (int8range(start_ip_int, end_ip_int, '[]')) + """ + + # Update planner statistics so the new index is used immediately + execute "ANALYZE geoip_blocks" + end + + def down do + execute "DROP INDEX CONCURRENTLY IF EXISTS geoip_blocks_ip_range_gist" + end +end diff --git a/priv/repo/migrations/20260404000002_drop_unused_check_indexes.exs b/priv/repo/migrations/20260404000002_drop_unused_check_indexes.exs new file mode 100644 index 00000000..f5eb9381 --- /dev/null +++ b/priv/repo/migrations/20260404000002_drop_unused_check_indexes.exs @@ -0,0 +1,43 @@ +defmodule Towerops.Repo.Migrations.DropUnusedCheckIndexes do + use Ecto.Migration + + @disable_ddl_transaction true + @disable_migration_lock true + + # These indexes have had 0 scans since stats reset (2026-03-08): + # + # - checks_source_type_index: superseded by the composite + # checks_source_type_source_id_index which covers all source_type queries + # + # - checks_enabled_index: queries that filter by enabled also filter by + # device_id or organization_id first, so the device_id/org index is used + # + # - checks_device_type_source_unique_index: unique partial index on + # (device_id, check_type, source_id) for auto_discovery rows; 0 scans + # since recreated in 20260325215632. NOTE: removes DB-level uniqueness + # enforcement for auto-discovery checks (Check changeset still validates). + + def up do + execute "DROP INDEX CONCURRENTLY IF EXISTS checks_source_type_index" + execute "DROP INDEX CONCURRENTLY IF EXISTS checks_enabled_index" + execute "DROP INDEX CONCURRENTLY IF EXISTS checks_device_type_source_unique_index" + end + + def down do + create_if_not_exists index(:checks, [:source_type], + name: :checks_source_type_index, + concurrently: true + ) + + create_if_not_exists index(:checks, [:enabled], + name: :checks_enabled_index, + concurrently: true + ) + + execute """ + CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS checks_device_type_source_unique_index + ON checks (device_id, check_type, source_id) + WHERE source_type = 'auto_discovery' AND source_id IS NOT NULL + """ + end +end diff --git a/priv/repo/migrations/20260404000003_tune_oban_jobs_autovacuum.exs b/priv/repo/migrations/20260404000003_tune_oban_jobs_autovacuum.exs new file mode 100644 index 00000000..09264dbb --- /dev/null +++ b/priv/repo/migrations/20260404000003_tune_oban_jobs_autovacuum.exs @@ -0,0 +1,30 @@ +defmodule Towerops.Repo.Migrations.TuneObanJobsAutovacuum do + use Ecto.Migration + + # oban_jobs has extreme churn (1.6B inserts, 5B updates, 1.6B deletes). + # Default autovacuum scale factors are too conservative for a table this busy, + # causing bloat and table growth. These settings trigger vacuum/analyze much + # more aggressively. + + def up do + execute """ + ALTER TABLE oban_jobs SET ( + autovacuum_vacuum_scale_factor = 0.01, + autovacuum_vacuum_threshold = 100, + autovacuum_analyze_scale_factor = 0.005, + autovacuum_vacuum_cost_delay = 0 + ) + """ + end + + def down do + execute """ + ALTER TABLE oban_jobs RESET ( + autovacuum_vacuum_scale_factor, + autovacuum_vacuum_threshold, + autovacuum_analyze_scale_factor, + autovacuum_vacuum_cost_delay + ) + """ + end +end diff --git a/priv/static/changelog.txt b/priv/static/changelog.txt index d487450a..7e8dd098 100644 --- a/priv/static/changelog.txt +++ b/priv/static/changelog.txt @@ -1,3 +1,7 @@ +2026-04-04 — Performance Improvements +* Significantly faster IP geolocation lookups +* Reduced database maintenance overhead for background job processing + 2026-03-27 — Stability Improvements * Fixed production deployment crash related to real-time messaging infrastructure * Fixed dashboard crashes when alerts change