graham/towerops
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

db-performance-migrations (#234)

Browse source 
Reviewed-on: graham/towerops-web#234
This commit is contained in:
Graham McIntire 2026-04-04 14:22:25 -05:00 • committed by graham
parent 70e07d6033
commit 978a72ad41
7 changed files with 126 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
CHANGELOG.txt
View file

@ -1,3 +1,17 @@
2026-04-04
perf: database performance and maintenance migrations
- Added GiST index on geoip_blocks using int8range for IP range containment queries
(geoip.ex now uses @> operator; expected to reduce 530ms seq scan to fast index scan)
- Enabled btree_gist extension required for GiST index on int8range expression
- Dropped unused checks indexes: checks_source_type_index, checks_enabled_index,
checks_device_type_source_unique_index (all 0 scans since 2026-03-08)
- Tuned oban_jobs per-table autovacuum: scale_factor=0.01, threshold=100,
analyze_scale_factor=0.005, cost_delay=0 (table has 1.6B inserts/deletes)
Files: priv/repo/migrations/20260404000001_add_geoip_gist_index.exs,
priv/repo/migrations/20260404000002_drop_unused_check_indexes.exs,
priv/repo/migrations/20260404000003_tune_oban_jobs_autovacuum.exs,
lib/towerops/geoip.ex
2026-03-27
test: simplify validator tests to match actual validation
- Removed tests expecting validation not present in current implementation

7
lib/towerops/geoip.ex
View file

@ -64,13 +64,14 @@ defmodule Towerops.GeoIP do
end
defp find_location(ip_int) do
# Query for IP block that contains this IP address
# Uses index on (start_ip_int, end_ip_int) for fast range lookup
# Query for IP block that contains this IP address.
# Uses GiST index on int8range(start_ip_int, end_ip_int, '[]') via the @>
# (contains) operator. The double-inequality form does not use the GiST index.
query =
from b in Block,
join: l in Location,
on: b.geoname_id == l.geoname_id,
where: b.start_ip_int <= ^ip_int and b.end_ip_int >= ^ip_int,
where: fragment("int8range(?, ?, '[]') @> ?::bigint", b.start_ip_int, b.end_ip_int, ^ip_int),
select: %{
country_code: l.country_code,
country_name: l.country_name,

8
mix.lock
View file

@ -1,5 +1,5 @@
%{
"absinthe": {:hex, :absinthe, "1.9.1", "19fe8614d5cdabefaf127ee224cb89eceea48314de4d709737451b43b5bdedd5", [:mix], [{:dataloader, "~> 1.0.0 or ~> 2.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2.1 or ~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d93e1aa61d68b974f48d5660104cb911ae045ee3a5d69954d251f91f3dbe2077"},
"absinthe": {:hex, :absinthe, "1.10.0", "58e4923c2a96bf12cba9aad9298f36f624533039f2e5305badf0d60904eee0a0", [:mix], [{:dataloader, "~> 1.0.0 or ~> 2.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2.1 or ~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6810d21f13a2b4783d1b7f468c63a5174d769a3035f188e3fc93f440128a46af"},
"absinthe_phoenix": {:hex, :absinthe_phoenix, "2.0.4", "f36999412fbd6a2339abb5b7e24a4cc9492bbc7909d5806deeef83b06f55c508", [:mix], [{:absinthe, "~> 1.5", [hex: :absinthe, repo: "hexpm", optional: false]}, {:absinthe_plug, "~> 1.5", [hex: :absinthe_plug, repo: "hexpm", optional: false]}, {:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.5", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.13 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}], "hexpm", "66617ee63b725256ca16264364148b10b19e2ecb177488cd6353584f2e6c1cf3"},
"absinthe_plug": {:hex, :absinthe_plug, "1.5.9", "4f66fd46aecf969b349dd94853e6132db6d832ae6a4b951312b6926ad4ee7ca3", [:mix], [{:absinthe, "~> 1.7", [hex: :absinthe, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "dcdc84334b0e9e2cd439bd2653678a822623f212c71088edf0a4a7d03f1fa225"},
"argon2_elixir": {:hex, :argon2_elixir, "4.1.3", "4f28318286f89453364d7fbb53e03d4563fd7ed2438a60237eba5e426e97785f", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "7c295b8d8e0eaf6f43641698f962526cdf87c6feb7d14bd21e599271b510608c"},
@ -43,13 +43,13 @@
"idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
"inet_cidr": {:hex, :inet_cidr, "1.0.9", "e0ef72a2942529da78c8e4147d53f2ef5f6f5293335c3637b0fdf83c012cc816", [:mix], [], "hexpm", "172da15ff7cf635b1feaf14f5818be28c811b37cc5fb7c5f7c01058c1c1066cc"},
"jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
"lazy_html": {:hex, :lazy_html, "0.1.10", "ffe42a0b4e70859cf21a33e12a251e0c76c1dff76391609bd56702a0ef5bc429", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9.0", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "50f67e5faa09d45a99c1ddf3fac004f051997877dc8974c5797bb5ccd8e27058"},
"lazy_html": {:hex, :lazy_html, "0.1.11", "136c8e9cd616b4f4e9c1562daa683880891120b759606dc4c3b6b18058ba5d79", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9.0", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "3b1be592929c31eca1a21673d25696e5c14cddfe922d9d1a3e3b48be4163883b"},
"libcluster": {:hex, :libcluster, "3.5.0", "5ee4cfde4bdf32b2fef271e33ce3241e89509f4344f6c6a8d4069937484866ba", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebf6561fcedd765a4cd43b4b8c04b1c87f4177b5fb3cbdfe40a780499d72f743"},
"logger_backends": {:hex, :logger_backends, "1.0.0", "09c4fad6202e08cb0fbd37f328282f16539aca380f512523ce9472b28edc6bdf", [:mix], [], "hexpm", "1faceb3e7ec3ef66a8f5746c5afd020e63996df6fd4eb8cdb789e5665ae6c9ce"},
"logger_file_backend": {:hex, :logger_file_backend, "0.0.14", "774bb661f1c3fed51b624d2859180c01e386eb1273dc22de4f4a155ef749a602", [:mix], [], "hexpm", "071354a18196468f3904ef09413af20971d55164267427f6257b52cfba03f9e6"},
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
"mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"},
"mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"},
"mimerl": {:hex, :mimerl, "1.5.0", "f35aca6f23242339b3666e0ac0702379e362b469d0aea167f6cc713547e777ed", [:rebar3], [], "hexpm", "db648ce065bae14ea84ca8b5dd123f42f49417cef693541110bf6f9e9be9ecc4"},
"mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"},
"mix_audit": {:hex, :mix_audit, "2.1.5", "c0f77cee6b4ef9d97e37772359a187a166c7a1e0e08b50edf5bf6959dfe5a016", [:make, :mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.11", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "87f9298e21da32f697af535475860dc1d3617a010e0b418d2ec6142bc8b42d69"},
"mix_test_watch": {:hex, :mix_test_watch, "1.4.0", "d88bcc4fbe3198871266e9d2f00cd8ae350938efbb11d3fa1da091586345adbb", [:mix], [{:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "2b4693e17c8ead2ef56d4f48a0329891e8c2d0d73752c0f09272a2b17dc38d1b"},
@ -82,7 +82,7 @@
"stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"},
"styler": {:hex, :styler, "1.11.0", "35010d970689a23c2bcc8e97bd8bf7d20e3561d60c49be84654df5c37d051a9c", [:mix], [], "hexpm", "70f36165d0cf238a32b7a456fdef6a9c72e77e657d7ac4a0ace33aeba3f2b8c0"},
"sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"},
"swoosh": {:hex, :swoosh, "1.24.0", "4df9645aeeef925a2eb10f7a588a6a09ddd6d370c5dfbd3e821b699c574bdf57", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:idna, "~> 6.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6ddd84550800468d0e2c15a8aaff924a64c014ed6cff90318077efd1672b8b3b"},
"swoosh": {:hex, :swoosh, "1.25.0", "d60dcba6d1ce538b1994f8712a3d55bc9519ffba4654cc4665a75683881d11dd", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:idna, "~> 6.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c59db3d838b595b95954a3d0a13782e56881cecfe7ba7b793b1a1a6775273a6e"},
"table_rex": {:hex, :table_rex, "4.1.0", "fbaa8b1ce154c9772012bf445bfb86b587430fb96f3b12022d3f35ee4a68c918", [:mix], [], "hexpm", "95932701df195d43bc2d1c6531178fc8338aa8f38c80f098504d529c43bc2601"},
"tailwind": {:hex, :tailwind, "0.4.1", "e7bcc222fe96a1e55f948e76d13dd84a1a7653fb051d2a167135db3b4b08d3e9", [:mix], [], "hexpm", "6249d4f9819052911120dbdbe9e532e6bd64ea23476056adb7f730aa25c220d1"},
"telemetry": {:hex, :telemetry, "1.4.1", "ab6de178e2b29b58e8256b92b382ea3f590a47152ca3651ea857a6cae05ac423", [:rebar3], [], "hexpm", "2172e05a27531d3d31dd9782841065c50dd5c3c7699d95266b2edd54c2dafa1c"},

27
priv/repo/migrations/20260404000001_add_geoip_gist_index.exs Normal file
View file

@ -0,0 +1,27 @@
defmodule Towerops.Repo.Migrations.AddGeoipGistIndex do
use Ecto.Migration
@disable_ddl_transaction true
@disable_migration_lock true
def up do
# btree_gist is required to create a GiST index on int8range expressions
execute "CREATE EXTENSION IF NOT EXISTS btree_gist"
# GiST index for IP range containment queries using the @> operator.
# Turns the full sequential scan (530ms, 22K calls) into a fast index scan.
# The companion query change in geoip.ex uses:
# WHERE int8range(start_ip_int, end_ip_int, '[]') @> $1::bigint
execute """
CREATE INDEX CONCURRENTLY IF NOT EXISTS geoip_blocks_ip_range_gist
ON geoip_blocks USING gist (int8range(start_ip_int, end_ip_int, '[]'))
"""
# Update planner statistics so the new index is used immediately
execute "ANALYZE geoip_blocks"
end
def down do
execute "DROP INDEX CONCURRENTLY IF EXISTS geoip_blocks_ip_range_gist"
end
end

43
priv/repo/migrations/20260404000002_drop_unused_check_indexes.exs Normal file
View file

@ -0,0 +1,43 @@
defmodule Towerops.Repo.Migrations.DropUnusedCheckIndexes do
use Ecto.Migration
@disable_ddl_transaction true
@disable_migration_lock true
# These indexes have had 0 scans since stats reset (2026-03-08):
#
# - checks_source_type_index: superseded by the composite
# checks_source_type_source_id_index which covers all source_type queries
#
# - checks_enabled_index: queries that filter by enabled also filter by
# device_id or organization_id first, so the device_id/org index is used
#
# - checks_device_type_source_unique_index: unique partial index on
# (device_id, check_type, source_id) for auto_discovery rows; 0 scans
# since recreated in 20260325215632. NOTE: removes DB-level uniqueness
# enforcement for auto-discovery checks (Check changeset still validates).
def up do
execute "DROP INDEX CONCURRENTLY IF EXISTS checks_source_type_index"
execute "DROP INDEX CONCURRENTLY IF EXISTS checks_enabled_index"
execute "DROP INDEX CONCURRENTLY IF EXISTS checks_device_type_source_unique_index"
end
def down do
create_if_not_exists index(:checks, [:source_type],
name: :checks_source_type_index,
concurrently: true
)
create_if_not_exists index(:checks, [:enabled],
name: :checks_enabled_index,
concurrently: true
)
execute """
CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS checks_device_type_source_unique_index
ON checks (device_id, check_type, source_id)
WHERE source_type = 'auto_discovery' AND source_id IS NOT NULL
"""
end
end

30
priv/repo/migrations/20260404000003_tune_oban_jobs_autovacuum.exs Normal file
View file

@ -0,0 +1,30 @@
defmodule Towerops.Repo.Migrations.TuneObanJobsAutovacuum do
use Ecto.Migration
# oban_jobs has extreme churn (1.6B inserts, 5B updates, 1.6B deletes).
# Default autovacuum scale factors are too conservative for a table this busy,
# causing bloat and table growth. These settings trigger vacuum/analyze much
# more aggressively.
def up do
execute """
ALTER TABLE oban_jobs SET (
autovacuum_vacuum_scale_factor = 0.01,
autovacuum_vacuum_threshold = 100,
autovacuum_analyze_scale_factor = 0.005,
autovacuum_vacuum_cost_delay = 0
)
"""
end
def down do
execute """
ALTER TABLE oban_jobs RESET (
autovacuum_vacuum_scale_factor,
autovacuum_vacuum_threshold,
autovacuum_analyze_scale_factor,
autovacuum_vacuum_cost_delay
)
"""
end
end

4
priv/static/changelog.txt
View file

@ -1,3 +1,7 @@
2026-04-04 — Performance Improvements
* Significantly faster IP geolocation lookups
* Reduced database maintenance overhead for background job processing
2026-03-27 — Stability Improvements
* Fixed production deployment crash related to real-time messaging infrastructure
* Fixed dashboard crashes when alerts change