Graham McIntire
b08eba2c6b
Convert the packets table to a time-partitioned table with daily partitions (PARTITION BY RANGE on received_at). This replaces the expensive CTE-based batch DELETE cleanup with instant DROP TABLE partition drops — no dead tuples, no WAL pressure, no VACUUM needed. - Add PartitionManager GenServer to create/drop daily partitions - Simplify PacketCleanupWorker to delegate to PartitionManager - Replace gridsquare/geocalc deps with local Maidenhead module - Remove gettext_pseudolocalize dependency - Fix Decimal comparison in packet consumer test
366 lines
12 KiB
Elixir
366 lines
12 KiB
Elixir
defmodule Aprsme.Repo.Migrations.PartitionPacketsTable do
|
|
use Ecto.Migration
|
|
|
|
def up do
|
|
# Disable statement timeout for DDL on large table with many indexes
|
|
execute("SET LOCAL statement_timeout = '0'")
|
|
|
|
# Drop existing non-partitioned table (data is ephemeral, refills from APRS-IS stream)
|
|
execute("DROP TABLE IF EXISTS packets CASCADE")
|
|
|
|
# Create partitioned table with composite PK required for partitioning
|
|
execute("""
|
|
CREATE TABLE packets (
|
|
id uuid NOT NULL,
|
|
base_callsign varchar,
|
|
data_type varchar,
|
|
destination varchar,
|
|
information_field text,
|
|
path text,
|
|
sender varchar,
|
|
ssid varchar,
|
|
data_extended jsonb,
|
|
inserted_at timestamp NOT NULL,
|
|
updated_at timestamp NOT NULL,
|
|
received_at timestamp NOT NULL,
|
|
region varchar,
|
|
lat numeric,
|
|
lon numeric,
|
|
has_position boolean DEFAULT false,
|
|
item_name text,
|
|
object_name text,
|
|
is_item boolean DEFAULT false,
|
|
is_object boolean DEFAULT false,
|
|
dao jsonb,
|
|
raw_packet text,
|
|
symbol_code varchar,
|
|
symbol_table_id varchar,
|
|
comment text,
|
|
"timestamp" varchar,
|
|
aprs_messaging boolean DEFAULT false,
|
|
temperature double precision,
|
|
humidity double precision,
|
|
wind_speed double precision,
|
|
wind_direction integer,
|
|
wind_gust double precision,
|
|
pressure double precision,
|
|
rain_1h double precision,
|
|
rain_24h double precision,
|
|
rain_since_midnight double precision,
|
|
manufacturer text,
|
|
equipment_type text,
|
|
course integer,
|
|
speed double precision,
|
|
altitude double precision,
|
|
addressee text,
|
|
message_text text,
|
|
message_number varchar,
|
|
location geometry(Point,4326),
|
|
device_identifier text,
|
|
luminosity integer,
|
|
snow double precision,
|
|
phg_power integer,
|
|
phg_height integer,
|
|
phg_gain integer,
|
|
phg_directivity integer,
|
|
has_weather boolean DEFAULT false,
|
|
rain_midnight double precision,
|
|
position_ambiguity integer,
|
|
posresolution double precision,
|
|
format text,
|
|
telemetry_seq integer,
|
|
telemetry_vals integer[],
|
|
telemetry_bits text,
|
|
radiorange text,
|
|
srccallsign varchar,
|
|
dstcallsign varchar,
|
|
body text,
|
|
origpacket text,
|
|
header text,
|
|
alive integer DEFAULT 1,
|
|
posambiguity integer,
|
|
symboltable varchar,
|
|
symbolcode varchar,
|
|
messaging integer,
|
|
PRIMARY KEY (id, received_at)
|
|
) PARTITION BY RANGE (received_at)
|
|
""")
|
|
|
|
# Default partition catches rows outside named partitions
|
|
execute("CREATE TABLE packets_default PARTITION OF packets DEFAULT")
|
|
|
|
# Create daily partitions for recent past + near future
|
|
today = Date.utc_today()
|
|
|
|
for offset <- -8..2 do
|
|
date = Date.add(today, offset)
|
|
name = "packets_#{Calendar.strftime(date, "%Y%m%d")}"
|
|
next_date = Date.add(date, 1)
|
|
|
|
execute(
|
|
"CREATE TABLE IF NOT EXISTS #{name} PARTITION OF packets FOR VALUES FROM ('#{date} 00:00:00') TO ('#{next_date} 00:00:00')"
|
|
)
|
|
end
|
|
|
|
create_indexes()
|
|
create_triggers("packets")
|
|
end
|
|
|
|
def down do
|
|
execute("DROP TABLE IF EXISTS packets CASCADE")
|
|
|
|
# Recreate as regular non-partitioned table
|
|
execute("""
|
|
CREATE TABLE packets (
|
|
id uuid NOT NULL PRIMARY KEY,
|
|
base_callsign varchar,
|
|
data_type varchar,
|
|
destination varchar,
|
|
information_field text,
|
|
path text,
|
|
sender varchar,
|
|
ssid varchar,
|
|
data_extended jsonb,
|
|
inserted_at timestamp NOT NULL,
|
|
updated_at timestamp NOT NULL,
|
|
received_at timestamp NOT NULL,
|
|
region varchar,
|
|
lat numeric,
|
|
lon numeric,
|
|
has_position boolean DEFAULT false,
|
|
item_name text,
|
|
object_name text,
|
|
is_item boolean DEFAULT false,
|
|
is_object boolean DEFAULT false,
|
|
dao jsonb,
|
|
raw_packet text,
|
|
symbol_code varchar,
|
|
symbol_table_id varchar,
|
|
comment text,
|
|
"timestamp" varchar,
|
|
aprs_messaging boolean DEFAULT false,
|
|
temperature double precision,
|
|
humidity double precision,
|
|
wind_speed double precision,
|
|
wind_direction integer,
|
|
wind_gust double precision,
|
|
pressure double precision,
|
|
rain_1h double precision,
|
|
rain_24h double precision,
|
|
rain_since_midnight double precision,
|
|
manufacturer text,
|
|
equipment_type text,
|
|
course integer,
|
|
speed double precision,
|
|
altitude double precision,
|
|
addressee text,
|
|
message_text text,
|
|
message_number varchar,
|
|
location geometry(Point,4326),
|
|
device_identifier text,
|
|
luminosity integer,
|
|
snow double precision,
|
|
phg_power integer,
|
|
phg_height integer,
|
|
phg_gain integer,
|
|
phg_directivity integer,
|
|
has_weather boolean DEFAULT false,
|
|
rain_midnight double precision,
|
|
position_ambiguity integer,
|
|
posresolution double precision,
|
|
format text,
|
|
telemetry_seq integer,
|
|
telemetry_vals integer[],
|
|
telemetry_bits text,
|
|
radiorange text,
|
|
srccallsign varchar,
|
|
dstcallsign varchar,
|
|
body text,
|
|
origpacket text,
|
|
header text,
|
|
alive integer DEFAULT 1,
|
|
posambiguity integer,
|
|
symboltable varchar,
|
|
symbolcode varchar,
|
|
messaging integer
|
|
)
|
|
""")
|
|
|
|
create_triggers("packets")
|
|
end
|
|
|
|
defp create_indexes do
|
|
# Time-series
|
|
execute("CREATE INDEX idx_packets_received_desc ON packets (received_at DESC)")
|
|
execute("CREATE INDEX idx_packets_received_brin ON packets USING brin (received_at)")
|
|
|
|
# Sender/Callsign
|
|
execute("CREATE INDEX idx_packets_sender_received ON packets (sender, received_at DESC)")
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_upper_sender ON packets (upper(sender::text), received_at DESC)"
|
|
)
|
|
|
|
execute("CREATE INDEX idx_packets_sender_pattern ON packets (sender text_pattern_ops)")
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_base_callsign_pattern ON packets (base_callsign text_pattern_ops)"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_base_callsign_time ON packets (base_callsign, received_at DESC)"
|
|
)
|
|
|
|
execute("CREATE INDEX idx_packets_sender_id_desc ON packets (sender, id DESC)")
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_upper_base_callsign ON packets (upper(base_callsign::text))"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_sender_trgm ON packets USING gin (sender gin_trgm_ops) WHERE sender IS NOT NULL"
|
|
)
|
|
|
|
# Position/Geographic
|
|
execute(
|
|
"CREATE INDEX idx_packets_positioned ON packets (received_at DESC, lat, lon, sender) WHERE has_position = true"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_position_time ON packets (has_position, received_at DESC) WHERE has_position = true"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_location ON packets USING gist (location) WHERE has_position = true"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_location_geography ON packets USING gist ((location::geography))"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_spatial_temporal ON packets (received_at DESC, base_callsign) WHERE has_position = true"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_sender_position ON packets (sender, received_at DESC) WHERE lat IS NOT NULL AND lon IS NOT NULL"
|
|
)
|
|
|
|
execute("CREATE INDEX idx_packets_lat ON packets (lat)")
|
|
execute("CREATE INDEX idx_packets_lon ON packets (lon)")
|
|
|
|
# Region
|
|
execute("CREATE INDEX idx_packets_region ON packets (region)")
|
|
execute("CREATE INDEX idx_packets_region_received ON packets (region, received_at)")
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_region_position_time ON packets (region, has_position, received_at DESC) WHERE has_position = true AND region IS NOT NULL"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_region_data_type ON packets (region, data_type, received_at)"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_region_position ON packets (region, has_position, received_at)"
|
|
)
|
|
|
|
# Weather
|
|
execute(
|
|
"CREATE INDEX idx_packets_weather ON packets (sender, received_at DESC) WHERE temperature IS NOT NULL OR humidity IS NOT NULL OR pressure IS NOT NULL OR wind_speed IS NOT NULL OR wind_direction IS NOT NULL OR rain_1h IS NOT NULL"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_has_weather ON packets (has_weather, sender, received_at DESC) WHERE has_weather = true"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_temperature ON packets (temperature) WHERE temperature IS NOT NULL"
|
|
)
|
|
|
|
execute("CREATE INDEX idx_packets_humidity ON packets (humidity) WHERE humidity IS NOT NULL")
|
|
execute("CREATE INDEX idx_packets_pressure ON packets (pressure) WHERE pressure IS NOT NULL")
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_wind_speed ON packets (wind_speed) WHERE wind_speed IS NOT NULL"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_received_temp ON packets (received_at, temperature) WHERE temperature IS NOT NULL"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_sender_temp ON packets (sender, temperature) WHERE temperature IS NOT NULL"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_weather_history ON packets (sender, data_type, received_at) WHERE data_type = 'weather'"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_weather_selective ON packets (received_at DESC) WHERE data_type IN ('weather', 'Weather', 'WX', 'wx')"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_weather_symbol ON packets (received_at DESC, lat, lon) WHERE (symbol_table_id = '/' AND symbol_code = '_') OR (symbol_table_id = E'\\\\' AND symbol_code = '_')"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_sender_weather_lookup ON packets (upper(sender::text), received_at DESC) INCLUDE (data_type, symbol_table_id, symbol_code) WHERE data_type = 'weather' OR (symbol_table_id = '/' AND symbol_code = '_') OR (symbol_table_id = E'\\\\' AND symbol_code = '_')"
|
|
)
|
|
|
|
# Data type & Device
|
|
execute(
|
|
"CREATE INDEX idx_packets_datatype_sender_time ON packets (data_type, sender, received_at DESC)"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_device_identifier ON packets (device_identifier) WHERE device_identifier IS NOT NULL"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_device_received ON packets (device_identifier, received_at) WHERE device_identifier IS NOT NULL"
|
|
)
|
|
|
|
# Path
|
|
execute(
|
|
"CREATE INDEX idx_packets_path_pattern ON packets (path text_pattern_ops) WHERE path IS NOT NULL AND path <> ''"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_path_trgm ON packets USING gin (path gin_trgm_ops) WHERE path IS NOT NULL AND path <> '' AND path !~ '^TCPIP' AND path !~ ',TCPIP'"
|
|
)
|
|
|
|
# Digipeater/stations heard
|
|
execute(
|
|
"CREATE INDEX idx_packets_digipeater ON packets (sender, received_at DESC) WHERE location IS NOT NULL"
|
|
)
|
|
|
|
execute(
|
|
"CREATE INDEX idx_packets_stations_heard ON packets (received_at DESC, sender, lat, lon) WHERE path IS NOT NULL AND path <> '' AND path !~ '^TCPIP' AND path !~ ',TCPIP' AND lat IS NOT NULL AND lon IS NOT NULL"
|
|
)
|
|
|
|
# Symbol
|
|
execute(
|
|
"CREATE INDEX idx_packets_symbol_time ON packets (symbol_table_id, symbol_code, received_at)"
|
|
)
|
|
end
|
|
|
|
defp create_triggers(table) do
|
|
execute(
|
|
"CREATE TRIGGER update_has_weather_trigger BEFORE INSERT OR UPDATE ON #{table} FOR EACH ROW EXECUTE FUNCTION update_has_weather()"
|
|
)
|
|
|
|
execute(
|
|
"CREATE TRIGGER packets_notify_insert AFTER INSERT ON #{table} FOR EACH ROW EXECUTE FUNCTION notify_packets_insert()"
|
|
)
|
|
|
|
execute(
|
|
"CREATE TRIGGER packet_insert_sequence AFTER INSERT ON #{table} FOR EACH ROW EXECUTE FUNCTION increment_packet_sequence()"
|
|
)
|
|
|
|
execute(
|
|
"CREATE TRIGGER packet_delete_sequence AFTER DELETE ON #{table} FOR EACH ROW EXECUTE FUNCTION decrement_packet_sequence()"
|
|
)
|
|
end
|
|
end
|