350 lines
12 KiB
Elixir
350 lines
12 KiB
Elixir
defmodule Aprsme.PromEx.Plugins.Aprsme do
|
|
@moduledoc """
|
|
PromEx plugin exposing app-specific telemetry as Prometheus metrics.
|
|
|
|
Mirrors the metrics already defined for LiveDashboard in
|
|
`AprsmeWeb.Telemetry.metrics/0`, grouping them by subsystem:
|
|
|
|
* GenStage packet pipeline (`[:aprsme, :packet_pipeline, :batch]`)
|
|
* Producer backpressure (`[:aprsme, :packet_producer, :backpressure]`)
|
|
* Spatial PubSub (`[:aprsme, :spatial_pubsub, ...]`)
|
|
* Database connection pool (`[:aprsme, :repo, :pool]`)
|
|
* Postgres database stats (`[:aprsme, :postgres, ...]`)
|
|
* Insert optimizer (`[:aprsme, :insert_optimizer, ...]`)
|
|
"""
|
|
|
|
use PromEx.Plugin
|
|
|
|
@duration_buckets [10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000]
|
|
|
|
@impl true
|
|
def event_metrics(_opts) do
|
|
[
|
|
packet_pipeline_metrics(),
|
|
producer_backpressure_metrics(),
|
|
spatial_pubsub_metrics(),
|
|
repo_pool_metrics(),
|
|
postgres_metrics(),
|
|
insert_optimizer_metrics(),
|
|
api_metrics()
|
|
]
|
|
end
|
|
|
|
defp packet_pipeline_metrics do
|
|
Event.build(
|
|
:aprsme_packet_pipeline_event_metrics,
|
|
[
|
|
counter(
|
|
[:aprsme, :packet_pipeline, :batch, :count],
|
|
event_name: [:aprsme, :packet_pipeline, :batch],
|
|
measurement: :count,
|
|
description: "Total number of packets processed in batches"
|
|
),
|
|
counter(
|
|
[:aprsme, :packet_pipeline, :batch, :success],
|
|
event_name: [:aprsme, :packet_pipeline, :batch],
|
|
measurement: :success,
|
|
description: "Total number of successful inserts"
|
|
),
|
|
counter(
|
|
[:aprsme, :packet_pipeline, :batch, :error],
|
|
event_name: [:aprsme, :packet_pipeline, :batch],
|
|
measurement: :error,
|
|
description: "Total number of insert errors"
|
|
),
|
|
distribution(
|
|
[:aprsme, :packet_pipeline, :batch, :duration_milliseconds],
|
|
event_name: [:aprsme, :packet_pipeline, :batch],
|
|
measurement: :duration_ms,
|
|
description: "Batch insert duration",
|
|
unit: :millisecond,
|
|
reporter_options: [buckets: @duration_buckets]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp producer_backpressure_metrics do
|
|
Event.build(
|
|
:aprsme_packet_producer_event_metrics,
|
|
[
|
|
counter(
|
|
[:aprsme, :packet_producer, :backpressure, :count],
|
|
event_name: [:aprsme, :packet_producer, :backpressure],
|
|
measurement: :count,
|
|
description: "Backpressure state changes in the packet producer"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp spatial_pubsub_metrics do
|
|
Event.build(
|
|
:aprsme_spatial_pubsub_event_metrics,
|
|
[
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :clients, :count],
|
|
event_name: [:aprsme, :spatial_pubsub, :clients],
|
|
measurement: :count,
|
|
description: "Number of connected spatial PubSub clients"
|
|
),
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :clients, :grid_cells],
|
|
event_name: [:aprsme, :spatial_pubsub, :clients],
|
|
measurement: :grid_cells,
|
|
description: "Number of active grid cells"
|
|
),
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :clients, :avg_clients_per_cell],
|
|
event_name: [:aprsme, :spatial_pubsub, :clients],
|
|
measurement: :avg_clients_per_cell,
|
|
description: "Average clients per grid cell"
|
|
),
|
|
counter(
|
|
[:aprsme, :spatial_pubsub, :broadcasts, :total],
|
|
event_name: [:aprsme, :spatial_pubsub, :broadcasts],
|
|
measurement: :total,
|
|
description: "Total spatial broadcasts sent"
|
|
),
|
|
counter(
|
|
[:aprsme, :spatial_pubsub, :broadcasts, :filtered],
|
|
event_name: [:aprsme, :spatial_pubsub, :broadcasts],
|
|
measurement: :filtered,
|
|
description: "Broadcasts filtered out by viewport"
|
|
),
|
|
counter(
|
|
[:aprsme, :spatial_pubsub, :broadcasts, :packets],
|
|
event_name: [:aprsme, :spatial_pubsub, :broadcasts],
|
|
measurement: :packets,
|
|
description: "Total packets processed by spatial PubSub"
|
|
),
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :efficiency, :ratio],
|
|
event_name: [:aprsme, :spatial_pubsub, :efficiency],
|
|
measurement: :ratio,
|
|
description: "Broadcast efficiency ratio (0-1)"
|
|
),
|
|
counter(
|
|
[:aprsme, :spatial_pubsub, :efficiency, :saved_broadcasts],
|
|
event_name: [:aprsme, :spatial_pubsub, :efficiency],
|
|
measurement: :saved_broadcasts,
|
|
description: "Number of broadcasts saved by viewport filtering"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp repo_pool_metrics do
|
|
Event.build(
|
|
:aprsme_repo_pool_event_metrics,
|
|
[
|
|
last_value(
|
|
[:aprsme, :repo, :pool, :size],
|
|
event_name: [:aprsme, :repo, :pool],
|
|
measurement: :size,
|
|
description: "Configured DB connection pool size"
|
|
),
|
|
last_value(
|
|
[:aprsme, :repo, :pool, :idle],
|
|
event_name: [:aprsme, :repo, :pool],
|
|
measurement: :idle,
|
|
description: "Idle DB connections"
|
|
),
|
|
last_value(
|
|
[:aprsme, :repo, :pool, :busy],
|
|
event_name: [:aprsme, :repo, :pool],
|
|
measurement: :busy,
|
|
description: "Busy DB connections"
|
|
),
|
|
last_value(
|
|
[:aprsme, :repo, :pool, :available],
|
|
event_name: [:aprsme, :repo, :pool],
|
|
measurement: :available,
|
|
description: "Available DB connections"
|
|
),
|
|
last_value(
|
|
[:aprsme, :repo, :pool, :queue_length],
|
|
event_name: [:aprsme, :repo, :pool],
|
|
measurement: :queue_length,
|
|
description: "Processes waiting on a DB connection"
|
|
),
|
|
last_value(
|
|
[:aprsme, :repo, :pool, :total],
|
|
event_name: [:aprsme, :repo, :pool],
|
|
measurement: :total,
|
|
description: "Total connections in the DB pool"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp postgres_metrics do
|
|
Event.build(
|
|
:aprsme_postgres_event_metrics,
|
|
[
|
|
last_value(
|
|
[:aprsme, :postgres, :database, :size_bytes],
|
|
event_name: [:aprsme, :postgres, :database],
|
|
measurement: :size_bytes,
|
|
description: "Total Postgres database size in bytes",
|
|
unit: :byte
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :connections, :total],
|
|
event_name: [:aprsme, :postgres, :connections],
|
|
measurement: :total,
|
|
description: "Total Postgres connections"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :connections, :active],
|
|
event_name: [:aprsme, :postgres, :connections],
|
|
measurement: :active,
|
|
description: "Active Postgres connections"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :connections, :idle],
|
|
event_name: [:aprsme, :postgres, :connections],
|
|
measurement: :idle,
|
|
description: "Idle Postgres connections"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :connections, :idle_in_transaction],
|
|
event_name: [:aprsme, :postgres, :connections],
|
|
measurement: :idle_in_transaction,
|
|
description: "Connections idle in transaction"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :connections, :waiting],
|
|
event_name: [:aprsme, :postgres, :connections],
|
|
measurement: :waiting,
|
|
description: "Postgres connections waiting on locks"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :packets_table, :live_tuples],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :live_tuples,
|
|
description: "Live rows in the packets table"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :packets_table, :dead_tuples],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :dead_tuples,
|
|
description: "Dead rows in the packets table"
|
|
),
|
|
counter(
|
|
[:aprsme, :postgres, :packets_table, :total_inserts],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :total_inserts,
|
|
description: "Total inserts into the packets table"
|
|
),
|
|
counter(
|
|
[:aprsme, :postgres, :packets_table, :total_updates],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :total_updates,
|
|
description: "Total updates on the packets table"
|
|
),
|
|
counter(
|
|
[:aprsme, :postgres, :packets_table, :total_deletes],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :total_deletes,
|
|
description: "Total deletes from the packets table"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :packets_table, :table_size_bytes],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :table_size_bytes,
|
|
description: "Packets table size in bytes",
|
|
unit: :byte
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :packets_table, :indexes_size_bytes],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :indexes_size_bytes,
|
|
description: "Packets indexes size in bytes",
|
|
unit: :byte
|
|
),
|
|
counter(
|
|
[:aprsme, :postgres, :query_stats, :total_calls],
|
|
event_name: [:aprsme, :postgres, :query_stats],
|
|
measurement: :total_calls,
|
|
description: "Total Postgres queries executed (pg_stat_statements)"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :query_stats, :total_time_ms],
|
|
event_name: [:aprsme, :postgres, :query_stats],
|
|
measurement: :total_time_ms,
|
|
description: "Total Postgres query execution time",
|
|
unit: :millisecond
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :query_stats, :avg_time_ms],
|
|
event_name: [:aprsme, :postgres, :query_stats],
|
|
measurement: :avg_time_ms,
|
|
description: "Average Postgres query execution time",
|
|
unit: :millisecond
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :query_stats, :max_time_ms],
|
|
event_name: [:aprsme, :postgres, :query_stats],
|
|
measurement: :max_time_ms,
|
|
description: "Maximum Postgres query execution time",
|
|
unit: :millisecond
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :replication, :lag_seconds],
|
|
event_name: [:aprsme, :postgres, :replication],
|
|
measurement: :lag_seconds,
|
|
description: "Postgres replication lag in seconds",
|
|
unit: :second
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp insert_optimizer_metrics do
|
|
Event.build(
|
|
:aprsme_insert_optimizer_event_metrics,
|
|
[
|
|
last_value(
|
|
[:aprsme, :insert_optimizer, :batch_size],
|
|
event_name: [:aprsme, :insert_optimizer, :batch_size],
|
|
measurement: :value,
|
|
description: "Current batch size chosen by the insert optimizer"
|
|
),
|
|
last_value(
|
|
[:aprsme, :insert_optimizer, :throughput],
|
|
event_name: [:aprsme, :insert_optimizer, :throughput],
|
|
measurement: :value,
|
|
description: "Average insert throughput observed by the optimizer"
|
|
),
|
|
last_value(
|
|
[:aprsme, :insert_optimizer, :duration],
|
|
event_name: [:aprsme, :insert_optimizer, :duration],
|
|
measurement: :value,
|
|
description: "Average insert duration observed by the optimizer"
|
|
),
|
|
counter(
|
|
[:aprsme, :insert_optimizer, :optimizations],
|
|
event_name: [:aprsme, :insert_optimizer, :optimizations],
|
|
measurement: :count,
|
|
description: "Number of times the optimizer adjusted batch parameters"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp api_metrics do
|
|
Event.build(
|
|
:aprsme_api_request_event_metrics,
|
|
[
|
|
counter(
|
|
[:aprsme, :api, :request, :total],
|
|
event_name: [:aprsme, :api, :request],
|
|
measurement: :count,
|
|
description: "Total API and WebSocket requests",
|
|
tags: [:event]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
end
|