Graham McIntire
eb389af64b
Some checks failed
Build and Push / Build and Push Docker Image (push) Failing after 2s
- PacketProducer: 15s periodic buffer depth telemetry, overflow events - MobileChannel: 500-packet ring buffer with overflow drops, payload validation (query max 20 chars, callsign max 20 chars, results capped at 200, bounds area max 1000 sq deg), rejection telemetry - PromEx: buffer depth/max/overflow metrics, mobile overlay, payload rejected - Tests: 2491 passed, 0 failures - Handoff document: marked reliability #1 and #2 as done
350 lines
12 KiB
Elixir
350 lines
12 KiB
Elixir
defmodule Aprsme.PromEx.Plugins.Aprsme do
|
|
@moduledoc """
|
|
PromEx plugin exposing app-specific telemetry as Prometheus metrics.
|
|
|
|
Mirrors the metrics already defined for LiveDashboard in
|
|
`AprsmeWeb.Telemetry.metrics/0`, grouping them by subsystem:
|
|
|
|
* GenStage packet pipeline (`[:aprsme, :packet_pipeline, :batch]`)
|
|
* Producer backpressure (`[:aprsme, :packet_producer, :backpressure]`)
|
|
* Spatial PubSub (`[:aprsme, :spatial_pubsub, ...]`)
|
|
* Database connection pool (`[:aprsme, :repo, :pool]`)
|
|
* Postgres database stats (`[:aprsme, :postgres, ...]`)
|
|
"""
|
|
|
|
use PromEx.Plugin
|
|
|
|
@duration_buckets [10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000]
|
|
|
|
@impl true
|
|
def event_metrics(_opts) do
|
|
[
|
|
packet_pipeline_metrics(),
|
|
buffer_metrics(),
|
|
producer_backpressure_metrics(),
|
|
spatial_pubsub_metrics(),
|
|
mobile_metrics(),
|
|
payload_metrics(),
|
|
repo_pool_metrics(),
|
|
postgres_metrics(),
|
|
api_metrics()
|
|
]
|
|
end
|
|
|
|
defp packet_pipeline_metrics do
|
|
Event.build(
|
|
:aprsme_packet_pipeline_event_metrics,
|
|
[
|
|
counter(
|
|
[:aprsme, :packet_pipeline, :batch, :count],
|
|
event_name: [:aprsme, :packet_pipeline, :batch],
|
|
measurement: :count,
|
|
description: "Total number of packets processed in batches"
|
|
),
|
|
counter(
|
|
[:aprsme, :packet_pipeline, :batch, :success],
|
|
event_name: [:aprsme, :packet_pipeline, :batch],
|
|
measurement: :success,
|
|
description: "Total number of successful inserts"
|
|
),
|
|
counter(
|
|
[:aprsme, :packet_pipeline, :batch, :error],
|
|
event_name: [:aprsme, :packet_pipeline, :batch],
|
|
measurement: :error,
|
|
description: "Total number of insert errors"
|
|
),
|
|
distribution(
|
|
[:aprsme, :packet_pipeline, :batch, :duration_milliseconds],
|
|
event_name: [:aprsme, :packet_pipeline, :batch],
|
|
measurement: :duration_ms,
|
|
description: "Batch insert duration",
|
|
unit: :millisecond,
|
|
reporter_options: [buckets: @duration_buckets]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp buffer_metrics do
|
|
Event.build(
|
|
:aprsme_buffer_event_metrics,
|
|
[
|
|
counter(
|
|
[:aprsme, :buffer, :overflow, :total],
|
|
event_name: [:aprsme, :buffer, :overflow],
|
|
measurement: :dropped,
|
|
description: "Packets dropped from the producer buffer due to overflow"
|
|
),
|
|
last_value(
|
|
[:aprsme, :buffer, :depth, :current],
|
|
event_name: [:aprsme, :buffer, :depth],
|
|
measurement: :current,
|
|
description: "Current packet producer buffer depth"
|
|
),
|
|
last_value(
|
|
[:aprsme, :buffer, :depth, :max],
|
|
event_name: [:aprsme, :buffer, :depth],
|
|
measurement: :max,
|
|
description: "Configured maximum packet producer buffer size"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp producer_backpressure_metrics do
|
|
Event.build(
|
|
:aprsme_packet_producer_event_metrics,
|
|
[
|
|
counter(
|
|
[:aprsme, :packet_producer, :backpressure, :count],
|
|
event_name: [:aprsme, :packet_producer, :backpressure],
|
|
measurement: :count,
|
|
description: "Backpressure state changes in the packet producer"
|
|
),
|
|
counter(
|
|
[:aprsme, :packet_producer, :buffer_overflow, :total],
|
|
event_name: [:aprsme, :packet_producer, :buffer_overflow],
|
|
measurement: :dropped,
|
|
description: "Packets dropped from the producer buffer due to overflow"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp spatial_pubsub_metrics do
|
|
Event.build(
|
|
:aprsme_spatial_pubsub_event_metrics,
|
|
[
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :clients, :count],
|
|
event_name: [:aprsme, :spatial_pubsub, :clients],
|
|
measurement: :count,
|
|
description: "Number of connected spatial PubSub clients"
|
|
),
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :clients, :grid_cells],
|
|
event_name: [:aprsme, :spatial_pubsub, :clients],
|
|
measurement: :grid_cells,
|
|
description: "Number of active grid cells"
|
|
),
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :clients, :avg_clients_per_cell],
|
|
event_name: [:aprsme, :spatial_pubsub, :clients],
|
|
measurement: :avg_clients_per_cell,
|
|
description: "Average clients per grid cell"
|
|
),
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :broadcasts, :total],
|
|
event_name: [:aprsme, :spatial_pubsub, :broadcasts],
|
|
measurement: :total,
|
|
description: "Total spatial broadcasts sent"
|
|
),
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :broadcasts, :filtered],
|
|
event_name: [:aprsme, :spatial_pubsub, :broadcasts],
|
|
measurement: :filtered,
|
|
description: "Broadcasts filtered out by viewport"
|
|
),
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :broadcasts, :packets],
|
|
event_name: [:aprsme, :spatial_pubsub, :broadcasts],
|
|
measurement: :packets,
|
|
description: "Total packets processed by spatial PubSub"
|
|
),
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :efficiency, :ratio],
|
|
event_name: [:aprsme, :spatial_pubsub, :efficiency],
|
|
measurement: :ratio,
|
|
description: "Broadcast efficiency ratio (0-1)"
|
|
),
|
|
last_value(
|
|
[:aprsme, :spatial_pubsub, :efficiency, :saved_broadcasts],
|
|
event_name: [:aprsme, :spatial_pubsub, :efficiency],
|
|
measurement: :saved_broadcasts,
|
|
description: "Number of broadcasts saved by viewport filtering"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp repo_pool_metrics do
|
|
Event.build(
|
|
:aprsme_repo_pool_event_metrics,
|
|
[
|
|
last_value(
|
|
[:aprsme, :repo, :pool, :size],
|
|
event_name: [:aprsme, :repo, :pool],
|
|
measurement: :size,
|
|
description: "Configured DB connection pool size"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp mobile_metrics do
|
|
Event.build(
|
|
:aprsme_mobile_event_metrics,
|
|
[
|
|
counter(
|
|
[:aprsme, :mobile, :buffer_overflow, :total],
|
|
event_name: [:aprsme, :mobile, :buffer_overflow],
|
|
measurement: :dropped,
|
|
description: "Packets dropped from mobile channel per-socket queue"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp payload_metrics do
|
|
Event.build(
|
|
:aprsme_payload_event_metrics,
|
|
[
|
|
counter(
|
|
[:aprsme, :payload, :rejected, :total],
|
|
event_name: [:aprsme, :payload, :rejected],
|
|
measurement: :count,
|
|
description: "Client payloads rejected due to size/validation limits",
|
|
tags: [:reason]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp postgres_metrics do
|
|
Event.build(
|
|
:aprsme_postgres_event_metrics,
|
|
[
|
|
last_value(
|
|
[:aprsme, :postgres, :database, :size_bytes],
|
|
event_name: [:aprsme, :postgres, :database],
|
|
measurement: :size_bytes,
|
|
description: "Total Postgres database size in bytes",
|
|
unit: :byte
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :connections, :total],
|
|
event_name: [:aprsme, :postgres, :connections],
|
|
measurement: :total,
|
|
description: "Total Postgres connections"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :connections, :active],
|
|
event_name: [:aprsme, :postgres, :connections],
|
|
measurement: :active,
|
|
description: "Active Postgres connections"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :connections, :idle],
|
|
event_name: [:aprsme, :postgres, :connections],
|
|
measurement: :idle,
|
|
description: "Idle Postgres connections"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :connections, :idle_in_transaction],
|
|
event_name: [:aprsme, :postgres, :connections],
|
|
measurement: :idle_in_transaction,
|
|
description: "Connections idle in transaction"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :connections, :waiting],
|
|
event_name: [:aprsme, :postgres, :connections],
|
|
measurement: :waiting,
|
|
description: "Postgres connections waiting on locks"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :packets_table, :live_tuples],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :live_tuples,
|
|
description: "Live rows in the packets table"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :packets_table, :dead_tuples],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :dead_tuples,
|
|
description: "Dead rows in the packets table"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :packets_table, :total_inserts],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :total_inserts,
|
|
description: "Total inserts into the packets table"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :packets_table, :total_updates],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :total_updates,
|
|
description: "Total updates on the packets table"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :packets_table, :total_deletes],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :total_deletes,
|
|
description: "Total deletes from the packets table"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :packets_table, :table_size_bytes],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :table_size_bytes,
|
|
description: "Packets table size in bytes",
|
|
unit: :byte
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :packets_table, :indexes_size_bytes],
|
|
event_name: [:aprsme, :postgres, :packets_table],
|
|
measurement: :indexes_size_bytes,
|
|
description: "Packets indexes size in bytes",
|
|
unit: :byte
|
|
),
|
|
counter(
|
|
[:aprsme, :postgres, :query_stats, :total_calls],
|
|
event_name: [:aprsme, :postgres, :query_stats],
|
|
measurement: :total_calls,
|
|
description: "Total Postgres queries executed (pg_stat_statements)"
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :query_stats, :total_time_ms],
|
|
event_name: [:aprsme, :postgres, :query_stats],
|
|
measurement: :total_time_ms,
|
|
description: "Total Postgres query execution time",
|
|
unit: :millisecond
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :query_stats, :avg_time_ms],
|
|
event_name: [:aprsme, :postgres, :query_stats],
|
|
measurement: :avg_time_ms,
|
|
description: "Average Postgres query execution time",
|
|
unit: :millisecond
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :query_stats, :max_time_ms],
|
|
event_name: [:aprsme, :postgres, :query_stats],
|
|
measurement: :max_time_ms,
|
|
description: "Maximum Postgres query execution time",
|
|
unit: :millisecond
|
|
),
|
|
last_value(
|
|
[:aprsme, :postgres, :replication, :lag_seconds],
|
|
event_name: [:aprsme, :postgres, :replication],
|
|
measurement: :lag_seconds,
|
|
description: "Postgres replication lag in seconds",
|
|
unit: :second
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp api_metrics do
|
|
Event.build(
|
|
:aprsme_api_request_event_metrics,
|
|
[
|
|
counter(
|
|
[:aprsme, :api, :request, :total],
|
|
event_name: [:aprsme, :api, :request],
|
|
measurement: :count,
|
|
description: "Total API and WebSocket requests",
|
|
tags: [:event]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
end
|