graham/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

updates

Browse source
This commit is contained in:
Graham McIntire 2025-08-02 11:14:56 -05:00
parent ed2d43f36d
commit c68fc9827d
No known key found for this signature in database
68 changed files with 3630 additions and 1270 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

107
ansible/create-vm-template.yml
View file

@ -1,107 +0,0 @@
---
# Create a cloud-init template on Proxmox for K8s nodes
# This playbook runs commands directly on the Proxmox host
- name: Create cloud-init template on Proxmox
hosts: proxmox[0]
become: yes
vars:
template_vmid: 9000
template_name: debian12-cloud-init
debian_image_url: https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2
storage: ceph
tasks:
- name: Check if template already exists
shell: |
qm status {{ template_vmid }} >/dev/null 2>&1 && echo "exists" || echo "missing"
register: template_check
changed_when: false
- name: Display template status
debug:
msg: "Template VM {{ template_vmid }} already exists. Skipping creation."
when: template_check.stdout == "exists"
- name: Install required packages
apt:
name:
- wget
- libguestfs-tools
state: present
update_cache: yes
when: template_check.stdout == "missing"
- name: Download Debian cloud image
get_url:
url: "{{ debian_image_url }}"
dest: /var/lib/vz/template/iso/debian-12-genericcloud-amd64.qcow2
mode: '0644'
when: template_check.stdout == "missing"
- name: Create VM for template
shell: |
qm create {{ template_vmid }} --name {{ template_name }} --memory 2048 --cores 2 --net0 virtio,bridge=vmbr0
args:
creates: /etc/pve/qemu-server/{{ template_vmid }}.conf
when: template_check.stdout == "missing"
- name: Import disk image to VM
shell: |
qm importdisk {{ template_vmid }} /var/lib/vz/template/iso/debian-12-genericcloud-amd64.qcow2 {{ storage }}
register: import_result
changed_when: "'successfully imported' in import_result.stdout"
when: template_check.stdout == "missing"
- name: Get imported disk name
shell: |
pvesm list {{ storage }} | grep "vm-{{ template_vmid }}-disk-" | awk '{print $1}' | head -1
register: disk_name
when: template_check.stdout == "missing"
- name: Attach imported disk to VM
shell: |
qm set {{ template_vmid }} --scsihw virtio-scsi-pci --scsi0 {{ disk_name.stdout }}
when:
- template_check.stdout == "missing"
- disk_name.stdout != ""
- name: Check if cloud-init drive exists
shell: |
qm config {{ template_vmid }} | grep -q 'ide2:.*cloudinit' && echo "exists" || echo "missing"
register: cloudinit_check
changed_when: false
when: template_check.stdout == "missing"
- name: Add cloud-init drive
shell: |
qm set {{ template_vmid }} --ide2 {{ storage }}:cloudinit
when:
- template_check.stdout == "missing"
- cloudinit_check.stdout == "missing"
- name: Set boot disk
shell: |
qm set {{ template_vmid }} --boot c --bootdisk scsi0
when: template_check.stdout == "missing"
- name: Add serial console
shell: |
qm set {{ template_vmid }} --serial0 socket --vga serial0
when: template_check.stdout == "missing"
- name: Enable QEMU guest agent
shell: |
qm set {{ template_vmid }} --agent enabled=1
when: template_check.stdout == "missing"
- name: Convert VM to template
shell: |
qm template {{ template_vmid }}
when: template_check.stdout == "missing"
- name: Clean up downloaded image
file:
path: /var/lib/vz/template/iso/debian-12-genericcloud-amd64.qcow2
state: absent
when: template_check.stdout == "missing"

8
ansible/host_vars/postgres.yml
View file

@ -9,10 +9,10 @@ ansible_python_interpreter: /usr/bin/python3.11
# shared_buffers: 8GB
# RAID configuration
# Comment out for auto-detection or specify actual device names
# raid_devices:
# - /dev/sda
# - /dev/sdb
# Two SSDs in RAID0 for performance
raid_devices:
- /dev/nvme0n1
- /dev/nvme1n1
# Network configuration
postgresql_listen_addresses: "'*'"

2
ansible/hosts
View file

@ -34,4 +34,4 @@ g.w5isp.com
g.w5isp.com
[postgresql_servers]
postgres ansible_host=10.0.16.230
postgres ansible_host=10.0.19.220

4
ansible/inventory.yml
View file

@ -1,11 +1,11 @@
all:
hosts:
postgres:
ansible_host: 10.0.16.230
ansible_host: 10.0.19.220
ansible_user: graham
ansible_become: yes
ansible_python_interpreter: /usr/bin/python3
children:
postgresql_servers:
postgres_servers:
hosts:
postgres:

74
ansible/postgres-playbook.yml
View file

@ -1,74 +0,0 @@
---
- name: Deploy PostgreSQL server on CM3588 with RAID0
hosts: postgres
become: yes
gather_facts: yes
pre_tasks:
- name: Set hostname
hostname:
name: postgres
- name: Update /etc/hosts
lineinfile:
path: /etc/hosts
regexp: '^127\.0\.1\.1'
line: '127.0.1.1 postgres'
state: present
- name: Update apt cache
apt:
update_cache: yes
cache_valid_time: 3600
- name: Install Python dependencies for Ansible
apt:
name:
- python3-apt
- python3-psycopg2
state: present
roles:
- raid
- postgresql
post_tasks:
- name: Display PostgreSQL connection information
debug:
msg: |
PostgreSQL has been configured on {{ inventory_hostname }}
Connection details:
- Host: {{ ansible_host }}
- Port: 5432 (PostgreSQL direct)
- Port: 6432 (PgBouncer pooled)
- Database: aprsme_prod
- User: aprsme
RAID0 array status:
- Device: /dev/md0
- Mount: /var/lib/postgresql
- Size: ~4TB (2x2TB in RAID0)
To connect from your k3s cluster:
psql -h {{ ansible_host }} -p 6432 -U aprsme -d aprsme_prod
Remember to:
1. Update your APRS.me DATABASE_URL to point to this server
2. Run migrations: mix ecto.migrate
3. Monitor disk usage and performance
4. Set up regular backups to external storage
- name: Test PostgreSQL connection
postgresql_ping:
db: aprsme_prod
login_host: localhost
login_port: 5432
login_user: aprsme
login_password: "{{ vault_postgresql_aprsme_password }}"
register: db_ping
ignore_errors: yes
- name: Show connection test result
debug:
msg: "Database connection test: {{ 'SUCCESS' if db_ping is succeeded else 'FAILED' }}"

50
ansible/postgres-with-1password.yml Normal file
View file

@ -0,0 +1,50 @@
---
# Example playbook for PostgreSQL setup with 1Password integration
# Usage: ansible-playbook -i inventory.yml postgres-with-1password.yml
- name: Configure PostgreSQL server with 1Password integration
hosts: postgres
become: yes
vars:
# Enable 1Password integration
use_1password: true
# Optionally specify 1Password account ID
# If not specified, defaults to YOOATCZZSVGH7AD6VABUVPORLI
# op_account_id: "YOUR_ACCOUNT_ID"
# Override any defaults here
postgresql_databases:
- name: aprsme_prod
owner: aprsme
postgresql_users:
- name: aprsme
# Password will be generated and stored in 1Password
priv: "aprsme_prod:ALL"
roles:
- base
- raid
- postgresql
post_tasks:
- name: Display connection information
debug:
msg: |
PostgreSQL is now configured!
Connection details:
- Host: {{ ansible_host }}
- Port: 5432 (PostgreSQL direct)
- Port: 6432 (PgBouncer pooled)
- Database: {{ postgresql_databases[0].name }}
- User: {{ postgresql_users[0].name }}
- Password: Stored in 1Password as "PostgreSQL {{ postgresql_users[0].name }} - {{ ansible_hostname }}"
Connection strings:
- Direct: postgresql://{{ postgresql_users[0].name }}:<password>@{{ ansible_host }}:5432/{{ postgresql_databases[0].name }}
- Pooled: postgresql://{{ postgresql_users[0].name }}:<password>@{{ ansible_host }}:6432/{{ postgresql_databases[0].name }}
To retrieve the password:
op item get "PostgreSQL {{ postgresql_users[0].name }} - {{ ansible_hostname }}" --fields password

47
ansible/roles/postgresql/defaults/main.yml
View file

@ -1,6 +1,17 @@
---
# PostgreSQL version
postgresql_version: 16
postgresql_version: 17
# Password management
use_1password: false # Set to true to use 1Password for password storage
op_account_id: "YOOATCZZSVGH7AD6VABUVPORLI" # Default 1Password account ID
# Storage configuration
postgresql_storage_setup_enabled: false # Set to true to configure storage
postgresql_storage_type: raid0 # Options: single_nvme, raid0
postgresql_raid_device: /dev/md0
postgresql_storage_device: "{{ postgresql_raid_device if postgresql_storage_type == 'raid0' else '/dev/nvme0n1p1' }}"
postgresql_mount_point: /mnt/pgdata
# PostgreSQL configuration
postgresql_data_directory: /mnt/pgdata/{{ postgresql_version }}/main
@ -18,33 +29,33 @@ postgresql_users:
password: "{{ vault_postgresql_aprsme_password }}" # Store in vault
priv: "aprsme_prod:ALL"
# Performance tuning for CM3588 with RK3588 (assuming 32GB RAM)
# Performance tuning for CM3588 with 16GB RAM
# These are optimized for APRS.me workload
postgresql_config:
# Connection settings
listen_addresses: "'*'"
max_connections: 200 # Increased for multiple app instances
max_connections: 200 # Increased for 16GB system
# Memory settings (for 32GB RAM system)
shared_buffers: 8GB # 25% of RAM
effective_cache_size: 24GB # 75% of RAM
maintenance_work_mem: 2GB
work_mem: 64MB # High for complex queries
# Memory settings (for 16GB RAM system)
shared_buffers: 4GB # 25% of RAM
effective_cache_size: 12GB # 75% of RAM
maintenance_work_mem: 1GB
work_mem: 64MB # Increased for available memory
# Disk settings optimized for RAID0 SSDs
random_page_cost: 1.1 # SSD optimization
effective_io_concurrency: 200 # Modern SSD
max_worker_processes: 8 # RK3588 has 8 cores
max_parallel_workers_per_gather: 4
max_parallel_workers: 8
max_parallel_maintenance_workers: 4
max_worker_processes: 4 # Pi 5 has 4 cores
max_parallel_workers_per_gather: 2
max_parallel_workers: 4
max_parallel_maintenance_workers: 2
# Write performance settings
wal_buffers: 64MB
wal_buffers: 32MB
checkpoint_completion_target: 0.9
checkpoint_timeout: 15min
max_wal_size: 16GB
min_wal_size: 2GB
checkpoint_timeout: 10min
max_wal_size: 4GB
min_wal_size: 512MB
# Logging
log_min_duration_statement: 1000 # Log queries over 1 second
@ -72,7 +83,7 @@ postgresql_config:
track_functions: all
# PostGIS specific
postgis.gdal_datapath: '/usr/share/gdal'
postgis.gdal_datapath: "'/usr/share/gdal'"
# Security settings
postgresql_hba_entries:
@ -94,7 +105,7 @@ postgresql_hba_entries:
user: all
address: ::1/128
auth_method: scram-sha-256
- type: host
- type: hostssl
database: aprsme_prod
user: aprsme
address: 10.0.0.0/8 # Adjust for your network

5
ansible/roles/postgresql/handlers/main.yml
View file

@ -20,4 +20,7 @@
daemon_reload: yes
- name: reload sysctl
command: sysctl -p /etc/sysctl.d/30-postgresql.conf
command: sysctl -p /etc/sysctl.d/30-postgresql.conf
- name: reload udev
command: udevadm control --reload-rules && udevadm trigger

55
ansible/roles/postgresql/tasks/1password.yml Normal file
View file

@ -0,0 +1,55 @@
---
# 1Password integration for PostgreSQL passwords
# This file handles password generation and storage in 1Password
- name: Check if 1Password CLI is available
command: which op
register: op_cli_check
changed_when: false
failed_when: false
delegate_to: localhost
- name: Fail if 1Password CLI is not installed
fail:
msg: "1Password CLI (op) is not installed. Please install it first."
when: op_cli_check.rc != 0
- name: Generate password for PostgreSQL user
set_fact:
generated_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}"
when: vault_postgresql_aprsme_password is not defined
no_log: true
- name: Create 1Password item for PostgreSQL password
shell: |
op item create \
--category=database \
--title="PostgreSQL {{ item.name }} - {{ ansible_hostname }}" \
--vault="Private" \
username="{{ item.name }}" \
password="{{ generated_password | default(vault_postgresql_aprsme_password) }}" \
hostname="{{ ansible_host }}" \
port="5432" \
database="{{ postgresql_databases[0].name }}" \
--account="{{ op_account_id | default('YOOATCZZSVGH7AD6VABUVPORLI') }}" \
--format=json || \
op item edit \
"PostgreSQL {{ item.name }} - {{ ansible_hostname }}" \
password="{{ generated_password | default(vault_postgresql_aprsme_password) }}" \
--account="{{ op_account_id | default('YOOATCZZSVGH7AD6VABUVPORLI') }}"
delegate_to: localhost
loop: "{{ postgresql_users }}"
when: use_1password | default(false)
no_log: true
register: op_items
- name: Set password fact from 1Password or generated
set_fact:
postgresql_users_with_passwords: >-
{%- set users = [] -%}
{%- for user in postgresql_users -%}
{%- set _ = user.update({'password': generated_password | default(vault_postgresql_aprsme_password)}) -%}
{%- set _ = users.append(user) -%}
{%- endfor -%}
{{ users }}
no_log: true

142
ansible/roles/postgresql/tasks/main.yml
View file

@ -1,4 +1,9 @@
---
# Set up storage before installing PostgreSQL
- name: Set up storage for PostgreSQL
include_tasks: storage.yml
when: postgresql_storage_setup_enabled | default(false)
- name: Add PostgreSQL GPG key
apt_key:
url: https://www.postgresql.org/media/keys/ACCC4CF8.asc
@ -41,6 +46,28 @@
path: /var/lib/postgresql/{{ postgresql_version }}/main
state: absent
- name: Install locales package
apt:
name: locales
state: present
- name: Generate en_US.UTF-8 locale
locale_gen:
name: en_US.UTF-8
state: present
- name: Create PostgreSQL config directory structure
file:
path: "{{ item }}"
state: directory
owner: postgres
group: postgres
mode: '0755'
loop:
- /etc/postgresql
- /etc/postgresql/{{ postgresql_version }}
- /etc/postgresql/{{ postgresql_version }}/main
- name: Create PostgreSQL cluster on RAID array
command: |
/usr/lib/postgresql/{{ postgresql_version }}/bin/initdb \
@ -48,15 +75,18 @@
--wal-segsize=64 \
--data-checksums \
--encoding=UTF8 \
--locale=en_US.UTF-8 \
--lc-collate=en_US.UTF-8 \
--lc-ctype=en_US.UTF-8
--locale=C.UTF-8 \
--lc-collate=C.UTF-8 \
--lc-ctype=C.UTF-8
args:
creates: "{{ postgresql_data_directory }}/PG_VERSION"
become_user: postgres
environment:
PGDATA: "{{ postgresql_data_directory }}"
- name: Set up SSL/TLS
include_tasks: ssl.yml
- name: Configure PostgreSQL
template:
src: postgresql.conf.j2
@ -97,28 +127,88 @@
- reload systemd
- restart postgresql
- name: Move WAL to separate directory on RAID
- name: Check if PostgreSQL cluster is properly initialized
stat:
path: "{{ postgresql_data_directory }}/PG_VERSION"
register: pg_initialized
- name: Reinitialize PostgreSQL if corrupted
block:
- name: Stop PostgreSQL if running
systemd:
name: postgresql
name: postgresql@{{ postgresql_version }}-main
state: stopped
ignore_errors: yes
- name: Move existing WAL directory
command: mv {{ postgresql_data_directory }}/pg_wal {{ postgresql_wal_directory }}
args:
creates: "{{ postgresql_wal_directory }}"
removes: "{{ postgresql_data_directory }}/pg_wal"
become_user: postgres
- name: Create symlink for WAL directory
- name: Remove corrupted data directory
file:
src: "{{ postgresql_wal_directory }}"
dest: "{{ postgresql_data_directory }}/pg_wal"
state: link
path: "{{ postgresql_data_directory }}"
state: absent
- name: Create data directory
file:
path: "{{ postgresql_data_directory }}"
state: directory
owner: postgres
group: postgres
mode: '0700'
- name: Clear WAL directory for reinit
file:
path: "{{ postgresql_wal_directory }}"
state: absent
- name: Create fresh WAL directory
file:
path: "{{ postgresql_wal_directory }}"
state: directory
owner: postgres
group: postgres
mode: '0700'
- name: Initialize new PostgreSQL cluster
command: |
/usr/lib/postgresql/{{ postgresql_version }}/bin/initdb \
-D {{ postgresql_data_directory }} \
--wal-segsize=64 \
--data-checksums \
--encoding=UTF8 \
--locale=C.UTF-8 \
--lc-collate=C.UTF-8 \
--lc-ctype=C.UTF-8 \
--waldir={{ postgresql_wal_directory }}
become_user: postgres
environment:
PGDATA: "{{ postgresql_data_directory }}"
when: pg_initialized.stat.exists and force_reinit | default(false)
- name: Ensure WAL directory is properly configured
block:
- name: Check if WAL directory exists
stat:
path: "{{ postgresql_wal_directory }}"
register: wal_dir_stat
- name: Create WAL directory if missing
file:
path: "{{ postgresql_wal_directory }}"
state: directory
owner: postgres
group: postgres
mode: '0700'
when: not wal_dir_stat.stat.exists
- name: Check if pg_wal is a symlink
stat:
path: "{{ postgresql_data_directory }}/pg_wal"
register: pg_wal_stat
- name: Ensure pg_wal symlink has correct ownership
file:
path: "{{ postgresql_data_directory }}/pg_wal"
owner: postgres
group: postgres
when: pg_wal_stat.stat.exists and pg_wal_stat.stat.islnk
- name: Start and enable PostgreSQL
systemd:
@ -133,6 +223,10 @@
host: localhost
delay: 5
- name: Handle password management with 1Password
include_tasks: 1password.yml
when: use_1password | default(false)
- name: Create PostgreSQL users
postgresql_user:
name: "{{ item.name }}"
@ -177,8 +271,8 @@
- name: Grant privileges
postgresql_privs:
database: "{{ item.split(':')[0] }}"
privs: "{{ item.split(':')[1] }}"
database: "{{ priv_item.split(':')[0] }}"
privs: "{{ priv_item.split(':')[1] }}"
type: database
role: "{{ user.name }}"
state: present
@ -190,6 +284,18 @@
vars:
user: "{{ postgresql_users | selectattr('priv', 'equalto', priv_item) | first }}"
- name: Retrieve SCRAM hashes for PgBouncer
postgresql_query:
db: postgres
query: "SELECT rolname, rolpassword FROM pg_authid WHERE rolname = %s"
positional_args:
- "{{ item.name }}"
become: yes
become_user: postgres
loop: "{{ postgresql_users }}"
register: scram_hashes
no_log: true
- name: Configure PgBouncer
template:
src: pgbouncer.ini.j2

70
ansible/roles/postgresql/tasks/ssl.yml Normal file
View file

@ -0,0 +1,70 @@
---
# SSL/TLS configuration for PostgreSQL
- name: Install OpenSSL
apt:
name: openssl
state: present
- name: Create SSL directory
file:
path: "{{ postgresql_data_directory }}/ssl"
state: directory
owner: postgres
group: postgres
mode: '0700'
- name: Check if SSL certificate exists
stat:
path: "{{ postgresql_data_directory }}/ssl/server.crt"
register: ssl_cert
- name: Generate private key for PostgreSQL
shell: |
openssl genrsa -out {{ postgresql_data_directory }}/ssl/server.key 4096
chmod 600 {{ postgresql_data_directory }}/ssl/server.key
chown postgres:postgres {{ postgresql_data_directory }}/ssl/server.key
when: not ssl_cert.stat.exists
- name: Generate self-signed certificate
shell: |
openssl req -new -x509 -days 3650 \
-key {{ postgresql_data_directory }}/ssl/server.key \
-out {{ postgresql_data_directory }}/ssl/server.crt \
-subj "/C=US/ST=State/L=City/O=Organization/CN={{ ansible_hostname }}" \
-addext "subjectAltName=DNS:{{ ansible_hostname }},DNS:{{ ansible_fqdn | default(ansible_hostname) }},IP:{{ ansible_default_ipv4.address }},IP:{{ ansible_host }}"
chmod 644 {{ postgresql_data_directory }}/ssl/server.crt
chown postgres:postgres {{ postgresql_data_directory }}/ssl/server.crt
when: not ssl_cert.stat.exists
- name: Create CA certificate (copy server cert as CA for self-signed)
copy:
src: "{{ postgresql_data_directory }}/ssl/server.crt"
dest: "{{ postgresql_data_directory }}/ssl/ca.crt"
remote_src: yes
owner: postgres
group: postgres
mode: '0644'
- name: Set proper permissions on SSL files
file:
path: "{{ item.path }}"
owner: postgres
group: postgres
mode: "{{ item.mode }}"
loop:
- { path: "{{ postgresql_data_directory }}/ssl/server.key", mode: '0600' }
- { path: "{{ postgresql_data_directory }}/ssl/server.crt", mode: '0644' }
- { path: "{{ postgresql_data_directory }}/ssl/ca.crt", mode: '0644' }
- name: Create symlinks in PostgreSQL config directory
file:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
state: link
owner: postgres
group: postgres
loop:
- { src: "{{ postgresql_data_directory }}/ssl/server.crt", dest: "{{ postgresql_config_directory }}/server.crt" }
- { src: "{{ postgresql_data_directory }}/ssl/server.key", dest: "{{ postgresql_config_directory }}/server.key" }
- { src: "{{ postgresql_data_directory }}/ssl/ca.crt", dest: "{{ postgresql_config_directory }}/ca.crt" }

110
ansible/roles/postgresql/tasks/storage.yml Normal file
View file

@ -0,0 +1,110 @@
---
# Storage setup for PostgreSQL on NVMe drives
- name: Install storage utilities
apt:
name:
- mdadm
- xfsprogs
- nvme-cli
- smartmontools
state: present
update_cache: yes
- name: Check for NVMe devices
shell: nvme list -o json
register: nvme_devices
changed_when: false
failed_when: false
- name: Create partition on NVMe drive
parted:
device: /dev/nvme0n1
number: 1
state: present
label: gpt
part_start: 0%
part_end: 100%
fs_type: ext4
- name: Create RAID0 array with single drive (for future expansion)
command: |
mdadm --create {{ postgresql_raid_device }} \
--level=0 \
--raid-devices=1 \
--force \
/dev/nvme0n1p1
args:
creates: "{{ postgresql_raid_device }}"
when: postgresql_storage_type == "raid0"
- name: Wait for RAID array to be ready
wait_for:
path: "{{ postgresql_raid_device }}"
state: present
when: postgresql_storage_type == "raid0"
- name: Create filesystem on storage device
filesystem:
fstype: xfs
dev: "{{ postgresql_storage_device }}"
opts: "-L pgdata"
when: postgresql_storage_device is defined
- name: Get filesystem UUID
command: blkid -s UUID -o value {{ postgresql_storage_device }}
register: fs_uuid
changed_when: false
when: postgresql_storage_device is defined
- name: Create mount points
file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: '0755'
loop:
- "{{ postgresql_mount_point }}"
- "{{ postgresql_data_directory }}"
- "{{ postgresql_wal_directory }}"
- name: Mount PostgreSQL storage
mount:
path: "{{ postgresql_mount_point }}"
src: "UUID={{ fs_uuid.stdout }}"
fstype: xfs
opts: "noatime,nodiratime"
state: mounted
when: fs_uuid.stdout is defined
- name: Save RAID configuration
shell: mdadm --detail --scan >> /etc/mdadm/mdadm.conf
when: postgresql_storage_type == "raid0"
- name: Update initramfs
command: update-initramfs -u
when: postgresql_storage_type == "raid0"
- name: Set optimal I/O scheduler for NVMe
copy:
content: |
ACTION=="add|change", KERNEL=="nvme[0-9]n[0-9]", ATTR{queue/scheduler}="none"
ACTION=="add|change", KERNEL=="nvme[0-9]n[0-9]", ATTR{queue/nr_requests}="1024"
dest: /etc/udev/rules.d/60-nvme-scheduler.rules
mode: '0644'
notify: reload udev
- name: Configure sysctl for PostgreSQL performance
sysctl:
name: "{{ item.name }}"
value: "{{ item.value }}"
state: present
reload: yes
loop:
- { name: 'vm.dirty_background_ratio', value: '5' }
- { name: 'vm.dirty_ratio', value: '10' }
- { name: 'vm.swappiness', value: '10' }
- { name: 'vm.overcommit_memory', value: '2' }
- { name: 'vm.overcommit_ratio', value: '95' }
ignore_errors: yes

2
ansible/roles/postgresql/templates/30-postgresql.conf.j2
View file

@ -1,5 +1,5 @@
# Kernel parameters for PostgreSQL optimization
# For CM3588 with 32GB RAM and RAID0 SSDs
# For CM3588 with 16GB RAM and RAID0 SSDs
# Shared memory settings
kernel.shmmax = 17179869184 # 16GB

9
ansible/roles/postgresql/templates/pgbouncer-userlist.txt.j2
View file

@ -1,9 +1,12 @@
# PgBouncer userlist file
# Format: "username" "password"
# Format: "username" "password_or_scram_hash"
# Managed by Ansible
# Note: When using SCRAM-SHA-256, this contains the SCRAM hash from pg_authid
{% for user in postgresql_users %}
"{{ user.name }}" "{{ user.password }}"
{% for result in scram_hashes.results %}
{% if result.query_result[0].rolpassword %}
"{{ result.query_result[0].rolname }}" "{{ result.query_result[0].rolpassword }}"
{% endif %}
{% endfor %}
# PgBouncer stats user

19
ansible/roles/postgresql/templates/pgbouncer.ini.j2
View file

@ -2,9 +2,6 @@
# Connection string for aprsme database
aprsme_prod = host=127.0.0.1 port=5432 dbname=aprsme_prod
# Add a pooled connection for PgBouncer statistics
pgbouncer = host=127.0.0.1 port=5432 dbname=pgbouncer auth_user=pgbouncer
[pgbouncer]
# Connection settings
listen_addr = *
@ -61,6 +58,16 @@ tcp_keepintvl = 10
dns_max_ttl = 15
dns_nxdomain_ttl = 15
# TLS settings (optional)
# client_tls_sslmode = prefer
# client_tls_ca_file = /etc/ssl/certs/ca-certificates.crt
# TLS settings for client connections
client_tls_sslmode = allow
client_tls_cert_file = /etc/postgresql/{{ postgresql_version }}/main/server.crt
client_tls_key_file = /etc/postgresql/{{ postgresql_version }}/main/server.key
client_tls_ca_file = /etc/postgresql/{{ postgresql_version }}/main/ca.crt
client_tls_protocols = secure
client_tls_ciphers = HIGH:MEDIUM:+3DES:!aNULL
# TLS settings for server connections
server_tls_sslmode = require
server_tls_ca_file = /etc/postgresql/{{ postgresql_version }}/main/ca.crt
server_tls_protocols = secure
server_tls_ciphers = HIGH:MEDIUM:+3DES:!aNULL

14
ansible/roles/postgresql/templates/postgresql.conf.j2
View file

@ -100,4 +100,16 @@ max_locks_per_transaction = 128
#------------------------------------------------------------------------------
exit_on_error = off
restart_after_crash = on
restart_after_crash = on
#------------------------------------------------------------------------------
# SSL/TLS CONFIGURATION
#------------------------------------------------------------------------------
ssl = on
ssl_cert_file = '/etc/postgresql/{{ postgresql_version }}/main/server.crt'
ssl_key_file = '/etc/postgresql/{{ postgresql_version }}/main/server.key'
ssl_ca_file = '/etc/postgresql/{{ postgresql_version }}/main/ca.crt'
ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL'
ssl_prefer_server_ciphers = on
ssl_min_protocol_version = 'TLSv1.2'

57
ansible/roles/raid/tasks/main.yml
View file

@ -33,24 +33,24 @@
set_fact:
auto_detect_devices: "{{ raid_devices is not defined or raid_devices | length == 0 }}"
- name: Auto-detect two largest unused disks
- name: Auto-detect largest unused disks
when: auto_detect_devices
block:
- name: Find unused block devices
shell: |
lsblk -dpno NAME,SIZE,TYPE,MOUNTPOINT | grep -E "disk.*$" | grep -v -E "/(|/boot)" | sort -k2 -h -r | head -2 | awk '{print $1}'
lsblk -dpno NAME,SIZE,TYPE,MOUNTPOINT | grep -E "disk.*$" | grep -v -E "/(|/boot)" | sort -k2 -h -r | head -5 | awk '{print $1}'
register: detected_devices
changed_when: false
- name: Set raid_devices from detected devices
set_fact:
raid_devices: "{{ detected_devices.stdout_lines }}"
when: detected_devices.stdout_lines | length >= 2
when: detected_devices.stdout_lines | length >= 1
- name: Fail if not enough devices detected
- name: Fail if no devices detected
fail:
msg: "Could not detect at least 2 unused block devices for RAID. Please specify raid_devices manually."
when: detected_devices.stdout_lines | length < 2
msg: "Could not detect any unused block devices for RAID. Please specify raid_devices manually."
when: detected_devices.stdout_lines | length < 1
- name: Display RAID devices to be used
debug:
@ -70,40 +70,32 @@
- name: Zero superblocks on disks
command: mdadm --zero-superblock {{ item }}
loop:
- /dev/nvme0n1
- /dev/nvme1n1
loop: "{{ raid_devices }}"
when: raid_status.rc == 0
when: force_raid_rebuild | default(false)
- name: Create partition table on first NVMe drive
- name: Create partition table on RAID devices
parted:
device: /dev/nvme0n1
device: "{{ item }}"
number: 1
state: present
part_type: primary
part_start: 0%
part_end: 100%
label: gpt
loop: "{{ raid_devices }}"
when: raid_status.rc != 0
- name: Create partition table on second NVMe drive
parted:
device: /dev/nvme1n1
number: 1
state: present
part_type: primary
part_start: 0%
part_end: 100%
label: gpt
when: raid_status.rc != 0
- name: Build RAID device list with partitions
set_fact:
raid_partitions: "{{ raid_devices | map('regex_replace', '$', 'p1') | list }}"
- name: Create RAID0 array
- name: Create RAID0 array with dynamic device count
command: >
mdadm --create /dev/md0
--level=0
--raid-devices=2
/dev/nvme0n1p1 /dev/nvme1n1p1
--raid-devices={{ raid_devices | length }}
{{ raid_partitions | join(' ') }}
--force
when: raid_status.rc != 0
register: raid_create
@ -120,16 +112,29 @@
shell: mdadm --detail --scan >> /etc/mdadm/mdadm.conf
when: raid_create is changed
- name: Check if update-initramfs exists
stat:
path: /usr/sbin/update-initramfs
register: initramfs_exists
- name: Update initramfs
command: update-initramfs -u
when: raid_create is changed
when:
- raid_create is changed
- initramfs_exists.stat.exists
- name: Check if filesystem exists on RAID array
command: blkid /dev/md0
register: filesystem_check
failed_when: false
changed_when: false
- name: Create filesystem on RAID array
filesystem:
fstype: ext4
dev: /dev/md0
opts: "-E lazy_itable_init=0,lazy_journal_init=0"
when: raid_create is changed
when: filesystem_check.rc != 0
- name: Create mount point for PostgreSQL data
file:

37
ansible/scripts/get-postgres-password.sh Executable file
View file

@ -0,0 +1,37 @@
#!/bin/bash
# Script to retrieve PostgreSQL password from 1Password
set -e
# Default values
DEFAULT_USER="aprsme"
DEFAULT_HOST="CM3588"
DEFAULT_ACCOUNT="YOOATCZZSVGH7AD6VABUVPORLI"
# Parse arguments
USER="${1:-$DEFAULT_USER}"
HOST="${2:-$DEFAULT_HOST}"
ACCOUNT="${3:-$DEFAULT_ACCOUNT}"
# Item name in 1Password
ITEM_NAME="PostgreSQL $USER - $HOST"
echo "Retrieving password for: $ITEM_NAME"
# Get password
PASSWORD=$(op item get "$ITEM_NAME" --fields password --reveal --account "$ACCOUNT" 2>/dev/null)
if [ $? -eq 0 ]; then
echo "Password retrieved successfully"
echo
echo "Connection details:"
echo " User: $USER"
echo " Password: $PASSWORD"
echo
echo "URL-encoded password (for connection strings):"
echo "$PASSWORD" | python3 -c "import urllib.parse; print(urllib.parse.quote(input(), safe=''))"
else
echo "Error: Could not retrieve password from 1Password"
echo "Make sure the item '$ITEM_NAME' exists in your vault"
exit 1
fi

62
ansible/setup-postgres-storage.yml Normal file
View file

@ -0,0 +1,62 @@
---
- name: Set up PostgreSQL with NVMe storage
hosts: postgres_servers
become: yes
vars:
postgresql_storage_setup_enabled: true
postgresql_storage_type: raid0
# Override data directory to ensure it's on the RAID array
postgresql_data_directory: /mnt/pgdata/16/main
postgresql_wal_directory: /mnt/pgdata/wal
postgresql_backup_directory: /mnt/pgdata/backups
# Set the APRS database password (you should use ansible-vault for this)
vault_postgresql_aprsme_password: "mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21"
tasks:
- name: Check current storage status
command: lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINT
register: storage_status
changed_when: false
- name: Display current storage
debug:
var: storage_status.stdout_lines
- name: Check if PostgreSQL is already installed
stat:
path: /usr/lib/postgresql/16/bin/postgres
register: postgres_installed
- name: Stop PostgreSQL if it's running
systemd:
name: postgresql
state: stopped
when: postgres_installed.stat.exists
ignore_errors: yes
- name: Include PostgreSQL role
include_role:
name: postgresql
- name: Verify PostgreSQL is running
systemd:
name: postgresql
state: started
register: postgres_status
- name: Show PostgreSQL status
debug:
var: postgres_status
- name: Test database connection
postgresql_ping:
db: aprsme_prod
login_user: aprsme
login_password: "{{ vault_postgresql_aprsme_password }}"
become_user: postgres
register: db_ping
ignore_errors: yes
- name: Show database connection status
debug:
var: db_ping

2
ansible/site.yml
View file

@ -3,6 +3,6 @@
hosts: postgres
become: yes
roles:
- common
- base
- raid
- postgresql

36
clusters/aprs/AUTO-HEALING-GUIDE.md
View file

@ -114,7 +114,41 @@ kubectl rollout restart statefulset/aprs -n aprs
## Applying the Configuration
To apply all auto-healing configurations:
### ⚠️ WARNING: Data Persistence
The enhanced deployments use different volume configurations. Applying them directly will create new volumes and lose existing data.
### Safe Application (Recommended)
For existing deployments with data:
```bash
cd /Users/graham/dev/infra/clusters/aprs
./apply-auto-healing-safe.sh
```
This applies only non-destructive changes:
- PodDisruptionBudgets
- Cleanup CronJob
- Creates patch files for gradual application
Then apply patches one at a time:
```bash
# PostgreSQL enhancements
kubectl patch deployment postgis -n aprs --patch-file postgis-deployment-patch.yaml
kubectl rollout status deployment/postgis -n aprs
# PgBouncer enhancements
kubectl patch deployment pgbouncer -n aprs --patch-file pgbouncer-deployment-patch.yaml
kubectl rollout status deployment/pgbouncer -n aprs
# APRS enhancements
kubectl patch statefulset aprs -n aprs --patch-file aprs-statefulset-patch.yaml
kubectl rollout status statefulset/aprs -n aprs
```
### Fresh Installation
For new deployments without existing data:
```bash
cd /Users/graham/dev/infra/clusters/aprs

148
clusters/aprs/apply-auto-healing-safe.sh Executable file
View file

@ -0,0 +1,148 @@
#!/bin/bash
set -e
echo "Applying auto-healing configurations safely..."
# Only apply configurations that don't affect data persistence
echo "Applying PodDisruptionBudgets..."
kubectl apply -f postgis-pdb.yaml
echo "Applying PostgreSQL cleanup CronJob..."
kubectl apply -f postgres-cleanup-cronjob.yaml
echo "Creating enhanced configurations as patches (not applied automatically)..."
# Create patch files instead of replacing deployments
cat > postgis-deployment-patch.yaml << 'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgis
namespace: aprs
spec:
template:
spec:
containers:
- name: postgis
args:
- postgres
- -c
- max_connections=200
- -c
- idle_in_transaction_session_timeout=30s
- -c
- statement_timeout=300s
- -c
- shared_buffers=256MB
resources:
limits:
memory: "2Gi"
cpu: "1000m"
requests:
memory: "1Gi"
cpu: "500m"
livenessProbe:
exec:
command:
- /bin/sh
- -c
- pg_isready -U $POSTGRES_USER -d $POSTGRES_DB
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
exec:
command:
- /bin/sh
- -c
- pg_isready -U $POSTGRES_USER -d $POSTGRES_DB
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 2
EOF
cat > pgbouncer-deployment-patch.yaml << 'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
name: pgbouncer
namespace: aprs
spec:
template:
spec:
containers:
- name: pgbouncer
env:
- name: PGBOUNCER_SERVER_IDLE_TIMEOUT
value: "600"
- name: PGBOUNCER_SERVER_LIFETIME
value: "3600"
- name: PGBOUNCER_QUERY_TIMEOUT
value: "300"
- name: PGBOUNCER_QUERY_WAIT_TIMEOUT
value: "120"
- name: PGBOUNCER_CLIENT_IDLE_TIMEOUT
value: "300"
- name: PGBOUNCER_CLIENT_LOGIN_TIMEOUT
value: "60"
- name: PGBOUNCER_SERVER_CONNECT_TIMEOUT
value: "15"
- name: PGBOUNCER_SERVER_LOGIN_RETRY
value: "15"
resources:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "256Mi"
cpu: "200m"
EOF
cat > aprs-statefulset-patch.yaml << 'EOF'
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: aprs
namespace: aprs
spec:
template:
spec:
containers:
- name: aprs
env:
- name: POOL_SIZE
value: "5"
- name: DATABASE_POOL_TIMEOUT
value: "60000"
- name: DATABASE_CONNECT_TIMEOUT
value: "30000"
resources:
limits:
memory: "1Gi"
cpu: "1000m"
requests:
memory: "512Mi"
cpu: "500m"
startupProbe:
httpGet:
path: /health
port: 4000
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 10
failureThreshold: 12
EOF
echo ""
echo "Safe auto-healing configurations applied!"
echo ""
echo "To apply the enhanced configurations gradually:"
echo "1. Apply PostgreSQL patch: kubectl patch deployment postgis -n aprs --patch-file postgis-deployment-patch.yaml"
echo "2. Wait for rollout: kubectl rollout status deployment/postgis -n aprs"
echo "3. Apply PgBouncer patch: kubectl patch deployment pgbouncer -n aprs --patch-file pgbouncer-deployment-patch.yaml"
echo "4. Wait for rollout: kubectl rollout status deployment/pgbouncer -n aprs"
echo "5. Apply APRS patch: kubectl patch statefulset aprs -n aprs --patch-file aprs-statefulset-patch.yaml"
echo ""
echo "Monitor with: ./monitor-postgres-health.sh"

101
clusters/aprs/aprs-deployment.yaml
View file

@ -1,101 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: aprs
namespace: aprs
spec:
replicas: 2
selector:
matchLabels:
app: aprs
template:
metadata:
labels:
app: aprs
spec:
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: aprs
image: ghcr.io/aprsme/aprs.me:latest
imagePullPolicy: Always
ports:
- containerPort: 4000
- containerPort: 4369
name: epmd
- containerPort: 9100
name: erlang
- containerPort: 45892
name: gossip
protocol: UDP
env:
- name: PHX_HOST
value: "aprs.me"
- name: PORT
value: "4000"
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: aprs-secret
key: database-url
- name: SECRET_KEY_BASE
valueFrom:
secretKeyRef:
name: aprs-secret
key: secret-key-base
- name: PHX_SERVER
value: "true"
- name: MIX_ENV
value: "prod"
- name: APRS_CALLSIGN
value: "W5ISP-1"
- name: APRS_FILTER
value: "r/33/-96/1000000000000"
- name: APRS_PASSCODE
value: "15748"
- name: APRS_PORT
value: "10152"
- name: APRS_SERVER
value: "dallas.aprs2.net"
- name: PACKET_RETENTION_DAYS
value: "7"
- name: POOL_SIZE
value: "20"
- name: CLUSTER_ENABLED
value: "true"
- name: RELEASE_COOKIE
valueFrom:
secretKeyRef:
name: aprs-secret
key: erlang-cookie
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
resources:
limits:
memory: "512Mi"
cpu: "500m"
requests:
memory: "256Mi"
cpu: "250m"
livenessProbe:
httpGet:
path: /health
port: 4000
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
httpGet:
path: /health
port: 4000
initialDelaySeconds: 5
periodSeconds: 5

21
clusters/aprs/aprs-headless-service.yaml
View file

@ -1,21 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: aprs-headless
namespace: aprs
spec:
clusterIP: None
selector:
app: aprs
ports:
- name: epmd
port: 4369
targetPort: 4369
- name: erlang
port: 9100
targetPort: 9100
- name: gossip
port: 45892
targetPort: 45892
protocol: UDP
publishNotReadyAddresses: true

17
clusters/aprs/aprs-secret.yaml Normal file
View file

@ -0,0 +1,17 @@
apiVersion: v1
kind: Secret
metadata:
name: aprs-secret
namespace: aprs
type: Opaque
stringData:
database-password: "rku23IEO9Xq1lZTl+DETeAGRn/VxdzXzuidIrHn5Gi4="
# Direct connection to PostgreSQL on CM3588 (with SSL)
database-url: "ecto://aprsme:rku23IEO9Xq1lZTl%2BDETeAGRn%2FVxdzXzuidIrHn5Gi4%3D@10.0.19.220:5432/aprsme_prod"
# SSL configuration
database-ssl: "true"
database-ssl-verify: "false"
# Connection via PgBouncer on CM3588 (recommended)
database-url-pgbouncer: "ecto://aprsme:rku23IEO9Xq1lZTl%2BDETeAGRn%2FVxdzXzuidIrHn5Gi4%3D@10.0.19.220:6432/aprsme_prod"
erlang-cookie: "aprs-cluster-cookie-ksdf8934jfklsdjf983jfklsdf"
secret-key-base: "GJxzOUW3YXBAF7xB4mH5VcXY6BN1NYLnoJYadlZyiCxcRUGBw65w4Sco7+sNiW2t"

132
clusters/aprs/aprs-statefulset-enhanced.yaml
View file

@ -1,132 +0,0 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: aprs
namespace: aprs
spec:
serviceName: aprs-headless
replicas: 2
selector:
matchLabels:
app: aprs
updateStrategy:
type: RollingUpdate
podManagementPolicy: Parallel
template:
metadata:
labels:
app: aprs
spec:
terminationGracePeriodSeconds: 60
initContainers:
- name: wait-for-db
image: busybox:1.36.1
command: ['sh', '-c', 'until nc -z pgbouncer 5432; do echo waiting for database; sleep 2; done']
- name: wait-for-redis
image: busybox:1.36.1
command: ['sh', '-c', 'until nc -z redis 6379; do echo waiting for redis; sleep 2; done']
containers:
- name: aprs
image: ghcr.io/aprsme/aprs.me:latest
imagePullPolicy: Always
ports:
- containerPort: 4000
protocol: TCP
env:
- name: SECRET_KEY_BASE
valueFrom:
secretKeyRef:
name: aprs-secret
key: secret-key-base
- name: DATABASE_URL
value: "ecto://aprs:$(DATABASE_PASSWORD)@pgbouncer:5432/aprs"
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: aprs-secret
key: database-password
- name: PHX_HOST
value: "aprs.me"
- name: PHX_SERVER
value: "true"
- name: POOL_SIZE
value: "5" # Reduced from default 10 to prevent connection exhaustion
- name: ECTO_IPV6
value: "false"
- name: ERLANG_COOKIE
valueFrom:
secretKeyRef:
name: aprs-secret
key: erlang-cookie
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: REDIS_URL
value: "redis://redis.aprs.svc.cluster.local:6379"
- name: DRAIN_TIMEOUT_MS
value: "45000"
- name: SKIP_DB_CREATE
value: "true"
# Enhanced database connection settings
- name: DATABASE_POOL_TIMEOUT
value: "60000" # 60 seconds
- name: DATABASE_QUEUE_TARGET
value: "50"
- name: DATABASE_QUEUE_INTERVAL
value: "1000"
- name: DATABASE_CONNECT_TIMEOUT
value: "30000" # 30 seconds
- name: DATABASE_IDLE_TIMEOUT
value: "900000" # 15 minutes
# Memory management
- name: ERL_MAX_ETS_TABLES
value: "5000"
- name: ERLANG_SCHEDULER_BUSY_WAIT_THRESHOLD
value: "none"
resources:
limits:
memory: "1Gi" # Increased from 512Mi
cpu: "1000m" # Increased from 500m
requests:
memory: "512Mi" # Increased from 256Mi
cpu: "500m" # Increased from 250m
livenessProbe:
httpGet:
path: /health
port: 4000
initialDelaySeconds: 60 # Increased from 30
periodSeconds: 10
timeoutSeconds: 10
failureThreshold: 3
readinessProbe:
httpGet:
path: /health
port: 4000
initialDelaySeconds: 30 # Increased from 5
periodSeconds: 5
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 3
startupProbe:
httpGet:
path: /health
port: 4000
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 10
failureThreshold: 12 # 120 seconds total
lifecycle:
preStop:
exec:
command: ["/bin/sh", "-c", "sleep 15"]
imagePullSecrets:
- name: ghcr-pull-secret

37
clusters/aprs/migrate-job.yaml Normal file
View file

@ -0,0 +1,37 @@
apiVersion: batch/v1
kind: Job
metadata:
name: aprs-migrate
namespace: aprs
spec:
template:
spec:
restartPolicy: Never
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: migrate
image: ghcr.io/aprsme/aprs.me:latest
env:
- name: DATABASE_URL
value: "ecto://aprsme:rku23IEO9Xq1lZTl%2BDETeAGRn%2FVxdzXzuidIrHn5Gi4%3D@10.0.19.220:5432/aprsme_prod"
- name: SECRET_KEY_BASE
valueFrom:
secretKeyRef:
name: aprs-secret
key: secret-key-base
- name: MIX_ENV
value: "prod"
- name: POOL_SIZE
value: "2"
- name: PHX_HOST
value: "aprs.me"
- name: PORT
value: "4000"
- name: SKIP_CLUSTERING
value: "true"
- name: CLUSTER_ENABLED
value: "false"
- name: REDIS_URL
value: "redis://redis.aprs.svc.cluster.local:6379"
command: ["/app/bin/migrate"]

72
clusters/aprs/migrate-to-external-db.sh Executable file
View file

@ -0,0 +1,72 @@
#!/bin/bash
set -e
echo "Migrating APRS to external PostgreSQL database"
echo "=============================================="
echo ""
# Test connection to external database first
echo "Testing connection to external PostgreSQL..."
PGPASSWORD="mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21" psql -h 10.0.19.220 -p 6432 -U aprsme -d aprsme_prod -c "SELECT version();" > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "✅ Successfully connected to external database"
else
echo "❌ Failed to connect to external database at 10.0.19.220:6432"
echo "Please ensure the PostgreSQL server is running and accessible"
exit 1
fi
echo ""
echo "Step 1: Updating database connection secret..."
kubectl apply -f aprs-secret-external-db.yaml
echo ""
echo "Step 2: Restarting APRS pods to use new database..."
kubectl rollout restart statefulset/aprs -n aprs
kubectl rollout status statefulset/aprs -n aprs --timeout=300s
echo ""
echo "Step 3: Verifying pods are running with new database..."
sleep 10
kubectl get pods -n aprs -l app=aprs
echo ""
echo "Step 4: Checking application health..."
for i in {1..5}; do
POD=$(kubectl get pods -n aprs -l app=aprs -o jsonpath='{.items[0].metadata.name}')
if kubectl exec -n aprs $POD -- curl -s http://localhost:4000/health > /dev/null; then
echo "✅ Application is healthy"
break
else
echo "Waiting for application to be ready... ($i/5)"
sleep 10
fi
done
echo ""
echo "Step 5: Removing local PostgreSQL resources..."
read -p "Are you sure you want to remove the local PostgreSQL deployment? (y/n): " confirm
if [ "$confirm" = "y" ]; then
echo "Removing PostgreSQL resources..."
kubectl delete deployment postgis -n aprs --ignore-not-found=true
kubectl delete service postgis -n aprs --ignore-not-found=true
kubectl delete deployment pgbouncer -n aprs --ignore-not-found=true
kubectl delete service pgbouncer -n aprs --ignore-not-found=true
kubectl delete cronjob postgres-cleanup -n aprs --ignore-not-found=true
kubectl delete job -l app=postgres-cleanup -n aprs --ignore-not-found=true
echo ""
echo "PostgreSQL PVC will be retained for backup purposes"
echo "To delete it later: kubectl delete pvc postgis-pvc-ceph -n aprs"
fi
echo ""
echo "Migration complete!"
echo ""
echo "Summary:"
echo "- APRS is now using external PostgreSQL at 10.0.19.220"
echo "- Connection is via PgBouncer on port 6432"
echo "- Local PostgreSQL resources have been removed"
echo ""
echo "To verify:"
echo "kubectl logs -n aprs -l app=aprs --tail=50"

244
clusters/aprs/monitoring/MONITORING-GUIDE.md Normal file
View file

@ -0,0 +1,244 @@
# Monitoring and Observability Guide
This guide explains the comprehensive monitoring stack for the APRS k3s cluster.
## Overview
The monitoring stack consists of:
- **Prometheus**: Metrics collection and storage
- **Grafana**: Visualization and dashboards
- **AlertManager**: Alert routing and notifications
- **Exporters**: PostgreSQL, PgBouncer, Redis, and application metrics
## Architecture
```
┌─────────────┐ ┌─────────────┐ ┌─────────────┐
│ APRS App │ │ PostgreSQL │ │ PgBouncer │
│ /metrics │ │ Exporter │ │ Exporter │
└──────┬──────┘ └──────┬──────┘ └──────┬──────┘
│ │ │
└───────────────────┴───────────────────┘
│
┌──────▼──────┐
│ Prometheus │
│ (scraping) │
└──────┬──────┘
│
┌──────────┴──────────┐
│ │
┌──────▼──────┐ ┌──────▼──────┐
│ Grafana │ │ AlertManager│
│ (dashboards)│ │ (alerts) │
└─────────────┘ └─────────────┘
```
## Installation
1. **Install the monitoring stack**:
```bash
cd /path/to/infra/clusters/aprs/monitoring
./install-monitoring.sh
```
2. **Apply the enhanced deployments** (if not already done):
```bash
cd /path/to/infra/clusters/aprs
./apply-auto-healing.sh
```
3. **Update Redis deployment** to include metrics:
```bash
kubectl apply -f redis-deployment-with-metrics.yaml
```
## Access
### Grafana
```bash
kubectl port-forward -n monitoring svc/kube-prometheus-stack-grafana 3000:80
```
Access at: http://localhost:3000
- Username: `admin`
- Password: `changeme` (change this!)
### Prometheus
```bash
kubectl port-forward -n monitoring svc/kube-prometheus-stack-prometheus 9090:9090
```
Access at: http://localhost:9090
### AlertManager
```bash
kubectl port-forward -n monitoring svc/kube-prometheus-stack-alertmanager 9093:9093
```
Access at: http://localhost:9093
## Metrics Available
### PostgreSQL Metrics
- `pg_connection_stats_total_connections` - Current total connections
- `pg_connection_stats_max_connections` - Maximum allowed connections
- `pg_connections_count` - Connections by state (active, idle, etc.)
- `pg_long_running_queries_count` - Queries running > 5 minutes
- `pg_database_size_bytes` - Database size
- Standard `pg_*` metrics from postgres_exporter
### PgBouncer Metrics
- `pgbouncer_pools_server_active_connections` - Active server connections
- `pgbouncer_pools_client_active_connections` - Active client connections
- `pgbouncer_pools_pool_size` - Configured pool size
- `pgbouncer_stats_*` - Various PgBouncer statistics
### Application Metrics
- `phoenix_endpoint_stop_duration_millisecond` - HTTP request duration
- `aprsme_repo_query_*` - Database query metrics
- `aprsme_packet_pipeline_*` - Packet processing metrics
- `aprsme_spatial_pubsub_*` - Real-time broadcast metrics
- `vm_memory_total` - Erlang VM memory usage
### Kubernetes Metrics
- `container_memory_working_set_bytes` - Container memory usage
- `container_cpu_usage_seconds_total` - Container CPU usage
- `kube_pod_container_status_restarts_total` - Pod restart count
## Dashboards
### Pre-installed Dashboards
1. **PostgreSQL Dashboard** (ID: 9628) - Comprehensive PostgreSQL metrics
2. **PgBouncer Dashboard** (ID: 15984) - Connection pooling metrics
3. **Kubernetes Cluster** (ID: 7249) - Cluster overview
### Custom Dashboards
1. **APRS Application Overview** - Application-specific metrics
2. **PostgreSQL Connection Analysis** - Detailed connection monitoring
## Alerts
### Critical Alerts
- **PostgreSQLConnectionsCritical**: > 95% connections used
- **PostgreSQLDown**: PostgreSQL not responding
- **PgBouncerDown**: PgBouncer not responding
- **APRSPodCrashLooping**: Pod restarting frequently
### Warning Alerts
- **PostgreSQLConnectionsHigh**: > 80% connections used
- **PostgreSQLIdleTransactionsHigh**: > 10 idle transactions
- **PostgreSQLLongRunningQueries**: Queries > 5 minutes
- **APRSHighMemoryUsage**: > 90% memory limit
- **APRSHighDatabaseConnectionQueueTime**: Queue time > 1 second
## Troubleshooting
### Check Metrics Collection
```bash
# Verify all exporters are running
kubectl get pods -n aprs | grep exporter
# Check ServiceMonitors
kubectl get servicemonitor -n aprs
# Verify Prometheus targets
kubectl port-forward -n monitoring svc/kube-prometheus-stack-prometheus 9090:9090
# Visit http://localhost:9090/targets
```
### Common Issues
1. **Metrics not appearing**:
- Check if exporter pods are running
- Verify ServiceMonitor labels match Prometheus selector
- Check Prometheus targets page for errors
2. **Dashboards empty**:
- Ensure data source is set to "Prometheus"
- Check metric names in queries
- Verify time range selection
3. **Alerts not firing**:
- Check PrometheusRule is loaded
- Verify alert expressions in Prometheus
- Check AlertManager configuration
## Adding New Metrics
### Application Metrics
Add to `/lib/aprsme_web/telemetry.ex`:
```elixir
counter("your_metric_name",
description: "Description",
tags: [:tag1, :tag2]
)
```
### Custom Queries
Add to postgres-exporter ConfigMap:
```yaml
your_custom_metric:
query: "SELECT ..."
metrics:
- column_name:
usage: "GAUGE"
description: "Description"
```
## Performance Tuning
### Prometheus Storage
Current settings:
- Retention: 30 days
- Storage: 20Gi
Adjust in `kube-prometheus-stack-values.yaml` if needed.
### Scrape Intervals
Default: 30 seconds
Adjust in ServiceMonitors for specific endpoints.
### Resource Usage
Monitor Prometheus and Grafana resource usage:
```bash
kubectl top pods -n monitoring
```
## Backup and Recovery
### Grafana Dashboards
Export dashboards as JSON from Grafana UI for backup.
### Prometheus Data
Data is stored in PVC. For backup:
```bash
kubectl exec -n monitoring prometheus-kube-prometheus-stack-prometheus-0 -- \
tar czf /tmp/prometheus-backup.tar.gz /prometheus
kubectl cp monitoring/prometheus-kube-prometheus-stack-prometheus-0:/tmp/prometheus-backup.tar.gz \
./prometheus-backup.tar.gz
```
## Integration with CI/CD
Add to your deployment pipeline:
```bash
# Check if metrics endpoint is healthy
curl -f http://your-app/metrics || exit 1
# Query Prometheus for deployment validation
curl -G http://prometheus:9090/api/v1/query \
--data-urlencode 'query=up{job="aprs-app"}' | jq '.data.result[0].value[1]'
```
## Security Considerations
1. **Change default passwords** in Grafana
2. **Configure network policies** to restrict access
3. **Enable TLS** for external access
4. **Set up authentication** for Prometheus and AlertManager
5. **Limit metric cardinality** to prevent resource exhaustion
## Next Steps
1. Configure AlertManager to send notifications (email, Slack, etc.)
2. Set up long-term storage with Thanos or Cortex
3. Implement SLO/SLI dashboards
4. Add distributed tracing with Jaeger
5. Set up log aggregation with Loki

114
clusters/aprs/monitoring/aprs-dashboard-configmap.yaml Normal file
View file

@ -0,0 +1,114 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: aprs-dashboard
namespace: monitoring
labels:
grafana_dashboard: "1"
app.kubernetes.io/name: grafana
app.kubernetes.io/instance: kube-prometheus-stack
data:
aprs-dashboard.json: |
{
"dashboard": {
"id": null,
"title": "APRS.me Application Metrics",
"tags": ["aprs", "amateur-radio", "real-time"],
"style": "dark",
"timezone": "browser",
"refresh": "30s",
"time": {
"from": "now-1h",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "APRS-IS Connection Status",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
"targets": [
{
"expr": "up{job=\"aprs-metrics\"}",
"legendFormat": "Service Up"
}
],
"fieldConfig": {
"defaults": {
"color": {"mode": "thresholds"},
"thresholds": {
"steps": [
{"color": "red", "value": 0},
{"color": "green", "value": 1}
]
}
}
}
},
{
"id": 2,
"title": "Packet Processing Rate",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 6, "y": 0},
"targets": [
{
"expr": "rate(aprsme_packet_pipeline_batch_count_total[5m])",
"legendFormat": "Packets/sec"
},
{
"expr": "rate(aprsme_packet_pipeline_batch_success_total[5m])",
"legendFormat": "Successful/sec"
},
{
"expr": "rate(aprsme_packet_pipeline_batch_error_total[5m])",
"legendFormat": "Errors/sec"
}
]
},
{
"id": 3,
"title": "Connected Clients",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
"targets": [
{
"expr": "aprsme_spatial_pubsub_clients_count",
"legendFormat": "Active Clients"
}
]
},
{
"id": 4,
"title": "Database Performance",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
"targets": [
{
"expr": "rate(aprsme_repo_query_total_time_sum[5m]) / rate(aprsme_repo_query_total_time_count[5m])",
"legendFormat": "Avg Query Time (ms)"
},
{
"expr": "rate(aprsme_repo_query_queue_time_sum[5m]) / rate(aprsme_repo_query_queue_time_count[5m])",
"legendFormat": "Avg Queue Time (ms)"
}
]
},
{
"id": 5,
"title": "Spatial Broadcast Metrics",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
"targets": [
{
"expr": "rate(aprsme_spatial_pubsub_broadcasts_total[5m])",
"legendFormat": "Total Broadcasts/sec"
},
{
"expr": "rate(aprsme_spatial_pubsub_broadcasts_filtered[5m])",
"legendFormat": "Filtered Broadcasts/sec"
}
]
}
]
}
}

332
clusters/aprs/monitoring/aprs-dashboard-enhanced.yaml Normal file
View file

@ -0,0 +1,332 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: aprs-dashboard-enhanced
namespace: monitoring
labels:
grafana_dashboard: "1"
app.kubernetes.io/name: grafana
app.kubernetes.io/instance: kube-prometheus-stack
data:
aprs-dashboard-enhanced.json: |
{
"dashboard": {
"id": null,
"title": "APRS.me Complete Metrics Dashboard",
"tags": ["aprs", "postgresql", "database", "real-time"],
"style": "dark",
"timezone": "browser",
"refresh": "10s",
"time": {
"from": "now-1h",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "Database Connection Pool",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"targets": [
{
"expr": "aprsme_repo_pool_size{job=\"aprs-metrics\"}",
"legendFormat": "Pool Size"
},
{
"expr": "aprsme_repo_pool_busy{job=\"aprs-metrics\"}",
"legendFormat": "Busy Connections"
},
{
"expr": "aprsme_repo_pool_idle{job=\"aprs-metrics\"}",
"legendFormat": "Idle Connections"
},
{
"expr": "aprsme_repo_pool_queue_length{job=\"aprs-metrics\"}",
"legendFormat": "Queue Length"
}
],
"yaxes": [
{
"format": "short",
"label": "Connections"
},
{
"format": "short"
}
]
},
{
"id": 2,
"title": "PostgreSQL Connections",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
"targets": [
{
"expr": "aprsme_postgres_connections_total{job=\"aprs-metrics\"}",
"legendFormat": "Total"
},
{
"expr": "aprsme_postgres_connections_active{job=\"aprs-metrics\"}",
"legendFormat": "Active"
},
{
"expr": "aprsme_postgres_connections_idle{job=\"aprs-metrics\"}",
"legendFormat": "Idle"
},
{
"expr": "aprsme_postgres_connections_idle_in_transaction{job=\"aprs-metrics\"}",
"legendFormat": "Idle in Transaction"
},
{
"expr": "aprsme_postgres_connections_waiting{job=\"aprs-metrics\"}",
"legendFormat": "Waiting"
}
],
"yaxes": [
{
"format": "short",
"label": "Connections"
},
{
"format": "short"
}
]
},
{
"id": 3,
"title": "Database Size",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 8},
"targets": [
{
"expr": "aprsme_postgres_database_size_bytes{job=\"aprs-metrics\"} / 1024 / 1024 / 1024",
"legendFormat": "Database Size"
}
],
"fieldConfig": {
"defaults": {
"unit": "GB",
"decimals": 2
}
}
},
{
"id": 4,
"title": "Packets Table Size",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 8},
"targets": [
{
"expr": "aprsme_postgres_packets_table_table_size_bytes{job=\"aprs-metrics\"} / 1024 / 1024",
"legendFormat": "Table Size"
}
],
"fieldConfig": {
"defaults": {
"unit": "MB",
"decimals": 2
}
}
},
{
"id": 5,
"title": "Packets Table Rows",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 12, "y": 8},
"targets": [
{
"expr": "aprsme_postgres_packets_table_live_tuples{job=\"aprs-metrics\"}",
"legendFormat": "Live Rows"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
}
}
},
{
"id": 6,
"title": "Dead Tuples Ratio",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 8},
"targets": [
{
"expr": "(aprsme_postgres_packets_table_dead_tuples{job=\"aprs-metrics\"} / (aprsme_postgres_packets_table_live_tuples{job=\"aprs-metrics\"} + aprsme_postgres_packets_table_dead_tuples{job=\"aprs-metrics\"})) * 100",
"legendFormat": "Dead Tuple %"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"decimals": 2,
"color": {"mode": "thresholds"},
"thresholds": {
"steps": [
{"color": "green", "value": 0},
{"color": "yellow", "value": 10},
{"color": "red", "value": 20}
]
}
}
}
},
{
"id": 7,
"title": "Query Performance",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
"targets": [
{
"expr": "aprsme_postgres_query_stats_avg_time_ms{job=\"aprs-metrics\"}",
"legendFormat": "Average Query Time"
},
{
"expr": "aprsme_postgres_query_stats_max_time_ms{job=\"aprs-metrics\"}",
"legendFormat": "Max Query Time"
}
],
"yaxes": [
{
"format": "ms",
"label": "Query Time"
},
{
"format": "short"
}
]
},
{
"id": 8,
"title": "Database Operations",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
"targets": [
{
"expr": "rate(aprsme_postgres_packets_table_total_inserts{job=\"aprs-metrics\"}[5m])",
"legendFormat": "Insert Rate"
},
{
"expr": "rate(aprsme_postgres_packets_table_total_updates{job=\"aprs-metrics\"}[5m])",
"legendFormat": "Update Rate"
},
{
"expr": "rate(aprsme_postgres_packets_table_total_deletes{job=\"aprs-metrics\"}[5m])",
"legendFormat": "Delete Rate"
}
],
"yaxes": [
{
"format": "ops",
"label": "Operations/sec"
},
{
"format": "short"
}
]
},
{
"id": 9,
"title": "Ecto Query Times",
"type": "heatmap",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 20},
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(aprsme_repo_query_total_time_bucket{job=\"aprs-metrics\"}[5m])) by (le))",
"format": "heatmap",
"legendFormat": "{{le}}"
}
],
"dataFormat": "tsbuckets",
"yAxis": {
"format": "ms",
"decimals": 0
}
},
{
"id": 10,
"title": "Connection Pool Queue Time",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 20},
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(aprsme_repo_query_queue_time_bucket{job=\"aprs-metrics\"}[5m])) by (le))",
"legendFormat": "95th percentile"
},
{
"expr": "histogram_quantile(0.99, sum(rate(aprsme_repo_query_queue_time_bucket{job=\"aprs-metrics\"}[5m])) by (le))",
"legendFormat": "99th percentile"
}
],
"yaxes": [
{
"format": "ms",
"label": "Queue Time"
},
{
"format": "short"
}
]
},
{
"id": 11,
"title": "APRS Packet Processing",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 28},
"targets": [
{
"expr": "rate(aprsme_packet_pipeline_batch_count{job=\"aprs-metrics\"}[5m])",
"legendFormat": "Packets/sec"
},
{
"expr": "rate(aprsme_packet_pipeline_batch_success{job=\"aprs-metrics\"}[5m])",
"legendFormat": "Success/sec"
},
{
"expr": "rate(aprsme_packet_pipeline_batch_error{job=\"aprs-metrics\"}[5m])",
"legendFormat": "Errors/sec"
}
],
"yaxes": [
{
"format": "short",
"label": "Packets/sec"
},
{
"format": "short"
}
]
},
{
"id": 12,
"title": "Spatial Broadcasting Efficiency",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 28},
"targets": [
{
"expr": "aprsme_spatial_pubsub_efficiency_ratio{job=\"aprs-metrics\"}",
"legendFormat": "Efficiency %"
},
{
"expr": "rate(aprsme_spatial_pubsub_efficiency_saved_broadcasts{job=\"aprs-metrics\"}[5m])",
"legendFormat": "Saved Broadcasts/sec",
"yaxis": 2
}
],
"yaxes": [
{
"format": "percent",
"label": "Efficiency"
},
{
"format": "short",
"label": "Saved/sec"
}
]
}
],
"schemaVersion": 27,
"version": 1
}
}

164
clusters/aprs/monitoring/aprs-dashboard.json Normal file
View file

@ -0,0 +1,164 @@
{
"dashboard": {
"id": null,
"title": "APRS.me Application Metrics",
"tags": ["aprs", "amateur-radio", "real-time"],
"style": "dark",
"timezone": "browser",
"refresh": "30s",
"time": {
"from": "now-1h",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "APRS-IS Connection Status",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
"targets": [
{
"expr": "up{job=\"aprs-metrics\"}",
"legendFormat": "Service Up"
}
],
"fieldConfig": {
"defaults": {
"color": {"mode": "thresholds"},
"thresholds": {
"steps": [
{"color": "red", "value": 0},
{"color": "green", "value": 1}
]
}
}
}
},
{
"id": 2,
"title": "Packet Processing Rate",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 6, "y": 0},
"targets": [
{
"expr": "rate(aprsme_packet_pipeline_batch_count_total[5m])",
"legendFormat": "Packets/sec"
},
{
"expr": "rate(aprsme_packet_pipeline_batch_success_total[5m])",
"legendFormat": "Successful/sec"
},
{
"expr": "rate(aprsme_packet_pipeline_batch_error_total[5m])",
"legendFormat": "Errors/sec"
}
],
"yAxes": [
{"label": "Packets/second", "min": 0}
]
},
{
"id": 3,
"title": "Connected Clients",
"type": "stat",
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
"targets": [
{
"expr": "aprsme_spatial_pubsub_clients_count",
"legendFormat": "Active Clients"
}
]
},
{
"id": 4,
"title": "Database Performance",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
"targets": [
{
"expr": "rate(aprsme_repo_query_total_time_sum[5m]) / rate(aprsme_repo_query_total_time_count[5m])",
"legendFormat": "Avg Query Time"
},
{
"expr": "rate(aprsme_repo_query_queue_time_sum[5m]) / rate(aprsme_repo_query_queue_time_count[5m])",
"legendFormat": "Avg Queue Time"
}
],
"yAxes": [
{"label": "Milliseconds", "min": 0}
]
},
{
"id": 5,
"title": "Spatial Broadcast Efficiency",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
"targets": [
{
"expr": "aprsme_spatial_pubsub_efficiency_ratio",
"legendFormat": "Efficiency %"
},
{
"expr": "rate(aprsme_spatial_pubsub_broadcasts_total[5m])",
"legendFormat": "Total Broadcasts/sec"
},
{
"expr": "rate(aprsme_spatial_pubsub_broadcasts_filtered[5m])",
"legendFormat": "Filtered Broadcasts/sec"
}
]
},
{
"id": 6,
"title": "HTTP Request Performance",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 16},
"targets": [
{
"expr": "rate(phoenix_router_dispatch_stop_duration_sum[5m]) / rate(phoenix_router_dispatch_stop_duration_count[5m])",
"legendFormat": "Avg Response Time"
},
{
"expr": "rate(phoenix_router_dispatch_stop_duration_count[5m])",
"legendFormat": "Requests/sec"
}
]
},
{
"id": 7,
"title": "Memory Usage",
"type": "graph",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 16},
"targets": [
{
"expr": "vm_memory_total",
"legendFormat": "Total Memory"
}
],
"yAxes": [
{"label": "Bytes", "min": 0}
]
},
{
"id": 8,
"title": "Packet Pipeline Batch Duration",
"type": "graph",
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 24},
"targets": [
{
"expr": "rate(aprsme_packet_pipeline_batch_duration_ms_sum[5m]) / rate(aprsme_packet_pipeline_batch_duration_ms_count[5m])",
"legendFormat": "Avg Batch Duration (ms)"
},
{
"expr": "histogram_quantile(0.95, rate(aprsme_packet_pipeline_batch_duration_ms_bucket[5m]))",
"legendFormat": "95th Percentile"
},
{
"expr": "histogram_quantile(0.99, rate(aprsme_packet_pipeline_batch_duration_ms_bucket[5m]))",
"legendFormat": "99th Percentile"
}
]
}
]
}
}

17
clusters/aprs/monitoring/aprs-metrics-service.yaml Normal file
View file

@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: aprs-prometheus-metrics
namespace: aprs
labels:
app: aprs
component: metrics
spec:
selector:
app: aprs
ports:
- name: metrics
port: 9568
targetPort: 9568
protocol: TCP
type: ClusterIP

37
clusters/aprs/monitoring/aprs-servicemonitor.yaml Normal file
View file

@ -0,0 +1,37 @@
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: aprs-metrics
namespace: monitoring
labels:
app: aprs
release: kube-prometheus-stack
spec:
selector:
matchLabels:
app: aprs
namespaceSelector:
matchNames:
- aprs
endpoints:
- port: metrics
path: /metrics
interval: 30s
scrapeTimeout: 10s
---
apiVersion: v1
kind: Service
metadata:
name: aprs-metrics-service
namespace: aprs
labels:
app: aprs
spec:
selector:
app: aprs
ports:
- name: http
port: 9568
targetPort: 9568
protocol: TCP
type: ClusterIP

49
clusters/aprs/monitoring/check-monitoring-status.sh Executable file
View file

@ -0,0 +1,49 @@
#!/bin/bash
echo "=== Monitoring Stack Status ==="
echo ""
# Check Prometheus
echo "Prometheus:"
kubectl get pods -n monitoring -l app.kubernetes.io/name=prometheus -o wide
echo ""
# Check Grafana
echo "Grafana:"
kubectl get pods -n monitoring -l app.kubernetes.io/name=grafana -o wide
echo ""
# Check AlertManager
echo "AlertManager:"
kubectl get pods -n monitoring -l app.kubernetes.io/name=alertmanager -o wide
echo ""
# Check Exporters
echo "Exporters in APRS namespace:"
kubectl get pods -n aprs | grep -E "(exporter|NAME)"
echo ""
# Check ServiceMonitors
echo "ServiceMonitors:"
kubectl get servicemonitor -A | grep -E "(aprs|NAME)"
echo ""
# Check PrometheusRules
echo "PrometheusRules:"
kubectl get prometheusrule -A | grep -E "(aprs|NAME)"
echo ""
# Check if metrics endpoints are accessible
echo "Checking metrics endpoints:"
echo -n "PostgreSQL Exporter: "
kubectl exec -n aprs deployment/postgres-exporter -- wget -q -O- http://localhost:9187/metrics | head -1 2>/dev/null && echo "OK" || echo "FAILED"
echo -n "PgBouncer Exporter: "
kubectl exec -n aprs deployment/pgbouncer-exporter -- wget -q -O- http://localhost:9127/metrics | head -1 2>/dev/null && echo "OK" || echo "FAILED"
echo -n "APRS Application: "
kubectl exec -n aprs aprs-0 -- wget -q -O- http://localhost:4000/metrics | head -1 2>/dev/null && echo "OK" || echo "FAILED"
echo ""
echo "To access Grafana: kubectl port-forward -n monitoring svc/kube-prometheus-stack-grafana 3000:80"
echo "To access Prometheus: kubectl port-forward -n monitoring svc/kube-prometheus-stack-prometheus 9090:9090"

181
clusters/aprs/monitoring/grafana-dashboards.yaml Normal file
View file

@ -0,0 +1,181 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: aprs-custom-dashboards
namespace: monitoring
labels:
grafana_dashboard: "1"
data:
aprs-overview.json: |
{
"dashboard": {
"title": "APRS Application Overview",
"uid": "aprs-overview",
"timezone": "browser",
"panels": [
{
"title": "Database Connection Usage",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"type": "graph",
"targets": [
{
"expr": "pg_connection_stats_total_connections",
"legendFormat": "Current Connections"
},
{
"expr": "pg_connection_stats_max_connections",
"legendFormat": "Max Connections"
}
]
},
{
"title": "Connection States",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
"type": "piechart",
"targets": [
{
"expr": "pg_connections_count",
"legendFormat": "{{ state }}"
}
]
},
{
"title": "Packet Processing Rate",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
"type": "graph",
"targets": [
{
"expr": "rate(aprsme_packet_pipeline_batch_success[5m])",
"legendFormat": "Successful Batches/sec"
},
{
"expr": "rate(aprsme_packet_pipeline_batch_error[5m])",
"legendFormat": "Failed Batches/sec"
}
]
},
{
"title": "Memory Usage",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
"type": "graph",
"targets": [
{
"expr": "container_memory_working_set_bytes{namespace=\"aprs\", pod=~\"aprs-.*\", container=\"aprs\"}",
"legendFormat": "{{ pod }}"
}
]
},
{
"title": "HTTP Request Duration (95th percentile)",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 16},
"type": "graph",
"targets": [
{
"expr": "histogram_quantile(0.95, phoenix_endpoint_stop_duration_millisecond)",
"legendFormat": "95th percentile"
}
]
},
{
"title": "Database Query Time",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 16},
"type": "graph",
"targets": [
{
"expr": "histogram_quantile(0.95, aprsme_repo_query_queue_time_millisecond)",
"legendFormat": "Queue Time (95th)"
},
{
"expr": "histogram_quantile(0.95, aprsme_repo_query_query_time_millisecond)",
"legendFormat": "Query Time (95th)"
}
]
}
]
}
}
postgres-connections.json: |
{
"dashboard": {
"title": "PostgreSQL Connection Analysis",
"uid": "postgres-connections",
"timezone": "browser",
"panels": [
{
"title": "Connection Usage Over Time",
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 0},
"type": "graph",
"targets": [
{
"expr": "(pg_connection_stats_total_connections / pg_connection_stats_max_connections) * 100",
"legendFormat": "Connection Usage %"
}
],
"alert": {
"conditions": [
{
"evaluator": {
"params": [80],
"type": "gt"
},
"query": {
"params": ["A", "5m", "now"]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
]
}
},
{
"title": "Connection States Breakdown",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
"type": "bargauge",
"targets": [
{
"expr": "pg_connection_stats_active_connections",
"legendFormat": "Active"
},
{
"expr": "pg_connection_stats_idle_connections",
"legendFormat": "Idle"
},
{
"expr": "pg_connection_stats_idle_in_transaction",
"legendFormat": "Idle in Transaction"
}
]
},
{
"title": "Long Running Queries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
"type": "stat",
"targets": [
{
"expr": "pg_long_running_queries_count",
"legendFormat": "Queries > 5 min"
}
]
},
{
"title": "PgBouncer Pool Usage",
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 16},
"type": "graph",
"targets": [
{
"expr": "pgbouncer_pools_server_active_connections",
"legendFormat": "Active Server Connections - {{ database }}"
},
{
"expr": "pgbouncer_pools_client_active_connections",
"legendFormat": "Active Client Connections - {{ database }}"
}
]
}
]
}
}

17
clusters/aprs/monitoring/grafana-loadbalancer.yaml Normal file
View file

@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: grafana-loadbalancer
namespace: monitoring
annotations:
metallb.universe.tf/loadBalancerIPs: 10.0.19.223
spec:
type: LoadBalancer
ports:
- port: 80
targetPort: 3000
protocol: TCP
name: http
selector:
app.kubernetes.io/name: grafana
app.kubernetes.io/instance: kube-prometheus-stack

41
clusters/aprs/monitoring/install-monitoring.sh Executable file
View file

@ -0,0 +1,41 @@
#!/bin/bash
set -e
echo "Installing monitoring stack for k3s..."
# Create namespace
echo "Creating monitoring namespace..."
kubectl apply -f namespace.yaml
# Add prometheus-community helm repo
echo "Adding Helm repositories..."
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo add stable https://charts.helm.sh/stable
helm repo update
# Install kube-prometheus-stack
echo "Installing kube-prometheus-stack..."
helm upgrade --install \
kube-prometheus-stack \
prometheus-community/kube-prometheus-stack \
--namespace monitoring \
--values kube-prometheus-stack-values.yaml \
--wait \
--timeout 10m
# Apply additional configurations
echo "Applying additional monitoring configurations..."
kubectl apply -f postgres-exporter.yaml
kubectl apply -f pgbouncer-exporter.yaml
kubectl apply -f servicemonitors.yaml
kubectl apply -f prometheusrules.yaml
echo "Monitoring stack installed successfully!"
echo ""
echo "Access Grafana at: http://localhost:3000 (after port-forward)"
echo " kubectl port-forward -n monitoring svc/kube-prometheus-stack-grafana 3000:80"
echo ""
echo "Access Prometheus at: http://localhost:9090 (after port-forward)"
echo " kubectl port-forward -n monitoring svc/kube-prometheus-stack-prometheus 9090:9090"
echo ""
echo "Default Grafana credentials: admin / changeme"

150
clusters/aprs/monitoring/kube-prometheus-stack-values.yaml Normal file
View file

@ -0,0 +1,150 @@
# Optimized values for k3s deployment
prometheus:
prometheusSpec:
# Resource limits for edge deployment
resources:
requests:
memory: 400Mi
cpu: 100m
limits:
memory: 2Gi
cpu: 1000m
# Retention and storage
retention: 30d
retentionSize: "10GB"
# Storage configuration
storageSpec:
volumeClaimTemplate:
spec:
storageClassName: ceph-rbd
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 200Gi
# Scrape configs
additionalScrapeConfigs:
- job_name: 'postgres-exporter'
static_configs:
- targets: ['postgres-exporter.aprs:9187']
- job_name: 'pgbouncer-exporter'
static_configs:
- targets: ['pgbouncer-exporter.aprs:9127']
alertmanager:
enabled: true
alertmanagerSpec:
resources:
requests:
memory: 100Mi
cpu: 50m
limits:
memory: 200Mi
cpu: 100m
storage:
volumeClaimTemplate:
spec:
storageClassName: ceph-rbd
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 1Gi
grafana:
enabled: true
adminPassword: "changeme" # Change this!
resources:
requests:
memory: 100Mi
cpu: 100m
limits:
memory: 200Mi
cpu: 200m
persistence:
enabled: true
size: 1Gi
storageClassName: ceph-rbd
# Pre-install dashboards
dashboardProviders:
dashboardproviders.yaml:
apiVersion: 1
providers:
- name: 'default'
orgId: 1
folder: ''
type: file
disableDeletion: false
editable: true
options:
path: /var/lib/grafana/dashboards/default
dashboards:
default:
postgresql:
gnetId: 9628
revision: 7
datasource: Prometheus
pgbouncer:
gnetId: 15984
revision: 1
datasource: Prometheus
kubernetes-cluster:
gnetId: 7249
revision: 1
datasource: Prometheus
aprs-application:
file: dashboards/aprs-dashboard.json
datasource: Prometheus
# Reduce resource usage for k3s
kubeStateMetrics:
resources:
requests:
memory: 50Mi
cpu: 10m
limits:
memory: 150Mi
cpu: 100m
nodeExporter:
resources:
requests:
memory: 30Mi
cpu: 10m
limits:
memory: 100Mi
cpu: 100m
prometheusOperator:
resources:
requests:
memory: 100Mi
cpu: 50m
limits:
memory: 200Mi
cpu: 200m
# Disable components not needed for k3s
kubeApiServer:
enabled: false
kubeControllerManager:
enabled: false
kubeScheduler:
enabled: false
kubeProxy:
enabled: false
kubeEtcd:
enabled: false
# Enable ServiceMonitor for k3s components
kubelet:
enabled: true
serviceMonitor:
https: true

4
clusters/aprs/monitoring/namespace.yaml Normal file
View file

@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: monitoring

110
clusters/aprs/monitoring/pgbouncer-exporter.yaml Normal file
View file

@ -0,0 +1,110 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: pgbouncer-config-exporter
namespace: aprs
data:
pgbouncer.ini: |
[databases]
pgbouncer = host=127.0.0.1 port=6432 dbname=pgbouncer auth_user=stats
[pgbouncer]
listen_port = 6433
listen_addr = 127.0.0.1
auth_type = trust
auth_file = /etc/pgbouncer/userlist.txt
admin_users = stats
stats_users = stats
pool_mode = session
max_client_conn = 10
userlist.txt: |
"stats" "trusted"
---
apiVersion: v1
kind: Service
metadata:
name: pgbouncer-exporter
namespace: aprs
labels:
app: pgbouncer-exporter
spec:
ports:
- port: 9127
targetPort: 9127
name: metrics
selector:
app: pgbouncer-exporter
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: pgbouncer-exporter
namespace: aprs
labels:
app: pgbouncer-exporter
spec:
replicas: 1
selector:
matchLabels:
app: pgbouncer-exporter
template:
metadata:
labels:
app: pgbouncer-exporter
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9127"
spec:
containers:
- name: pgbouncer-stats-proxy
image: bitnami/pgbouncer:1.22.1
ports:
- containerPort: 6433
env:
- name: PGBOUNCER_INI_FILE
value: /etc/pgbouncer/pgbouncer.ini
volumeMounts:
- name: config
mountPath: /etc/pgbouncer
resources:
requests:
memory: 32Mi
cpu: 10m
limits:
memory: 64Mi
cpu: 50m
- name: pgbouncer-exporter
image: prometheuscommunity/pgbouncer-exporter:v0.7.0
ports:
- containerPort: 9127
name: metrics
env:
- name: PGBOUNCER_EXPORTER_CONNECTION_STRING
value: "postgres://stats:trusted@pgbouncer:5432/pgbouncer?sslmode=disable"
- name: PGBOUNCER_EXPORTER_PORT
value: "9127"
resources:
requests:
memory: 50Mi
cpu: 50m
limits:
memory: 100Mi
cpu: 100m
livenessProbe:
httpGet:
path: /metrics
port: 9127
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
httpGet:
path: /metrics
port: 9127
initialDelaySeconds: 5
periodSeconds: 5
volumes:
- name: config
configMap:
name: pgbouncer-config-exporter

160
clusters/aprs/monitoring/postgres-exporter.yaml Normal file
View file

@ -0,0 +1,160 @@
apiVersion: v1
kind: Service
metadata:
name: postgres-exporter
namespace: aprs
labels:
app: postgres-exporter
spec:
ports:
- port: 9187
targetPort: 9187
name: metrics
selector:
app: postgres-exporter
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgres-exporter
namespace: aprs
labels:
app: postgres-exporter
spec:
replicas: 1
selector:
matchLabels:
app: postgres-exporter
template:
metadata:
labels:
app: postgres-exporter
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9187"
spec:
containers:
- name: postgres-exporter
image: prometheuscommunity/postgres-exporter:v0.15.0
ports:
- containerPort: 9187
name: metrics
env:
- name: DATA_SOURCE_NAME
value: "postgresql://aprs:$(DATABASE_PASSWORD)@postgis:5432/aprs_prod?sslmode=disable"
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: aprs-secret
key: database-password
- name: PG_EXPORTER_AUTO_DISCOVER_DATABASES
value: "true"
- name: PG_EXPORTER_INCLUDE_DATABASES
value: "aprs_prod"
- name: PG_EXPORTER_EXTEND_QUERY_PATH
value: "/etc/postgres_exporter/queries.yaml"
volumeMounts:
- name: queries
mountPath: /etc/postgres_exporter
resources:
requests:
memory: 50Mi
cpu: 50m
limits:
memory: 150Mi
cpu: 200m
livenessProbe:
httpGet:
path: /metrics
port: 9187
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
httpGet:
path: /metrics
port: 9187
initialDelaySeconds: 5
periodSeconds: 5
volumes:
- name: queries
configMap:
name: postgres-exporter-queries
---
apiVersion: v1
kind: ConfigMap
metadata:
name: postgres-exporter-queries
namespace: aprs
data:
queries.yaml: |
pg_connections:
query: |
SELECT
datname,
state,
COUNT(*) as count
FROM pg_stat_activity
WHERE datname IS NOT NULL
GROUP BY datname, state
metrics:
- datname:
usage: "LABEL"
description: "Database name"
- state:
usage: "LABEL"
description: "Connection state"
- count:
usage: "GAUGE"
description: "Number of connections"
pg_connection_stats:
query: |
SELECT
(SELECT COUNT(*) FROM pg_stat_activity) as total_connections,
(SELECT setting::int FROM pg_settings WHERE name = 'max_connections') as max_connections,
(SELECT COUNT(*) FROM pg_stat_activity WHERE state = 'active') as active_connections,
(SELECT COUNT(*) FROM pg_stat_activity WHERE state = 'idle') as idle_connections,
(SELECT COUNT(*) FROM pg_stat_activity WHERE state = 'idle in transaction') as idle_in_transaction
metrics:
- total_connections:
usage: "GAUGE"
description: "Total number of connections"
- max_connections:
usage: "GAUGE"
description: "Maximum allowed connections"
- active_connections:
usage: "GAUGE"
description: "Number of active connections"
- idle_connections:
usage: "GAUGE"
description: "Number of idle connections"
- idle_in_transaction:
usage: "GAUGE"
description: "Number of idle in transaction connections"
pg_long_running_queries:
query: |
SELECT
COUNT(*) as count
FROM pg_stat_activity
WHERE state = 'active'
AND query_start < NOW() - INTERVAL '5 minutes'
metrics:
- count:
usage: "GAUGE"
description: "Number of queries running longer than 5 minutes"
pg_database_size:
query: |
SELECT
datname,
pg_database_size(datname) as size_bytes
FROM pg_database
WHERE datname NOT IN ('template0', 'template1', 'postgres')
metrics:
- datname:
usage: "LABEL"
description: "Database name"
- size_bytes:
usage: "GAUGE"
description: "Database size in bytes"

149
clusters/aprs/monitoring/prometheusrules.yaml Normal file
View file

@ -0,0 +1,149 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: postgres-alerts
namespace: aprs
labels:
prometheus: kube-prometheus
role: alert-rules
spec:
groups:
- name: postgresql
interval: 30s
rules:
- alert: PostgreSQLConnectionsHigh
expr: pg_connection_stats_total_connections / pg_connection_stats_max_connections > 0.8
for: 5m
labels:
severity: warning
annotations:
summary: "PostgreSQL connection usage is high ({{ $value | humanizePercentage }})"
description: "PostgreSQL on {{ $labels.instance }} is using {{ $value | humanizePercentage }} of available connections"
- alert: PostgreSQLConnectionsCritical
expr: pg_connection_stats_total_connections / pg_connection_stats_max_connections > 0.95
for: 2m
labels:
severity: critical
annotations:
summary: "PostgreSQL is almost out of connections ({{ $value | humanizePercentage }})"
description: "PostgreSQL on {{ $labels.instance }} is using {{ $value | humanizePercentage }} of available connections. Connection exhaustion imminent!"
- alert: PostgreSQLIdleTransactionsHigh
expr: pg_connection_stats_idle_in_transaction > 10
for: 5m
labels:
severity: warning
annotations:
summary: "High number of idle transactions ({{ $value }})"
description: "PostgreSQL has {{ $value }} idle transactions, which may be blocking other queries"
- alert: PostgreSQLLongRunningQueries
expr: pg_long_running_queries_count > 5
for: 5m
labels:
severity: warning
annotations:
summary: "Long running queries detected ({{ $value }})"
description: "PostgreSQL has {{ $value }} queries running longer than 5 minutes"
- alert: PostgreSQLDown
expr: up{job="postgres-exporter"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: "PostgreSQL is down"
description: "PostgreSQL exporter on {{ $labels.instance }} is not responding"
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: pgbouncer-alerts
namespace: aprs
labels:
prometheus: kube-prometheus
role: alert-rules
spec:
groups:
- name: pgbouncer
interval: 30s
rules:
- alert: PgBouncerDown
expr: up{job="pgbouncer-exporter"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: "PgBouncer is down"
description: "PgBouncer exporter on {{ $labels.instance }} is not responding"
- alert: PgBouncerPoolExhausted
expr: pgbouncer_pools_server_active_connections / pgbouncer_pools_pool_size > 0.9
for: 5m
labels:
severity: warning
annotations:
summary: "PgBouncer pool almost exhausted"
description: "PgBouncer pool {{ $labels.database }} is using {{ $value | humanizePercentage }} of available connections"
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: aprs-app-alerts
namespace: aprs
labels:
prometheus: kube-prometheus
role: alert-rules
spec:
groups:
- name: aprs-application
interval: 30s
rules:
- alert: APRSHighMemoryUsage
expr: |
(container_memory_working_set_bytes{namespace="aprs", pod=~"aprs-.*", container="aprs"}
/ container_spec_memory_limit_bytes{namespace="aprs", pod=~"aprs-.*", container="aprs"}) > 0.9
for: 5m
labels:
severity: warning
annotations:
summary: "APRS pod memory usage is high"
description: "APRS pod {{ $labels.pod }} is using {{ $value | humanizePercentage }} of memory limit"
- alert: APRSHighCPUUsage
expr: |
(rate(container_cpu_usage_seconds_total{namespace="aprs", pod=~"aprs-.*", container="aprs"}[5m])
/ container_spec_cpu_quota{namespace="aprs", pod=~"aprs-.*", container="aprs"} * 100000) > 80
for: 5m
labels:
severity: warning
annotations:
summary: "APRS pod CPU usage is high"
description: "APRS pod {{ $labels.pod }} is using {{ $value }}% of CPU limit"
- alert: APRSPodCrashLooping
expr: rate(kube_pod_container_status_restarts_total{namespace="aprs", pod=~"aprs-.*"}[15m]) > 0
labels:
severity: critical
annotations:
summary: "APRS pod is crash looping"
description: "APRS pod {{ $labels.pod }} has restarted {{ $value }} times in the last 15 minutes"
- alert: APRSHighDatabaseConnectionQueueTime
expr: histogram_quantile(0.95, aprsme_repo_query_queue_time_millisecond) > 1000
for: 5m
labels:
severity: warning
annotations:
summary: "High database connection queue time"
description: "95th percentile database queue time is {{ $value }}ms, indicating connection pool exhaustion"
- alert: APRSPacketProcessingBacklog
expr: rate(aprsme_packet_pipeline_batch_error[5m]) > 0.1
for: 5m
labels:
severity: warning
annotations:
summary: "Packet processing errors detected"
description: "Packet pipeline is experiencing {{ $value }} errors per second"

67
clusters/aprs/monitoring/servicemonitors.yaml Normal file
View file

@ -0,0 +1,67 @@
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: postgres-exporter
namespace: aprs
labels:
app: postgres-exporter
prometheus: kube-prometheus
spec:
selector:
matchLabels:
app: postgres-exporter
endpoints:
- port: metrics
interval: 30s
path: /metrics
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: pgbouncer-exporter
namespace: aprs
labels:
app: pgbouncer-exporter
prometheus: kube-prometheus
spec:
selector:
matchLabels:
app: pgbouncer-exporter
endpoints:
- port: metrics
interval: 30s
path: /metrics
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: aprs-app
namespace: aprs
labels:
app: aprs
prometheus: kube-prometheus
spec:
selector:
matchLabels:
app: aprs
endpoints:
- port: http
interval: 30s
path: /metrics
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: redis
namespace: aprs
labels:
app: redis
prometheus: kube-prometheus
spec:
selector:
matchLabels:
app: redis
endpoints:
- port: metrics
interval: 30s
path: /metrics

50
clusters/aprs/pgbouncer-configmap.yaml
View file

@ -1,50 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: pgbouncer-config
namespace: aprs
data:
pgbouncer.ini: |
[databases]
aprs = host=postgis port=5432 dbname=aprs
[pgbouncer]
listen_addr = 0.0.0.0
listen_port = 6432
auth_type = md5
auth_file = /etc/pgbouncer/userlist.txt
pool_mode = transaction
max_client_conn = 1000
default_pool_size = 25
reserve_pool_size = 5
reserve_pool_timeout = 3
server_lifetime = 3600
server_idle_timeout = 600
log_connections = 0
log_disconnections = 0
stats_period = 60
# Important for prepared statements
server_reset_query = DISCARD ALL
server_reset_query_always = 1
# Connection limits
min_pool_size = 10
max_db_connections = 100
# Timeouts
query_timeout = 0
query_wait_timeout = 120
client_idle_timeout = 0
client_login_timeout = 60
# For better performance
pkt_buf = 8192
sbuf_loopcnt = 2
tcp_keepalive = 1
tcp_keepidle = 1
tcp_keepintvl = 1
tcp_keepcnt = 1
# Ignore certain PostgreSQL parameters
ignore_startup_parameters = extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout

131
clusters/aprs/pgbouncer-deployment-enhanced.yaml
View file

@ -1,131 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: pgbouncer
namespace: aprs
labels:
app: pgbouncer
spec:
ports:
- port: 5432
targetPort: 6432
name: pgbouncer
selector:
app: pgbouncer
type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: pgbouncer
namespace: aprs
labels:
app: pgbouncer
spec:
replicas: 2
selector:
matchLabels:
app: pgbouncer
template:
metadata:
labels:
app: pgbouncer
spec:
containers:
- name: pgbouncer
image: bitnami/pgbouncer:1.22.1
ports:
- containerPort: 6432
name: pgbouncer
env:
- name: POSTGRESQL_HOST
value: "postgis"
- name: POSTGRESQL_PORT
value: "5432"
- name: POSTGRESQL_DATABASE
value: "aprs"
- name: POSTGRESQL_USERNAME
value: "aprs"
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: aprs-secret
key: database-password
- name: PGBOUNCER_DATABASE
value: "aprs"
- name: PGBOUNCER_PORT
value: "6432"
- name: PGBOUNCER_POOL_MODE
value: "transaction"
- name: PGBOUNCER_MAX_CLIENT_CONN
value: "1000"
- name: PGBOUNCER_DEFAULT_POOL_SIZE
value: "25"
- name: PGBOUNCER_MIN_POOL_SIZE
value: "5"
- name: PGBOUNCER_RESERVE_POOL_SIZE
value: "5"
- name: PGBOUNCER_SERVER_RESET_QUERY
value: "DISCARD ALL"
- name: PGBOUNCER_STATS_USERS
value: "aprs"
- name: PGBOUNCER_AUTH_TYPE
value: "md5"
- name: PGBOUNCER_IGNORE_STARTUP_PARAMETERS
value: "extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout"
# Enhanced connection handling
- name: PGBOUNCER_SERVER_IDLE_TIMEOUT
value: "600" # 10 minutes
- name: PGBOUNCER_SERVER_LIFETIME
value: "3600" # 1 hour
- name: PGBOUNCER_QUERY_TIMEOUT
value: "300" # 5 minutes
- name: PGBOUNCER_QUERY_WAIT_TIMEOUT
value: "120" # 2 minutes
- name: PGBOUNCER_CLIENT_IDLE_TIMEOUT
value: "300" # 5 minutes
- name: PGBOUNCER_CLIENT_LOGIN_TIMEOUT
value: "60" # 1 minute
# Connection retry settings
- name: PGBOUNCER_SERVER_CONNECT_TIMEOUT
value: "15"
- name: PGBOUNCER_SERVER_LOGIN_RETRY
value: "15" # Retry for 15 seconds before failing
- name: PGBOUNCER_SERVER_ROUND_ROBIN
value: "1" # Enable round-robin
- name: PGBOUNCER_LOG_CONNECTIONS
value: "1"
- name: PGBOUNCER_LOG_DISCONNECTIONS
value: "1"
- name: PGBOUNCER_LOG_POOLER_ERRORS
value: "1"
resources:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "256Mi"
cpu: "200m"
livenessProbe:
tcpSocket:
port: 6432
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
exec:
command:
- /bin/sh
- -c
- |
nc -z localhost 6432 && \
echo "SHOW POOLS;" | psql -h localhost -p 6432 -U pgbouncer pgbouncer || exit 0
initialDelaySeconds: 5
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 2
lifecycle:
preStop:
exec:
command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 10"]

96
clusters/aprs/pgbouncer-deployment.yaml
View file

@ -1,96 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: pgbouncer
namespace: aprs
labels:
app: pgbouncer
spec:
ports:
- port: 5432
targetPort: 6432
name: pgbouncer
selector:
app: pgbouncer
type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: pgbouncer
namespace: aprs
labels:
app: pgbouncer
spec:
replicas: 2
selector:
matchLabels:
app: pgbouncer
template:
metadata:
labels:
app: pgbouncer
spec:
containers:
- name: pgbouncer
image: bitnami/pgbouncer:1.22.1
ports:
- containerPort: 6432
name: pgbouncer
env:
- name: POSTGRESQL_HOST
value: "postgis"
- name: POSTGRESQL_PORT
value: "5432"
- name: POSTGRESQL_DATABASE
value: "aprs"
- name: POSTGRESQL_USERNAME
value: "aprs"
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: aprs-secret
key: database-password
- name: PGBOUNCER_DATABASE
value: "aprs"
- name: PGBOUNCER_PORT
value: "6432"
- name: PGBOUNCER_POOL_MODE
value: "transaction"
- name: PGBOUNCER_MAX_CLIENT_CONN
value: "1000"
- name: PGBOUNCER_DEFAULT_POOL_SIZE
value: "25"
- name: PGBOUNCER_MIN_POOL_SIZE
value: "10"
- name: PGBOUNCER_RESERVE_POOL_SIZE
value: "5"
- name: PGBOUNCER_SERVER_RESET_QUERY
value: "DISCARD ALL"
- name: PGBOUNCER_STATS_USERS
value: "aprs"
- name: PGBOUNCER_AUTH_TYPE
value: "md5"
- name: PGBOUNCER_IGNORE_STARTUP_PARAMETERS
value: "extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout"
resources:
requests:
memory: "64Mi"
cpu: "50m"
limits:
memory: "128Mi"
cpu: "100m"
livenessProbe:
tcpSocket:
port: 6432
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
tcpSocket:
port: 6432
initialDelaySeconds: 5
periodSeconds: 5
lifecycle:
preStop:
exec:
command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 10"]

53
clusters/aprs/postgis-deployment-ceph.yaml
View file

@ -1,53 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgis
namespace: aprs
spec:
replicas: 1
selector:
matchLabels:
app: postgis
template:
metadata:
labels:
app: postgis
spec:
initContainers:
- name: volume-permissions
image: busybox:1.36.1
command: ['sh', '-c', 'chmod -R 777 /var/lib/postgresql/data || true']
volumeMounts:
- name: postgis-storage
mountPath: /var/lib/postgresql/data
containers:
- name: postgis
image: postgis/postgis:17-3.4
ports:
- containerPort: 5432
env:
- name: POSTGRES_DB
value: aprs_prod
- name: POSTGRES_USER
value: aprs
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgis-secret
key: postgres-password
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
volumeMounts:
- name: postgis-storage
mountPath: /var/lib/postgresql/data
resources:
limits:
memory: "1Gi"
cpu: "500m"
requests:
memory: "512Mi"
cpu: "250m"
volumes:
- name: postgis-storage
persistentVolumeClaim:
claimName: postgis-pvc-ceph

110
clusters/aprs/postgis-deployment-enhanced.yaml
View file

@ -1,110 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgis
namespace: aprs
spec:
replicas: 1
selector:
matchLabels:
app: postgis
template:
metadata:
labels:
app: postgis
spec:
initContainers:
- name: volume-permissions
image: busybox:1.36.1
command: ['sh', '-c', 'chmod -R 777 /var/lib/postgresql/data || true']
volumeMounts:
- name: postgis-storage
mountPath: /var/lib/postgresql/data
containers:
- name: postgis
image: postgis/postgis:17-3.4
ports:
- containerPort: 5432
env:
- name: POSTGRES_DB
value: aprs_prod
- name: POSTGRES_USER
value: aprs
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgis-secret
key: postgres-password
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
# PostgreSQL configuration for better connection handling
- name: POSTGRES_INITDB_ARGS
value: "--encoding=UTF8 --locale=en_US.utf8"
args:
- -c
- max_connections=200 # Increased from default 100
- -c
- shared_buffers=256MB
- -c
- effective_cache_size=1GB
- -c
- maintenance_work_mem=64MB
- -c
- checkpoint_completion_target=0.7
- -c
- wal_buffers=16MB
- -c
- default_statistics_target=100
- -c
- random_page_cost=1.1
- -c
- effective_io_concurrency=200
- -c
- work_mem=4MB
- -c
- min_wal_size=1GB
- -c
- max_wal_size=2GB
- -c
- idle_in_transaction_session_timeout=30s # Kill idle transactions
- -c
- statement_timeout=300s # 5 minute statement timeout
volumeMounts:
- name: postgis-storage
mountPath: /var/lib/postgresql/data
resources:
limits:
memory: "2Gi" # Increased from 1Gi
cpu: "1000m" # Increased from 500m
requests:
memory: "1Gi" # Increased from 512Mi
cpu: "500m" # Increased from 250m
# Health checks to detect connection issues
livenessProbe:
exec:
command:
- /bin/sh
- -c
- pg_isready -U $POSTGRES_USER -d $POSTGRES_DB
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
exec:
command:
- /bin/sh
- -c
- |
pg_isready -U $POSTGRES_USER -d $POSTGRES_DB && \
CONN_COUNT=$(psql -U $POSTGRES_USER -d $POSTGRES_DB -t -c "SELECT count(*) FROM pg_stat_activity WHERE state != 'idle';") && \
[ $CONN_COUNT -lt 180 ] # Fail readiness if too many connections
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 2
volumes:
- name: postgis-storage
hostPath:
path: /var/lib/rancher/k3s/storage/postgis
type: DirectoryOrCreate

54
clusters/aprs/postgis-deployment.yaml
View file

@ -1,54 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgis
namespace: aprs
spec:
replicas: 1
selector:
matchLabels:
app: postgis
template:
metadata:
labels:
app: postgis
spec:
initContainers:
- name: volume-permissions
image: busybox:1.36.1
command: ['sh', '-c', 'chmod -R 777 /var/lib/postgresql/data || true']
volumeMounts:
- name: postgis-storage
mountPath: /var/lib/postgresql/data
containers:
- name: postgis
image: postgis/postgis:17-3.4
ports:
- containerPort: 5432
env:
- name: POSTGRES_DB
value: aprs_prod
- name: POSTGRES_USER
value: aprs
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgis-secret
key: postgres-password
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
volumeMounts:
- name: postgis-storage
mountPath: /var/lib/postgresql/data
resources:
limits:
memory: "1Gi"
cpu: "500m"
requests:
memory: "512Mi"
cpu: "250m"
volumes:
- name: postgis-storage
hostPath:
path: /var/lib/rancher/k3s/storage/postgis
type: DirectoryOrCreate

21
clusters/aprs/postgis-pdb.yaml
View file

@ -1,21 +0,0 @@
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: postgis-pdb
namespace: aprs
spec:
minAvailable: 1
selector:
matchLabels:
app: postgis
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: pgbouncer-pdb
namespace: aprs
spec:
minAvailable: 1
selector:
matchLabels:
app: pgbouncer

12
clusters/aprs/postgis-pvc-ceph.yaml
View file

@ -1,12 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: postgis-pvc-ceph
namespace: aprs
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 300Gi
storageClassName: ceph-rbd

12
clusters/aprs/postgis-service.yaml
View file

@ -1,12 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: postgis
namespace: aprs
spec:
selector:
app: postgis
ports:
- port: 5432
targetPort: 5432
type: ClusterIP

107
clusters/aprs/postgres-cleanup-cronjob.yaml
View file

@ -1,107 +0,0 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: postgres-cleanup
namespace: aprs
spec:
schedule: "*/15 * * * *" # Every 15 minutes
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
spec:
template:
spec:
restartPolicy: OnFailure
containers:
- name: postgres-cleanup
image: postgres:17-alpine
env:
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: postgis-secret
key: postgres-password
- name: PGHOST
value: postgis
- name: PGUSER
value: aprs
- name: PGDATABASE
value: aprs_prod
command:
- /bin/sh
- -c
- |
echo "Starting PostgreSQL connection cleanup..."
# Kill idle connections older than 30 minutes
psql -c "
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname = 'aprs_prod'
AND pid <> pg_backend_pid()
AND state = 'idle'
AND state_change < NOW() - INTERVAL '30 minutes';"
# Kill idle in transaction connections older than 5 minutes
psql -c "
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname = 'aprs_prod'
AND pid <> pg_backend_pid()
AND state = 'idle in transaction'
AND state_change < NOW() - INTERVAL '5 minutes';"
# Log connection stats
psql -c "
SELECT
state,
COUNT(*) as count,
MAX(NOW() - state_change) as max_duration
FROM pg_stat_activity
WHERE datname = 'aprs_prod'
GROUP BY state
ORDER BY count DESC;"
# Check total connections vs max
psql -c "
SELECT
(SELECT COUNT(*) FROM pg_stat_activity) as current_connections,
(SELECT setting::int FROM pg_settings WHERE name = 'max_connections') as max_connections,
ROUND(100.0 * COUNT(*) / (SELECT setting::int FROM pg_settings WHERE name = 'max_connections'), 2) as percentage_used
FROM pg_stat_activity;"
echo "Cleanup completed successfully"
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: postgres-cleanup
namespace: aprs
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: postgres-cleanup
namespace: aprs
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: postgres-cleanup
namespace: aprs
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: postgres-cleanup
subjects:
- kind: ServiceAccount
name: postgres-cleanup
namespace: aprs

96
clusters/aprs/redis-deployment-with-metrics.yaml Normal file
View file

@ -0,0 +1,96 @@
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: aprs
labels:
app: redis
spec:
ports:
- port: 6379
targetPort: 6379
name: redis
- port: 9121
targetPort: 9121
name: metrics
selector:
app: redis
type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: redis
namespace: aprs
labels:
app: redis
spec:
replicas: 1
selector:
matchLabels:
app: redis
template:
metadata:
labels:
app: redis
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9121"
spec:
containers:
- name: redis
image: redis:7-alpine
ports:
- containerPort: 6379
name: redis
command:
- redis-server
- --maxmemory
- "256mb"
- --maxmemory-policy
- allkeys-lru
- --save
- ""
- --appendonly
- "no"
resources:
requests:
memory: "256Mi"
cpu: "100m"
limits:
memory: "512Mi"
cpu: "500m"
livenessProbe:
tcpSocket:
port: 6379
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
exec:
command:
- redis-cli
- ping
initialDelaySeconds: 5
periodSeconds: 5
- name: redis-exporter
image: oliver006/redis_exporter:v1.55.0
ports:
- containerPort: 9121
name: metrics
env:
- name: REDIS_ADDR
value: "redis://localhost:6379"
resources:
requests:
memory: 32Mi
cpu: 10m
limits:
memory: 64Mi
cpu: 50m
livenessProbe:
httpGet:
path: /metrics
port: 9121
initialDelaySeconds: 30
periodSeconds: 10

66
clusters/aprs/redis-deployment.yaml
View file

@ -1,66 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: aprs
labels:
app: redis
spec:
ports:
- port: 6379
targetPort: 6379
name: redis
selector:
app: redis
type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: redis
namespace: aprs
labels:
app: redis
spec:
replicas: 1
selector:
matchLabels:
app: redis
template:
metadata:
labels:
app: redis
spec:
containers:
- name: redis
image: redis:7-alpine
ports:
- containerPort: 6379
name: redis
resources:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "256Mi"
cpu: "200m"
livenessProbe:
exec:
command:
- redis-cli
- ping
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
exec:
command:
- redis-cli
- ping
initialDelaySeconds: 5
periodSeconds: 5
volumeMounts:
- name: redis-data
mountPath: /data
volumes:
- name: redis-data
emptyDir: {}

39
clusters/aprs/secrets.yaml
View file

@ -1,39 +0,0 @@
apiVersion: v1
kind: Secret
metadata:
name: postgis-secret
namespace: aprs
type: Opaque
stringData:
postgres-password: "mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21"
---
apiVersion: v1
kind: Secret
metadata:
name: aprs-secret
namespace: aprs
type: Opaque
stringData:
database-url: "ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@postgis:5432/aprs"
database-url-pgbouncer: "ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@pgbouncer:5432/aprs"
database-password: "mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21"
secret-key-base: "GJxzOUW3YXBAF7xB4mH5VcXY6BN1NYLnoJYadlZyiCxcRUGBw65w4Sco7+sNiW2t"
erlang-cookie: "aprs-cluster-cookie-ksdf8934jfklsdjf983jfklsdf"
---
apiVersion: v1
kind: Secret
metadata:
name: ghcr-pull-secret
namespace: aprs
type: kubernetes.io/dockerconfigjson
stringData:
.dockerconfigjson: |
{
"auths": {
"ghcr.io": {
"username": "gmcintire",
"password": "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",
"auth": "Z21jaW50aXJlOmdocF9rOFhBZkJMSmhHdzExMDZiZkZXQVhkYjdYN1E4OUkxdjlhQWk="
}
}
}

5
clusters/aprs/tailscale/00-namespace.yaml Normal file
View file

@ -0,0 +1,5 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: tailscale

76
clusters/aprs/tailscale/01-rbac.yaml Normal file
View file

@ -0,0 +1,76 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: operator
namespace: tailscale
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: tailscale-operator
rules:
- apiGroups: [""]
resources: ["events", "services", "services/status"]
verbs: ["*"]
- apiGroups: ["networking.k8s.io"]
resources: ["ingresses", "ingresses/status"]
verbs: ["*"]
- apiGroups: ["networking.k8s.io"]
resources: ["ingressclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["tailscale.com"]
resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["*"]
- apiGroups: [""]
resources: ["serviceaccounts", "configmaps"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
resources: ["statefulsets", "deployments"]
verbs: ["*"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: tailscale-operator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: tailscale-operator
subjects:
- kind: ServiceAccount
name: operator
namespace: tailscale
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: operator
namespace: tailscale
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["*"]
- apiGroups: ["apps"]
resources: ["statefulsets"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: operator
namespace: tailscale
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: operator
subjects:
- kind: ServiceAccount
name: operator
namespace: tailscale

296
clusters/aprs/tailscale/02-crds.yaml Normal file
View file

@ -0,0 +1,296 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: connectors.tailscale.com
spec:
group: tailscale.com
versions:
- name: v1alpha1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
properties:
tags:
type: array
items:
type: string
hostname:
type: string
proxyClass:
type: string
exitNode:
type: boolean
subnetRouter:
type: object
properties:
routes:
type: array
items:
type: string
status:
type: object
properties:
conditions:
type: array
items:
type: object
properties:
type:
type: string
status:
type: string
reason:
type: string
message:
type: string
lastTransitionTime:
type: string
subnetRouter:
type: object
properties:
routes:
type: array
items:
type: string
isExitNode:
type: boolean
subresources:
status: {}
scope: Cluster
names:
plural: connectors
singular: connector
kind: Connector
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: proxyclasses.tailscale.com
spec:
group: tailscale.com
versions:
- name: v1alpha1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
properties:
metrics:
type: object
properties:
enable:
type: boolean
statefulSet:
type: object
properties:
labels:
type: object
additionalProperties:
type: string
annotations:
type: object
additionalProperties:
type: string
pod:
type: object
properties:
labels:
type: object
additionalProperties:
type: string
annotations:
type: object
additionalProperties:
type: string
tailscaleContainer:
type: object
properties:
env:
type: array
items:
type: object
properties:
name:
type: string
value:
type: string
imagePullPolicy:
type: string
enum: ["Always", "Never", "IfNotPresent"]
resources:
type: object
properties:
requests:
type: object
additionalProperties:
type: string
limits:
type: object
additionalProperties:
type: string
securityContext:
type: object
properties:
privileged:
type: boolean
allowPrivilegeEscalation:
type: boolean
readOnlyRootFilesystem:
type: boolean
runAsNonRoot:
type: boolean
runAsUser:
type: integer
runAsGroup:
type: integer
capabilities:
type: object
properties:
add:
type: array
items:
type: string
drop:
type: array
items:
type: string
tailscaleInitContainer:
type: object
properties:
env:
type: array
items:
type: object
properties:
name:
type: string
value:
type: string
imagePullPolicy:
type: string
enum: ["Always", "Never", "IfNotPresent"]
resources:
type: object
properties:
requests:
type: object
additionalProperties:
type: string
limits:
type: object
additionalProperties:
type: string
securityContext:
type: object
properties:
privileged:
type: boolean
allowPrivilegeEscalation:
type: boolean
readOnlyRootFilesystem:
type: boolean
runAsNonRoot:
type: boolean
runAsUser:
type: integer
runAsGroup:
type: integer
capabilities:
type: object
properties:
add:
type: array
items:
type: string
drop:
type: array
items:
type: string
imagePullSecrets:
type: array
items:
type: object
properties:
name:
type: string
nodeName:
type: string
nodeSelector:
type: object
additionalProperties:
type: string
affinity:
type: object
tolerations:
type: array
items:
type: object
properties:
key:
type: string
operator:
type: string
value:
type: string
effect:
type: string
tolerationSeconds:
type: integer
hostNetwork:
type: boolean
serviceAccountName:
type: string
securityContext:
type: object
properties:
runAsNonRoot:
type: boolean
runAsUser:
type: integer
runAsGroup:
type: integer
fsGroup:
type: integer
service:
type: object
properties:
type:
type: string
enum: ["ClusterIP", "LoadBalancer", "NodePort"]
status:
type: object
properties:
conditions:
type: array
items:
type: object
properties:
type:
type: string
status:
type: string
reason:
type: string
message:
type: string
observedGeneration:
type: integer
lastTransitionTime:
type: string
subresources:
status: {}
scope: Cluster
names:
plural: proxyclasses
singular: proxyclass
kind: ProxyClass

29
clusters/aprs/tailscale/03-auth-secret.yaml.example Normal file
View file

@ -0,0 +1,29 @@
# IMPORTANT: Create your actual secret file as 03-auth-secret.yaml
# Do not commit the actual secret to git!
#
# To create a Tailscale auth key:
# 1. Go to https://login.tailscale.com/admin/settings/keys
# 2. Generate an auth key (reusable, with appropriate expiry)
# 3. Create the secret:
#
# kubectl create secret generic operator-oauth \
# --namespace=tailscale \
# --from-literal=client_id="<your-oauth-client-id>" \
# --from-literal=client_secret="<your-oauth-client-secret>"
#
# OR for auth key:
#
# kubectl create secret generic operator \
# --namespace=tailscale \
# --from-literal=authkey="<your-auth-key>"
#
# Example YAML format:
---
apiVersion: v1
kind: Secret
metadata:
name: operator
namespace: tailscale
type: Opaque
stringData:
authkey: "tskey-auth-YOUR-KEY-HERE"

44
clusters/aprs/tailscale/04-operator-authkey.yaml Normal file
View file

@ -0,0 +1,44 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: operator
namespace: tailscale
spec:
replicas: 1
selector:
matchLabels:
app: operator
template:
metadata:
labels:
app: operator
spec:
serviceAccountName: operator
containers:
- name: operator
image: tailscale/k8s-operator:stable
env:
- name: OPERATOR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: OPERATOR_SECRET
value: operator
- name: PROXY_IMAGE
value: tailscale/tailscale:stable
- name: PROXY_TAGS
value: tag:k8s
- name: APISERVER_PROXY
value: "false" # Set to true if you want to expose the K8s API
- name: PROXY_FIREWALL_MODE
value: auto
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 256Mi
nodeSelector:
kubernetes.io/os: linux

48
clusters/aprs/tailscale/04-operator.yaml Normal file
View file

@ -0,0 +1,48 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: operator
namespace: tailscale
spec:
replicas: 1
selector:
matchLabels:
app: operator
template:
metadata:
labels:
app: operator
spec:
serviceAccountName: operator
containers:
- name: operator
image: tailscale/k8s-operator:stable
env:
- name: OPERATOR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CLIENT_ID_FILE
value: /oauth/client_id
- name: CLIENT_SECRET_FILE
value: /oauth/client_secret
- name: PROXY_IMAGE
value: tailscale/tailscale:stable
- name: PROXY_TAGS
value: tag:k8s
- name: APISERVER_PROXY
value: "true"
- name: PROXY_FIREWALL_MODE
value: auto
volumeMounts:
- name: oauth
mountPath: /oauth
readOnly: true
volumes:
- name: oauth
secret:
secretName: operator-oauth
optional: true
nodeSelector:
kubernetes.io/os: linux

45
clusters/aprs/tailscale/05-proxy-example.yaml Normal file
View file

@ -0,0 +1,45 @@
# Example: Expose a service via Tailscale proxy
# This creates a Tailscale device that forwards traffic to a Kubernetes service
---
apiVersion: v1
kind: Service
metadata:
name: grafana-tailscale
namespace: monitoring
annotations:
tailscale.com/expose: "true"
tailscale.com/hostname: "aprs-grafana"
tailscale.com/tags: "tag:k8s,tag:monitoring"
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: grafana
ports:
- port: 80
targetPort: 3000
protocol: TCP
---
# Alternative: Using Ingress
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: grafana-tailscale
namespace: monitoring
annotations:
tailscale.com/ingress-class: "tailscale"
spec:
ingressClassName: tailscale
rules:
- host: grafana
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: grafana
port:
number: 80
tls:
- hosts:
- grafana

128
clusters/aprs/tailscale/README.md Normal file
View file

@ -0,0 +1,128 @@
# Tailscale Operator for K3s
This directory contains the Tailscale operator configuration for the k3s cluster.
## Setup Instructions
1. **Apply the base configuration:**
```bash
kubectl apply -f 00-namespace.yaml
kubectl apply -f 01-rbac.yaml
kubectl apply -f 02-crds.yaml
```
2. **Create the authentication secret:**
Option A - Using OAuth (recommended for production):
```bash
# Get OAuth credentials from https://login.tailscale.com/admin/settings/oauth
kubectl create secret generic operator-oauth \
--namespace=tailscale \
--from-literal=client_id="<your-oauth-client-id>" \
--from-literal=client_secret="<your-oauth-client-secret>"
```
Option B - Using auth key (simpler, good for testing):
```bash
# Get auth key from https://login.tailscale.com/admin/settings/keys
kubectl create secret generic operator \
--namespace=tailscale \
--from-literal=authkey="<your-auth-key>"
```
3. **Deploy the operator:**
```bash
kubectl apply -f 04-operator.yaml
```
4. **Verify the operator is running:**
```bash
kubectl get pods -n tailscale
kubectl logs -n tailscale deployment/operator
```
## Exposing Services
There are several ways to expose services via Tailscale:
### Method 1: Service Annotations
```yaml
apiVersion: v1
kind: Service
metadata:
name: my-service
annotations:
tailscale.com/expose: "true"
tailscale.com/hostname: "my-app"
tailscale.com/tags: "tag:k8s"
```
### Method 2: Ingress
```yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: my-ingress
spec:
ingressClassName: tailscale
rules:
- host: my-app
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: my-service
port:
number: 80
```
### Method 3: Connector (Subnet Router/Exit Node)
```yaml
apiVersion: tailscale.com/v1alpha1
kind: Connector
metadata:
name: subnet-router
spec:
hostname: k3s-subnet
exitNode: false
subnetRouter:
routes:
- "10.0.19.0/24" # Your cluster subnet
tags:
- "tag:k8s"
- "tag:subnet-router"
```
## Accessing the API Server
The operator can expose the Kubernetes API server on your tailnet:
```bash
# This is enabled by default with APISERVER_PROXY=true
# Access via: https://tailscale-operator.tailnet-name.ts.net:443
```
## Troubleshooting
1. Check operator logs:
```bash
kubectl logs -n tailscale deployment/operator
```
2. Check proxy pods:
```bash
kubectl get pods -A | grep ts-
```
3. Verify CRDs are installed:
```bash
kubectl get crds | grep tailscale
```
## Security Notes
- Never commit auth keys or OAuth credentials to git
- Use appropriate ACL tags in your Tailscale admin panel
- Consider using OAuth for production deployments
- Auth keys should be reusable and have appropriate expiry times

20
clusters/aprs/tailscale/grafana-tailscale.yaml Normal file
View file

@ -0,0 +1,20 @@
---
apiVersion: v1
kind: Service
metadata:
name: grafana-tailscale
namespace: monitoring
annotations:
tailscale.com/expose: "true"
tailscale.com/hostname: "aprs-grafana"
tailscale.com/tags: "tag:k8s"
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: grafana
app.kubernetes.io/part-of: kube-prometheus
ports:
- port: 80
targetPort: 3000
protocol: TCP
name: http

76
clusters/aprs/tailscale/setup.sh Executable file
View file

@ -0,0 +1,76 @@
#!/bin/bash
set -e
echo "Tailscale Operator Setup for K3s"
echo "================================"
echo ""
# Check if kubectl is available
if ! command -v kubectl &> /dev/null; then
echo "Error: kubectl is not installed or not in PATH"
exit 1
fi
# Check if we can connect to the cluster
if ! kubectl cluster-info &> /dev/null; then
echo "Error: Cannot connect to Kubernetes cluster"
exit 1
fi
echo "Step 1: Applying base configuration..."
kubectl apply -f 00-namespace.yaml
kubectl apply -f 01-rbac.yaml
kubectl apply -f 02-crds.yaml
echo ""
echo "Step 2: Authentication setup"
echo "You need to create an auth key from: https://login.tailscale.com/admin/settings/keys"
echo ""
echo "Choose one option:"
echo "1) I have an auth key ready"
echo "2) I'll set it up later"
echo ""
read -p "Enter your choice (1 or 2): " choice
if [ "$choice" = "1" ]; then
read -p "Enter your Tailscale auth key: " authkey
kubectl create secret generic operator \
--namespace=tailscale \
--from-literal=authkey="$authkey" \
--dry-run=client -o yaml | kubectl apply -f -
echo "Auth key secret created/updated"
echo ""
echo "Step 3: Deploying operator..."
kubectl apply -f 04-operator-authkey.yaml
echo ""
echo "Waiting for operator to start..."
sleep 5
kubectl wait --for=condition=ready pod -l app=operator -n tailscale --timeout=60s
echo ""
echo "Operator status:"
kubectl get pods -n tailscale
echo ""
echo "Step 4: Would you like to expose Grafana via Tailscale? (y/n)"
read -p "Choice: " expose_grafana
if [ "$expose_grafana" = "y" ]; then
kubectl apply -f grafana-tailscale.yaml
echo "Grafana will be available at: https://aprs-grafana.<your-tailnet-name>.ts.net"
fi
else
echo ""
echo "Skipping auth key setup. To complete the setup later:"
echo "1. Create an auth key at https://login.tailscale.com/admin/settings/keys"
echo "2. Run: kubectl create secret generic operator --namespace=tailscale --from-literal=authkey='<your-key>'"
echo "3. Apply the operator: kubectl apply -f 04-operator-authkey.yaml"
fi
echo ""
echo "Setup complete!"
echo ""
echo "To check the operator status: kubectl logs -n tailscale deployment/operator"
echo "To see Tailscale proxies: kubectl get pods -A | grep ts-"

10
terraform/dns_w5isp.tf
View file

@ -1,7 +1,7 @@
resource "dnsimple_zone_record" "w5isp_root" {
zone_name = "w5isp.com"
name = ""
value = "204.110.191.210"
value = "204.110.191.8"
type = "A"
ttl = 3600
}
@ -54,6 +54,14 @@ resource "dnsimple_zone_record" "w5isp_skippy" {
ttl = 3600
}
resource "dnsimple_zone_record" "w5isp_ntfy" {
zone_name = "w5isp.com"
name = "ntfy"
value = "204.110.191.8"
type = "A"
ttl = 3600
}
resource "dnsimple_zone_record" "w5isp_jellyfin" {
zone_name = "w5isp.com"
name = "jellyfin"