updates

ansible/bootstrap.yml

@@ -5,6 +5,4 @@
   roles:
     - base
   become: yes
-  # become_method: "{{ 'su' if (ansible_distribution | default('')) == 'Debian' else 'sudo' }}"
+  become_method: sudo
-  # become_method: "sudo"
-  become_method: "su"

ansible/host_vars/postgres.yml

@@ -0,0 +1,28 @@
+---
+# Host-specific variables for postgres server (CM3588)
+
+# Python interpreter
+ansible_python_interpreter: /usr/bin/python3.11
+
+# Override PostgreSQL memory settings if needed
+# postgresql_config:
+#   shared_buffers: 8GB
+
+# RAID configuration
+# Comment out for auto-detection or specify actual device names
+# raid_devices:
+#   - /dev/sda
+#   - /dev/sdb
+
+# Network configuration
+postgresql_listen_addresses: "'*'"
+postgresql_allowed_networks:
+  - 10.0.0.0/8  # Adjust to your network
+
+# Backup configuration
+postgresql_backup_retention_days: 7
+postgresql_backup_compress: true
+
+# Monitoring
+enable_monitoring: true
+monitoring_email: root@localhost

ansible/hosts

@@ -32,3 +32,6 @@ g.w5isp.com
 
 [caddy_servers]
 g.w5isp.com
+
+[postgresql_servers]
+postgres ansible_host=10.0.16.230

ansible/inventory.yml

@@ -0,0 +1,11 @@
+all:
+  hosts:
+    postgres:
+      ansible_host: 10.0.16.230
+      ansible_user: graham
+      ansible_become: yes
+      ansible_python_interpreter: /usr/bin/python3
+  children:
+    postgresql_servers:
+      hosts:
+        postgres:

ansible/playbook.yml

@@ -103,6 +103,42 @@
   roles:
     - caddy
 
+- name: Configure PostgreSQL servers
+  hosts: postgresql_servers
+  become: true
+  gather_facts: true
+  tags:
+    - postgresql
+    - database
+  
+  pre_tasks:
+    - name: Set hostname
+      hostname:
+        name: "{{ inventory_hostname }}"
+    
+    - name: Update /etc/hosts
+      lineinfile:
+        path: /etc/hosts
+        regexp: '^127\.0\.1\.1'
+        line: "127.0.1.1 {{ inventory_hostname }}"
+        state: present
+    
+    - name: Update apt cache
+      apt:
+        update_cache: yes
+        cache_valid_time: 3600
+    
+    - name: Install Python dependencies for Ansible
+      apt:
+        name:
+          - python3-apt
+          - python3-psycopg2
+        state: present
+  
+  roles:
+    - raid
+    - postgresql
+
 # - import_playbook: baseline/proxmox/main.yml
 # - name: Tailscale Home
 #   hosts: tailscale_home

ansible/postgres-playbook.yml

@@ -0,0 +1,74 @@
+---
+- name: Deploy PostgreSQL server on CM3588 with RAID0
+  hosts: postgres
+  become: yes
+  gather_facts: yes
+  
+  pre_tasks:
+    - name: Set hostname
+      hostname:
+        name: postgres
+    
+    - name: Update /etc/hosts
+      lineinfile:
+        path: /etc/hosts
+        regexp: '^127\.0\.1\.1'
+        line: '127.0.1.1 postgres'
+        state: present
+    
+    - name: Update apt cache
+      apt:
+        update_cache: yes
+        cache_valid_time: 3600
+    
+    - name: Install Python dependencies for Ansible
+      apt:
+        name:
+          - python3-apt
+          - python3-psycopg2
+        state: present
+
+  roles:
+    - raid
+    - postgresql
+  
+  post_tasks:
+    - name: Display PostgreSQL connection information
+      debug:
+        msg: |
+          PostgreSQL has been configured on {{ inventory_hostname }}
+          
+          Connection details:
+          - Host: {{ ansible_host }}
+          - Port: 5432 (PostgreSQL direct)
+          - Port: 6432 (PgBouncer pooled)
+          - Database: aprsme_prod
+          - User: aprsme
+          
+          RAID0 array status:
+          - Device: /dev/md0
+          - Mount: /var/lib/postgresql
+          - Size: ~4TB (2x2TB in RAID0)
+          
+          To connect from your k3s cluster:
+          psql -h {{ ansible_host }} -p 6432 -U aprsme -d aprsme_prod
+          
+          Remember to:
+          1. Update your APRS.me DATABASE_URL to point to this server
+          2. Run migrations: mix ecto.migrate
+          3. Monitor disk usage and performance
+          4. Set up regular backups to external storage
+
+    - name: Test PostgreSQL connection
+      postgresql_ping:
+        db: aprsme_prod
+        login_host: localhost
+        login_port: 5432
+        login_user: aprsme
+        login_password: "{{ vault_postgresql_aprsme_password }}"
+      register: db_ping
+      ignore_errors: yes
+
+    - name: Show connection test result
+      debug:
+        msg: "Database connection test: {{ 'SUCCESS' if db_ping is succeeded else 'FAILED' }}"

ansible/postgres-readme.md

@@ -0,0 +1,182 @@
+# PostgreSQL Server Configuration for APRS.me
+
+This Ansible configuration deploys a high-performance PostgreSQL server on the CM3588 with RAID0 storage configuration, optimized for the APRS.me workload.
+
+## Hardware
+
+- **Server**: CM3588 with RK3588 SoC
+- **Storage**: 2x 2TB NVMe SSDs in RAID0 (4TB total capacity)
+- **Network**: 2.5 Gigabit Ethernet
+- **RAM**: 32GB (assumed)
+
+## Features
+
+- Software RAID0 configuration for maximum performance
+- PostgreSQL 16 with PostGIS for spatial data
+- PgBouncer for connection pooling
+- Optimized for high insert rates and spatial queries
+- Automated backups with retention policy
+- Health monitoring and alerting
+- Kernel and systemd tuning for database workloads
+
+## Prerequisites
+
+1. Ubuntu 22.04 or later installed on the CM3588
+2. SSH access to the server with the user 'graham'
+3. Ansible installed on your control machine
+
+## Initial Setup
+
+**Note**: This playbook assumes a fresh PostgreSQL installation. All data will be stored on the RAID array from the start.
+
+1. First, create a vault file for sensitive data:
+```bash
+ansible-vault create group_vars/all/vault.yml
+```
+
+Add the following to the vault file:
+```yaml
+vault_postgresql_aprsme_password: "your-secure-password-here"
+```
+
+2. Bootstrap the server (first time only):
+```bash
+ansible-playbook -i inventory.yml bootstrap.yml --ask-pass --ask-become-pass
+```
+
+3. Deploy PostgreSQL:
+```bash
+ansible-playbook -i inventory.yml postgres-playbook.yml --ask-vault-pass
+```
+
+The playbook will:
+- Create RAID0 array from both NVMe drives
+- Initialize PostgreSQL cluster directly on the RAID array
+- Configure all PostgreSQL data, WAL, archives, and backups on the RAID array
+- No data migration needed since this is a fresh installation
+
+## Configuration Details
+
+### RAID0 Configuration
+- Creates `/dev/md0` from `/dev/nvme0n1` and `/dev/nvme1n1`
+- Formatted as ext4 with optimizations for SSDs
+- Mounted at `/mnt/pgdata` with noatime,nodiratime options
+- All PostgreSQL data stored on RAID array:
+  - Data directory: `/mnt/pgdata/16/main`
+  - WAL directory: `/mnt/pgdata/wal`
+  - Archive directory: `/mnt/pgdata/archive`
+  - Backup directory: `/mnt/pgdata/backups`
+
+### PostgreSQL Optimization
+- **shared_buffers**: 8GB (25% of RAM)
+- **effective_cache_size**: 24GB (75% of RAM)
+- **work_mem**: 64MB (for complex spatial queries)
+- **max_connections**: 200
+- **autovacuum**: Tuned for high insert rate
+- **checkpoint**: Optimized for write performance
+
+### PgBouncer Settings
+- **pool_mode**: transaction (optimal for APRS.me)
+- **default_pool_size**: 25
+- **max_client_conn**: 1000
+
+### Security
+- PostgreSQL listening on all interfaces (configure firewall accordingly)
+- SCRAM-SHA-256 authentication
+- Access restricted to 10.0.0.0/8 network (adjust as needed)
+
+## Maintenance
+
+### Check RAID Status
+```bash
+ssh postgres@10.0.16.230 'sudo mdadm --detail /dev/md0'
+```
+
+### PostgreSQL Logs
+```bash
+ssh postgres@10.0.16.230 'sudo journalctl -u postgresql -f'
+```
+
+### Backup Status
+```bash
+ssh postgres@10.0.16.230 'ls -la /mnt/pgdata/backups/'
+```
+
+### Performance Monitoring
+```bash
+# Connect to PostgreSQL
+psql -h 10.0.16.230 -p 5432 -U postgres -d aprsme_prod
+
+# Check slow queries
+SELECT * FROM pg_stat_statements ORDER BY total_time DESC LIMIT 10;
+
+# Check table sizes
+SELECT schemaname, tablename, pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS size
+FROM pg_tables
+ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC;
+
+# Check index usage
+SELECT schemaname, tablename, indexname, idx_scan
+FROM pg_stat_user_indexes
+ORDER BY idx_scan;
+```
+
+## Connecting from APRS.me
+
+Update your APRS.me configuration:
+
+```elixir
+# config/runtime.exs
+config :aprsme, Aprsme.Repo,
+  url: "postgresql://aprsme:password@10.0.16.230:6432/aprsme_prod",
+  pool_size: 5
+```
+
+Or use environment variable:
+```bash
+DATABASE_URL=postgresql://aprsme:password@10.0.16.230:6432/aprsme_prod
+```
+
+## Disaster Recovery
+
+1. **RAID Failure**: While RAID0 provides no redundancy, the backup script runs nightly
+2. **Backup Restore**:
+   ```bash
+   pg_restore -h localhost -U postgres -d aprsme_prod_restore /mnt/pgdata/backups/aprsme_prod_20240126_020000.sql.gz
+   ```
+3. **Replication**: Consider setting up streaming replication to a standby server
+
+## Performance Expectations
+
+With this configuration, expect:
+- Insert rate: 1000+ packets/second sustained
+- Spatial query response: <10ms for indexed queries
+- Connection overhead: Minimal with PgBouncer
+- Storage capacity: ~4TB usable (years of APRS data)
+
+## Troubleshooting
+
+### RAID Issues
+```bash
+# Check RAID status
+cat /proc/mdstat
+mdadm --detail /dev/md0
+
+# Check disk health
+smartctl -a /dev/nvme0n1
+smartctl -a /dev/nvme1n1
+```
+
+### PostgreSQL Performance
+```bash
+# Check current connections
+SELECT count(*) FROM pg_stat_activity;
+
+# Check cache hit ratio
+SELECT sum(heap_blks_hit) / (sum(heap_blks_hit) + sum(heap_blks_read)) AS cache_hit_ratio FROM pg_statio_user_tables;
+```
+
+### PgBouncer Stats
+```bash
+psql -h 10.0.16.230 -p 6432 -U postgres pgbouncer -c "SHOW STATS;"
+```

ansible/roles/postgresql/defaults/main.yml

@@ -0,0 +1,111 @@
+---
+# PostgreSQL version
+postgresql_version: 16
+
+# PostgreSQL configuration
+postgresql_data_directory: /mnt/pgdata/{{ postgresql_version }}/main
+postgresql_wal_directory: /mnt/pgdata/wal
+postgresql_archive_directory: /mnt/pgdata/archive
+postgresql_config_directory: /etc/postgresql/{{ postgresql_version }}/main
+
+# Database settings for APRS.me
+postgresql_databases:
+  - name: aprsme_prod
+    owner: aprsme
+
+postgresql_users:
+  - name: aprsme
+    password: "{{ vault_postgresql_aprsme_password }}"  # Store in vault
+    priv: "aprsme_prod:ALL"
+
+# Performance tuning for CM3588 with RK3588 (assuming 32GB RAM)
+# These are optimized for APRS.me workload
+postgresql_config:
+  # Connection settings
+  listen_addresses: "'*'"
+  max_connections: 200  # Increased for multiple app instances
+  
+  # Memory settings (for 32GB RAM system)
+  shared_buffers: 8GB  # 25% of RAM
+  effective_cache_size: 24GB  # 75% of RAM
+  maintenance_work_mem: 2GB
+  work_mem: 64MB  # High for complex queries
+  
+  # Disk settings optimized for RAID0 SSDs
+  random_page_cost: 1.1  # SSD optimization
+  effective_io_concurrency: 200  # Modern SSD
+  max_worker_processes: 8  # RK3588 has 8 cores
+  max_parallel_workers_per_gather: 4
+  max_parallel_workers: 8
+  max_parallel_maintenance_workers: 4
+  
+  # Write performance settings
+  wal_buffers: 64MB
+  checkpoint_completion_target: 0.9
+  checkpoint_timeout: 15min
+  max_wal_size: 16GB
+  min_wal_size: 2GB
+  
+  # Logging
+  log_min_duration_statement: 1000  # Log queries over 1 second
+  log_checkpoints: on
+  log_connections: on
+  log_disconnections: on
+  log_line_prefix: "'%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '"
+  log_timezone: 'UTC'
+  
+  # Query tuning
+  default_statistics_target: 100
+  
+  # Autovacuum tuning for high insert rate
+  autovacuum: on
+  autovacuum_max_workers: 4
+  autovacuum_naptime: 30s  # More frequent checks
+  autovacuum_vacuum_threshold: 50
+  autovacuum_analyze_threshold: 50
+  autovacuum_vacuum_scale_factor: 0.1  # 10% for more frequent vacuums
+  autovacuum_analyze_scale_factor: 0.05  # 5% for more frequent analyzes
+  
+  # Enable query planning statistics
+  track_activity_query_size: 2048
+  track_io_timing: on
+  track_functions: all
+  
+  # PostGIS specific
+  postgis.gdal_datapath: '/usr/share/gdal'
+  
+# Security settings
+postgresql_hba_entries:
+  - type: local
+    database: all
+    user: postgres
+    auth_method: peer
+  - type: local
+    database: all
+    user: all
+    auth_method: peer
+  - type: host
+    database: all
+    user: all
+    address: 127.0.0.1/32
+    auth_method: scram-sha-256
+  - type: host
+    database: all
+    user: all
+    address: ::1/128
+    auth_method: scram-sha-256
+  - type: host
+    database: aprsme_prod
+    user: aprsme
+    address: 10.0.0.0/8  # Adjust for your network
+    auth_method: scram-sha-256
+
+# Backup settings
+postgresql_backup_enabled: true
+postgresql_backup_directory: /mnt/pgdata/backups
+postgresql_backup_schedule:
+  minute: "0"
+  hour: "2"
+  day: "*"
+  month: "*"
+  weekday: "*"

ansible/roles/postgresql/handlers/main.yml

@@ -0,0 +1,23 @@
+---
+- name: restart postgresql
+  systemd:
+    name: postgresql
+    state: restarted
+    daemon_reload: yes
+
+- name: reload postgresql
+  systemd:
+    name: postgresql
+    state: reloaded
+
+- name: restart pgbouncer
+  systemd:
+    name: pgbouncer
+    state: restarted
+
+- name: reload systemd
+  systemd:
+    daemon_reload: yes
+
+- name: reload sysctl
+  command: sysctl -p /etc/sysctl.d/30-postgresql.conf

ansible/roles/postgresql/tasks/main.yml

@@ -0,0 +1,261 @@
+---
+- name: Add PostgreSQL GPG key
+  apt_key:
+    url: https://www.postgresql.org/media/keys/ACCC4CF8.asc
+    state: present
+
+- name: Add PostgreSQL repository
+  apt_repository:
+    repo: "deb http://apt.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg main"
+    state: present
+    filename: pgdg
+
+- name: Install PostgreSQL and PostGIS
+  apt:
+    name:
+      - postgresql-{{ postgresql_version }}
+      - postgresql-client-{{ postgresql_version }}
+      - postgresql-contrib-{{ postgresql_version }}
+      - postgresql-{{ postgresql_version }}-postgis-3
+      - postgresql-{{ postgresql_version }}-postgis-3-scripts
+      - postgis
+      - python3-psycopg2
+      - libpq-dev
+      - pgbouncer
+    state: present
+    update_cache: yes
+
+- name: Stop PostgreSQL service if running
+  systemd:
+    name: postgresql
+    state: stopped
+  ignore_errors: yes
+
+- name: Disable PostgreSQL from starting automatically (we'll configure it first)
+  systemd:
+    name: postgresql
+    enabled: no
+
+- name: Remove any default PostgreSQL data directory
+  file:
+    path: /var/lib/postgresql/{{ postgresql_version }}/main
+    state: absent
+
+- name: Create PostgreSQL cluster on RAID array
+  command: |
+    /usr/lib/postgresql/{{ postgresql_version }}/bin/initdb \
+      -D {{ postgresql_data_directory }} \
+      --wal-segsize=64 \
+      --data-checksums \
+      --encoding=UTF8 \
+      --locale=en_US.UTF-8 \
+      --lc-collate=en_US.UTF-8 \
+      --lc-ctype=en_US.UTF-8
+  args:
+    creates: "{{ postgresql_data_directory }}/PG_VERSION"
+  become_user: postgres
+  environment:
+    PGDATA: "{{ postgresql_data_directory }}"
+
+- name: Configure PostgreSQL
+  template:
+    src: postgresql.conf.j2
+    dest: "{{ postgresql_config_directory }}/postgresql.conf"
+    owner: postgres
+    group: postgres
+    mode: '0644'
+  notify: restart postgresql
+
+- name: Configure PostgreSQL authentication
+  template:
+    src: pg_hba.conf.j2
+    dest: "{{ postgresql_config_directory }}/pg_hba.conf"
+    owner: postgres
+    group: postgres
+    mode: '0640'
+  notify: restart postgresql
+
+- name: Create sysctl configuration for PostgreSQL
+  template:
+    src: 30-postgresql.conf.j2
+    dest: /etc/sysctl.d/30-postgresql.conf
+    mode: '0644'
+  notify: reload sysctl
+
+- name: Create systemd override directory
+  file:
+    path: /etc/systemd/system/postgresql@.service.d
+    state: directory
+    mode: '0755'
+
+- name: Configure systemd limits for PostgreSQL
+  template:
+    src: postgresql-override.conf.j2
+    dest: /etc/systemd/system/postgresql@.service.d/override.conf
+    mode: '0644'
+  notify:
+    - reload systemd
+    - restart postgresql
+
+- name: Move WAL to separate directory on RAID
+  block:
+    - name: Stop PostgreSQL if running
+      systemd:
+        name: postgresql
+        state: stopped
+      ignore_errors: yes
+    
+    - name: Move existing WAL directory
+      command: mv {{ postgresql_data_directory }}/pg_wal {{ postgresql_wal_directory }}
+      args:
+        creates: "{{ postgresql_wal_directory }}"
+        removes: "{{ postgresql_data_directory }}/pg_wal"
+      become_user: postgres
+    
+    - name: Create symlink for WAL directory
+      file:
+        src: "{{ postgresql_wal_directory }}"
+        dest: "{{ postgresql_data_directory }}/pg_wal"
+        state: link
+        owner: postgres
+        group: postgres
+
+- name: Start and enable PostgreSQL
+  systemd:
+    name: postgresql
+    state: started
+    enabled: yes
+    daemon_reload: yes
+
+- name: Wait for PostgreSQL to start
+  wait_for:
+    port: 5432
+    host: localhost
+    delay: 5
+
+- name: Create PostgreSQL users
+  postgresql_user:
+    name: "{{ item.name }}"
+    password: "{{ item.password }}"
+    encrypted: yes
+    state: present
+  become: yes
+  become_user: postgres
+  loop: "{{ postgresql_users }}"
+  no_log: true
+
+- name: Create databases
+  postgresql_db:
+    name: "{{ item.name }}"
+    owner: "{{ item.owner }}"
+    encoding: UTF-8
+    lc_collate: en_US.UTF-8
+    lc_ctype: en_US.UTF-8
+    template: template0
+    state: present
+  become: yes
+  become_user: postgres
+  loop: "{{ postgresql_databases }}"
+
+- name: Enable PostGIS extension
+  postgresql_ext:
+    name: postgis
+    db: "{{ item.name }}"
+    state: present
+  become: yes
+  become_user: postgres
+  loop: "{{ postgresql_databases }}"
+
+- name: Enable additional extensions
+  postgresql_ext:
+    name: "{{ item[1] }}"
+    db: "{{ item[0].name }}"
+    state: present
+  become: yes
+  become_user: postgres
+  loop: "{{ postgresql_databases | product(['pg_stat_statements', 'btree_gist', 'btree_gin']) | list }}"
+
+- name: Grant privileges
+  postgresql_privs:
+    database: "{{ item.split(':')[0] }}"
+    privs: "{{ item.split(':')[1] }}"
+    type: database
+    role: "{{ user.name }}"
+    state: present
+  become: yes
+  become_user: postgres
+  loop: "{{ postgresql_users | selectattr('priv', 'defined') | map(attribute='priv') | list }}"
+  loop_control:
+    loop_var: priv_item
+  vars:
+    user: "{{ postgresql_users | selectattr('priv', 'equalto', priv_item) | first }}"
+
+- name: Configure PgBouncer
+  template:
+    src: pgbouncer.ini.j2
+    dest: /etc/pgbouncer/pgbouncer.ini
+    owner: postgres
+    group: postgres
+    mode: '0640'
+  notify: restart pgbouncer
+
+- name: Configure PgBouncer authentication
+  template:
+    src: pgbouncer-userlist.txt.j2
+    dest: /etc/pgbouncer/userlist.txt
+    owner: postgres
+    group: postgres
+    mode: '0640'
+  notify: restart pgbouncer
+
+- name: Enable and start PgBouncer
+  systemd:
+    name: pgbouncer
+    state: started
+    enabled: yes
+
+- name: Set up backup script
+  template:
+    src: backup-postgresql.sh.j2
+    dest: /usr/local/bin/backup-postgresql
+    mode: '0755'
+  when: postgresql_backup_enabled
+
+- name: Create backup directory
+  file:
+    path: "{{ postgresql_backup_directory }}"
+    state: directory
+    owner: postgres
+    group: postgres
+    mode: '0755'
+  when: postgresql_backup_enabled
+
+- name: Set up backup cron job
+  cron:
+    name: "PostgreSQL backup"
+    minute: "{{ postgresql_backup_schedule.minute }}"
+    hour: "{{ postgresql_backup_schedule.hour }}"
+    day: "{{ postgresql_backup_schedule.day }}"
+    month: "{{ postgresql_backup_schedule.month }}"
+    weekday: "{{ postgresql_backup_schedule.weekday }}"
+    user: postgres
+    job: "/usr/local/bin/backup-postgresql"
+  when: postgresql_backup_enabled
+
+- name: Configure log rotation
+  template:
+    src: postgresql-logrotate.j2
+    dest: /etc/logrotate.d/postgresql-custom
+    mode: '0644'
+
+- name: Install monitoring script
+  template:
+    src: check-postgresql.sh.j2
+    dest: /usr/local/bin/check-postgresql
+    mode: '0755'
+
+- name: Set up monitoring cron job
+  cron:
+    name: "Check PostgreSQL health"
+    minute: "*/5"
+    job: "/usr/local/bin/check-postgresql || echo 'PostgreSQL health check failed' | logger -t postgresql-health"

ansible/roles/postgresql/templates/30-postgresql.conf.j2

@@ -0,0 +1,34 @@
+# Kernel parameters for PostgreSQL optimization
+# For CM3588 with 32GB RAM and RAID0 SSDs
+
+# Shared memory settings
+kernel.shmmax = 17179869184  # 16GB
+kernel.shmall = 4194304      # 16GB / 4096 (page size)
+kernel.shmmni = 4096
+
+# Semaphores: semmsl, semmns, semopm, semmni
+kernel.sem = 250 32000 100 128
+
+# Virtual memory
+vm.swappiness = 10
+vm.dirty_ratio = 15
+vm.dirty_background_ratio = 5
+vm.overcommit_memory = 2
+vm.overcommit_ratio = 95
+
+# Huge pages (optional, can be enabled if needed)
+# vm.nr_hugepages = 4096  # 8GB of huge pages
+
+# Network optimization for 2.5Gbit connection
+net.core.rmem_default = 262144
+net.core.rmem_max = 4194304
+net.core.wmem_default = 262144
+net.core.wmem_max = 4194304
+net.ipv4.tcp_rmem = 4096 87380 4194304
+net.ipv4.tcp_wmem = 4096 65536 4194304
+net.ipv4.tcp_keepalive_time = 60
+net.ipv4.tcp_keepalive_intvl = 10
+net.ipv4.tcp_keepalive_probes = 6
+
+# File system
+fs.file-max = 65536

ansible/roles/postgresql/templates/backup-postgresql.sh.j2

@@ -0,0 +1,48 @@
+#!/bin/bash
+# PostgreSQL backup script for APRS.me
+
+set -euo pipefail
+
+# Configuration
+BACKUP_DIR="{{ postgresql_backup_directory }}"
+DB_NAME="aprsme_prod"
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+BACKUP_FILE="${BACKUP_DIR}/${DB_NAME}_${TIMESTAMP}.sql.gz"
+KEEP_DAYS=7
+
+# Create backup directory if it doesn't exist
+mkdir -p "${BACKUP_DIR}"
+
+# Log backup start
+echo "[$(date)] Starting backup of ${DB_NAME}" | logger -t postgresql-backup
+
+# Perform backup with compression
+if pg_dump -Fc -Z9 "${DB_NAME}" > "${BACKUP_FILE}.tmp"; then
+    mv "${BACKUP_FILE}.tmp" "${BACKUP_FILE}"
+    echo "[$(date)] Backup completed: ${BACKUP_FILE}" | logger -t postgresql-backup
+    
+    # Calculate backup size
+    SIZE=$(du -h "${BACKUP_FILE}" | cut -f1)
+    echo "[$(date)] Backup size: ${SIZE}" | logger -t postgresql-backup
+else
+    echo "[$(date)] Backup failed for ${DB_NAME}" | logger -t postgresql-backup -p user.err
+    rm -f "${BACKUP_FILE}.tmp"
+    exit 1
+fi
+
+# Clean up old backups
+echo "[$(date)] Cleaning up backups older than ${KEEP_DAYS} days" | logger -t postgresql-backup
+find "${BACKUP_DIR}" -name "${DB_NAME}_*.sql.gz" -type f -mtime +${KEEP_DAYS} -delete
+
+# Verify backup integrity
+if pg_restore -l "${BACKUP_FILE}" > /dev/null 2>&1; then
+    echo "[$(date)] Backup verification successful" | logger -t postgresql-backup
+else
+    echo "[$(date)] Backup verification failed" | logger -t postgresql-backup -p user.err
+    exit 1
+fi
+
+# Optional: Copy to remote backup location
+# rsync -av "${BACKUP_FILE}" user@backup-server:/path/to/backups/
+
+exit 0

ansible/roles/postgresql/templates/check-postgresql.sh.j2

@@ -0,0 +1,60 @@
+#!/bin/bash
+# PostgreSQL health check script
+
+set -euo pipefail
+
+# Check if PostgreSQL is running
+if ! systemctl is-active --quiet postgresql; then
+    echo "PostgreSQL service is not running"
+    exit 1
+fi
+
+# Check if we can connect to PostgreSQL
+if ! pg_isready -h localhost -p 5432 -U postgres -q; then
+    echo "Cannot connect to PostgreSQL"
+    exit 1
+fi
+
+# Check database connection
+if ! psql -h localhost -p 5432 -U postgres -d aprsme_prod -c "SELECT 1" > /dev/null 2>&1; then
+    echo "Cannot connect to aprsme_prod database"
+    exit 1
+fi
+
+# Check PgBouncer
+if ! pg_isready -h localhost -p 6432 -U postgres -q; then
+    echo "PgBouncer is not responding"
+    exit 1
+fi
+
+# Check replication lag (if applicable)
+LAG=$(psql -h localhost -p 5432 -U postgres -t -c "SELECT EXTRACT(EPOCH FROM (NOW() - pg_last_xact_replay_timestamp()))::INT" 2>/dev/null || echo "0")
+if [ "${LAG}" -gt "60" ]; then
+    echo "Replication lag is too high: ${LAG} seconds"
+    exit 1
+fi
+
+# Check disk space
+DISK_USAGE=$(df -h /var/lib/postgresql | tail -1 | awk '{print $5}' | sed 's/%//')
+if [ "${DISK_USAGE}" -gt "90" ]; then
+    echo "Disk usage is critical: ${DISK_USAGE}%"
+    exit 1
+fi
+
+# Check connection count
+CONNECTIONS=$(psql -h localhost -p 5432 -U postgres -t -c "SELECT COUNT(*) FROM pg_stat_activity" 2>/dev/null || echo "0")
+MAX_CONNECTIONS=$(psql -h localhost -p 5432 -U postgres -t -c "SHOW max_connections" 2>/dev/null || echo "200")
+if [ "${CONNECTIONS}" -gt "$(( ${MAX_CONNECTIONS} * 90 / 100 ))" ]; then
+    echo "Connection count is high: ${CONNECTIONS}/${MAX_CONNECTIONS}"
+    exit 1
+fi
+
+# Check for long running queries
+LONG_QUERIES=$(psql -h localhost -p 5432 -U postgres -t -c "SELECT COUNT(*) FROM pg_stat_activity WHERE state != 'idle' AND query_start < NOW() - INTERVAL '5 minutes'" 2>/dev/null || echo "0")
+if [ "${LONG_QUERIES}" -gt "5" ]; then
+    echo "Too many long running queries: ${LONG_QUERIES}"
+    exit 1
+fi
+
+echo "PostgreSQL health check passed"
+exit 0

ansible/roles/postgresql/templates/pg_hba.conf.j2

@@ -0,0 +1,9 @@
+# PostgreSQL Client Authentication Configuration File
+# ===================================================
+# Managed by Ansible - Do not edit manually
+
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+
+{% for entry in postgresql_hba_entries %}
+{{ entry.type }}   {{ entry.database }}   {{ entry.user }}   {{ entry.address | default('') }}   {{ entry.auth_method }}
+{% endfor %}

ansible/roles/postgresql/templates/pgbouncer-userlist.txt.j2

@@ -0,0 +1,10 @@
+# PgBouncer userlist file
+# Format: "username" "password"
+# Managed by Ansible
+
+{% for user in postgresql_users %}
+"{{ user.name }}" "{{ user.password }}"
+{% endfor %}
+
+# PgBouncer stats user
+"pgbouncer" "pgbouncer"

ansible/roles/postgresql/templates/pgbouncer.ini.j2

@@ -0,0 +1,66 @@
+[databases]
+# Connection string for aprsme database
+aprsme_prod = host=127.0.0.1 port=5432 dbname=aprsme_prod
+
+# Add a pooled connection for PgBouncer statistics
+pgbouncer = host=127.0.0.1 port=5432 dbname=pgbouncer auth_user=pgbouncer
+
+[pgbouncer]
+# Connection settings
+listen_addr = *
+listen_port = 6432
+auth_type = scram-sha-256
+auth_file = /etc/pgbouncer/userlist.txt
+
+# Pool settings optimized for APRS.me
+pool_mode = transaction
+max_client_conn = 1000
+default_pool_size = 25
+min_pool_size = 5
+reserve_pool_size = 5
+reserve_pool_timeout = 3
+
+# Performance settings
+max_db_connections = 100
+max_user_connections = 100
+server_lifetime = 3600
+server_idle_timeout = 600
+server_connect_timeout = 15
+server_login_retry = 15
+query_timeout = 0
+query_wait_timeout = 120
+client_idle_timeout = 0
+client_login_timeout = 60
+
+# Logging
+log_connections = 1
+log_disconnections = 1
+log_pooler_errors = 1
+stats_period = 60
+
+# Admin settings
+admin_users = postgres
+stats_users = postgres, aprsme
+
+# Security
+ignore_startup_parameters = extra_float_digits
+
+# Resource limits
+pkt_buf = 4096
+max_packet_size = 2147483647
+listen_backlog = 128
+sbuf_loopcnt = 5
+tcp_defer_accept = 0
+tcp_socket_buffer = 0
+tcp_keepalive = 1
+tcp_keepcnt = 3
+tcp_keepidle = 60
+tcp_keepintvl = 10
+
+# DNS settings
+dns_max_ttl = 15
+dns_nxdomain_ttl = 15
+
+# TLS settings (optional)
+# client_tls_sslmode = prefer
+# client_tls_ca_file = /etc/ssl/certs/ca-certificates.crt

ansible/roles/postgresql/templates/postgresql-logrotate.j2

@@ -0,0 +1,16 @@
+/var/log/postgresql/*.log {
+    daily
+    rotate 7
+    missingok
+    compress
+    delaycompress
+    notifempty
+    create 640 postgres postgres
+    sharedscripts
+    postrotate
+        # Signal PostgreSQL to reopen log files
+        if [ -f /var/run/postgresql/{{ postgresql_version }}-main.pid ]; then
+            /usr/bin/pg_ctlcluster {{ postgresql_version }} main reload > /dev/null
+        fi
+    endscript
+}

ansible/roles/postgresql/templates/postgresql-override.conf.j2

@@ -0,0 +1,17 @@
+[Service]
+# Increase limits for PostgreSQL
+LimitNOFILE=65536
+LimitNPROC=4096
+
+# Disable OOM killer for PostgreSQL
+OOMScoreAdjust=-1000
+
+# CPU affinity - use all cores
+CPUAffinity=0-7
+
+# Nice level for better priority
+Nice=-5
+
+# I/O scheduling class and priority
+IOSchedulingClass=best-effort
+IOSchedulingPriority=0

ansible/roles/postgresql/templates/postgresql.conf.j2

@@ -0,0 +1,103 @@
+# PostgreSQL configuration for APRS.me on CM3588
+# Optimized for RK3588 with RAID0 SSDs
+
+#------------------------------------------------------------------------------
+# FILE LOCATIONS
+#------------------------------------------------------------------------------
+
+data_directory = '{{ postgresql_data_directory }}'
+hba_file = '{{ postgresql_config_directory }}/pg_hba.conf'
+ident_file = '{{ postgresql_config_directory }}/pg_ident.conf'
+external_pid_file = '/var/run/postgresql/{{ postgresql_version }}-main.pid'
+
+#------------------------------------------------------------------------------
+# CONNECTIONS AND AUTHENTICATION
+#------------------------------------------------------------------------------
+
+{% for key, value in postgresql_config.items() %}
+{{ key }} = {{ value }}
+{% endfor %}
+
+#------------------------------------------------------------------------------
+# RESOURCE USAGE (except WAL)
+#------------------------------------------------------------------------------
+
+# These settings are configured above in postgresql_config
+
+#------------------------------------------------------------------------------
+# WRITE-AHEAD LOG
+#------------------------------------------------------------------------------
+
+# WAL settings configured above
+
+#------------------------------------------------------------------------------
+# REPLICATION
+#------------------------------------------------------------------------------
+
+wal_level = replica
+archive_mode = on
+archive_command = 'test ! -f {{ postgresql_archive_directory }}/%f && cp %p {{ postgresql_archive_directory }}/%f'
+max_wal_senders = 3
+wal_keep_size = 1GB
+
+#------------------------------------------------------------------------------
+# QUERY TUNING
+#------------------------------------------------------------------------------
+
+# Planner settings for PostGIS and complex queries
+enable_partitionwise_join = on
+enable_partitionwise_aggregate = on
+jit = on
+
+#------------------------------------------------------------------------------
+# REPORTING AND LOGGING
+#------------------------------------------------------------------------------
+
+# Logging configuration set above
+
+#------------------------------------------------------------------------------
+# STATISTICS
+#------------------------------------------------------------------------------
+
+# Stats configuration set above
+shared_preload_libraries = 'pg_stat_statements'
+pg_stat_statements.max = 10000
+pg_stat_statements.track = all
+
+#------------------------------------------------------------------------------
+# AUTOVACUUM
+#------------------------------------------------------------------------------
+
+# Autovacuum settings configured above
+
+#------------------------------------------------------------------------------
+# CLIENT CONNECTION DEFAULTS
+#------------------------------------------------------------------------------
+
+datestyle = 'iso, mdy'
+timezone = 'UTC'
+lc_messages = 'en_US.UTF-8'
+lc_monetary = 'en_US.UTF-8'
+lc_numeric = 'en_US.UTF-8'
+lc_time = 'en_US.UTF-8'
+default_text_search_config = 'pg_catalog.english'
+
+#------------------------------------------------------------------------------
+# LOCK MANAGEMENT
+#------------------------------------------------------------------------------
+
+deadlock_timeout = 1s
+max_locks_per_transaction = 128
+
+#------------------------------------------------------------------------------
+# VERSION/PLATFORM COMPATIBILITY
+#------------------------------------------------------------------------------
+
+# No specific settings needed
+
+#------------------------------------------------------------------------------
+# ERROR HANDLING
+#------------------------------------------------------------------------------
+
+exit_on_error = off
+restart_after_crash = on

ansible/roles/raid/tasks/main.yml

@@ -0,0 +1,210 @@
+---
+- name: Install RAID management tools
+  apt:
+    name:
+      - mdadm
+      - smartmontools
+      - parted
+      - nvme-cli
+      - lsscsi
+    state: present
+    update_cache: yes
+
+- name: Gather disk information
+  shell: |
+    echo "=== Block devices ==="
+    lsblk -d -o NAME,SIZE,TYPE,MODEL
+    echo "=== NVMe devices ==="
+    nvme list 2>/dev/null || echo "No NVMe devices found"
+    echo "=== SCSI devices ==="
+    lsscsi -s 2>/dev/null || echo "No SCSI devices found"
+    echo "=== All disk devices ==="
+    ls -la /dev/sd* /dev/nvme* /dev/vd* /dev/mmcblk* 2>/dev/null | grep -E "^b" || echo "Checking other paths..."
+    echo "=== Detailed block device tree ==="
+    lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINT,MODEL
+  register: disk_info
+  changed_when: false
+
+- name: Display disk information
+  debug:
+    msg: "{{ disk_info.stdout_lines }}"
+
+- name: Check if we should auto-detect RAID devices
+  set_fact:
+    auto_detect_devices: "{{ raid_devices is not defined or raid_devices | length == 0 }}"
+
+- name: Auto-detect two largest unused disks
+  when: auto_detect_devices
+  block:
+    - name: Find unused block devices
+      shell: |
+        lsblk -dpno NAME,SIZE,TYPE,MOUNTPOINT | grep -E "disk.*$" | grep -v -E "/(|/boot)" | sort -k2 -h -r | head -2 | awk '{print $1}'
+      register: detected_devices
+      changed_when: false
+
+    - name: Set raid_devices from detected devices
+      set_fact:
+        raid_devices: "{{ detected_devices.stdout_lines }}"
+      when: detected_devices.stdout_lines | length >= 2
+
+    - name: Fail if not enough devices detected
+      fail:
+        msg: "Could not detect at least 2 unused block devices for RAID. Please specify raid_devices manually."
+      when: detected_devices.stdout_lines | length < 2
+
+- name: Display RAID devices to be used
+  debug:
+    msg: "Will use these devices for RAID0: {{ raid_devices }}"
+
+- name: Check if RAID array already exists
+  command: mdadm --detail /dev/md0
+  register: raid_status
+  failed_when: false
+  changed_when: false
+
+- name: Stop and remove existing RAID array if it exists
+  block:
+    - name: Stop RAID array
+      command: mdadm --stop /dev/md0
+      when: raid_status.rc == 0
+
+    - name: Zero superblocks on disks
+      command: mdadm --zero-superblock {{ item }}
+      loop:
+        - /dev/nvme0n1
+        - /dev/nvme1n1
+      when: raid_status.rc == 0
+  when: force_raid_rebuild | default(false)
+
+- name: Create partition table on first NVMe drive
+  parted:
+    device: /dev/nvme0n1
+    number: 1
+    state: present
+    part_type: primary
+    part_start: 0%
+    part_end: 100%
+    label: gpt
+  when: raid_status.rc != 0
+
+- name: Create partition table on second NVMe drive
+  parted:
+    device: /dev/nvme1n1
+    number: 1
+    state: present
+    part_type: primary
+    part_start: 0%
+    part_end: 100%
+    label: gpt
+  when: raid_status.rc != 0
+
+- name: Create RAID0 array
+  command: >
+    mdadm --create /dev/md0
+    --level=0
+    --raid-devices=2
+    /dev/nvme0n1p1 /dev/nvme1n1p1
+    --force
+  when: raid_status.rc != 0
+  register: raid_create
+
+- name: Wait for RAID array to be ready
+  command: mdadm --detail /dev/md0
+  register: raid_ready
+  until: raid_ready.rc == 0
+  retries: 10
+  delay: 5
+  when: raid_create is changed
+
+- name: Create mdadm configuration
+  shell: mdadm --detail --scan >> /etc/mdadm/mdadm.conf
+  when: raid_create is changed
+
+- name: Update initramfs
+  command: update-initramfs -u
+  when: raid_create is changed
+
+- name: Create filesystem on RAID array
+  filesystem:
+    fstype: ext4
+    dev: /dev/md0
+    opts: "-E lazy_itable_init=0,lazy_journal_init=0"
+  when: raid_create is changed
+
+- name: Create mount point for PostgreSQL data
+  file:
+    path: /mnt/pgdata
+    state: directory
+    mode: '0755'
+
+- name: Mount RAID array
+  mount:
+    path: /mnt/pgdata
+    src: /dev/md0
+    fstype: ext4
+    opts: noatime,nodiratime,nobarrier
+    state: mounted
+
+- name: Create PostgreSQL user and group
+  block:
+    - name: Create postgres group
+      group:
+        name: postgres
+        state: present
+        system: yes
+
+    - name: Create postgres user
+      user:
+        name: postgres
+        group: postgres
+        home: /var/lib/postgresql
+        shell: /bin/bash
+        system: yes
+        state: present
+
+- name: Ensure PostgreSQL directories exist on RAID
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: postgres
+    group: postgres
+    mode: '0700'
+  loop:
+    - /mnt/pgdata
+    - /mnt/pgdata/16
+    - /mnt/pgdata/16/main
+    - /mnt/pgdata/wal
+    - /mnt/pgdata/archive
+    - /mnt/pgdata/backups
+
+- name: Set up RAID monitoring
+  lineinfile:
+    path: /etc/mdadm/mdadm.conf
+    line: "MAILADDR root"
+    insertafter: EOF
+
+- name: Enable mdadm monitoring service
+  systemd:
+    name: mdmonitor
+    enabled: yes
+    state: started
+
+- name: Create RAID monitoring script
+  copy:
+    content: |
+      #!/bin/bash
+      # RAID health check script
+      RAID_STATUS=$(mdadm --detail /dev/md0 | grep "State :" | awk '{print $3}')
+      if [ "$RAID_STATUS" != "clean" ] && [ "$RAID_STATUS" != "active" ]; then
+          echo "RAID array /dev/md0 is in state: $RAID_STATUS"
+          exit 1
+      fi
+      exit 0
+    dest: /usr/local/bin/check-raid-health
+    mode: '0755'
+
+- name: Set up RAID health check cron job
+  cron:
+    name: "Check RAID health"
+    minute: "*/5"
+    job: "/usr/local/bin/check-raid-health || echo 'RAID health check failed' | logger -t raid-health"

ansible/site.yml

@@ -0,0 +1,8 @@
+---
+- name: Configure PostgreSQL server on CM3588
+  hosts: postgres
+  become: yes
+  roles:
+    - common
+    - raid
+    - postgresql

clusters/aprs/AUTO-HEALING-GUIDE.md

@@ -0,0 +1,154 @@
+# PostgreSQL Auto-Healing Configuration Guide
+
+This guide explains the auto-healing mechanisms implemented to prevent and recover from PostgreSQL connection exhaustion issues.
+
+## Overview
+
+The auto-healing system consists of several components working together:
+
+1. **Enhanced PostgreSQL Configuration**
+2. **Improved PgBouncer Connection Pooling**
+3. **Health Checks and Automatic Restarts**
+4. **Periodic Connection Cleanup**
+5. **Resource Limits and Scaling**
+
+## Components
+
+### 1. PostgreSQL Enhancements (`postgis-deployment-enhanced.yaml`)
+
+- **Increased max_connections**: From 100 to 200
+- **Connection timeouts**:
+  - `idle_in_transaction_session_timeout=30s`: Kills idle transactions after 30 seconds
+  - `statement_timeout=300s`: Kills queries running longer than 5 minutes
+- **Readiness probe**: Fails if connection count exceeds 180 (90% of max)
+- **Increased resources**: 2Gi memory, 1 CPU
+
+### 2. PgBouncer Enhancements (`pgbouncer-deployment-enhanced.yaml`)
+
+- **Connection pooling settings**:
+  - Pool mode: `transaction` (releases connections after each transaction)
+  - Default pool size: 25 connections per database/user pair
+  - Server idle timeout: 10 minutes
+  - Query timeout: 5 minutes
+- **Retry logic**:
+  - `server_login_retry=15`: Retries failed logins for 15 seconds
+  - `server_round_robin=1`: Distributes connections evenly
+- **Enhanced logging**: Tracks connections, disconnections, and errors
+
+### 3. APRS Application Enhancements (`aprs-statefulset-enhanced.yaml`)
+
+- **Reduced connection pool**: From 10 to 5 connections per pod
+- **Connection timeouts**:
+  - Pool timeout: 60 seconds
+  - Connect timeout: 30 seconds
+  - Idle timeout: 15 minutes
+- **Startup probe**: Allows up to 120 seconds for application startup
+- **Increased resources**: 1Gi memory, 1 CPU
+
+### 4. Automatic Connection Cleanup (`postgres-cleanup-cronjob.yaml`)
+
+Runs every 15 minutes to:
+- Kill idle connections older than 30 minutes
+- Kill idle-in-transaction connections older than 5 minutes
+- Log connection statistics
+- Monitor connection usage percentage
+
+### 5. Pod Disruption Budgets (`postgis-pdb.yaml`)
+
+Ensures at least 1 replica of PostgreSQL and PgBouncer remain available during:
+- Cluster upgrades
+- Node maintenance
+- Voluntary disruptions
+
+## How Auto-Healing Works
+
+### Connection Exhaustion Prevention
+
+1. **Application-level**: Reduced pool size prevents each pod from using too many connections
+2. **PgBouncer-level**: Connection pooling multiplexes many client connections over fewer server connections
+3. **PostgreSQL-level**: Timeouts automatically close stale connections
+
+### Automatic Recovery
+
+1. **Health checks**: Pods restart automatically when:
+   - PostgreSQL has too many connections (>180)
+   - PgBouncer can't connect to PostgreSQL
+   - Application can't connect to the database
+
+2. **CronJob cleanup**: Every 15 minutes, forcefully closes:
+   - Idle connections wasting resources
+   - Stuck transactions blocking others
+
+3. **Connection retry**: PgBouncer retries failed connections for 15 seconds before giving up
+
+## Monitoring
+
+Use the provided monitoring script to check system health:
+
+```bash
+./monitor-postgres-health.sh
+```
+
+This shows:
+- Current connection count and states
+- Connection usage percentage
+- Recent errors from logs
+- Last cleanup job execution
+
+## Manual Intervention
+
+If automatic healing fails, you can manually intervene:
+
+```bash
+# Check connection details
+kubectl exec -it deployment/postgis -n aprs -- psql -U aprs -d aprs_prod -c "SELECT * FROM pg_stat_activity WHERE datname = 'aprs_prod';"
+
+# Kill all connections (emergency)
+kubectl exec -it deployment/postgis -n aprs -- psql -U aprs -d aprs_prod -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'aprs_prod' AND pid <> pg_backend_pid();"
+
+# Restart components
+kubectl rollout restart deployment/postgis -n aprs
+kubectl rollout restart deployment/pgbouncer -n aprs
+kubectl rollout restart statefulset/aprs -n aprs
+```
+
+## Applying the Configuration
+
+To apply all auto-healing configurations:
+
+```bash
+cd /Users/graham/dev/infra/clusters/aprs
+./apply-auto-healing.sh
+```
+
+This script:
+1. Backs up current configurations
+2. Applies all enhanced configurations
+3. Waits for rollouts to complete
+4. Provides a summary of changes
+
+## Tuning Parameters
+
+You may need to adjust these based on your workload:
+
+1. **PostgreSQL max_connections**: Currently 200, increase if needed
+2. **PgBouncer pool sizes**: Currently 25, adjust based on connection patterns
+3. **Application POOL_SIZE**: Currently 5, can be increased if connections are available
+4. **Cleanup intervals**: Currently 15 minutes, can be more or less frequent
+
+## Troubleshooting
+
+Common issues and solutions:
+
+1. **"too many clients already"**: Cleanup job should fix this within 15 minutes
+2. **"server login has been failing"**: PgBouncer is in retry mode, will recover in 15 seconds
+3. **Pods in CrashLoopBackOff**: Check logs for specific errors
+4. **High memory usage**: Consider increasing resource limits
+
+## Future Improvements
+
+Consider implementing:
+1. Prometheus metrics for connection monitoring
+2. Alerting when connection usage exceeds 80%
+3. Horizontal pod autoscaling based on connection metrics
+4. Connection pooling at the application level using a library like `poolboy`

clusters/aprs/apply-auto-healing.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+set -e
+
+echo "Applying PostgreSQL and PgBouncer auto-healing configurations..."
+
+# Backup current deployments
+echo "Creating backups of current deployments..."
+kubectl get deployment postgis -n aprs -o yaml > postgis-deployment-backup.yaml || true
+kubectl get deployment pgbouncer -n aprs -o yaml > pgbouncer-deployment-backup.yaml || true
+kubectl get statefulset aprs -n aprs -o yaml > aprs-statefulset-backup.yaml || true
+
+# Apply enhanced configurations
+echo "Applying enhanced PostgreSQL deployment..."
+kubectl apply -f postgis-deployment-enhanced.yaml
+
+echo "Applying enhanced PgBouncer deployment..."
+kubectl apply -f pgbouncer-deployment-enhanced.yaml
+
+echo "Applying PodDisruptionBudgets..."
+kubectl apply -f postgis-pdb.yaml
+
+echo "Applying enhanced APRS StatefulSet..."
+kubectl apply -f aprs-statefulset-enhanced.yaml
+
+echo "Applying PostgreSQL cleanup CronJob..."
+kubectl apply -f postgres-cleanup-cronjob.yaml
+
+# Wait for rollouts
+echo "Waiting for PostgreSQL rollout..."
+kubectl rollout status deployment/postgis -n aprs --timeout=300s
+
+echo "Waiting for PgBouncer rollout..."
+kubectl rollout status deployment/pgbouncer -n aprs --timeout=300s
+
+echo "Waiting for APRS rollout..."
+kubectl rollout status statefulset/aprs -n aprs --timeout=300s
+
+echo "Auto-healing configurations applied successfully!"
+echo ""
+echo "Summary of changes:"
+echo "1. PostgreSQL: Increased max_connections to 200, added connection timeouts"
+echo "2. PgBouncer: Enhanced retry logic and connection pooling"
+echo "3. APRS pods: Reduced pool size to 5, added startup probes"
+echo "4. Added health checks that restart pods when connections are exhausted"
+echo "5. CronJob runs every 15 minutes to clean up stale connections"
+echo ""
+echo "Monitor the deployment with:"
+echo "  kubectl get pods -n aprs -w"
+echo ""
+echo "Check PostgreSQL connections with:"
+echo "  kubectl exec -it deployment/postgis -n aprs -- psql -U aprs -d aprs_prod -c 'SELECT state, count(*) FROM pg_stat_activity GROUP BY state;'"

clusters/aprs/aprs-pdb.yaml

@@ -0,0 +1,11 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: aprs-pdb
+  namespace: aprs
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: aprs
+  unhealthyPodEvictionPolicy: AlwaysAllow

clusters/aprs/aprs-service.yaml

@@ -11,5 +11,7 @@ spec:
   ports:
     - port: 80
       targetPort: 4000
+      name: http
   type: LoadBalancer
-  loadBalancerIP: 10.0.19.221
+  loadBalancerIP: 10.0.19.221
+  sessionAffinity: None

clusters/aprs/aprs-statefulset-enhanced.yaml

@@ -0,0 +1,132 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: aprs
+  namespace: aprs
+spec:
+  serviceName: aprs-headless
+  replicas: 2
+  selector:
+    matchLabels:
+      app: aprs
+  updateStrategy:
+    type: RollingUpdate
+  podManagementPolicy: Parallel
+  template:
+    metadata:
+      labels:
+        app: aprs
+    spec:
+      terminationGracePeriodSeconds: 60
+      initContainers:
+        - name: wait-for-db
+          image: busybox:1.36.1
+          command: ['sh', '-c', 'until nc -z pgbouncer 5432; do echo waiting for database; sleep 2; done']
+        - name: wait-for-redis
+          image: busybox:1.36.1
+          command: ['sh', '-c', 'until nc -z redis 6379; do echo waiting for redis; sleep 2; done']
+      containers:
+        - name: aprs
+          image: ghcr.io/aprsme/aprs.me:latest
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 4000
+              protocol: TCP
+          env:
+            - name: SECRET_KEY_BASE
+              valueFrom:
+                secretKeyRef:
+                  name: aprs-secret
+                  key: secret-key-base
+            - name: DATABASE_URL
+              value: "ecto://aprs:$(DATABASE_PASSWORD)@pgbouncer:5432/aprs"
+            - name: DATABASE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: aprs-secret
+                  key: database-password
+            - name: PHX_HOST
+              value: "aprs.me"
+            - name: PHX_SERVER
+              value: "true"
+            - name: POOL_SIZE
+              value: "5"  # Reduced from default 10 to prevent connection exhaustion
+            - name: ECTO_IPV6
+              value: "false"
+            - name: ERLANG_COOKIE
+              valueFrom:
+                secretKeyRef:
+                  name: aprs-secret
+                  key: erlang-cookie
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: REDIS_URL
+              value: "redis://redis.aprs.svc.cluster.local:6379"
+            - name: DRAIN_TIMEOUT_MS
+              value: "45000"
+            - name: SKIP_DB_CREATE
+              value: "true"
+            # Enhanced database connection settings
+            - name: DATABASE_POOL_TIMEOUT
+              value: "60000"  # 60 seconds
+            - name: DATABASE_QUEUE_TARGET
+              value: "50"
+            - name: DATABASE_QUEUE_INTERVAL
+              value: "1000"
+            - name: DATABASE_CONNECT_TIMEOUT
+              value: "30000"  # 30 seconds
+            - name: DATABASE_IDLE_TIMEOUT
+              value: "900000"  # 15 minutes
+            # Memory management
+            - name: ERL_MAX_ETS_TABLES
+              value: "5000"
+            - name: ERLANG_SCHEDULER_BUSY_WAIT_THRESHOLD
+              value: "none"
+          resources:
+            limits:
+              memory: "1Gi"   # Increased from 512Mi
+              cpu: "1000m"     # Increased from 500m
+            requests:
+              memory: "512Mi"  # Increased from 256Mi
+              cpu: "500m"      # Increased from 250m
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 4000
+            initialDelaySeconds: 60  # Increased from 30
+            periodSeconds: 10
+            timeoutSeconds: 10
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 4000
+            initialDelaySeconds: 30  # Increased from 5
+            periodSeconds: 5
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+          startupProbe:
+            httpGet:
+              path: /health
+              port: 4000
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 10
+            failureThreshold: 12  # 120 seconds total
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "sleep 15"]
+      imagePullSecrets:
+        - name: ghcr-pull-secret

clusters/aprs/aprs-statefulset.yaml

@@ -28,6 +28,11 @@ metadata:
 spec:
   serviceName: aprs-headless
   replicas: 2
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      partition: 0
+  podManagementPolicy: Parallel
   selector:
     matchLabels:
       app: aprs
@@ -36,6 +41,7 @@ spec:
       labels:
         app: aprs
     spec:
+      terminationGracePeriodSeconds: 60
       serviceAccountName: aprs-service-account
       imagePullSecrets:
         - name: ghcr-pull-secret
@@ -61,7 +67,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: aprs-secret
-                  key: database-url
+                  key: database-url-pgbouncer
             - name: SECRET_KEY_BASE
               valueFrom:
                 secretKeyRef:
@@ -84,7 +90,7 @@ spec:
             - name: PACKET_RETENTION_DAYS
               value: "7"
             - name: POOL_SIZE
-              value: "20"
+              value: "5"
             - name: CLUSTER_ENABLED
               value: "true"
             - name: RELEASE_COOKIE
@@ -104,6 +110,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: REDIS_URL
+              value: "redis://redis.aprs.svc.cluster.local:6379"
+            - name: DRAIN_TIMEOUT_MS
+              value: "45000"
+            - name: SKIP_DB_CREATE
+              value: "true"
           resources:
             limits:
               memory: "512Mi"
@@ -122,4 +134,8 @@ spec:
               path: /health
               port: 4000
             initialDelaySeconds: 5
-            periodSeconds: 5
+            periodSeconds: 5
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "sleep 15"]

clusters/aprs/generate-secrets-from-1password.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# Generate Kubernetes secrets from 1Password
+
+# Ensure we're logged in to 1Password
+if ! op whoami &>/dev/null; then
+    echo "Please log in to 1Password first: eval \$(op signin)"
+    exit 1
+fi
+
+# Get PostgreSQL password
+POSTGRES_PASSWORD=$(op item get "APRS PostgreSQL Password" --fields password)
+
+# Get APRS application secrets
+DATABASE_URL=$(op item get "APRS Application Secrets" --fields database.url)
+DATABASE_URL_PGBOUNCER=$(op item get "APRS Application Secrets" --fields database.url.pgbouncer)
+DATABASE_PASSWORD=$(op item get "APRS Application Secrets" --fields password)
+SECRET_KEY_BASE=$(op item get "APRS Application Secrets" --fields secret.key.base)
+ERLANG_COOKIE=$(op item get "APRS Application Secrets" --fields erlang.cookie)
+
+# Get GitHub Container Registry credentials
+GHCR_USERNAME=$(op item get "GitHub Container Registry" --fields username)
+GHCR_PASSWORD=$(op item get "GitHub Container Registry" --fields password)
+GHCR_AUTH=$(echo -n "${GHCR_USERNAME}:${GHCR_PASSWORD}" | base64)
+
+# Create PostgreSQL secret
+kubectl create secret generic postgis-secret \
+  --namespace=aprs \
+  --from-literal=postgres-password="${POSTGRES_PASSWORD}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# Create APRS application secret
+kubectl create secret generic aprs-secret \
+  --namespace=aprs \
+  --from-literal=database-url="${DATABASE_URL}" \
+  --from-literal=database-url-pgbouncer="${DATABASE_URL_PGBOUNCER}" \
+  --from-literal=database-password="${DATABASE_PASSWORD}" \
+  --from-literal=secret-key-base="${SECRET_KEY_BASE}" \
+  --from-literal=erlang-cookie="${ERLANG_COOKIE}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# Create Docker registry secret
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ghcr-pull-secret
+  namespace: aprs
+type: kubernetes.io/dockerconfigjson
+stringData:
+  .dockerconfigjson: |
+    {
+      "auths": {
+        "ghcr.io": {
+          "username": "${GHCR_USERNAME}",
+          "password": "${GHCR_PASSWORD}",
+          "auth": "${GHCR_AUTH}"
+        }
+      }
+    }
+EOF
+
+echo "Secrets successfully created/updated from 1Password"

clusters/aprs/monitor-postgres-health.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+echo "=== PostgreSQL Connection Health Check ==="
+echo ""
+
+# Check pod status
+echo "Pod Status:"
+kubectl get pods -n aprs | grep -E "(NAME|postgis|pgbouncer|aprs-)"
+echo ""
+
+# Check PostgreSQL connections
+echo "PostgreSQL Connection Stats:"
+kubectl exec deployment/postgis -n aprs -- psql -U aprs -d aprs_prod -c "
+SELECT 
+    state,
+    COUNT(*) as connections,
+    ROUND(AVG(EXTRACT(EPOCH FROM (NOW() - state_change))), 2) as avg_duration_seconds
+FROM pg_stat_activity
+WHERE datname = 'aprs_prod'
+GROUP BY state
+ORDER BY connections DESC;" 2>/dev/null || echo "Failed to query PostgreSQL"
+echo ""
+
+# Check connection percentage
+echo "Connection Usage:"
+kubectl exec deployment/postgis -n aprs -- psql -U aprs -d aprs_prod -c "
+SELECT 
+    (SELECT COUNT(*) FROM pg_stat_activity) as current,
+    (SELECT setting::int FROM pg_settings WHERE name = 'max_connections') as max,
+    ROUND(100.0 * COUNT(*) / (SELECT setting::int FROM pg_settings WHERE name = 'max_connections'), 2) || '%' as usage
+FROM pg_stat_activity;" 2>/dev/null || echo "Failed to query PostgreSQL"
+echo ""
+
+# Check PgBouncer logs for errors
+echo "Recent PgBouncer Errors (last 10):"
+kubectl logs -n aprs -l app=pgbouncer --tail=50 | grep -i "error\|warning" | tail -10
+echo ""
+
+# Check APRS pod logs for connection errors
+echo "Recent APRS Connection Errors (last 10):"
+kubectl logs -n aprs -l app=aprs --tail=100 | grep -i "postgrex\|connection\|protocol_violation" | tail -10
+echo ""
+
+# Show last cleanup job run
+echo "Last Cleanup Job Run:"
+kubectl get jobs -n aprs | grep postgres-cleanup | tail -1

clusters/aprs/pgbouncer-configmap.yaml

@@ -0,0 +1,50 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pgbouncer-config
+  namespace: aprs
+data:
+  pgbouncer.ini: |
+    [databases]
+    aprs = host=postgis port=5432 dbname=aprs
+
+    [pgbouncer]
+    listen_addr = 0.0.0.0
+    listen_port = 6432
+    auth_type = md5
+    auth_file = /etc/pgbouncer/userlist.txt
+    pool_mode = transaction
+    max_client_conn = 1000
+    default_pool_size = 25
+    reserve_pool_size = 5
+    reserve_pool_timeout = 3
+    server_lifetime = 3600
+    server_idle_timeout = 600
+    log_connections = 0
+    log_disconnections = 0
+    stats_period = 60
+    
+    # Important for prepared statements
+    server_reset_query = DISCARD ALL
+    server_reset_query_always = 1
+    
+    # Connection limits
+    min_pool_size = 10
+    max_db_connections = 100
+    
+    # Timeouts
+    query_timeout = 0
+    query_wait_timeout = 120
+    client_idle_timeout = 0
+    client_login_timeout = 60
+    
+    # For better performance
+    pkt_buf = 8192
+    sbuf_loopcnt = 2
+    tcp_keepalive = 1
+    tcp_keepidle = 1
+    tcp_keepintvl = 1
+    tcp_keepcnt = 1
+    
+    # Ignore certain PostgreSQL parameters
+    ignore_startup_parameters = extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout

clusters/aprs/pgbouncer-deployment-enhanced.yaml

@@ -0,0 +1,131 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: pgbouncer
+  namespace: aprs
+  labels:
+    app: pgbouncer
+spec:
+  ports:
+  - port: 5432
+    targetPort: 6432
+    name: pgbouncer
+  selector:
+    app: pgbouncer
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pgbouncer
+  namespace: aprs
+  labels:
+    app: pgbouncer
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: pgbouncer
+  template:
+    metadata:
+      labels:
+        app: pgbouncer
+    spec:
+      containers:
+      - name: pgbouncer
+        image: bitnami/pgbouncer:1.22.1
+        ports:
+        - containerPort: 6432
+          name: pgbouncer
+        env:
+        - name: POSTGRESQL_HOST
+          value: "postgis"
+        - name: POSTGRESQL_PORT
+          value: "5432"
+        - name: POSTGRESQL_DATABASE
+          value: "aprs"
+        - name: POSTGRESQL_USERNAME
+          value: "aprs"
+        - name: POSTGRESQL_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: aprs-secret
+              key: database-password
+        - name: PGBOUNCER_DATABASE
+          value: "aprs"
+        - name: PGBOUNCER_PORT
+          value: "6432"
+        - name: PGBOUNCER_POOL_MODE
+          value: "transaction"
+        - name: PGBOUNCER_MAX_CLIENT_CONN
+          value: "1000"
+        - name: PGBOUNCER_DEFAULT_POOL_SIZE
+          value: "25"
+        - name: PGBOUNCER_MIN_POOL_SIZE
+          value: "5"
+        - name: PGBOUNCER_RESERVE_POOL_SIZE
+          value: "5"
+        - name: PGBOUNCER_SERVER_RESET_QUERY
+          value: "DISCARD ALL"
+        - name: PGBOUNCER_STATS_USERS
+          value: "aprs"
+        - name: PGBOUNCER_AUTH_TYPE
+          value: "md5"
+        - name: PGBOUNCER_IGNORE_STARTUP_PARAMETERS
+          value: "extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout"
+        # Enhanced connection handling
+        - name: PGBOUNCER_SERVER_IDLE_TIMEOUT
+          value: "600"  # 10 minutes
+        - name: PGBOUNCER_SERVER_LIFETIME
+          value: "3600"  # 1 hour
+        - name: PGBOUNCER_QUERY_TIMEOUT
+          value: "300"  # 5 minutes
+        - name: PGBOUNCER_QUERY_WAIT_TIMEOUT
+          value: "120"  # 2 minutes  
+        - name: PGBOUNCER_CLIENT_IDLE_TIMEOUT
+          value: "300"  # 5 minutes
+        - name: PGBOUNCER_CLIENT_LOGIN_TIMEOUT
+          value: "60"  # 1 minute
+        # Connection retry settings
+        - name: PGBOUNCER_SERVER_CONNECT_TIMEOUT
+          value: "15"  
+        - name: PGBOUNCER_SERVER_LOGIN_RETRY
+          value: "15"  # Retry for 15 seconds before failing
+        - name: PGBOUNCER_SERVER_ROUND_ROBIN
+          value: "1"  # Enable round-robin
+        - name: PGBOUNCER_LOG_CONNECTIONS
+          value: "1"
+        - name: PGBOUNCER_LOG_DISCONNECTIONS
+          value: "1"
+        - name: PGBOUNCER_LOG_POOLER_ERRORS
+          value: "1"
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "100m"
+          limits:
+            memory: "256Mi"
+            cpu: "200m"
+        livenessProbe:
+          tcpSocket:
+            port: 6432
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 3
+        readinessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - |
+              nc -z localhost 6432 && \
+              echo "SHOW POOLS;" | psql -h localhost -p 6432 -U pgbouncer pgbouncer || exit 0
+          initialDelaySeconds: 5
+          periodSeconds: 5
+          timeoutSeconds: 5
+          failureThreshold: 2
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 10"]

clusters/aprs/pgbouncer-deployment.yaml

@@ -0,0 +1,96 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: pgbouncer
+  namespace: aprs
+  labels:
+    app: pgbouncer
+spec:
+  ports:
+  - port: 5432
+    targetPort: 6432
+    name: pgbouncer
+  selector:
+    app: pgbouncer
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pgbouncer
+  namespace: aprs
+  labels:
+    app: pgbouncer
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: pgbouncer
+  template:
+    metadata:
+      labels:
+        app: pgbouncer
+    spec:
+      containers:
+      - name: pgbouncer
+        image: bitnami/pgbouncer:1.22.1
+        ports:
+        - containerPort: 6432
+          name: pgbouncer
+        env:
+        - name: POSTGRESQL_HOST
+          value: "postgis"
+        - name: POSTGRESQL_PORT
+          value: "5432"
+        - name: POSTGRESQL_DATABASE
+          value: "aprs"
+        - name: POSTGRESQL_USERNAME
+          value: "aprs"
+        - name: POSTGRESQL_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: aprs-secret
+              key: database-password
+        - name: PGBOUNCER_DATABASE
+          value: "aprs"
+        - name: PGBOUNCER_PORT
+          value: "6432"
+        - name: PGBOUNCER_POOL_MODE
+          value: "transaction"
+        - name: PGBOUNCER_MAX_CLIENT_CONN
+          value: "1000"
+        - name: PGBOUNCER_DEFAULT_POOL_SIZE
+          value: "25"
+        - name: PGBOUNCER_MIN_POOL_SIZE
+          value: "10"
+        - name: PGBOUNCER_RESERVE_POOL_SIZE
+          value: "5"
+        - name: PGBOUNCER_SERVER_RESET_QUERY
+          value: "DISCARD ALL"
+        - name: PGBOUNCER_STATS_USERS
+          value: "aprs"
+        - name: PGBOUNCER_AUTH_TYPE
+          value: "md5"
+        - name: PGBOUNCER_IGNORE_STARTUP_PARAMETERS
+          value: "extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout"
+        resources:
+          requests:
+            memory: "64Mi"
+            cpu: "50m"
+          limits:
+            memory: "128Mi"
+            cpu: "100m"
+        livenessProbe:
+          tcpSocket:
+            port: 6432
+          initialDelaySeconds: 30
+          periodSeconds: 10
+        readinessProbe:
+          tcpSocket:
+            port: 6432
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 10"]

clusters/aprs/postgis-deployment-enhanced.yaml

@@ -0,0 +1,110 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgis
+  namespace: aprs
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgis
+  template:
+    metadata:
+      labels:
+        app: postgis
+    spec:
+      initContainers:
+      - name: volume-permissions
+        image: busybox:1.36.1
+        command: ['sh', '-c', 'chmod -R 777 /var/lib/postgresql/data || true']
+        volumeMounts:
+        - name: postgis-storage
+          mountPath: /var/lib/postgresql/data
+      containers:
+      - name: postgis
+        image: postgis/postgis:17-3.4
+        ports:
+        - containerPort: 5432
+        env:
+        - name: POSTGRES_DB
+          value: aprs_prod
+        - name: POSTGRES_USER
+          value: aprs
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: postgis-secret
+              key: postgres-password
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        # PostgreSQL configuration for better connection handling
+        - name: POSTGRES_INITDB_ARGS
+          value: "--encoding=UTF8 --locale=en_US.utf8"
+        args:
+        - -c
+        - max_connections=200  # Increased from default 100
+        - -c
+        - shared_buffers=256MB
+        - -c
+        - effective_cache_size=1GB
+        - -c
+        - maintenance_work_mem=64MB
+        - -c
+        - checkpoint_completion_target=0.7
+        - -c
+        - wal_buffers=16MB
+        - -c
+        - default_statistics_target=100
+        - -c
+        - random_page_cost=1.1
+        - -c
+        - effective_io_concurrency=200
+        - -c
+        - work_mem=4MB
+        - -c
+        - min_wal_size=1GB
+        - -c
+        - max_wal_size=2GB
+        - -c
+        - idle_in_transaction_session_timeout=30s  # Kill idle transactions
+        - -c
+        - statement_timeout=300s  # 5 minute statement timeout
+        volumeMounts:
+        - name: postgis-storage
+          mountPath: /var/lib/postgresql/data
+        resources:
+          limits:
+            memory: "2Gi"  # Increased from 1Gi
+            cpu: "1000m"   # Increased from 500m
+          requests:
+            memory: "1Gi"   # Increased from 512Mi
+            cpu: "500m"     # Increased from 250m
+        # Health checks to detect connection issues
+        livenessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - pg_isready -U $POSTGRES_USER -d $POSTGRES_DB
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 3
+        readinessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - |
+              pg_isready -U $POSTGRES_USER -d $POSTGRES_DB && \
+              CONN_COUNT=$(psql -U $POSTGRES_USER -d $POSTGRES_DB -t -c "SELECT count(*) FROM pg_stat_activity WHERE state != 'idle';") && \
+              [ $CONN_COUNT -lt 180 ]  # Fail readiness if too many connections
+          initialDelaySeconds: 5
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 2
+      volumes:
+      - name: postgis-storage
+        hostPath:
+          path: /var/lib/rancher/k3s/storage/postgis
+          type: DirectoryOrCreate

clusters/aprs/postgis-pdb.yaml

@@ -0,0 +1,21 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: postgis-pdb
+  namespace: aprs
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: postgis
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: pgbouncer-pdb
+  namespace: aprs
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: pgbouncer

clusters/aprs/postgres-cleanup-cronjob.yaml

@@ -0,0 +1,107 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: postgres-cleanup
+  namespace: aprs
+spec:
+  schedule: "*/15 * * * *"  # Every 15 minutes
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: postgres-cleanup
+            image: postgres:17-alpine
+            env:
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: postgis-secret
+                  key: postgres-password
+            - name: PGHOST
+              value: postgis
+            - name: PGUSER
+              value: aprs
+            - name: PGDATABASE
+              value: aprs_prod
+            command:
+            - /bin/sh
+            - -c
+            - |
+              echo "Starting PostgreSQL connection cleanup..."
+              
+              # Kill idle connections older than 30 minutes
+              psql -c "
+              SELECT pg_terminate_backend(pid)
+              FROM pg_stat_activity
+              WHERE datname = 'aprs_prod'
+                AND pid <> pg_backend_pid()
+                AND state = 'idle'
+                AND state_change < NOW() - INTERVAL '30 minutes';"
+              
+              # Kill idle in transaction connections older than 5 minutes
+              psql -c "
+              SELECT pg_terminate_backend(pid)
+              FROM pg_stat_activity
+              WHERE datname = 'aprs_prod'
+                AND pid <> pg_backend_pid()
+                AND state = 'idle in transaction'
+                AND state_change < NOW() - INTERVAL '5 minutes';"
+              
+              # Log connection stats
+              psql -c "
+              SELECT 
+                state,
+                COUNT(*) as count,
+                MAX(NOW() - state_change) as max_duration
+              FROM pg_stat_activity
+              WHERE datname = 'aprs_prod'
+              GROUP BY state
+              ORDER BY count DESC;"
+              
+              # Check total connections vs max
+              psql -c "
+              SELECT 
+                (SELECT COUNT(*) FROM pg_stat_activity) as current_connections,
+                (SELECT setting::int FROM pg_settings WHERE name = 'max_connections') as max_connections,
+                ROUND(100.0 * COUNT(*) / (SELECT setting::int FROM pg_settings WHERE name = 'max_connections'), 2) as percentage_used
+              FROM pg_stat_activity;"
+              
+              echo "Cleanup completed successfully"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: postgres-cleanup
+  namespace: aprs
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: postgres-cleanup
+  namespace: aprs
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: postgres-cleanup
+  namespace: aprs
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: postgres-cleanup
+subjects:
+- kind: ServiceAccount
+  name: postgres-cleanup
+  namespace: aprs

clusters/aprs/redis-deployment.yaml

@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+  namespace: aprs
+  labels:
+    app: redis
+spec:
+  ports:
+  - port: 6379
+    targetPort: 6379
+    name: redis
+  selector:
+    app: redis
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+  namespace: aprs
+  labels:
+    app: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - name: redis
+        image: redis:7-alpine
+        ports:
+        - containerPort: 6379
+          name: redis
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "100m"
+          limits:
+            memory: "256Mi"
+            cpu: "200m"
+        livenessProbe:
+          exec:
+            command:
+            - redis-cli
+            - ping
+          initialDelaySeconds: 30
+          periodSeconds: 10
+        readinessProbe:
+          exec:
+            command:
+            - redis-cli
+            - ping
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        volumeMounts:
+        - name: redis-data
+          mountPath: /data
+      volumes:
+      - name: redis-data
+        emptyDir: {}

clusters/aprs/secrets.yaml

@@ -15,6 +15,8 @@ metadata:
 type: Opaque
 stringData:
   database-url: "ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@postgis:5432/aprs"
+  database-url-pgbouncer: "ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@pgbouncer:5432/aprs"
+  database-password: "mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21"
   secret-key-base: "GJxzOUW3YXBAF7xB4mH5VcXY6BN1NYLnoJYadlZyiCxcRUGBw65w4Sco7+sNiW2t"
   erlang-cookie: "aprs-cluster-cookie-ksdf8934jfklsdjf983jfklsdf"
 ---

clusters/aprs/store-secrets-1password.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Create 1Password items for APRS secrets
+
+# PostgreSQL password
+op item create \
+  --category=password \
+  --title="APRS PostgreSQL Password" \
+  --vault="Private" \
+  password="mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21"
+
+# APRS application secrets
+op item create \
+  --category=login \
+  --title="APRS Application Secrets" \
+  --vault="Private" \
+  username="aprs" \
+  password="mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21" \
+  database.url="ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@postgis:5432/aprs" \
+  database.url.pgbouncer="ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@pgbouncer:5432/aprs" \
+  secret.key.base="GJxzOUW3YXBAF7xB4mH5VcXY6BN1NYLnoJYadlZyiCxcRUGBw65w4Sco7+sNiW2t" \
+  erlang.cookie="aprs-cluster-cookie-ksdf8934jfklsdjf983jfklsdf"
+
+# GitHub Container Registry credentials
+op item create \
+  --category=login \
+  --title="GitHub Container Registry" \
+  --vault="Private" \
+  username="gmcintire" \
+  password="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" \
+  url="ghcr.io"