diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml index be175f5..9d49e58 100644 --- a/ansible/bootstrap.yml +++ b/ansible/bootstrap.yml @@ -5,6 +5,4 @@ roles: - base become: yes - # become_method: "{{ 'su' if (ansible_distribution | default('')) == 'Debian' else 'sudo' }}" - # become_method: "sudo" - become_method: "su" + become_method: sudo diff --git a/ansible/host_vars/postgres.yml b/ansible/host_vars/postgres.yml new file mode 100644 index 0000000..b4d5974 --- /dev/null +++ b/ansible/host_vars/postgres.yml @@ -0,0 +1,28 @@ +--- +# Host-specific variables for postgres server (CM3588) + +# Python interpreter +ansible_python_interpreter: /usr/bin/python3.11 + +# Override PostgreSQL memory settings if needed +# postgresql_config: +# shared_buffers: 8GB + +# RAID configuration +# Comment out for auto-detection or specify actual device names +# raid_devices: +# - /dev/sda +# - /dev/sdb + +# Network configuration +postgresql_listen_addresses: "'*'" +postgresql_allowed_networks: + - 10.0.0.0/8 # Adjust to your network + +# Backup configuration +postgresql_backup_retention_days: 7 +postgresql_backup_compress: true + +# Monitoring +enable_monitoring: true +monitoring_email: root@localhost \ No newline at end of file diff --git a/ansible/hosts b/ansible/hosts index 47775ad..2b7d2fe 100755 --- a/ansible/hosts +++ b/ansible/hosts @@ -32,3 +32,6 @@ g.w5isp.com [caddy_servers] g.w5isp.com + +[postgresql_servers] +postgres ansible_host=10.0.16.230 diff --git a/ansible/inventory.yml b/ansible/inventory.yml new file mode 100644 index 0000000..e9b3255 --- /dev/null +++ b/ansible/inventory.yml @@ -0,0 +1,11 @@ +all: + hosts: + postgres: + ansible_host: 10.0.16.230 + ansible_user: graham + ansible_become: yes + ansible_python_interpreter: /usr/bin/python3 + children: + postgresql_servers: + hosts: + postgres: \ No newline at end of file diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 47da4f2..7125881 100755 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -103,6 +103,42 @@ roles: - caddy +- name: Configure PostgreSQL servers + hosts: postgresql_servers + become: true + gather_facts: true + tags: + - postgresql + - database + + pre_tasks: + - name: Set hostname + hostname: + name: "{{ inventory_hostname }}" + + - name: Update /etc/hosts + lineinfile: + path: /etc/hosts + regexp: '^127\.0\.1\.1' + line: "127.0.1.1 {{ inventory_hostname }}" + state: present + + - name: Update apt cache + apt: + update_cache: yes + cache_valid_time: 3600 + + - name: Install Python dependencies for Ansible + apt: + name: + - python3-apt + - python3-psycopg2 + state: present + + roles: + - raid + - postgresql + # - import_playbook: baseline/proxmox/main.yml # - name: Tailscale Home # hosts: tailscale_home diff --git a/ansible/postgres-playbook.yml b/ansible/postgres-playbook.yml new file mode 100644 index 0000000..5c852f5 --- /dev/null +++ b/ansible/postgres-playbook.yml @@ -0,0 +1,74 @@ +--- +- name: Deploy PostgreSQL server on CM3588 with RAID0 + hosts: postgres + become: yes + gather_facts: yes + + pre_tasks: + - name: Set hostname + hostname: + name: postgres + + - name: Update /etc/hosts + lineinfile: + path: /etc/hosts + regexp: '^127\.0\.1\.1' + line: '127.0.1.1 postgres' + state: present + + - name: Update apt cache + apt: + update_cache: yes + cache_valid_time: 3600 + + - name: Install Python dependencies for Ansible + apt: + name: + - python3-apt + - python3-psycopg2 + state: present + + roles: + - raid + - postgresql + + post_tasks: + - name: Display PostgreSQL connection information + debug: + msg: | + PostgreSQL has been configured on {{ inventory_hostname }} + + Connection details: + - Host: {{ ansible_host }} + - Port: 5432 (PostgreSQL direct) + - Port: 6432 (PgBouncer pooled) + - Database: aprsme_prod + - User: aprsme + + RAID0 array status: + - Device: /dev/md0 + - Mount: /var/lib/postgresql + - Size: ~4TB (2x2TB in RAID0) + + To connect from your k3s cluster: + psql -h {{ ansible_host }} -p 6432 -U aprsme -d aprsme_prod + + Remember to: + 1. Update your APRS.me DATABASE_URL to point to this server + 2. Run migrations: mix ecto.migrate + 3. Monitor disk usage and performance + 4. Set up regular backups to external storage + + - name: Test PostgreSQL connection + postgresql_ping: + db: aprsme_prod + login_host: localhost + login_port: 5432 + login_user: aprsme + login_password: "{{ vault_postgresql_aprsme_password }}" + register: db_ping + ignore_errors: yes + + - name: Show connection test result + debug: + msg: "Database connection test: {{ 'SUCCESS' if db_ping is succeeded else 'FAILED' }}" \ No newline at end of file diff --git a/ansible/postgres-readme.md b/ansible/postgres-readme.md new file mode 100644 index 0000000..71bc183 --- /dev/null +++ b/ansible/postgres-readme.md @@ -0,0 +1,182 @@ +# PostgreSQL Server Configuration for APRS.me + +This Ansible configuration deploys a high-performance PostgreSQL server on the CM3588 with RAID0 storage configuration, optimized for the APRS.me workload. + +## Hardware + +- **Server**: CM3588 with RK3588 SoC +- **Storage**: 2x 2TB NVMe SSDs in RAID0 (4TB total capacity) +- **Network**: 2.5 Gigabit Ethernet +- **RAM**: 32GB (assumed) + +## Features + +- Software RAID0 configuration for maximum performance +- PostgreSQL 16 with PostGIS for spatial data +- PgBouncer for connection pooling +- Optimized for high insert rates and spatial queries +- Automated backups with retention policy +- Health monitoring and alerting +- Kernel and systemd tuning for database workloads + +## Prerequisites + +1. Ubuntu 22.04 or later installed on the CM3588 +2. SSH access to the server with the user 'graham' +3. Ansible installed on your control machine + +## Initial Setup + +**Note**: This playbook assumes a fresh PostgreSQL installation. All data will be stored on the RAID array from the start. + +1. First, create a vault file for sensitive data: +```bash +ansible-vault create group_vars/all/vault.yml +``` + +Add the following to the vault file: +```yaml +vault_postgresql_aprsme_password: "your-secure-password-here" +``` + +2. Bootstrap the server (first time only): +```bash +ansible-playbook -i inventory.yml bootstrap.yml --ask-pass --ask-become-pass +``` + +3. Deploy PostgreSQL: +```bash +ansible-playbook -i inventory.yml postgres-playbook.yml --ask-vault-pass +``` + +The playbook will: +- Create RAID0 array from both NVMe drives +- Initialize PostgreSQL cluster directly on the RAID array +- Configure all PostgreSQL data, WAL, archives, and backups on the RAID array +- No data migration needed since this is a fresh installation + +## Configuration Details + +### RAID0 Configuration +- Creates `/dev/md0` from `/dev/nvme0n1` and `/dev/nvme1n1` +- Formatted as ext4 with optimizations for SSDs +- Mounted at `/mnt/pgdata` with noatime,nodiratime options +- All PostgreSQL data stored on RAID array: + - Data directory: `/mnt/pgdata/16/main` + - WAL directory: `/mnt/pgdata/wal` + - Archive directory: `/mnt/pgdata/archive` + - Backup directory: `/mnt/pgdata/backups` + +### PostgreSQL Optimization +- **shared_buffers**: 8GB (25% of RAM) +- **effective_cache_size**: 24GB (75% of RAM) +- **work_mem**: 64MB (for complex spatial queries) +- **max_connections**: 200 +- **autovacuum**: Tuned for high insert rate +- **checkpoint**: Optimized for write performance + +### PgBouncer Settings +- **pool_mode**: transaction (optimal for APRS.me) +- **default_pool_size**: 25 +- **max_client_conn**: 1000 + +### Security +- PostgreSQL listening on all interfaces (configure firewall accordingly) +- SCRAM-SHA-256 authentication +- Access restricted to 10.0.0.0/8 network (adjust as needed) + +## Maintenance + +### Check RAID Status +```bash +ssh postgres@10.0.16.230 'sudo mdadm --detail /dev/md0' +``` + +### PostgreSQL Logs +```bash +ssh postgres@10.0.16.230 'sudo journalctl -u postgresql -f' +``` + +### Backup Status +```bash +ssh postgres@10.0.16.230 'ls -la /mnt/pgdata/backups/' +``` + +### Performance Monitoring +```bash +# Connect to PostgreSQL +psql -h 10.0.16.230 -p 5432 -U postgres -d aprsme_prod + +# Check slow queries +SELECT * FROM pg_stat_statements ORDER BY total_time DESC LIMIT 10; + +# Check table sizes +SELECT schemaname, tablename, pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS size +FROM pg_tables +ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC; + +# Check index usage +SELECT schemaname, tablename, indexname, idx_scan +FROM pg_stat_user_indexes +ORDER BY idx_scan; +``` + +## Connecting from APRS.me + +Update your APRS.me configuration: + +```elixir +# config/runtime.exs +config :aprsme, Aprsme.Repo, + url: "postgresql://aprsme:password@10.0.16.230:6432/aprsme_prod", + pool_size: 5 +``` + +Or use environment variable: +```bash +DATABASE_URL=postgresql://aprsme:password@10.0.16.230:6432/aprsme_prod +``` + +## Disaster Recovery + +1. **RAID Failure**: While RAID0 provides no redundancy, the backup script runs nightly +2. **Backup Restore**: + ```bash + pg_restore -h localhost -U postgres -d aprsme_prod_restore /mnt/pgdata/backups/aprsme_prod_20240126_020000.sql.gz + ``` +3. **Replication**: Consider setting up streaming replication to a standby server + +## Performance Expectations + +With this configuration, expect: +- Insert rate: 1000+ packets/second sustained +- Spatial query response: <10ms for indexed queries +- Connection overhead: Minimal with PgBouncer +- Storage capacity: ~4TB usable (years of APRS data) + +## Troubleshooting + +### RAID Issues +```bash +# Check RAID status +cat /proc/mdstat +mdadm --detail /dev/md0 + +# Check disk health +smartctl -a /dev/nvme0n1 +smartctl -a /dev/nvme1n1 +``` + +### PostgreSQL Performance +```bash +# Check current connections +SELECT count(*) FROM pg_stat_activity; + +# Check cache hit ratio +SELECT sum(heap_blks_hit) / (sum(heap_blks_hit) + sum(heap_blks_read)) AS cache_hit_ratio FROM pg_statio_user_tables; +``` + +### PgBouncer Stats +```bash +psql -h 10.0.16.230 -p 6432 -U postgres pgbouncer -c "SHOW STATS;" +``` \ No newline at end of file diff --git a/ansible/roles/postgresql/defaults/main.yml b/ansible/roles/postgresql/defaults/main.yml new file mode 100644 index 0000000..4c7e29a --- /dev/null +++ b/ansible/roles/postgresql/defaults/main.yml @@ -0,0 +1,111 @@ +--- +# PostgreSQL version +postgresql_version: 16 + +# PostgreSQL configuration +postgresql_data_directory: /mnt/pgdata/{{ postgresql_version }}/main +postgresql_wal_directory: /mnt/pgdata/wal +postgresql_archive_directory: /mnt/pgdata/archive +postgresql_config_directory: /etc/postgresql/{{ postgresql_version }}/main + +# Database settings for APRS.me +postgresql_databases: + - name: aprsme_prod + owner: aprsme + +postgresql_users: + - name: aprsme + password: "{{ vault_postgresql_aprsme_password }}" # Store in vault + priv: "aprsme_prod:ALL" + +# Performance tuning for CM3588 with RK3588 (assuming 32GB RAM) +# These are optimized for APRS.me workload +postgresql_config: + # Connection settings + listen_addresses: "'*'" + max_connections: 200 # Increased for multiple app instances + + # Memory settings (for 32GB RAM system) + shared_buffers: 8GB # 25% of RAM + effective_cache_size: 24GB # 75% of RAM + maintenance_work_mem: 2GB + work_mem: 64MB # High for complex queries + + # Disk settings optimized for RAID0 SSDs + random_page_cost: 1.1 # SSD optimization + effective_io_concurrency: 200 # Modern SSD + max_worker_processes: 8 # RK3588 has 8 cores + max_parallel_workers_per_gather: 4 + max_parallel_workers: 8 + max_parallel_maintenance_workers: 4 + + # Write performance settings + wal_buffers: 64MB + checkpoint_completion_target: 0.9 + checkpoint_timeout: 15min + max_wal_size: 16GB + min_wal_size: 2GB + + # Logging + log_min_duration_statement: 1000 # Log queries over 1 second + log_checkpoints: on + log_connections: on + log_disconnections: on + log_line_prefix: "'%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '" + log_timezone: 'UTC' + + # Query tuning + default_statistics_target: 100 + + # Autovacuum tuning for high insert rate + autovacuum: on + autovacuum_max_workers: 4 + autovacuum_naptime: 30s # More frequent checks + autovacuum_vacuum_threshold: 50 + autovacuum_analyze_threshold: 50 + autovacuum_vacuum_scale_factor: 0.1 # 10% for more frequent vacuums + autovacuum_analyze_scale_factor: 0.05 # 5% for more frequent analyzes + + # Enable query planning statistics + track_activity_query_size: 2048 + track_io_timing: on + track_functions: all + + # PostGIS specific + postgis.gdal_datapath: '/usr/share/gdal' + +# Security settings +postgresql_hba_entries: + - type: local + database: all + user: postgres + auth_method: peer + - type: local + database: all + user: all + auth_method: peer + - type: host + database: all + user: all + address: 127.0.0.1/32 + auth_method: scram-sha-256 + - type: host + database: all + user: all + address: ::1/128 + auth_method: scram-sha-256 + - type: host + database: aprsme_prod + user: aprsme + address: 10.0.0.0/8 # Adjust for your network + auth_method: scram-sha-256 + +# Backup settings +postgresql_backup_enabled: true +postgresql_backup_directory: /mnt/pgdata/backups +postgresql_backup_schedule: + minute: "0" + hour: "2" + day: "*" + month: "*" + weekday: "*" \ No newline at end of file diff --git a/ansible/roles/postgresql/handlers/main.yml b/ansible/roles/postgresql/handlers/main.yml new file mode 100644 index 0000000..df6d558 --- /dev/null +++ b/ansible/roles/postgresql/handlers/main.yml @@ -0,0 +1,23 @@ +--- +- name: restart postgresql + systemd: + name: postgresql + state: restarted + daemon_reload: yes + +- name: reload postgresql + systemd: + name: postgresql + state: reloaded + +- name: restart pgbouncer + systemd: + name: pgbouncer + state: restarted + +- name: reload systemd + systemd: + daemon_reload: yes + +- name: reload sysctl + command: sysctl -p /etc/sysctl.d/30-postgresql.conf \ No newline at end of file diff --git a/ansible/roles/postgresql/tasks/main.yml b/ansible/roles/postgresql/tasks/main.yml new file mode 100644 index 0000000..c8a7a71 --- /dev/null +++ b/ansible/roles/postgresql/tasks/main.yml @@ -0,0 +1,261 @@ +--- +- name: Add PostgreSQL GPG key + apt_key: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + state: present + +- name: Add PostgreSQL repository + apt_repository: + repo: "deb http://apt.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg main" + state: present + filename: pgdg + +- name: Install PostgreSQL and PostGIS + apt: + name: + - postgresql-{{ postgresql_version }} + - postgresql-client-{{ postgresql_version }} + - postgresql-contrib-{{ postgresql_version }} + - postgresql-{{ postgresql_version }}-postgis-3 + - postgresql-{{ postgresql_version }}-postgis-3-scripts + - postgis + - python3-psycopg2 + - libpq-dev + - pgbouncer + state: present + update_cache: yes + +- name: Stop PostgreSQL service if running + systemd: + name: postgresql + state: stopped + ignore_errors: yes + +- name: Disable PostgreSQL from starting automatically (we'll configure it first) + systemd: + name: postgresql + enabled: no + +- name: Remove any default PostgreSQL data directory + file: + path: /var/lib/postgresql/{{ postgresql_version }}/main + state: absent + +- name: Create PostgreSQL cluster on RAID array + command: | + /usr/lib/postgresql/{{ postgresql_version }}/bin/initdb \ + -D {{ postgresql_data_directory }} \ + --wal-segsize=64 \ + --data-checksums \ + --encoding=UTF8 \ + --locale=en_US.UTF-8 \ + --lc-collate=en_US.UTF-8 \ + --lc-ctype=en_US.UTF-8 + args: + creates: "{{ postgresql_data_directory }}/PG_VERSION" + become_user: postgres + environment: + PGDATA: "{{ postgresql_data_directory }}" + +- name: Configure PostgreSQL + template: + src: postgresql.conf.j2 + dest: "{{ postgresql_config_directory }}/postgresql.conf" + owner: postgres + group: postgres + mode: '0644' + notify: restart postgresql + +- name: Configure PostgreSQL authentication + template: + src: pg_hba.conf.j2 + dest: "{{ postgresql_config_directory }}/pg_hba.conf" + owner: postgres + group: postgres + mode: '0640' + notify: restart postgresql + +- name: Create sysctl configuration for PostgreSQL + template: + src: 30-postgresql.conf.j2 + dest: /etc/sysctl.d/30-postgresql.conf + mode: '0644' + notify: reload sysctl + +- name: Create systemd override directory + file: + path: /etc/systemd/system/postgresql@.service.d + state: directory + mode: '0755' + +- name: Configure systemd limits for PostgreSQL + template: + src: postgresql-override.conf.j2 + dest: /etc/systemd/system/postgresql@.service.d/override.conf + mode: '0644' + notify: + - reload systemd + - restart postgresql + +- name: Move WAL to separate directory on RAID + block: + - name: Stop PostgreSQL if running + systemd: + name: postgresql + state: stopped + ignore_errors: yes + + - name: Move existing WAL directory + command: mv {{ postgresql_data_directory }}/pg_wal {{ postgresql_wal_directory }} + args: + creates: "{{ postgresql_wal_directory }}" + removes: "{{ postgresql_data_directory }}/pg_wal" + become_user: postgres + + - name: Create symlink for WAL directory + file: + src: "{{ postgresql_wal_directory }}" + dest: "{{ postgresql_data_directory }}/pg_wal" + state: link + owner: postgres + group: postgres + +- name: Start and enable PostgreSQL + systemd: + name: postgresql + state: started + enabled: yes + daemon_reload: yes + +- name: Wait for PostgreSQL to start + wait_for: + port: 5432 + host: localhost + delay: 5 + +- name: Create PostgreSQL users + postgresql_user: + name: "{{ item.name }}" + password: "{{ item.password }}" + encrypted: yes + state: present + become: yes + become_user: postgres + loop: "{{ postgresql_users }}" + no_log: true + +- name: Create databases + postgresql_db: + name: "{{ item.name }}" + owner: "{{ item.owner }}" + encoding: UTF-8 + lc_collate: en_US.UTF-8 + lc_ctype: en_US.UTF-8 + template: template0 + state: present + become: yes + become_user: postgres + loop: "{{ postgresql_databases }}" + +- name: Enable PostGIS extension + postgresql_ext: + name: postgis + db: "{{ item.name }}" + state: present + become: yes + become_user: postgres + loop: "{{ postgresql_databases }}" + +- name: Enable additional extensions + postgresql_ext: + name: "{{ item[1] }}" + db: "{{ item[0].name }}" + state: present + become: yes + become_user: postgres + loop: "{{ postgresql_databases | product(['pg_stat_statements', 'btree_gist', 'btree_gin']) | list }}" + +- name: Grant privileges + postgresql_privs: + database: "{{ item.split(':')[0] }}" + privs: "{{ item.split(':')[1] }}" + type: database + role: "{{ user.name }}" + state: present + become: yes + become_user: postgres + loop: "{{ postgresql_users | selectattr('priv', 'defined') | map(attribute='priv') | list }}" + loop_control: + loop_var: priv_item + vars: + user: "{{ postgresql_users | selectattr('priv', 'equalto', priv_item) | first }}" + +- name: Configure PgBouncer + template: + src: pgbouncer.ini.j2 + dest: /etc/pgbouncer/pgbouncer.ini + owner: postgres + group: postgres + mode: '0640' + notify: restart pgbouncer + +- name: Configure PgBouncer authentication + template: + src: pgbouncer-userlist.txt.j2 + dest: /etc/pgbouncer/userlist.txt + owner: postgres + group: postgres + mode: '0640' + notify: restart pgbouncer + +- name: Enable and start PgBouncer + systemd: + name: pgbouncer + state: started + enabled: yes + +- name: Set up backup script + template: + src: backup-postgresql.sh.j2 + dest: /usr/local/bin/backup-postgresql + mode: '0755' + when: postgresql_backup_enabled + +- name: Create backup directory + file: + path: "{{ postgresql_backup_directory }}" + state: directory + owner: postgres + group: postgres + mode: '0755' + when: postgresql_backup_enabled + +- name: Set up backup cron job + cron: + name: "PostgreSQL backup" + minute: "{{ postgresql_backup_schedule.minute }}" + hour: "{{ postgresql_backup_schedule.hour }}" + day: "{{ postgresql_backup_schedule.day }}" + month: "{{ postgresql_backup_schedule.month }}" + weekday: "{{ postgresql_backup_schedule.weekday }}" + user: postgres + job: "/usr/local/bin/backup-postgresql" + when: postgresql_backup_enabled + +- name: Configure log rotation + template: + src: postgresql-logrotate.j2 + dest: /etc/logrotate.d/postgresql-custom + mode: '0644' + +- name: Install monitoring script + template: + src: check-postgresql.sh.j2 + dest: /usr/local/bin/check-postgresql + mode: '0755' + +- name: Set up monitoring cron job + cron: + name: "Check PostgreSQL health" + minute: "*/5" + job: "/usr/local/bin/check-postgresql || echo 'PostgreSQL health check failed' | logger -t postgresql-health" \ No newline at end of file diff --git a/ansible/roles/postgresql/templates/30-postgresql.conf.j2 b/ansible/roles/postgresql/templates/30-postgresql.conf.j2 new file mode 100644 index 0000000..c7fed29 --- /dev/null +++ b/ansible/roles/postgresql/templates/30-postgresql.conf.j2 @@ -0,0 +1,34 @@ +# Kernel parameters for PostgreSQL optimization +# For CM3588 with 32GB RAM and RAID0 SSDs + +# Shared memory settings +kernel.shmmax = 17179869184 # 16GB +kernel.shmall = 4194304 # 16GB / 4096 (page size) +kernel.shmmni = 4096 + +# Semaphores: semmsl, semmns, semopm, semmni +kernel.sem = 250 32000 100 128 + +# Virtual memory +vm.swappiness = 10 +vm.dirty_ratio = 15 +vm.dirty_background_ratio = 5 +vm.overcommit_memory = 2 +vm.overcommit_ratio = 95 + +# Huge pages (optional, can be enabled if needed) +# vm.nr_hugepages = 4096 # 8GB of huge pages + +# Network optimization for 2.5Gbit connection +net.core.rmem_default = 262144 +net.core.rmem_max = 4194304 +net.core.wmem_default = 262144 +net.core.wmem_max = 4194304 +net.ipv4.tcp_rmem = 4096 87380 4194304 +net.ipv4.tcp_wmem = 4096 65536 4194304 +net.ipv4.tcp_keepalive_time = 60 +net.ipv4.tcp_keepalive_intvl = 10 +net.ipv4.tcp_keepalive_probes = 6 + +# File system +fs.file-max = 65536 \ No newline at end of file diff --git a/ansible/roles/postgresql/templates/backup-postgresql.sh.j2 b/ansible/roles/postgresql/templates/backup-postgresql.sh.j2 new file mode 100644 index 0000000..874649c --- /dev/null +++ b/ansible/roles/postgresql/templates/backup-postgresql.sh.j2 @@ -0,0 +1,48 @@ +#!/bin/bash +# PostgreSQL backup script for APRS.me + +set -euo pipefail + +# Configuration +BACKUP_DIR="{{ postgresql_backup_directory }}" +DB_NAME="aprsme_prod" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +BACKUP_FILE="${BACKUP_DIR}/${DB_NAME}_${TIMESTAMP}.sql.gz" +KEEP_DAYS=7 + +# Create backup directory if it doesn't exist +mkdir -p "${BACKUP_DIR}" + +# Log backup start +echo "[$(date)] Starting backup of ${DB_NAME}" | logger -t postgresql-backup + +# Perform backup with compression +if pg_dump -Fc -Z9 "${DB_NAME}" > "${BACKUP_FILE}.tmp"; then + mv "${BACKUP_FILE}.tmp" "${BACKUP_FILE}" + echo "[$(date)] Backup completed: ${BACKUP_FILE}" | logger -t postgresql-backup + + # Calculate backup size + SIZE=$(du -h "${BACKUP_FILE}" | cut -f1) + echo "[$(date)] Backup size: ${SIZE}" | logger -t postgresql-backup +else + echo "[$(date)] Backup failed for ${DB_NAME}" | logger -t postgresql-backup -p user.err + rm -f "${BACKUP_FILE}.tmp" + exit 1 +fi + +# Clean up old backups +echo "[$(date)] Cleaning up backups older than ${KEEP_DAYS} days" | logger -t postgresql-backup +find "${BACKUP_DIR}" -name "${DB_NAME}_*.sql.gz" -type f -mtime +${KEEP_DAYS} -delete + +# Verify backup integrity +if pg_restore -l "${BACKUP_FILE}" > /dev/null 2>&1; then + echo "[$(date)] Backup verification successful" | logger -t postgresql-backup +else + echo "[$(date)] Backup verification failed" | logger -t postgresql-backup -p user.err + exit 1 +fi + +# Optional: Copy to remote backup location +# rsync -av "${BACKUP_FILE}" user@backup-server:/path/to/backups/ + +exit 0 \ No newline at end of file diff --git a/ansible/roles/postgresql/templates/check-postgresql.sh.j2 b/ansible/roles/postgresql/templates/check-postgresql.sh.j2 new file mode 100644 index 0000000..4d3e862 --- /dev/null +++ b/ansible/roles/postgresql/templates/check-postgresql.sh.j2 @@ -0,0 +1,60 @@ +#!/bin/bash +# PostgreSQL health check script + +set -euo pipefail + +# Check if PostgreSQL is running +if ! systemctl is-active --quiet postgresql; then + echo "PostgreSQL service is not running" + exit 1 +fi + +# Check if we can connect to PostgreSQL +if ! pg_isready -h localhost -p 5432 -U postgres -q; then + echo "Cannot connect to PostgreSQL" + exit 1 +fi + +# Check database connection +if ! psql -h localhost -p 5432 -U postgres -d aprsme_prod -c "SELECT 1" > /dev/null 2>&1; then + echo "Cannot connect to aprsme_prod database" + exit 1 +fi + +# Check PgBouncer +if ! pg_isready -h localhost -p 6432 -U postgres -q; then + echo "PgBouncer is not responding" + exit 1 +fi + +# Check replication lag (if applicable) +LAG=$(psql -h localhost -p 5432 -U postgres -t -c "SELECT EXTRACT(EPOCH FROM (NOW() - pg_last_xact_replay_timestamp()))::INT" 2>/dev/null || echo "0") +if [ "${LAG}" -gt "60" ]; then + echo "Replication lag is too high: ${LAG} seconds" + exit 1 +fi + +# Check disk space +DISK_USAGE=$(df -h /var/lib/postgresql | tail -1 | awk '{print $5}' | sed 's/%//') +if [ "${DISK_USAGE}" -gt "90" ]; then + echo "Disk usage is critical: ${DISK_USAGE}%" + exit 1 +fi + +# Check connection count +CONNECTIONS=$(psql -h localhost -p 5432 -U postgres -t -c "SELECT COUNT(*) FROM pg_stat_activity" 2>/dev/null || echo "0") +MAX_CONNECTIONS=$(psql -h localhost -p 5432 -U postgres -t -c "SHOW max_connections" 2>/dev/null || echo "200") +if [ "${CONNECTIONS}" -gt "$(( ${MAX_CONNECTIONS} * 90 / 100 ))" ]; then + echo "Connection count is high: ${CONNECTIONS}/${MAX_CONNECTIONS}" + exit 1 +fi + +# Check for long running queries +LONG_QUERIES=$(psql -h localhost -p 5432 -U postgres -t -c "SELECT COUNT(*) FROM pg_stat_activity WHERE state != 'idle' AND query_start < NOW() - INTERVAL '5 minutes'" 2>/dev/null || echo "0") +if [ "${LONG_QUERIES}" -gt "5" ]; then + echo "Too many long running queries: ${LONG_QUERIES}" + exit 1 +fi + +echo "PostgreSQL health check passed" +exit 0 \ No newline at end of file diff --git a/ansible/roles/postgresql/templates/pg_hba.conf.j2 b/ansible/roles/postgresql/templates/pg_hba.conf.j2 new file mode 100644 index 0000000..d8a6fc5 --- /dev/null +++ b/ansible/roles/postgresql/templates/pg_hba.conf.j2 @@ -0,0 +1,9 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# Managed by Ansible - Do not edit manually + +# TYPE DATABASE USER ADDRESS METHOD + +{% for entry in postgresql_hba_entries %} +{{ entry.type }} {{ entry.database }} {{ entry.user }} {{ entry.address | default('') }} {{ entry.auth_method }} +{% endfor %} \ No newline at end of file diff --git a/ansible/roles/postgresql/templates/pgbouncer-userlist.txt.j2 b/ansible/roles/postgresql/templates/pgbouncer-userlist.txt.j2 new file mode 100644 index 0000000..6b4d52d --- /dev/null +++ b/ansible/roles/postgresql/templates/pgbouncer-userlist.txt.j2 @@ -0,0 +1,10 @@ +# PgBouncer userlist file +# Format: "username" "password" +# Managed by Ansible + +{% for user in postgresql_users %} +"{{ user.name }}" "{{ user.password }}" +{% endfor %} + +# PgBouncer stats user +"pgbouncer" "pgbouncer" \ No newline at end of file diff --git a/ansible/roles/postgresql/templates/pgbouncer.ini.j2 b/ansible/roles/postgresql/templates/pgbouncer.ini.j2 new file mode 100644 index 0000000..8e90fac --- /dev/null +++ b/ansible/roles/postgresql/templates/pgbouncer.ini.j2 @@ -0,0 +1,66 @@ +[databases] +# Connection string for aprsme database +aprsme_prod = host=127.0.0.1 port=5432 dbname=aprsme_prod + +# Add a pooled connection for PgBouncer statistics +pgbouncer = host=127.0.0.1 port=5432 dbname=pgbouncer auth_user=pgbouncer + +[pgbouncer] +# Connection settings +listen_addr = * +listen_port = 6432 +auth_type = scram-sha-256 +auth_file = /etc/pgbouncer/userlist.txt + +# Pool settings optimized for APRS.me +pool_mode = transaction +max_client_conn = 1000 +default_pool_size = 25 +min_pool_size = 5 +reserve_pool_size = 5 +reserve_pool_timeout = 3 + +# Performance settings +max_db_connections = 100 +max_user_connections = 100 +server_lifetime = 3600 +server_idle_timeout = 600 +server_connect_timeout = 15 +server_login_retry = 15 +query_timeout = 0 +query_wait_timeout = 120 +client_idle_timeout = 0 +client_login_timeout = 60 + +# Logging +log_connections = 1 +log_disconnections = 1 +log_pooler_errors = 1 +stats_period = 60 + +# Admin settings +admin_users = postgres +stats_users = postgres, aprsme + +# Security +ignore_startup_parameters = extra_float_digits + +# Resource limits +pkt_buf = 4096 +max_packet_size = 2147483647 +listen_backlog = 128 +sbuf_loopcnt = 5 +tcp_defer_accept = 0 +tcp_socket_buffer = 0 +tcp_keepalive = 1 +tcp_keepcnt = 3 +tcp_keepidle = 60 +tcp_keepintvl = 10 + +# DNS settings +dns_max_ttl = 15 +dns_nxdomain_ttl = 15 + +# TLS settings (optional) +# client_tls_sslmode = prefer +# client_tls_ca_file = /etc/ssl/certs/ca-certificates.crt \ No newline at end of file diff --git a/ansible/roles/postgresql/templates/postgresql-logrotate.j2 b/ansible/roles/postgresql/templates/postgresql-logrotate.j2 new file mode 100644 index 0000000..1dc4c04 --- /dev/null +++ b/ansible/roles/postgresql/templates/postgresql-logrotate.j2 @@ -0,0 +1,16 @@ +/var/log/postgresql/*.log { + daily + rotate 7 + missingok + compress + delaycompress + notifempty + create 640 postgres postgres + sharedscripts + postrotate + # Signal PostgreSQL to reopen log files + if [ -f /var/run/postgresql/{{ postgresql_version }}-main.pid ]; then + /usr/bin/pg_ctlcluster {{ postgresql_version }} main reload > /dev/null + fi + endscript +} \ No newline at end of file diff --git a/ansible/roles/postgresql/templates/postgresql-override.conf.j2 b/ansible/roles/postgresql/templates/postgresql-override.conf.j2 new file mode 100644 index 0000000..cc3e0e5 --- /dev/null +++ b/ansible/roles/postgresql/templates/postgresql-override.conf.j2 @@ -0,0 +1,17 @@ +[Service] +# Increase limits for PostgreSQL +LimitNOFILE=65536 +LimitNPROC=4096 + +# Disable OOM killer for PostgreSQL +OOMScoreAdjust=-1000 + +# CPU affinity - use all cores +CPUAffinity=0-7 + +# Nice level for better priority +Nice=-5 + +# I/O scheduling class and priority +IOSchedulingClass=best-effort +IOSchedulingPriority=0 \ No newline at end of file diff --git a/ansible/roles/postgresql/templates/postgresql.conf.j2 b/ansible/roles/postgresql/templates/postgresql.conf.j2 new file mode 100644 index 0000000..419de64 --- /dev/null +++ b/ansible/roles/postgresql/templates/postgresql.conf.j2 @@ -0,0 +1,103 @@ +# PostgreSQL configuration for APRS.me on CM3588 +# Optimized for RK3588 with RAID0 SSDs + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +data_directory = '{{ postgresql_data_directory }}' +hba_file = '{{ postgresql_config_directory }}/pg_hba.conf' +ident_file = '{{ postgresql_config_directory }}/pg_ident.conf' +external_pid_file = '/var/run/postgresql/{{ postgresql_version }}-main.pid' + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +{% for key, value in postgresql_config.items() %} +{{ key }} = {{ value }} +{% endfor %} + +#------------------------------------------------------------------------------ +# RESOURCE USAGE (except WAL) +#------------------------------------------------------------------------------ + +# These settings are configured above in postgresql_config + +#------------------------------------------------------------------------------ +# WRITE-AHEAD LOG +#------------------------------------------------------------------------------ + +# WAL settings configured above + +#------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------ + +wal_level = replica +archive_mode = on +archive_command = 'test ! -f {{ postgresql_archive_directory }}/%f && cp %p {{ postgresql_archive_directory }}/%f' +max_wal_senders = 3 +wal_keep_size = 1GB + +#------------------------------------------------------------------------------ +# QUERY TUNING +#------------------------------------------------------------------------------ + +# Planner settings for PostGIS and complex queries +enable_partitionwise_join = on +enable_partitionwise_aggregate = on +jit = on + +#------------------------------------------------------------------------------ +# REPORTING AND LOGGING +#------------------------------------------------------------------------------ + +# Logging configuration set above + +#------------------------------------------------------------------------------ +# STATISTICS +#------------------------------------------------------------------------------ + +# Stats configuration set above +shared_preload_libraries = 'pg_stat_statements' +pg_stat_statements.max = 10000 +pg_stat_statements.track = all + +#------------------------------------------------------------------------------ +# AUTOVACUUM +#------------------------------------------------------------------------------ + +# Autovacuum settings configured above + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +datestyle = 'iso, mdy' +timezone = 'UTC' +lc_messages = 'en_US.UTF-8' +lc_monetary = 'en_US.UTF-8' +lc_numeric = 'en_US.UTF-8' +lc_time = 'en_US.UTF-8' +default_text_search_config = 'pg_catalog.english' + +#------------------------------------------------------------------------------ +# LOCK MANAGEMENT +#------------------------------------------------------------------------------ + +deadlock_timeout = 1s +max_locks_per_transaction = 128 + +#------------------------------------------------------------------------------ +# VERSION/PLATFORM COMPATIBILITY +#------------------------------------------------------------------------------ + +# No specific settings needed + +#------------------------------------------------------------------------------ +# ERROR HANDLING +#------------------------------------------------------------------------------ + +exit_on_error = off +restart_after_crash = on \ No newline at end of file diff --git a/ansible/roles/raid/tasks/main.yml b/ansible/roles/raid/tasks/main.yml new file mode 100644 index 0000000..83f6a41 --- /dev/null +++ b/ansible/roles/raid/tasks/main.yml @@ -0,0 +1,210 @@ +--- +- name: Install RAID management tools + apt: + name: + - mdadm + - smartmontools + - parted + - nvme-cli + - lsscsi + state: present + update_cache: yes + +- name: Gather disk information + shell: | + echo "=== Block devices ===" + lsblk -d -o NAME,SIZE,TYPE,MODEL + echo "=== NVMe devices ===" + nvme list 2>/dev/null || echo "No NVMe devices found" + echo "=== SCSI devices ===" + lsscsi -s 2>/dev/null || echo "No SCSI devices found" + echo "=== All disk devices ===" + ls -la /dev/sd* /dev/nvme* /dev/vd* /dev/mmcblk* 2>/dev/null | grep -E "^b" || echo "Checking other paths..." + echo "=== Detailed block device tree ===" + lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINT,MODEL + register: disk_info + changed_when: false + +- name: Display disk information + debug: + msg: "{{ disk_info.stdout_lines }}" + +- name: Check if we should auto-detect RAID devices + set_fact: + auto_detect_devices: "{{ raid_devices is not defined or raid_devices | length == 0 }}" + +- name: Auto-detect two largest unused disks + when: auto_detect_devices + block: + - name: Find unused block devices + shell: | + lsblk -dpno NAME,SIZE,TYPE,MOUNTPOINT | grep -E "disk.*$" | grep -v -E "/(|/boot)" | sort -k2 -h -r | head -2 | awk '{print $1}' + register: detected_devices + changed_when: false + + - name: Set raid_devices from detected devices + set_fact: + raid_devices: "{{ detected_devices.stdout_lines }}" + when: detected_devices.stdout_lines | length >= 2 + + - name: Fail if not enough devices detected + fail: + msg: "Could not detect at least 2 unused block devices for RAID. Please specify raid_devices manually." + when: detected_devices.stdout_lines | length < 2 + +- name: Display RAID devices to be used + debug: + msg: "Will use these devices for RAID0: {{ raid_devices }}" + +- name: Check if RAID array already exists + command: mdadm --detail /dev/md0 + register: raid_status + failed_when: false + changed_when: false + +- name: Stop and remove existing RAID array if it exists + block: + - name: Stop RAID array + command: mdadm --stop /dev/md0 + when: raid_status.rc == 0 + + - name: Zero superblocks on disks + command: mdadm --zero-superblock {{ item }} + loop: + - /dev/nvme0n1 + - /dev/nvme1n1 + when: raid_status.rc == 0 + when: force_raid_rebuild | default(false) + +- name: Create partition table on first NVMe drive + parted: + device: /dev/nvme0n1 + number: 1 + state: present + part_type: primary + part_start: 0% + part_end: 100% + label: gpt + when: raid_status.rc != 0 + +- name: Create partition table on second NVMe drive + parted: + device: /dev/nvme1n1 + number: 1 + state: present + part_type: primary + part_start: 0% + part_end: 100% + label: gpt + when: raid_status.rc != 0 + +- name: Create RAID0 array + command: > + mdadm --create /dev/md0 + --level=0 + --raid-devices=2 + /dev/nvme0n1p1 /dev/nvme1n1p1 + --force + when: raid_status.rc != 0 + register: raid_create + +- name: Wait for RAID array to be ready + command: mdadm --detail /dev/md0 + register: raid_ready + until: raid_ready.rc == 0 + retries: 10 + delay: 5 + when: raid_create is changed + +- name: Create mdadm configuration + shell: mdadm --detail --scan >> /etc/mdadm/mdadm.conf + when: raid_create is changed + +- name: Update initramfs + command: update-initramfs -u + when: raid_create is changed + +- name: Create filesystem on RAID array + filesystem: + fstype: ext4 + dev: /dev/md0 + opts: "-E lazy_itable_init=0,lazy_journal_init=0" + when: raid_create is changed + +- name: Create mount point for PostgreSQL data + file: + path: /mnt/pgdata + state: directory + mode: '0755' + +- name: Mount RAID array + mount: + path: /mnt/pgdata + src: /dev/md0 + fstype: ext4 + opts: noatime,nodiratime,nobarrier + state: mounted + +- name: Create PostgreSQL user and group + block: + - name: Create postgres group + group: + name: postgres + state: present + system: yes + + - name: Create postgres user + user: + name: postgres + group: postgres + home: /var/lib/postgresql + shell: /bin/bash + system: yes + state: present + +- name: Ensure PostgreSQL directories exist on RAID + file: + path: "{{ item }}" + state: directory + owner: postgres + group: postgres + mode: '0700' + loop: + - /mnt/pgdata + - /mnt/pgdata/16 + - /mnt/pgdata/16/main + - /mnt/pgdata/wal + - /mnt/pgdata/archive + - /mnt/pgdata/backups + +- name: Set up RAID monitoring + lineinfile: + path: /etc/mdadm/mdadm.conf + line: "MAILADDR root" + insertafter: EOF + +- name: Enable mdadm monitoring service + systemd: + name: mdmonitor + enabled: yes + state: started + +- name: Create RAID monitoring script + copy: + content: | + #!/bin/bash + # RAID health check script + RAID_STATUS=$(mdadm --detail /dev/md0 | grep "State :" | awk '{print $3}') + if [ "$RAID_STATUS" != "clean" ] && [ "$RAID_STATUS" != "active" ]; then + echo "RAID array /dev/md0 is in state: $RAID_STATUS" + exit 1 + fi + exit 0 + dest: /usr/local/bin/check-raid-health + mode: '0755' + +- name: Set up RAID health check cron job + cron: + name: "Check RAID health" + minute: "*/5" + job: "/usr/local/bin/check-raid-health || echo 'RAID health check failed' | logger -t raid-health" \ No newline at end of file diff --git a/ansible/site.yml b/ansible/site.yml new file mode 100644 index 0000000..804ce57 --- /dev/null +++ b/ansible/site.yml @@ -0,0 +1,8 @@ +--- +- name: Configure PostgreSQL server on CM3588 + hosts: postgres + become: yes + roles: + - common + - raid + - postgresql \ No newline at end of file diff --git a/clusters/aprs/AUTO-HEALING-GUIDE.md b/clusters/aprs/AUTO-HEALING-GUIDE.md new file mode 100644 index 0000000..8efe43a --- /dev/null +++ b/clusters/aprs/AUTO-HEALING-GUIDE.md @@ -0,0 +1,154 @@ +# PostgreSQL Auto-Healing Configuration Guide + +This guide explains the auto-healing mechanisms implemented to prevent and recover from PostgreSQL connection exhaustion issues. + +## Overview + +The auto-healing system consists of several components working together: + +1. **Enhanced PostgreSQL Configuration** +2. **Improved PgBouncer Connection Pooling** +3. **Health Checks and Automatic Restarts** +4. **Periodic Connection Cleanup** +5. **Resource Limits and Scaling** + +## Components + +### 1. PostgreSQL Enhancements (`postgis-deployment-enhanced.yaml`) + +- **Increased max_connections**: From 100 to 200 +- **Connection timeouts**: + - `idle_in_transaction_session_timeout=30s`: Kills idle transactions after 30 seconds + - `statement_timeout=300s`: Kills queries running longer than 5 minutes +- **Readiness probe**: Fails if connection count exceeds 180 (90% of max) +- **Increased resources**: 2Gi memory, 1 CPU + +### 2. PgBouncer Enhancements (`pgbouncer-deployment-enhanced.yaml`) + +- **Connection pooling settings**: + - Pool mode: `transaction` (releases connections after each transaction) + - Default pool size: 25 connections per database/user pair + - Server idle timeout: 10 minutes + - Query timeout: 5 minutes +- **Retry logic**: + - `server_login_retry=15`: Retries failed logins for 15 seconds + - `server_round_robin=1`: Distributes connections evenly +- **Enhanced logging**: Tracks connections, disconnections, and errors + +### 3. APRS Application Enhancements (`aprs-statefulset-enhanced.yaml`) + +- **Reduced connection pool**: From 10 to 5 connections per pod +- **Connection timeouts**: + - Pool timeout: 60 seconds + - Connect timeout: 30 seconds + - Idle timeout: 15 minutes +- **Startup probe**: Allows up to 120 seconds for application startup +- **Increased resources**: 1Gi memory, 1 CPU + +### 4. Automatic Connection Cleanup (`postgres-cleanup-cronjob.yaml`) + +Runs every 15 minutes to: +- Kill idle connections older than 30 minutes +- Kill idle-in-transaction connections older than 5 minutes +- Log connection statistics +- Monitor connection usage percentage + +### 5. Pod Disruption Budgets (`postgis-pdb.yaml`) + +Ensures at least 1 replica of PostgreSQL and PgBouncer remain available during: +- Cluster upgrades +- Node maintenance +- Voluntary disruptions + +## How Auto-Healing Works + +### Connection Exhaustion Prevention + +1. **Application-level**: Reduced pool size prevents each pod from using too many connections +2. **PgBouncer-level**: Connection pooling multiplexes many client connections over fewer server connections +3. **PostgreSQL-level**: Timeouts automatically close stale connections + +### Automatic Recovery + +1. **Health checks**: Pods restart automatically when: + - PostgreSQL has too many connections (>180) + - PgBouncer can't connect to PostgreSQL + - Application can't connect to the database + +2. **CronJob cleanup**: Every 15 minutes, forcefully closes: + - Idle connections wasting resources + - Stuck transactions blocking others + +3. **Connection retry**: PgBouncer retries failed connections for 15 seconds before giving up + +## Monitoring + +Use the provided monitoring script to check system health: + +```bash +./monitor-postgres-health.sh +``` + +This shows: +- Current connection count and states +- Connection usage percentage +- Recent errors from logs +- Last cleanup job execution + +## Manual Intervention + +If automatic healing fails, you can manually intervene: + +```bash +# Check connection details +kubectl exec -it deployment/postgis -n aprs -- psql -U aprs -d aprs_prod -c "SELECT * FROM pg_stat_activity WHERE datname = 'aprs_prod';" + +# Kill all connections (emergency) +kubectl exec -it deployment/postgis -n aprs -- psql -U aprs -d aprs_prod -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'aprs_prod' AND pid <> pg_backend_pid();" + +# Restart components +kubectl rollout restart deployment/postgis -n aprs +kubectl rollout restart deployment/pgbouncer -n aprs +kubectl rollout restart statefulset/aprs -n aprs +``` + +## Applying the Configuration + +To apply all auto-healing configurations: + +```bash +cd /Users/graham/dev/infra/clusters/aprs +./apply-auto-healing.sh +``` + +This script: +1. Backs up current configurations +2. Applies all enhanced configurations +3. Waits for rollouts to complete +4. Provides a summary of changes + +## Tuning Parameters + +You may need to adjust these based on your workload: + +1. **PostgreSQL max_connections**: Currently 200, increase if needed +2. **PgBouncer pool sizes**: Currently 25, adjust based on connection patterns +3. **Application POOL_SIZE**: Currently 5, can be increased if connections are available +4. **Cleanup intervals**: Currently 15 minutes, can be more or less frequent + +## Troubleshooting + +Common issues and solutions: + +1. **"too many clients already"**: Cleanup job should fix this within 15 minutes +2. **"server login has been failing"**: PgBouncer is in retry mode, will recover in 15 seconds +3. **Pods in CrashLoopBackOff**: Check logs for specific errors +4. **High memory usage**: Consider increasing resource limits + +## Future Improvements + +Consider implementing: +1. Prometheus metrics for connection monitoring +2. Alerting when connection usage exceeds 80% +3. Horizontal pod autoscaling based on connection metrics +4. Connection pooling at the application level using a library like `poolboy` \ No newline at end of file diff --git a/clusters/aprs/apply-auto-healing.sh b/clusters/aprs/apply-auto-healing.sh new file mode 100755 index 0000000..51e3d4e --- /dev/null +++ b/clusters/aprs/apply-auto-healing.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e + +echo "Applying PostgreSQL and PgBouncer auto-healing configurations..." + +# Backup current deployments +echo "Creating backups of current deployments..." +kubectl get deployment postgis -n aprs -o yaml > postgis-deployment-backup.yaml || true +kubectl get deployment pgbouncer -n aprs -o yaml > pgbouncer-deployment-backup.yaml || true +kubectl get statefulset aprs -n aprs -o yaml > aprs-statefulset-backup.yaml || true + +# Apply enhanced configurations +echo "Applying enhanced PostgreSQL deployment..." +kubectl apply -f postgis-deployment-enhanced.yaml + +echo "Applying enhanced PgBouncer deployment..." +kubectl apply -f pgbouncer-deployment-enhanced.yaml + +echo "Applying PodDisruptionBudgets..." +kubectl apply -f postgis-pdb.yaml + +echo "Applying enhanced APRS StatefulSet..." +kubectl apply -f aprs-statefulset-enhanced.yaml + +echo "Applying PostgreSQL cleanup CronJob..." +kubectl apply -f postgres-cleanup-cronjob.yaml + +# Wait for rollouts +echo "Waiting for PostgreSQL rollout..." +kubectl rollout status deployment/postgis -n aprs --timeout=300s + +echo "Waiting for PgBouncer rollout..." +kubectl rollout status deployment/pgbouncer -n aprs --timeout=300s + +echo "Waiting for APRS rollout..." +kubectl rollout status statefulset/aprs -n aprs --timeout=300s + +echo "Auto-healing configurations applied successfully!" +echo "" +echo "Summary of changes:" +echo "1. PostgreSQL: Increased max_connections to 200, added connection timeouts" +echo "2. PgBouncer: Enhanced retry logic and connection pooling" +echo "3. APRS pods: Reduced pool size to 5, added startup probes" +echo "4. Added health checks that restart pods when connections are exhausted" +echo "5. CronJob runs every 15 minutes to clean up stale connections" +echo "" +echo "Monitor the deployment with:" +echo " kubectl get pods -n aprs -w" +echo "" +echo "Check PostgreSQL connections with:" +echo " kubectl exec -it deployment/postgis -n aprs -- psql -U aprs -d aprs_prod -c 'SELECT state, count(*) FROM pg_stat_activity GROUP BY state;'" \ No newline at end of file diff --git a/clusters/aprs/aprs-pdb.yaml b/clusters/aprs/aprs-pdb.yaml new file mode 100644 index 0000000..cab2fd5 --- /dev/null +++ b/clusters/aprs/aprs-pdb.yaml @@ -0,0 +1,11 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: aprs-pdb + namespace: aprs +spec: + minAvailable: 1 + selector: + matchLabels: + app: aprs + unhealthyPodEvictionPolicy: AlwaysAllow \ No newline at end of file diff --git a/clusters/aprs/aprs-service.yaml b/clusters/aprs/aprs-service.yaml index baf8fe9..6e0d444 100644 --- a/clusters/aprs/aprs-service.yaml +++ b/clusters/aprs/aprs-service.yaml @@ -11,5 +11,7 @@ spec: ports: - port: 80 targetPort: 4000 + name: http type: LoadBalancer - loadBalancerIP: 10.0.19.221 \ No newline at end of file + loadBalancerIP: 10.0.19.221 + sessionAffinity: None \ No newline at end of file diff --git a/clusters/aprs/aprs-statefulset-enhanced.yaml b/clusters/aprs/aprs-statefulset-enhanced.yaml new file mode 100644 index 0000000..ff1afe6 --- /dev/null +++ b/clusters/aprs/aprs-statefulset-enhanced.yaml @@ -0,0 +1,132 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: aprs + namespace: aprs +spec: + serviceName: aprs-headless + replicas: 2 + selector: + matchLabels: + app: aprs + updateStrategy: + type: RollingUpdate + podManagementPolicy: Parallel + template: + metadata: + labels: + app: aprs + spec: + terminationGracePeriodSeconds: 60 + initContainers: + - name: wait-for-db + image: busybox:1.36.1 + command: ['sh', '-c', 'until nc -z pgbouncer 5432; do echo waiting for database; sleep 2; done'] + - name: wait-for-redis + image: busybox:1.36.1 + command: ['sh', '-c', 'until nc -z redis 6379; do echo waiting for redis; sleep 2; done'] + containers: + - name: aprs + image: ghcr.io/aprsme/aprs.me:latest + imagePullPolicy: Always + ports: + - containerPort: 4000 + protocol: TCP + env: + - name: SECRET_KEY_BASE + valueFrom: + secretKeyRef: + name: aprs-secret + key: secret-key-base + - name: DATABASE_URL + value: "ecto://aprs:$(DATABASE_PASSWORD)@pgbouncer:5432/aprs" + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: aprs-secret + key: database-password + - name: PHX_HOST + value: "aprs.me" + - name: PHX_SERVER + value: "true" + - name: POOL_SIZE + value: "5" # Reduced from default 10 to prevent connection exhaustion + - name: ECTO_IPV6 + value: "false" + - name: ERLANG_COOKIE + valueFrom: + secretKeyRef: + name: aprs-secret + key: erlang-cookie + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: REDIS_URL + value: "redis://redis.aprs.svc.cluster.local:6379" + - name: DRAIN_TIMEOUT_MS + value: "45000" + - name: SKIP_DB_CREATE + value: "true" + # Enhanced database connection settings + - name: DATABASE_POOL_TIMEOUT + value: "60000" # 60 seconds + - name: DATABASE_QUEUE_TARGET + value: "50" + - name: DATABASE_QUEUE_INTERVAL + value: "1000" + - name: DATABASE_CONNECT_TIMEOUT + value: "30000" # 30 seconds + - name: DATABASE_IDLE_TIMEOUT + value: "900000" # 15 minutes + # Memory management + - name: ERL_MAX_ETS_TABLES + value: "5000" + - name: ERLANG_SCHEDULER_BUSY_WAIT_THRESHOLD + value: "none" + resources: + limits: + memory: "1Gi" # Increased from 512Mi + cpu: "1000m" # Increased from 500m + requests: + memory: "512Mi" # Increased from 256Mi + cpu: "500m" # Increased from 250m + livenessProbe: + httpGet: + path: /health + port: 4000 + initialDelaySeconds: 60 # Increased from 30 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 4000 + initialDelaySeconds: 30 # Increased from 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /health + port: 4000 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 12 # 120 seconds total + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "sleep 15"] + imagePullSecrets: + - name: ghcr-pull-secret \ No newline at end of file diff --git a/clusters/aprs/aprs-statefulset.yaml b/clusters/aprs/aprs-statefulset.yaml index 6c6dacc..839c9ff 100644 --- a/clusters/aprs/aprs-statefulset.yaml +++ b/clusters/aprs/aprs-statefulset.yaml @@ -28,6 +28,11 @@ metadata: spec: serviceName: aprs-headless replicas: 2 + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 0 + podManagementPolicy: Parallel selector: matchLabels: app: aprs @@ -36,6 +41,7 @@ spec: labels: app: aprs spec: + terminationGracePeriodSeconds: 60 serviceAccountName: aprs-service-account imagePullSecrets: - name: ghcr-pull-secret @@ -61,7 +67,7 @@ spec: valueFrom: secretKeyRef: name: aprs-secret - key: database-url + key: database-url-pgbouncer - name: SECRET_KEY_BASE valueFrom: secretKeyRef: @@ -84,7 +90,7 @@ spec: - name: PACKET_RETENTION_DAYS value: "7" - name: POOL_SIZE - value: "20" + value: "5" - name: CLUSTER_ENABLED value: "true" - name: RELEASE_COOKIE @@ -104,6 +110,12 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: REDIS_URL + value: "redis://redis.aprs.svc.cluster.local:6379" + - name: DRAIN_TIMEOUT_MS + value: "45000" + - name: SKIP_DB_CREATE + value: "true" resources: limits: memory: "512Mi" @@ -122,4 +134,8 @@ spec: path: /health port: 4000 initialDelaySeconds: 5 - periodSeconds: 5 \ No newline at end of file + periodSeconds: 5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "sleep 15"] \ No newline at end of file diff --git a/clusters/aprs/generate-secrets-from-1password.sh b/clusters/aprs/generate-secrets-from-1password.sh new file mode 100755 index 0000000..3cf1821 --- /dev/null +++ b/clusters/aprs/generate-secrets-from-1password.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Generate Kubernetes secrets from 1Password + +# Ensure we're logged in to 1Password +if ! op whoami &>/dev/null; then + echo "Please log in to 1Password first: eval \$(op signin)" + exit 1 +fi + +# Get PostgreSQL password +POSTGRES_PASSWORD=$(op item get "APRS PostgreSQL Password" --fields password) + +# Get APRS application secrets +DATABASE_URL=$(op item get "APRS Application Secrets" --fields database.url) +DATABASE_URL_PGBOUNCER=$(op item get "APRS Application Secrets" --fields database.url.pgbouncer) +DATABASE_PASSWORD=$(op item get "APRS Application Secrets" --fields password) +SECRET_KEY_BASE=$(op item get "APRS Application Secrets" --fields secret.key.base) +ERLANG_COOKIE=$(op item get "APRS Application Secrets" --fields erlang.cookie) + +# Get GitHub Container Registry credentials +GHCR_USERNAME=$(op item get "GitHub Container Registry" --fields username) +GHCR_PASSWORD=$(op item get "GitHub Container Registry" --fields password) +GHCR_AUTH=$(echo -n "${GHCR_USERNAME}:${GHCR_PASSWORD}" | base64) + +# Create PostgreSQL secret +kubectl create secret generic postgis-secret \ + --namespace=aprs \ + --from-literal=postgres-password="${POSTGRES_PASSWORD}" \ + --dry-run=client -o yaml | kubectl apply -f - + +# Create APRS application secret +kubectl create secret generic aprs-secret \ + --namespace=aprs \ + --from-literal=database-url="${DATABASE_URL}" \ + --from-literal=database-url-pgbouncer="${DATABASE_URL_PGBOUNCER}" \ + --from-literal=database-password="${DATABASE_PASSWORD}" \ + --from-literal=secret-key-base="${SECRET_KEY_BASE}" \ + --from-literal=erlang-cookie="${ERLANG_COOKIE}" \ + --dry-run=client -o yaml | kubectl apply -f - + +# Create Docker registry secret +cat </dev/null || echo "Failed to query PostgreSQL" +echo "" + +# Check connection percentage +echo "Connection Usage:" +kubectl exec deployment/postgis -n aprs -- psql -U aprs -d aprs_prod -c " +SELECT + (SELECT COUNT(*) FROM pg_stat_activity) as current, + (SELECT setting::int FROM pg_settings WHERE name = 'max_connections') as max, + ROUND(100.0 * COUNT(*) / (SELECT setting::int FROM pg_settings WHERE name = 'max_connections'), 2) || '%' as usage +FROM pg_stat_activity;" 2>/dev/null || echo "Failed to query PostgreSQL" +echo "" + +# Check PgBouncer logs for errors +echo "Recent PgBouncer Errors (last 10):" +kubectl logs -n aprs -l app=pgbouncer --tail=50 | grep -i "error\|warning" | tail -10 +echo "" + +# Check APRS pod logs for connection errors +echo "Recent APRS Connection Errors (last 10):" +kubectl logs -n aprs -l app=aprs --tail=100 | grep -i "postgrex\|connection\|protocol_violation" | tail -10 +echo "" + +# Show last cleanup job run +echo "Last Cleanup Job Run:" +kubectl get jobs -n aprs | grep postgres-cleanup | tail -1 \ No newline at end of file diff --git a/clusters/aprs/pgbouncer-configmap.yaml b/clusters/aprs/pgbouncer-configmap.yaml new file mode 100644 index 0000000..5107d2e --- /dev/null +++ b/clusters/aprs/pgbouncer-configmap.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pgbouncer-config + namespace: aprs +data: + pgbouncer.ini: | + [databases] + aprs = host=postgis port=5432 dbname=aprs + + [pgbouncer] + listen_addr = 0.0.0.0 + listen_port = 6432 + auth_type = md5 + auth_file = /etc/pgbouncer/userlist.txt + pool_mode = transaction + max_client_conn = 1000 + default_pool_size = 25 + reserve_pool_size = 5 + reserve_pool_timeout = 3 + server_lifetime = 3600 + server_idle_timeout = 600 + log_connections = 0 + log_disconnections = 0 + stats_period = 60 + + # Important for prepared statements + server_reset_query = DISCARD ALL + server_reset_query_always = 1 + + # Connection limits + min_pool_size = 10 + max_db_connections = 100 + + # Timeouts + query_timeout = 0 + query_wait_timeout = 120 + client_idle_timeout = 0 + client_login_timeout = 60 + + # For better performance + pkt_buf = 8192 + sbuf_loopcnt = 2 + tcp_keepalive = 1 + tcp_keepidle = 1 + tcp_keepintvl = 1 + tcp_keepcnt = 1 + + # Ignore certain PostgreSQL parameters + ignore_startup_parameters = extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout \ No newline at end of file diff --git a/clusters/aprs/pgbouncer-deployment-enhanced.yaml b/clusters/aprs/pgbouncer-deployment-enhanced.yaml new file mode 100644 index 0000000..5dcded6 --- /dev/null +++ b/clusters/aprs/pgbouncer-deployment-enhanced.yaml @@ -0,0 +1,131 @@ +apiVersion: v1 +kind: Service +metadata: + name: pgbouncer + namespace: aprs + labels: + app: pgbouncer +spec: + ports: + - port: 5432 + targetPort: 6432 + name: pgbouncer + selector: + app: pgbouncer + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pgbouncer + namespace: aprs + labels: + app: pgbouncer +spec: + replicas: 2 + selector: + matchLabels: + app: pgbouncer + template: + metadata: + labels: + app: pgbouncer + spec: + containers: + - name: pgbouncer + image: bitnami/pgbouncer:1.22.1 + ports: + - containerPort: 6432 + name: pgbouncer + env: + - name: POSTGRESQL_HOST + value: "postgis" + - name: POSTGRESQL_PORT + value: "5432" + - name: POSTGRESQL_DATABASE + value: "aprs" + - name: POSTGRESQL_USERNAME + value: "aprs" + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + name: aprs-secret + key: database-password + - name: PGBOUNCER_DATABASE + value: "aprs" + - name: PGBOUNCER_PORT + value: "6432" + - name: PGBOUNCER_POOL_MODE + value: "transaction" + - name: PGBOUNCER_MAX_CLIENT_CONN + value: "1000" + - name: PGBOUNCER_DEFAULT_POOL_SIZE + value: "25" + - name: PGBOUNCER_MIN_POOL_SIZE + value: "5" + - name: PGBOUNCER_RESERVE_POOL_SIZE + value: "5" + - name: PGBOUNCER_SERVER_RESET_QUERY + value: "DISCARD ALL" + - name: PGBOUNCER_STATS_USERS + value: "aprs" + - name: PGBOUNCER_AUTH_TYPE + value: "md5" + - name: PGBOUNCER_IGNORE_STARTUP_PARAMETERS + value: "extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout" + # Enhanced connection handling + - name: PGBOUNCER_SERVER_IDLE_TIMEOUT + value: "600" # 10 minutes + - name: PGBOUNCER_SERVER_LIFETIME + value: "3600" # 1 hour + - name: PGBOUNCER_QUERY_TIMEOUT + value: "300" # 5 minutes + - name: PGBOUNCER_QUERY_WAIT_TIMEOUT + value: "120" # 2 minutes + - name: PGBOUNCER_CLIENT_IDLE_TIMEOUT + value: "300" # 5 minutes + - name: PGBOUNCER_CLIENT_LOGIN_TIMEOUT + value: "60" # 1 minute + # Connection retry settings + - name: PGBOUNCER_SERVER_CONNECT_TIMEOUT + value: "15" + - name: PGBOUNCER_SERVER_LOGIN_RETRY + value: "15" # Retry for 15 seconds before failing + - name: PGBOUNCER_SERVER_ROUND_ROBIN + value: "1" # Enable round-robin + - name: PGBOUNCER_LOG_CONNECTIONS + value: "1" + - name: PGBOUNCER_LOG_DISCONNECTIONS + value: "1" + - name: PGBOUNCER_LOG_POOLER_ERRORS + value: "1" + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + livenessProbe: + tcpSocket: + port: 6432 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - | + nc -z localhost 6432 && \ + echo "SHOW POOLS;" | psql -h localhost -p 6432 -U pgbouncer pgbouncer || exit 0 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 2 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 10"] \ No newline at end of file diff --git a/clusters/aprs/pgbouncer-deployment.yaml b/clusters/aprs/pgbouncer-deployment.yaml new file mode 100644 index 0000000..3bee875 --- /dev/null +++ b/clusters/aprs/pgbouncer-deployment.yaml @@ -0,0 +1,96 @@ +apiVersion: v1 +kind: Service +metadata: + name: pgbouncer + namespace: aprs + labels: + app: pgbouncer +spec: + ports: + - port: 5432 + targetPort: 6432 + name: pgbouncer + selector: + app: pgbouncer + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pgbouncer + namespace: aprs + labels: + app: pgbouncer +spec: + replicas: 2 + selector: + matchLabels: + app: pgbouncer + template: + metadata: + labels: + app: pgbouncer + spec: + containers: + - name: pgbouncer + image: bitnami/pgbouncer:1.22.1 + ports: + - containerPort: 6432 + name: pgbouncer + env: + - name: POSTGRESQL_HOST + value: "postgis" + - name: POSTGRESQL_PORT + value: "5432" + - name: POSTGRESQL_DATABASE + value: "aprs" + - name: POSTGRESQL_USERNAME + value: "aprs" + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + name: aprs-secret + key: database-password + - name: PGBOUNCER_DATABASE + value: "aprs" + - name: PGBOUNCER_PORT + value: "6432" + - name: PGBOUNCER_POOL_MODE + value: "transaction" + - name: PGBOUNCER_MAX_CLIENT_CONN + value: "1000" + - name: PGBOUNCER_DEFAULT_POOL_SIZE + value: "25" + - name: PGBOUNCER_MIN_POOL_SIZE + value: "10" + - name: PGBOUNCER_RESERVE_POOL_SIZE + value: "5" + - name: PGBOUNCER_SERVER_RESET_QUERY + value: "DISCARD ALL" + - name: PGBOUNCER_STATS_USERS + value: "aprs" + - name: PGBOUNCER_AUTH_TYPE + value: "md5" + - name: PGBOUNCER_IGNORE_STARTUP_PARAMETERS + value: "extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout" + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "100m" + livenessProbe: + tcpSocket: + port: 6432 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + tcpSocket: + port: 6432 + initialDelaySeconds: 5 + periodSeconds: 5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 10"] \ No newline at end of file diff --git a/clusters/aprs/postgis-deployment-enhanced.yaml b/clusters/aprs/postgis-deployment-enhanced.yaml new file mode 100644 index 0000000..b435cb9 --- /dev/null +++ b/clusters/aprs/postgis-deployment-enhanced.yaml @@ -0,0 +1,110 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgis + namespace: aprs +spec: + replicas: 1 + selector: + matchLabels: + app: postgis + template: + metadata: + labels: + app: postgis + spec: + initContainers: + - name: volume-permissions + image: busybox:1.36.1 + command: ['sh', '-c', 'chmod -R 777 /var/lib/postgresql/data || true'] + volumeMounts: + - name: postgis-storage + mountPath: /var/lib/postgresql/data + containers: + - name: postgis + image: postgis/postgis:17-3.4 + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: aprs_prod + - name: POSTGRES_USER + value: aprs + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgis-secret + key: postgres-password + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + # PostgreSQL configuration for better connection handling + - name: POSTGRES_INITDB_ARGS + value: "--encoding=UTF8 --locale=en_US.utf8" + args: + - -c + - max_connections=200 # Increased from default 100 + - -c + - shared_buffers=256MB + - -c + - effective_cache_size=1GB + - -c + - maintenance_work_mem=64MB + - -c + - checkpoint_completion_target=0.7 + - -c + - wal_buffers=16MB + - -c + - default_statistics_target=100 + - -c + - random_page_cost=1.1 + - -c + - effective_io_concurrency=200 + - -c + - work_mem=4MB + - -c + - min_wal_size=1GB + - -c + - max_wal_size=2GB + - -c + - idle_in_transaction_session_timeout=30s # Kill idle transactions + - -c + - statement_timeout=300s # 5 minute statement timeout + volumeMounts: + - name: postgis-storage + mountPath: /var/lib/postgresql/data + resources: + limits: + memory: "2Gi" # Increased from 1Gi + cpu: "1000m" # Increased from 500m + requests: + memory: "1Gi" # Increased from 512Mi + cpu: "500m" # Increased from 250m + # Health checks to detect connection issues + livenessProbe: + exec: + command: + - /bin/sh + - -c + - pg_isready -U $POSTGRES_USER -d $POSTGRES_DB + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - | + pg_isready -U $POSTGRES_USER -d $POSTGRES_DB && \ + CONN_COUNT=$(psql -U $POSTGRES_USER -d $POSTGRES_DB -t -c "SELECT count(*) FROM pg_stat_activity WHERE state != 'idle';") && \ + [ $CONN_COUNT -lt 180 ] # Fail readiness if too many connections + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 2 + volumes: + - name: postgis-storage + hostPath: + path: /var/lib/rancher/k3s/storage/postgis + type: DirectoryOrCreate \ No newline at end of file diff --git a/clusters/aprs/postgis-pdb.yaml b/clusters/aprs/postgis-pdb.yaml new file mode 100644 index 0000000..6bb06da --- /dev/null +++ b/clusters/aprs/postgis-pdb.yaml @@ -0,0 +1,21 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: postgis-pdb + namespace: aprs +spec: + minAvailable: 1 + selector: + matchLabels: + app: postgis +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: pgbouncer-pdb + namespace: aprs +spec: + minAvailable: 1 + selector: + matchLabels: + app: pgbouncer \ No newline at end of file diff --git a/clusters/aprs/postgres-cleanup-cronjob.yaml b/clusters/aprs/postgres-cleanup-cronjob.yaml new file mode 100644 index 0000000..7ec5505 --- /dev/null +++ b/clusters/aprs/postgres-cleanup-cronjob.yaml @@ -0,0 +1,107 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: postgres-cleanup + namespace: aprs +spec: + schedule: "*/15 * * * *" # Every 15 minutes + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: postgres-cleanup + image: postgres:17-alpine + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: postgis-secret + key: postgres-password + - name: PGHOST + value: postgis + - name: PGUSER + value: aprs + - name: PGDATABASE + value: aprs_prod + command: + - /bin/sh + - -c + - | + echo "Starting PostgreSQL connection cleanup..." + + # Kill idle connections older than 30 minutes + psql -c " + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = 'aprs_prod' + AND pid <> pg_backend_pid() + AND state = 'idle' + AND state_change < NOW() - INTERVAL '30 minutes';" + + # Kill idle in transaction connections older than 5 minutes + psql -c " + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = 'aprs_prod' + AND pid <> pg_backend_pid() + AND state = 'idle in transaction' + AND state_change < NOW() - INTERVAL '5 minutes';" + + # Log connection stats + psql -c " + SELECT + state, + COUNT(*) as count, + MAX(NOW() - state_change) as max_duration + FROM pg_stat_activity + WHERE datname = 'aprs_prod' + GROUP BY state + ORDER BY count DESC;" + + # Check total connections vs max + psql -c " + SELECT + (SELECT COUNT(*) FROM pg_stat_activity) as current_connections, + (SELECT setting::int FROM pg_settings WHERE name = 'max_connections') as max_connections, + ROUND(100.0 * COUNT(*) / (SELECT setting::int FROM pg_settings WHERE name = 'max_connections'), 2) as percentage_used + FROM pg_stat_activity;" + + echo "Cleanup completed successfully" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: postgres-cleanup + namespace: aprs +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: postgres-cleanup + namespace: aprs +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] +- apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: postgres-cleanup + namespace: aprs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: postgres-cleanup +subjects: +- kind: ServiceAccount + name: postgres-cleanup + namespace: aprs \ No newline at end of file diff --git a/clusters/aprs/redis-deployment.yaml b/clusters/aprs/redis-deployment.yaml new file mode 100644 index 0000000..ec11d74 --- /dev/null +++ b/clusters/aprs/redis-deployment.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: aprs + labels: + app: redis +spec: + ports: + - port: 6379 + targetPort: 6379 + name: redis + selector: + app: redis + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: aprs + labels: + app: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:7-alpine + ports: + - containerPort: 6379 + name: redis + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + volumeMounts: + - name: redis-data + mountPath: /data + volumes: + - name: redis-data + emptyDir: {} \ No newline at end of file diff --git a/clusters/aprs/secrets.yaml b/clusters/aprs/secrets.yaml index 0d60f70..d8b02d6 100644 --- a/clusters/aprs/secrets.yaml +++ b/clusters/aprs/secrets.yaml @@ -15,6 +15,8 @@ metadata: type: Opaque stringData: database-url: "ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@postgis:5432/aprs" + database-url-pgbouncer: "ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@pgbouncer:5432/aprs" + database-password: "mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21" secret-key-base: "GJxzOUW3YXBAF7xB4mH5VcXY6BN1NYLnoJYadlZyiCxcRUGBw65w4Sco7+sNiW2t" erlang-cookie: "aprs-cluster-cookie-ksdf8934jfklsdjf983jfklsdf" --- diff --git a/clusters/aprs/store-secrets-1password.sh b/clusters/aprs/store-secrets-1password.sh new file mode 100755 index 0000000..ef4231c --- /dev/null +++ b/clusters/aprs/store-secrets-1password.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Create 1Password items for APRS secrets + +# PostgreSQL password +op item create \ + --category=password \ + --title="APRS PostgreSQL Password" \ + --vault="Private" \ + password="mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21" + +# APRS application secrets +op item create \ + --category=login \ + --title="APRS Application Secrets" \ + --vault="Private" \ + username="aprs" \ + password="mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21" \ + database.url="ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@postgis:5432/aprs" \ + database.url.pgbouncer="ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@pgbouncer:5432/aprs" \ + secret.key.base="GJxzOUW3YXBAF7xB4mH5VcXY6BN1NYLnoJYadlZyiCxcRUGBw65w4Sco7+sNiW2t" \ + erlang.cookie="aprs-cluster-cookie-ksdf8934jfklsdjf983jfklsdf" + +# GitHub Container Registry credentials +op item create \ + --category=login \ + --title="GitHub Container Registry" \ + --vault="Private" \ + username="gmcintire" \ + password="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" \ + url="ghcr.io" \ No newline at end of file