diff --git a/ansible/create-vm-template.yml b/ansible/create-vm-template.yml deleted file mode 100644 index 6fcc688..0000000 --- a/ansible/create-vm-template.yml +++ /dev/null @@ -1,107 +0,0 @@ ---- -# Create a cloud-init template on Proxmox for K8s nodes -# This playbook runs commands directly on the Proxmox host - -- name: Create cloud-init template on Proxmox - hosts: proxmox[0] - become: yes - vars: - template_vmid: 9000 - template_name: debian12-cloud-init - debian_image_url: https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2 - storage: ceph - - tasks: - - name: Check if template already exists - shell: | - qm status {{ template_vmid }} >/dev/null 2>&1 && echo "exists" || echo "missing" - register: template_check - changed_when: false - - - name: Display template status - debug: - msg: "Template VM {{ template_vmid }} already exists. Skipping creation." - when: template_check.stdout == "exists" - - - name: Install required packages - apt: - name: - - wget - - libguestfs-tools - state: present - update_cache: yes - when: template_check.stdout == "missing" - - - name: Download Debian cloud image - get_url: - url: "{{ debian_image_url }}" - dest: /var/lib/vz/template/iso/debian-12-genericcloud-amd64.qcow2 - mode: '0644' - when: template_check.stdout == "missing" - - - name: Create VM for template - shell: | - qm create {{ template_vmid }} --name {{ template_name }} --memory 2048 --cores 2 --net0 virtio,bridge=vmbr0 - args: - creates: /etc/pve/qemu-server/{{ template_vmid }}.conf - when: template_check.stdout == "missing" - - - name: Import disk image to VM - shell: | - qm importdisk {{ template_vmid }} /var/lib/vz/template/iso/debian-12-genericcloud-amd64.qcow2 {{ storage }} - register: import_result - changed_when: "'successfully imported' in import_result.stdout" - when: template_check.stdout == "missing" - - - name: Get imported disk name - shell: | - pvesm list {{ storage }} | grep "vm-{{ template_vmid }}-disk-" | awk '{print $1}' | head -1 - register: disk_name - when: template_check.stdout == "missing" - - - name: Attach imported disk to VM - shell: | - qm set {{ template_vmid }} --scsihw virtio-scsi-pci --scsi0 {{ disk_name.stdout }} - when: - - template_check.stdout == "missing" - - disk_name.stdout != "" - - - name: Check if cloud-init drive exists - shell: | - qm config {{ template_vmid }} | grep -q 'ide2:.*cloudinit' && echo "exists" || echo "missing" - register: cloudinit_check - changed_when: false - when: template_check.stdout == "missing" - - - name: Add cloud-init drive - shell: | - qm set {{ template_vmid }} --ide2 {{ storage }}:cloudinit - when: - - template_check.stdout == "missing" - - cloudinit_check.stdout == "missing" - - - name: Set boot disk - shell: | - qm set {{ template_vmid }} --boot c --bootdisk scsi0 - when: template_check.stdout == "missing" - - - name: Add serial console - shell: | - qm set {{ template_vmid }} --serial0 socket --vga serial0 - when: template_check.stdout == "missing" - - - name: Enable QEMU guest agent - shell: | - qm set {{ template_vmid }} --agent enabled=1 - when: template_check.stdout == "missing" - - - name: Convert VM to template - shell: | - qm template {{ template_vmid }} - when: template_check.stdout == "missing" - - - name: Clean up downloaded image - file: - path: /var/lib/vz/template/iso/debian-12-genericcloud-amd64.qcow2 - state: absent - when: template_check.stdout == "missing" \ No newline at end of file diff --git a/ansible/host_vars/postgres.yml b/ansible/host_vars/postgres.yml index b4d5974..42911ee 100644 --- a/ansible/host_vars/postgres.yml +++ b/ansible/host_vars/postgres.yml @@ -9,10 +9,10 @@ ansible_python_interpreter: /usr/bin/python3.11 # shared_buffers: 8GB # RAID configuration -# Comment out for auto-detection or specify actual device names -# raid_devices: -# - /dev/sda -# - /dev/sdb +# Two SSDs in RAID0 for performance +raid_devices: + - /dev/nvme0n1 + - /dev/nvme1n1 # Network configuration postgresql_listen_addresses: "'*'" diff --git a/ansible/hosts b/ansible/hosts index 2b7d2fe..8cea8c0 100755 --- a/ansible/hosts +++ b/ansible/hosts @@ -34,4 +34,4 @@ g.w5isp.com g.w5isp.com [postgresql_servers] -postgres ansible_host=10.0.16.230 +postgres ansible_host=10.0.19.220 diff --git a/ansible/inventory.yml b/ansible/inventory.yml index e9b3255..9a86594 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -1,11 +1,11 @@ all: hosts: postgres: - ansible_host: 10.0.16.230 + ansible_host: 10.0.19.220 ansible_user: graham ansible_become: yes ansible_python_interpreter: /usr/bin/python3 children: - postgresql_servers: + postgres_servers: hosts: postgres: \ No newline at end of file diff --git a/ansible/postgres-playbook.yml b/ansible/postgres-playbook.yml deleted file mode 100644 index 5c852f5..0000000 --- a/ansible/postgres-playbook.yml +++ /dev/null @@ -1,74 +0,0 @@ ---- -- name: Deploy PostgreSQL server on CM3588 with RAID0 - hosts: postgres - become: yes - gather_facts: yes - - pre_tasks: - - name: Set hostname - hostname: - name: postgres - - - name: Update /etc/hosts - lineinfile: - path: /etc/hosts - regexp: '^127\.0\.1\.1' - line: '127.0.1.1 postgres' - state: present - - - name: Update apt cache - apt: - update_cache: yes - cache_valid_time: 3600 - - - name: Install Python dependencies for Ansible - apt: - name: - - python3-apt - - python3-psycopg2 - state: present - - roles: - - raid - - postgresql - - post_tasks: - - name: Display PostgreSQL connection information - debug: - msg: | - PostgreSQL has been configured on {{ inventory_hostname }} - - Connection details: - - Host: {{ ansible_host }} - - Port: 5432 (PostgreSQL direct) - - Port: 6432 (PgBouncer pooled) - - Database: aprsme_prod - - User: aprsme - - RAID0 array status: - - Device: /dev/md0 - - Mount: /var/lib/postgresql - - Size: ~4TB (2x2TB in RAID0) - - To connect from your k3s cluster: - psql -h {{ ansible_host }} -p 6432 -U aprsme -d aprsme_prod - - Remember to: - 1. Update your APRS.me DATABASE_URL to point to this server - 2. Run migrations: mix ecto.migrate - 3. Monitor disk usage and performance - 4. Set up regular backups to external storage - - - name: Test PostgreSQL connection - postgresql_ping: - db: aprsme_prod - login_host: localhost - login_port: 5432 - login_user: aprsme - login_password: "{{ vault_postgresql_aprsme_password }}" - register: db_ping - ignore_errors: yes - - - name: Show connection test result - debug: - msg: "Database connection test: {{ 'SUCCESS' if db_ping is succeeded else 'FAILED' }}" \ No newline at end of file diff --git a/ansible/postgres-with-1password.yml b/ansible/postgres-with-1password.yml new file mode 100644 index 0000000..fbaf083 --- /dev/null +++ b/ansible/postgres-with-1password.yml @@ -0,0 +1,50 @@ +--- +# Example playbook for PostgreSQL setup with 1Password integration +# Usage: ansible-playbook -i inventory.yml postgres-with-1password.yml + +- name: Configure PostgreSQL server with 1Password integration + hosts: postgres + become: yes + vars: + # Enable 1Password integration + use_1password: true + + # Optionally specify 1Password account ID + # If not specified, defaults to YOOATCZZSVGH7AD6VABUVPORLI + # op_account_id: "YOUR_ACCOUNT_ID" + + # Override any defaults here + postgresql_databases: + - name: aprsme_prod + owner: aprsme + + postgresql_users: + - name: aprsme + # Password will be generated and stored in 1Password + priv: "aprsme_prod:ALL" + + roles: + - base + - raid + - postgresql + + post_tasks: + - name: Display connection information + debug: + msg: | + PostgreSQL is now configured! + + Connection details: + - Host: {{ ansible_host }} + - Port: 5432 (PostgreSQL direct) + - Port: 6432 (PgBouncer pooled) + - Database: {{ postgresql_databases[0].name }} + - User: {{ postgresql_users[0].name }} + - Password: Stored in 1Password as "PostgreSQL {{ postgresql_users[0].name }} - {{ ansible_hostname }}" + + Connection strings: + - Direct: postgresql://{{ postgresql_users[0].name }}:@{{ ansible_host }}:5432/{{ postgresql_databases[0].name }} + - Pooled: postgresql://{{ postgresql_users[0].name }}:@{{ ansible_host }}:6432/{{ postgresql_databases[0].name }} + + To retrieve the password: + op item get "PostgreSQL {{ postgresql_users[0].name }} - {{ ansible_hostname }}" --fields password \ No newline at end of file diff --git a/ansible/roles/postgresql/defaults/main.yml b/ansible/roles/postgresql/defaults/main.yml index 4c7e29a..91a586b 100644 --- a/ansible/roles/postgresql/defaults/main.yml +++ b/ansible/roles/postgresql/defaults/main.yml @@ -1,6 +1,17 @@ --- # PostgreSQL version -postgresql_version: 16 +postgresql_version: 17 + +# Password management +use_1password: false # Set to true to use 1Password for password storage +op_account_id: "YOOATCZZSVGH7AD6VABUVPORLI" # Default 1Password account ID + +# Storage configuration +postgresql_storage_setup_enabled: false # Set to true to configure storage +postgresql_storage_type: raid0 # Options: single_nvme, raid0 +postgresql_raid_device: /dev/md0 +postgresql_storage_device: "{{ postgresql_raid_device if postgresql_storage_type == 'raid0' else '/dev/nvme0n1p1' }}" +postgresql_mount_point: /mnt/pgdata # PostgreSQL configuration postgresql_data_directory: /mnt/pgdata/{{ postgresql_version }}/main @@ -18,33 +29,33 @@ postgresql_users: password: "{{ vault_postgresql_aprsme_password }}" # Store in vault priv: "aprsme_prod:ALL" -# Performance tuning for CM3588 with RK3588 (assuming 32GB RAM) +# Performance tuning for CM3588 with 16GB RAM # These are optimized for APRS.me workload postgresql_config: # Connection settings listen_addresses: "'*'" - max_connections: 200 # Increased for multiple app instances + max_connections: 200 # Increased for 16GB system - # Memory settings (for 32GB RAM system) - shared_buffers: 8GB # 25% of RAM - effective_cache_size: 24GB # 75% of RAM - maintenance_work_mem: 2GB - work_mem: 64MB # High for complex queries + # Memory settings (for 16GB RAM system) + shared_buffers: 4GB # 25% of RAM + effective_cache_size: 12GB # 75% of RAM + maintenance_work_mem: 1GB + work_mem: 64MB # Increased for available memory # Disk settings optimized for RAID0 SSDs random_page_cost: 1.1 # SSD optimization effective_io_concurrency: 200 # Modern SSD - max_worker_processes: 8 # RK3588 has 8 cores - max_parallel_workers_per_gather: 4 - max_parallel_workers: 8 - max_parallel_maintenance_workers: 4 + max_worker_processes: 4 # Pi 5 has 4 cores + max_parallel_workers_per_gather: 2 + max_parallel_workers: 4 + max_parallel_maintenance_workers: 2 # Write performance settings - wal_buffers: 64MB + wal_buffers: 32MB checkpoint_completion_target: 0.9 - checkpoint_timeout: 15min - max_wal_size: 16GB - min_wal_size: 2GB + checkpoint_timeout: 10min + max_wal_size: 4GB + min_wal_size: 512MB # Logging log_min_duration_statement: 1000 # Log queries over 1 second @@ -72,7 +83,7 @@ postgresql_config: track_functions: all # PostGIS specific - postgis.gdal_datapath: '/usr/share/gdal' + postgis.gdal_datapath: "'/usr/share/gdal'" # Security settings postgresql_hba_entries: @@ -94,7 +105,7 @@ postgresql_hba_entries: user: all address: ::1/128 auth_method: scram-sha-256 - - type: host + - type: hostssl database: aprsme_prod user: aprsme address: 10.0.0.0/8 # Adjust for your network diff --git a/ansible/roles/postgresql/handlers/main.yml b/ansible/roles/postgresql/handlers/main.yml index df6d558..0b62248 100644 --- a/ansible/roles/postgresql/handlers/main.yml +++ b/ansible/roles/postgresql/handlers/main.yml @@ -20,4 +20,7 @@ daemon_reload: yes - name: reload sysctl - command: sysctl -p /etc/sysctl.d/30-postgresql.conf \ No newline at end of file + command: sysctl -p /etc/sysctl.d/30-postgresql.conf + +- name: reload udev + command: udevadm control --reload-rules && udevadm trigger \ No newline at end of file diff --git a/ansible/roles/postgresql/tasks/1password.yml b/ansible/roles/postgresql/tasks/1password.yml new file mode 100644 index 0000000..cd63471 --- /dev/null +++ b/ansible/roles/postgresql/tasks/1password.yml @@ -0,0 +1,55 @@ +--- +# 1Password integration for PostgreSQL passwords +# This file handles password generation and storage in 1Password + +- name: Check if 1Password CLI is available + command: which op + register: op_cli_check + changed_when: false + failed_when: false + delegate_to: localhost + +- name: Fail if 1Password CLI is not installed + fail: + msg: "1Password CLI (op) is not installed. Please install it first." + when: op_cli_check.rc != 0 + +- name: Generate password for PostgreSQL user + set_fact: + generated_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + when: vault_postgresql_aprsme_password is not defined + no_log: true + +- name: Create 1Password item for PostgreSQL password + shell: | + op item create \ + --category=database \ + --title="PostgreSQL {{ item.name }} - {{ ansible_hostname }}" \ + --vault="Private" \ + username="{{ item.name }}" \ + password="{{ generated_password | default(vault_postgresql_aprsme_password) }}" \ + hostname="{{ ansible_host }}" \ + port="5432" \ + database="{{ postgresql_databases[0].name }}" \ + --account="{{ op_account_id | default('YOOATCZZSVGH7AD6VABUVPORLI') }}" \ + --format=json || \ + op item edit \ + "PostgreSQL {{ item.name }} - {{ ansible_hostname }}" \ + password="{{ generated_password | default(vault_postgresql_aprsme_password) }}" \ + --account="{{ op_account_id | default('YOOATCZZSVGH7AD6VABUVPORLI') }}" + delegate_to: localhost + loop: "{{ postgresql_users }}" + when: use_1password | default(false) + no_log: true + register: op_items + +- name: Set password fact from 1Password or generated + set_fact: + postgresql_users_with_passwords: >- + {%- set users = [] -%} + {%- for user in postgresql_users -%} + {%- set _ = user.update({'password': generated_password | default(vault_postgresql_aprsme_password)}) -%} + {%- set _ = users.append(user) -%} + {%- endfor -%} + {{ users }} + no_log: true \ No newline at end of file diff --git a/ansible/roles/postgresql/tasks/main.yml b/ansible/roles/postgresql/tasks/main.yml index c8a7a71..ebce864 100644 --- a/ansible/roles/postgresql/tasks/main.yml +++ b/ansible/roles/postgresql/tasks/main.yml @@ -1,4 +1,9 @@ --- +# Set up storage before installing PostgreSQL +- name: Set up storage for PostgreSQL + include_tasks: storage.yml + when: postgresql_storage_setup_enabled | default(false) + - name: Add PostgreSQL GPG key apt_key: url: https://www.postgresql.org/media/keys/ACCC4CF8.asc @@ -41,6 +46,28 @@ path: /var/lib/postgresql/{{ postgresql_version }}/main state: absent +- name: Install locales package + apt: + name: locales + state: present + +- name: Generate en_US.UTF-8 locale + locale_gen: + name: en_US.UTF-8 + state: present + +- name: Create PostgreSQL config directory structure + file: + path: "{{ item }}" + state: directory + owner: postgres + group: postgres + mode: '0755' + loop: + - /etc/postgresql + - /etc/postgresql/{{ postgresql_version }} + - /etc/postgresql/{{ postgresql_version }}/main + - name: Create PostgreSQL cluster on RAID array command: | /usr/lib/postgresql/{{ postgresql_version }}/bin/initdb \ @@ -48,15 +75,18 @@ --wal-segsize=64 \ --data-checksums \ --encoding=UTF8 \ - --locale=en_US.UTF-8 \ - --lc-collate=en_US.UTF-8 \ - --lc-ctype=en_US.UTF-8 + --locale=C.UTF-8 \ + --lc-collate=C.UTF-8 \ + --lc-ctype=C.UTF-8 args: creates: "{{ postgresql_data_directory }}/PG_VERSION" become_user: postgres environment: PGDATA: "{{ postgresql_data_directory }}" +- name: Set up SSL/TLS + include_tasks: ssl.yml + - name: Configure PostgreSQL template: src: postgresql.conf.j2 @@ -97,28 +127,88 @@ - reload systemd - restart postgresql -- name: Move WAL to separate directory on RAID +- name: Check if PostgreSQL cluster is properly initialized + stat: + path: "{{ postgresql_data_directory }}/PG_VERSION" + register: pg_initialized + +- name: Reinitialize PostgreSQL if corrupted block: - name: Stop PostgreSQL if running systemd: - name: postgresql + name: postgresql@{{ postgresql_version }}-main state: stopped ignore_errors: yes - - name: Move existing WAL directory - command: mv {{ postgresql_data_directory }}/pg_wal {{ postgresql_wal_directory }} - args: - creates: "{{ postgresql_wal_directory }}" - removes: "{{ postgresql_data_directory }}/pg_wal" - become_user: postgres - - - name: Create symlink for WAL directory + - name: Remove corrupted data directory file: - src: "{{ postgresql_wal_directory }}" - dest: "{{ postgresql_data_directory }}/pg_wal" - state: link + path: "{{ postgresql_data_directory }}" + state: absent + + - name: Create data directory + file: + path: "{{ postgresql_data_directory }}" + state: directory owner: postgres group: postgres + mode: '0700' + + - name: Clear WAL directory for reinit + file: + path: "{{ postgresql_wal_directory }}" + state: absent + + - name: Create fresh WAL directory + file: + path: "{{ postgresql_wal_directory }}" + state: directory + owner: postgres + group: postgres + mode: '0700' + + - name: Initialize new PostgreSQL cluster + command: | + /usr/lib/postgresql/{{ postgresql_version }}/bin/initdb \ + -D {{ postgresql_data_directory }} \ + --wal-segsize=64 \ + --data-checksums \ + --encoding=UTF8 \ + --locale=C.UTF-8 \ + --lc-collate=C.UTF-8 \ + --lc-ctype=C.UTF-8 \ + --waldir={{ postgresql_wal_directory }} + become_user: postgres + environment: + PGDATA: "{{ postgresql_data_directory }}" + when: pg_initialized.stat.exists and force_reinit | default(false) + +- name: Ensure WAL directory is properly configured + block: + - name: Check if WAL directory exists + stat: + path: "{{ postgresql_wal_directory }}" + register: wal_dir_stat + + - name: Create WAL directory if missing + file: + path: "{{ postgresql_wal_directory }}" + state: directory + owner: postgres + group: postgres + mode: '0700' + when: not wal_dir_stat.stat.exists + + - name: Check if pg_wal is a symlink + stat: + path: "{{ postgresql_data_directory }}/pg_wal" + register: pg_wal_stat + + - name: Ensure pg_wal symlink has correct ownership + file: + path: "{{ postgresql_data_directory }}/pg_wal" + owner: postgres + group: postgres + when: pg_wal_stat.stat.exists and pg_wal_stat.stat.islnk - name: Start and enable PostgreSQL systemd: @@ -133,6 +223,10 @@ host: localhost delay: 5 +- name: Handle password management with 1Password + include_tasks: 1password.yml + when: use_1password | default(false) + - name: Create PostgreSQL users postgresql_user: name: "{{ item.name }}" @@ -177,8 +271,8 @@ - name: Grant privileges postgresql_privs: - database: "{{ item.split(':')[0] }}" - privs: "{{ item.split(':')[1] }}" + database: "{{ priv_item.split(':')[0] }}" + privs: "{{ priv_item.split(':')[1] }}" type: database role: "{{ user.name }}" state: present @@ -190,6 +284,18 @@ vars: user: "{{ postgresql_users | selectattr('priv', 'equalto', priv_item) | first }}" +- name: Retrieve SCRAM hashes for PgBouncer + postgresql_query: + db: postgres + query: "SELECT rolname, rolpassword FROM pg_authid WHERE rolname = %s" + positional_args: + - "{{ item.name }}" + become: yes + become_user: postgres + loop: "{{ postgresql_users }}" + register: scram_hashes + no_log: true + - name: Configure PgBouncer template: src: pgbouncer.ini.j2 diff --git a/ansible/roles/postgresql/tasks/ssl.yml b/ansible/roles/postgresql/tasks/ssl.yml new file mode 100644 index 0000000..3b4bae0 --- /dev/null +++ b/ansible/roles/postgresql/tasks/ssl.yml @@ -0,0 +1,70 @@ +--- +# SSL/TLS configuration for PostgreSQL + +- name: Install OpenSSL + apt: + name: openssl + state: present + +- name: Create SSL directory + file: + path: "{{ postgresql_data_directory }}/ssl" + state: directory + owner: postgres + group: postgres + mode: '0700' + +- name: Check if SSL certificate exists + stat: + path: "{{ postgresql_data_directory }}/ssl/server.crt" + register: ssl_cert + +- name: Generate private key for PostgreSQL + shell: | + openssl genrsa -out {{ postgresql_data_directory }}/ssl/server.key 4096 + chmod 600 {{ postgresql_data_directory }}/ssl/server.key + chown postgres:postgres {{ postgresql_data_directory }}/ssl/server.key + when: not ssl_cert.stat.exists + +- name: Generate self-signed certificate + shell: | + openssl req -new -x509 -days 3650 \ + -key {{ postgresql_data_directory }}/ssl/server.key \ + -out {{ postgresql_data_directory }}/ssl/server.crt \ + -subj "/C=US/ST=State/L=City/O=Organization/CN={{ ansible_hostname }}" \ + -addext "subjectAltName=DNS:{{ ansible_hostname }},DNS:{{ ansible_fqdn | default(ansible_hostname) }},IP:{{ ansible_default_ipv4.address }},IP:{{ ansible_host }}" + chmod 644 {{ postgresql_data_directory }}/ssl/server.crt + chown postgres:postgres {{ postgresql_data_directory }}/ssl/server.crt + when: not ssl_cert.stat.exists + +- name: Create CA certificate (copy server cert as CA for self-signed) + copy: + src: "{{ postgresql_data_directory }}/ssl/server.crt" + dest: "{{ postgresql_data_directory }}/ssl/ca.crt" + remote_src: yes + owner: postgres + group: postgres + mode: '0644' + +- name: Set proper permissions on SSL files + file: + path: "{{ item.path }}" + owner: postgres + group: postgres + mode: "{{ item.mode }}" + loop: + - { path: "{{ postgresql_data_directory }}/ssl/server.key", mode: '0600' } + - { path: "{{ postgresql_data_directory }}/ssl/server.crt", mode: '0644' } + - { path: "{{ postgresql_data_directory }}/ssl/ca.crt", mode: '0644' } + +- name: Create symlinks in PostgreSQL config directory + file: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + state: link + owner: postgres + group: postgres + loop: + - { src: "{{ postgresql_data_directory }}/ssl/server.crt", dest: "{{ postgresql_config_directory }}/server.crt" } + - { src: "{{ postgresql_data_directory }}/ssl/server.key", dest: "{{ postgresql_config_directory }}/server.key" } + - { src: "{{ postgresql_data_directory }}/ssl/ca.crt", dest: "{{ postgresql_config_directory }}/ca.crt" } \ No newline at end of file diff --git a/ansible/roles/postgresql/tasks/storage.yml b/ansible/roles/postgresql/tasks/storage.yml new file mode 100644 index 0000000..d6723ac --- /dev/null +++ b/ansible/roles/postgresql/tasks/storage.yml @@ -0,0 +1,110 @@ +--- +# Storage setup for PostgreSQL on NVMe drives + +- name: Install storage utilities + apt: + name: + - mdadm + - xfsprogs + - nvme-cli + - smartmontools + state: present + update_cache: yes + +- name: Check for NVMe devices + shell: nvme list -o json + register: nvme_devices + changed_when: false + failed_when: false + +- name: Create partition on NVMe drive + parted: + device: /dev/nvme0n1 + number: 1 + state: present + label: gpt + part_start: 0% + part_end: 100% + fs_type: ext4 + +- name: Create RAID0 array with single drive (for future expansion) + command: | + mdadm --create {{ postgresql_raid_device }} \ + --level=0 \ + --raid-devices=1 \ + --force \ + /dev/nvme0n1p1 + args: + creates: "{{ postgresql_raid_device }}" + when: postgresql_storage_type == "raid0" + +- name: Wait for RAID array to be ready + wait_for: + path: "{{ postgresql_raid_device }}" + state: present + when: postgresql_storage_type == "raid0" + +- name: Create filesystem on storage device + filesystem: + fstype: xfs + dev: "{{ postgresql_storage_device }}" + opts: "-L pgdata" + when: postgresql_storage_device is defined + +- name: Get filesystem UUID + command: blkid -s UUID -o value {{ postgresql_storage_device }} + register: fs_uuid + changed_when: false + when: postgresql_storage_device is defined + +- name: Create mount points + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - "{{ postgresql_mount_point }}" + - "{{ postgresql_data_directory }}" + - "{{ postgresql_wal_directory }}" + +- name: Mount PostgreSQL storage + mount: + path: "{{ postgresql_mount_point }}" + src: "UUID={{ fs_uuid.stdout }}" + fstype: xfs + opts: "noatime,nodiratime" + state: mounted + when: fs_uuid.stdout is defined + +- name: Save RAID configuration + shell: mdadm --detail --scan >> /etc/mdadm/mdadm.conf + when: postgresql_storage_type == "raid0" + +- name: Update initramfs + command: update-initramfs -u + when: postgresql_storage_type == "raid0" + +- name: Set optimal I/O scheduler for NVMe + copy: + content: | + ACTION=="add|change", KERNEL=="nvme[0-9]n[0-9]", ATTR{queue/scheduler}="none" + ACTION=="add|change", KERNEL=="nvme[0-9]n[0-9]", ATTR{queue/nr_requests}="1024" + dest: /etc/udev/rules.d/60-nvme-scheduler.rules + mode: '0644' + notify: reload udev + +- name: Configure sysctl for PostgreSQL performance + sysctl: + name: "{{ item.name }}" + value: "{{ item.value }}" + state: present + reload: yes + loop: + - { name: 'vm.dirty_background_ratio', value: '5' } + - { name: 'vm.dirty_ratio', value: '10' } + - { name: 'vm.swappiness', value: '10' } + - { name: 'vm.overcommit_memory', value: '2' } + - { name: 'vm.overcommit_ratio', value: '95' } + ignore_errors: yes \ No newline at end of file diff --git a/ansible/roles/postgresql/templates/30-postgresql.conf.j2 b/ansible/roles/postgresql/templates/30-postgresql.conf.j2 index c7fed29..51c18da 100644 --- a/ansible/roles/postgresql/templates/30-postgresql.conf.j2 +++ b/ansible/roles/postgresql/templates/30-postgresql.conf.j2 @@ -1,5 +1,5 @@ # Kernel parameters for PostgreSQL optimization -# For CM3588 with 32GB RAM and RAID0 SSDs +# For CM3588 with 16GB RAM and RAID0 SSDs # Shared memory settings kernel.shmmax = 17179869184 # 16GB diff --git a/ansible/roles/postgresql/templates/pgbouncer-userlist.txt.j2 b/ansible/roles/postgresql/templates/pgbouncer-userlist.txt.j2 index 6b4d52d..2c65f43 100644 --- a/ansible/roles/postgresql/templates/pgbouncer-userlist.txt.j2 +++ b/ansible/roles/postgresql/templates/pgbouncer-userlist.txt.j2 @@ -1,9 +1,12 @@ # PgBouncer userlist file -# Format: "username" "password" +# Format: "username" "password_or_scram_hash" # Managed by Ansible +# Note: When using SCRAM-SHA-256, this contains the SCRAM hash from pg_authid -{% for user in postgresql_users %} -"{{ user.name }}" "{{ user.password }}" +{% for result in scram_hashes.results %} +{% if result.query_result[0].rolpassword %} +"{{ result.query_result[0].rolname }}" "{{ result.query_result[0].rolpassword }}" +{% endif %} {% endfor %} # PgBouncer stats user diff --git a/ansible/roles/postgresql/templates/pgbouncer.ini.j2 b/ansible/roles/postgresql/templates/pgbouncer.ini.j2 index 8e90fac..16c3954 100644 --- a/ansible/roles/postgresql/templates/pgbouncer.ini.j2 +++ b/ansible/roles/postgresql/templates/pgbouncer.ini.j2 @@ -2,9 +2,6 @@ # Connection string for aprsme database aprsme_prod = host=127.0.0.1 port=5432 dbname=aprsme_prod -# Add a pooled connection for PgBouncer statistics -pgbouncer = host=127.0.0.1 port=5432 dbname=pgbouncer auth_user=pgbouncer - [pgbouncer] # Connection settings listen_addr = * @@ -61,6 +58,16 @@ tcp_keepintvl = 10 dns_max_ttl = 15 dns_nxdomain_ttl = 15 -# TLS settings (optional) -# client_tls_sslmode = prefer -# client_tls_ca_file = /etc/ssl/certs/ca-certificates.crt \ No newline at end of file +# TLS settings for client connections +client_tls_sslmode = allow +client_tls_cert_file = /etc/postgresql/{{ postgresql_version }}/main/server.crt +client_tls_key_file = /etc/postgresql/{{ postgresql_version }}/main/server.key +client_tls_ca_file = /etc/postgresql/{{ postgresql_version }}/main/ca.crt +client_tls_protocols = secure +client_tls_ciphers = HIGH:MEDIUM:+3DES:!aNULL + +# TLS settings for server connections +server_tls_sslmode = require +server_tls_ca_file = /etc/postgresql/{{ postgresql_version }}/main/ca.crt +server_tls_protocols = secure +server_tls_ciphers = HIGH:MEDIUM:+3DES:!aNULL \ No newline at end of file diff --git a/ansible/roles/postgresql/templates/postgresql.conf.j2 b/ansible/roles/postgresql/templates/postgresql.conf.j2 index 419de64..23866e3 100644 --- a/ansible/roles/postgresql/templates/postgresql.conf.j2 +++ b/ansible/roles/postgresql/templates/postgresql.conf.j2 @@ -100,4 +100,16 @@ max_locks_per_transaction = 128 #------------------------------------------------------------------------------ exit_on_error = off -restart_after_crash = on \ No newline at end of file +restart_after_crash = on + +#------------------------------------------------------------------------------ +# SSL/TLS CONFIGURATION +#------------------------------------------------------------------------------ + +ssl = on +ssl_cert_file = '/etc/postgresql/{{ postgresql_version }}/main/server.crt' +ssl_key_file = '/etc/postgresql/{{ postgresql_version }}/main/server.key' +ssl_ca_file = '/etc/postgresql/{{ postgresql_version }}/main/ca.crt' +ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' +ssl_prefer_server_ciphers = on +ssl_min_protocol_version = 'TLSv1.2' \ No newline at end of file diff --git a/ansible/roles/raid/tasks/main.yml b/ansible/roles/raid/tasks/main.yml index 83f6a41..a12599e 100644 --- a/ansible/roles/raid/tasks/main.yml +++ b/ansible/roles/raid/tasks/main.yml @@ -33,24 +33,24 @@ set_fact: auto_detect_devices: "{{ raid_devices is not defined or raid_devices | length == 0 }}" -- name: Auto-detect two largest unused disks +- name: Auto-detect largest unused disks when: auto_detect_devices block: - name: Find unused block devices shell: | - lsblk -dpno NAME,SIZE,TYPE,MOUNTPOINT | grep -E "disk.*$" | grep -v -E "/(|/boot)" | sort -k2 -h -r | head -2 | awk '{print $1}' + lsblk -dpno NAME,SIZE,TYPE,MOUNTPOINT | grep -E "disk.*$" | grep -v -E "/(|/boot)" | sort -k2 -h -r | head -5 | awk '{print $1}' register: detected_devices changed_when: false - name: Set raid_devices from detected devices set_fact: raid_devices: "{{ detected_devices.stdout_lines }}" - when: detected_devices.stdout_lines | length >= 2 + when: detected_devices.stdout_lines | length >= 1 - - name: Fail if not enough devices detected + - name: Fail if no devices detected fail: - msg: "Could not detect at least 2 unused block devices for RAID. Please specify raid_devices manually." - when: detected_devices.stdout_lines | length < 2 + msg: "Could not detect any unused block devices for RAID. Please specify raid_devices manually." + when: detected_devices.stdout_lines | length < 1 - name: Display RAID devices to be used debug: @@ -70,40 +70,32 @@ - name: Zero superblocks on disks command: mdadm --zero-superblock {{ item }} - loop: - - /dev/nvme0n1 - - /dev/nvme1n1 + loop: "{{ raid_devices }}" when: raid_status.rc == 0 when: force_raid_rebuild | default(false) -- name: Create partition table on first NVMe drive +- name: Create partition table on RAID devices parted: - device: /dev/nvme0n1 + device: "{{ item }}" number: 1 state: present part_type: primary part_start: 0% part_end: 100% label: gpt + loop: "{{ raid_devices }}" when: raid_status.rc != 0 -- name: Create partition table on second NVMe drive - parted: - device: /dev/nvme1n1 - number: 1 - state: present - part_type: primary - part_start: 0% - part_end: 100% - label: gpt - when: raid_status.rc != 0 +- name: Build RAID device list with partitions + set_fact: + raid_partitions: "{{ raid_devices | map('regex_replace', '$', 'p1') | list }}" -- name: Create RAID0 array +- name: Create RAID0 array with dynamic device count command: > mdadm --create /dev/md0 --level=0 - --raid-devices=2 - /dev/nvme0n1p1 /dev/nvme1n1p1 + --raid-devices={{ raid_devices | length }} + {{ raid_partitions | join(' ') }} --force when: raid_status.rc != 0 register: raid_create @@ -120,16 +112,29 @@ shell: mdadm --detail --scan >> /etc/mdadm/mdadm.conf when: raid_create is changed +- name: Check if update-initramfs exists + stat: + path: /usr/sbin/update-initramfs + register: initramfs_exists + - name: Update initramfs command: update-initramfs -u - when: raid_create is changed + when: + - raid_create is changed + - initramfs_exists.stat.exists + +- name: Check if filesystem exists on RAID array + command: blkid /dev/md0 + register: filesystem_check + failed_when: false + changed_when: false - name: Create filesystem on RAID array filesystem: fstype: ext4 dev: /dev/md0 opts: "-E lazy_itable_init=0,lazy_journal_init=0" - when: raid_create is changed + when: filesystem_check.rc != 0 - name: Create mount point for PostgreSQL data file: diff --git a/ansible/scripts/get-postgres-password.sh b/ansible/scripts/get-postgres-password.sh new file mode 100755 index 0000000..6af537d --- /dev/null +++ b/ansible/scripts/get-postgres-password.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Script to retrieve PostgreSQL password from 1Password + +set -e + +# Default values +DEFAULT_USER="aprsme" +DEFAULT_HOST="CM3588" +DEFAULT_ACCOUNT="YOOATCZZSVGH7AD6VABUVPORLI" + +# Parse arguments +USER="${1:-$DEFAULT_USER}" +HOST="${2:-$DEFAULT_HOST}" +ACCOUNT="${3:-$DEFAULT_ACCOUNT}" + +# Item name in 1Password +ITEM_NAME="PostgreSQL $USER - $HOST" + +echo "Retrieving password for: $ITEM_NAME" + +# Get password +PASSWORD=$(op item get "$ITEM_NAME" --fields password --reveal --account "$ACCOUNT" 2>/dev/null) + +if [ $? -eq 0 ]; then + echo "Password retrieved successfully" + echo + echo "Connection details:" + echo " User: $USER" + echo " Password: $PASSWORD" + echo + echo "URL-encoded password (for connection strings):" + echo "$PASSWORD" | python3 -c "import urllib.parse; print(urllib.parse.quote(input(), safe=''))" +else + echo "Error: Could not retrieve password from 1Password" + echo "Make sure the item '$ITEM_NAME' exists in your vault" + exit 1 +fi \ No newline at end of file diff --git a/ansible/setup-postgres-storage.yml b/ansible/setup-postgres-storage.yml new file mode 100644 index 0000000..42c08c4 --- /dev/null +++ b/ansible/setup-postgres-storage.yml @@ -0,0 +1,62 @@ +--- +- name: Set up PostgreSQL with NVMe storage + hosts: postgres_servers + become: yes + vars: + postgresql_storage_setup_enabled: true + postgresql_storage_type: raid0 + # Override data directory to ensure it's on the RAID array + postgresql_data_directory: /mnt/pgdata/16/main + postgresql_wal_directory: /mnt/pgdata/wal + postgresql_backup_directory: /mnt/pgdata/backups + # Set the APRS database password (you should use ansible-vault for this) + vault_postgresql_aprsme_password: "mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21" + + tasks: + - name: Check current storage status + command: lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINT + register: storage_status + changed_when: false + + - name: Display current storage + debug: + var: storage_status.stdout_lines + + - name: Check if PostgreSQL is already installed + stat: + path: /usr/lib/postgresql/16/bin/postgres + register: postgres_installed + + - name: Stop PostgreSQL if it's running + systemd: + name: postgresql + state: stopped + when: postgres_installed.stat.exists + ignore_errors: yes + + - name: Include PostgreSQL role + include_role: + name: postgresql + + - name: Verify PostgreSQL is running + systemd: + name: postgresql + state: started + register: postgres_status + + - name: Show PostgreSQL status + debug: + var: postgres_status + + - name: Test database connection + postgresql_ping: + db: aprsme_prod + login_user: aprsme + login_password: "{{ vault_postgresql_aprsme_password }}" + become_user: postgres + register: db_ping + ignore_errors: yes + + - name: Show database connection status + debug: + var: db_ping \ No newline at end of file diff --git a/ansible/site.yml b/ansible/site.yml index 804ce57..dbb81c1 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -3,6 +3,6 @@ hosts: postgres become: yes roles: - - common + - base - raid - postgresql \ No newline at end of file diff --git a/clusters/aprs/AUTO-HEALING-GUIDE.md b/clusters/aprs/AUTO-HEALING-GUIDE.md index 8efe43a..a51e48f 100644 --- a/clusters/aprs/AUTO-HEALING-GUIDE.md +++ b/clusters/aprs/AUTO-HEALING-GUIDE.md @@ -114,7 +114,41 @@ kubectl rollout restart statefulset/aprs -n aprs ## Applying the Configuration -To apply all auto-healing configurations: +### ⚠️ WARNING: Data Persistence +The enhanced deployments use different volume configurations. Applying them directly will create new volumes and lose existing data. + +### Safe Application (Recommended) + +For existing deployments with data: + +```bash +cd /Users/graham/dev/infra/clusters/aprs +./apply-auto-healing-safe.sh +``` + +This applies only non-destructive changes: +- PodDisruptionBudgets +- Cleanup CronJob +- Creates patch files for gradual application + +Then apply patches one at a time: +```bash +# PostgreSQL enhancements +kubectl patch deployment postgis -n aprs --patch-file postgis-deployment-patch.yaml +kubectl rollout status deployment/postgis -n aprs + +# PgBouncer enhancements +kubectl patch deployment pgbouncer -n aprs --patch-file pgbouncer-deployment-patch.yaml +kubectl rollout status deployment/pgbouncer -n aprs + +# APRS enhancements +kubectl patch statefulset aprs -n aprs --patch-file aprs-statefulset-patch.yaml +kubectl rollout status statefulset/aprs -n aprs +``` + +### Fresh Installation + +For new deployments without existing data: ```bash cd /Users/graham/dev/infra/clusters/aprs diff --git a/clusters/aprs/apply-auto-healing-safe.sh b/clusters/aprs/apply-auto-healing-safe.sh new file mode 100755 index 0000000..f059610 --- /dev/null +++ b/clusters/aprs/apply-auto-healing-safe.sh @@ -0,0 +1,148 @@ +#!/bin/bash +set -e + +echo "Applying auto-healing configurations safely..." + +# Only apply configurations that don't affect data persistence +echo "Applying PodDisruptionBudgets..." +kubectl apply -f postgis-pdb.yaml + +echo "Applying PostgreSQL cleanup CronJob..." +kubectl apply -f postgres-cleanup-cronjob.yaml + +echo "Creating enhanced configurations as patches (not applied automatically)..." + +# Create patch files instead of replacing deployments +cat > postgis-deployment-patch.yaml << 'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgis + namespace: aprs +spec: + template: + spec: + containers: + - name: postgis + args: + - postgres + - -c + - max_connections=200 + - -c + - idle_in_transaction_session_timeout=30s + - -c + - statement_timeout=300s + - -c + - shared_buffers=256MB + resources: + limits: + memory: "2Gi" + cpu: "1000m" + requests: + memory: "1Gi" + cpu: "500m" + livenessProbe: + exec: + command: + - /bin/sh + - -c + - pg_isready -U $POSTGRES_USER -d $POSTGRES_DB + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - pg_isready -U $POSTGRES_USER -d $POSTGRES_DB + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 2 +EOF + +cat > pgbouncer-deployment-patch.yaml << 'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pgbouncer + namespace: aprs +spec: + template: + spec: + containers: + - name: pgbouncer + env: + - name: PGBOUNCER_SERVER_IDLE_TIMEOUT + value: "600" + - name: PGBOUNCER_SERVER_LIFETIME + value: "3600" + - name: PGBOUNCER_QUERY_TIMEOUT + value: "300" + - name: PGBOUNCER_QUERY_WAIT_TIMEOUT + value: "120" + - name: PGBOUNCER_CLIENT_IDLE_TIMEOUT + value: "300" + - name: PGBOUNCER_CLIENT_LOGIN_TIMEOUT + value: "60" + - name: PGBOUNCER_SERVER_CONNECT_TIMEOUT + value: "15" + - name: PGBOUNCER_SERVER_LOGIN_RETRY + value: "15" + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" +EOF + +cat > aprs-statefulset-patch.yaml << 'EOF' +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: aprs + namespace: aprs +spec: + template: + spec: + containers: + - name: aprs + env: + - name: POOL_SIZE + value: "5" + - name: DATABASE_POOL_TIMEOUT + value: "60000" + - name: DATABASE_CONNECT_TIMEOUT + value: "30000" + resources: + limits: + memory: "1Gi" + cpu: "1000m" + requests: + memory: "512Mi" + cpu: "500m" + startupProbe: + httpGet: + path: /health + port: 4000 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 12 +EOF + +echo "" +echo "Safe auto-healing configurations applied!" +echo "" +echo "To apply the enhanced configurations gradually:" +echo "1. Apply PostgreSQL patch: kubectl patch deployment postgis -n aprs --patch-file postgis-deployment-patch.yaml" +echo "2. Wait for rollout: kubectl rollout status deployment/postgis -n aprs" +echo "3. Apply PgBouncer patch: kubectl patch deployment pgbouncer -n aprs --patch-file pgbouncer-deployment-patch.yaml" +echo "4. Wait for rollout: kubectl rollout status deployment/pgbouncer -n aprs" +echo "5. Apply APRS patch: kubectl patch statefulset aprs -n aprs --patch-file aprs-statefulset-patch.yaml" +echo "" +echo "Monitor with: ./monitor-postgres-health.sh" \ No newline at end of file diff --git a/clusters/aprs/aprs-deployment.yaml b/clusters/aprs/aprs-deployment.yaml deleted file mode 100644 index 018befd..0000000 --- a/clusters/aprs/aprs-deployment.yaml +++ /dev/null @@ -1,101 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: aprs - namespace: aprs -spec: - replicas: 2 - selector: - matchLabels: - app: aprs - template: - metadata: - labels: - app: aprs - spec: - imagePullSecrets: - - name: ghcr-pull-secret - containers: - - name: aprs - image: ghcr.io/aprsme/aprs.me:latest - imagePullPolicy: Always - ports: - - containerPort: 4000 - - containerPort: 4369 - name: epmd - - containerPort: 9100 - name: erlang - - containerPort: 45892 - name: gossip - protocol: UDP - env: - - name: PHX_HOST - value: "aprs.me" - - name: PORT - value: "4000" - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: aprs-secret - key: database-url - - name: SECRET_KEY_BASE - valueFrom: - secretKeyRef: - name: aprs-secret - key: secret-key-base - - name: PHX_SERVER - value: "true" - - name: MIX_ENV - value: "prod" - - name: APRS_CALLSIGN - value: "W5ISP-1" - - name: APRS_FILTER - value: "r/33/-96/1000000000000" - - name: APRS_PASSCODE - value: "15748" - - name: APRS_PORT - value: "10152" - - name: APRS_SERVER - value: "dallas.aprs2.net" - - name: PACKET_RETENTION_DAYS - value: "7" - - name: POOL_SIZE - value: "20" - - name: CLUSTER_ENABLED - value: "true" - - name: RELEASE_COOKIE - valueFrom: - secretKeyRef: - name: aprs-secret - key: erlang-cookie - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - resources: - limits: - memory: "512Mi" - cpu: "500m" - requests: - memory: "256Mi" - cpu: "250m" - livenessProbe: - httpGet: - path: /health - port: 4000 - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /health - port: 4000 - initialDelaySeconds: 5 - periodSeconds: 5 diff --git a/clusters/aprs/aprs-headless-service.yaml b/clusters/aprs/aprs-headless-service.yaml deleted file mode 100644 index a4f30ac..0000000 --- a/clusters/aprs/aprs-headless-service.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: aprs-headless - namespace: aprs -spec: - clusterIP: None - selector: - app: aprs - ports: - - name: epmd - port: 4369 - targetPort: 4369 - - name: erlang - port: 9100 - targetPort: 9100 - - name: gossip - port: 45892 - targetPort: 45892 - protocol: UDP - publishNotReadyAddresses: true \ No newline at end of file diff --git a/clusters/aprs/aprs-secret.yaml b/clusters/aprs/aprs-secret.yaml new file mode 100644 index 0000000..6e455d2 --- /dev/null +++ b/clusters/aprs/aprs-secret.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Secret +metadata: + name: aprs-secret + namespace: aprs +type: Opaque +stringData: + database-password: "rku23IEO9Xq1lZTl+DETeAGRn/VxdzXzuidIrHn5Gi4=" + # Direct connection to PostgreSQL on CM3588 (with SSL) + database-url: "ecto://aprsme:rku23IEO9Xq1lZTl%2BDETeAGRn%2FVxdzXzuidIrHn5Gi4%3D@10.0.19.220:5432/aprsme_prod" + # SSL configuration + database-ssl: "true" + database-ssl-verify: "false" + # Connection via PgBouncer on CM3588 (recommended) + database-url-pgbouncer: "ecto://aprsme:rku23IEO9Xq1lZTl%2BDETeAGRn%2FVxdzXzuidIrHn5Gi4%3D@10.0.19.220:6432/aprsme_prod" + erlang-cookie: "aprs-cluster-cookie-ksdf8934jfklsdjf983jfklsdf" + secret-key-base: "GJxzOUW3YXBAF7xB4mH5VcXY6BN1NYLnoJYadlZyiCxcRUGBw65w4Sco7+sNiW2t" \ No newline at end of file diff --git a/clusters/aprs/aprs-statefulset-enhanced.yaml b/clusters/aprs/aprs-statefulset-enhanced.yaml deleted file mode 100644 index ff1afe6..0000000 --- a/clusters/aprs/aprs-statefulset-enhanced.yaml +++ /dev/null @@ -1,132 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: aprs - namespace: aprs -spec: - serviceName: aprs-headless - replicas: 2 - selector: - matchLabels: - app: aprs - updateStrategy: - type: RollingUpdate - podManagementPolicy: Parallel - template: - metadata: - labels: - app: aprs - spec: - terminationGracePeriodSeconds: 60 - initContainers: - - name: wait-for-db - image: busybox:1.36.1 - command: ['sh', '-c', 'until nc -z pgbouncer 5432; do echo waiting for database; sleep 2; done'] - - name: wait-for-redis - image: busybox:1.36.1 - command: ['sh', '-c', 'until nc -z redis 6379; do echo waiting for redis; sleep 2; done'] - containers: - - name: aprs - image: ghcr.io/aprsme/aprs.me:latest - imagePullPolicy: Always - ports: - - containerPort: 4000 - protocol: TCP - env: - - name: SECRET_KEY_BASE - valueFrom: - secretKeyRef: - name: aprs-secret - key: secret-key-base - - name: DATABASE_URL - value: "ecto://aprs:$(DATABASE_PASSWORD)@pgbouncer:5432/aprs" - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: aprs-secret - key: database-password - - name: PHX_HOST - value: "aprs.me" - - name: PHX_SERVER - value: "true" - - name: POOL_SIZE - value: "5" # Reduced from default 10 to prevent connection exhaustion - - name: ECTO_IPV6 - value: "false" - - name: ERLANG_COOKIE - valueFrom: - secretKeyRef: - name: aprs-secret - key: erlang-cookie - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: REDIS_URL - value: "redis://redis.aprs.svc.cluster.local:6379" - - name: DRAIN_TIMEOUT_MS - value: "45000" - - name: SKIP_DB_CREATE - value: "true" - # Enhanced database connection settings - - name: DATABASE_POOL_TIMEOUT - value: "60000" # 60 seconds - - name: DATABASE_QUEUE_TARGET - value: "50" - - name: DATABASE_QUEUE_INTERVAL - value: "1000" - - name: DATABASE_CONNECT_TIMEOUT - value: "30000" # 30 seconds - - name: DATABASE_IDLE_TIMEOUT - value: "900000" # 15 minutes - # Memory management - - name: ERL_MAX_ETS_TABLES - value: "5000" - - name: ERLANG_SCHEDULER_BUSY_WAIT_THRESHOLD - value: "none" - resources: - limits: - memory: "1Gi" # Increased from 512Mi - cpu: "1000m" # Increased from 500m - requests: - memory: "512Mi" # Increased from 256Mi - cpu: "500m" # Increased from 250m - livenessProbe: - httpGet: - path: /health - port: 4000 - initialDelaySeconds: 60 # Increased from 30 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /health - port: 4000 - initialDelaySeconds: 30 # Increased from 5 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - startupProbe: - httpGet: - path: /health - port: 4000 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 12 # 120 seconds total - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "sleep 15"] - imagePullSecrets: - - name: ghcr-pull-secret \ No newline at end of file diff --git a/clusters/aprs/migrate-job.yaml b/clusters/aprs/migrate-job.yaml new file mode 100644 index 0000000..98ba852 --- /dev/null +++ b/clusters/aprs/migrate-job.yaml @@ -0,0 +1,37 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: aprs-migrate + namespace: aprs +spec: + template: + spec: + restartPolicy: Never + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: migrate + image: ghcr.io/aprsme/aprs.me:latest + env: + - name: DATABASE_URL + value: "ecto://aprsme:rku23IEO9Xq1lZTl%2BDETeAGRn%2FVxdzXzuidIrHn5Gi4%3D@10.0.19.220:5432/aprsme_prod" + - name: SECRET_KEY_BASE + valueFrom: + secretKeyRef: + name: aprs-secret + key: secret-key-base + - name: MIX_ENV + value: "prod" + - name: POOL_SIZE + value: "2" + - name: PHX_HOST + value: "aprs.me" + - name: PORT + value: "4000" + - name: SKIP_CLUSTERING + value: "true" + - name: CLUSTER_ENABLED + value: "false" + - name: REDIS_URL + value: "redis://redis.aprs.svc.cluster.local:6379" + command: ["/app/bin/migrate"] \ No newline at end of file diff --git a/clusters/aprs/migrate-to-external-db.sh b/clusters/aprs/migrate-to-external-db.sh new file mode 100755 index 0000000..006ab4a --- /dev/null +++ b/clusters/aprs/migrate-to-external-db.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -e + +echo "Migrating APRS to external PostgreSQL database" +echo "==============================================" +echo "" + +# Test connection to external database first +echo "Testing connection to external PostgreSQL..." +PGPASSWORD="mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21" psql -h 10.0.19.220 -p 6432 -U aprsme -d aprsme_prod -c "SELECT version();" > /dev/null 2>&1 +if [ $? -eq 0 ]; then + echo "✅ Successfully connected to external database" +else + echo "❌ Failed to connect to external database at 10.0.19.220:6432" + echo "Please ensure the PostgreSQL server is running and accessible" + exit 1 +fi + +echo "" +echo "Step 1: Updating database connection secret..." +kubectl apply -f aprs-secret-external-db.yaml + +echo "" +echo "Step 2: Restarting APRS pods to use new database..." +kubectl rollout restart statefulset/aprs -n aprs +kubectl rollout status statefulset/aprs -n aprs --timeout=300s + +echo "" +echo "Step 3: Verifying pods are running with new database..." +sleep 10 +kubectl get pods -n aprs -l app=aprs + +echo "" +echo "Step 4: Checking application health..." +for i in {1..5}; do + POD=$(kubectl get pods -n aprs -l app=aprs -o jsonpath='{.items[0].metadata.name}') + if kubectl exec -n aprs $POD -- curl -s http://localhost:4000/health > /dev/null; then + echo "✅ Application is healthy" + break + else + echo "Waiting for application to be ready... ($i/5)" + sleep 10 + fi +done + +echo "" +echo "Step 5: Removing local PostgreSQL resources..." +read -p "Are you sure you want to remove the local PostgreSQL deployment? (y/n): " confirm +if [ "$confirm" = "y" ]; then + echo "Removing PostgreSQL resources..." + kubectl delete deployment postgis -n aprs --ignore-not-found=true + kubectl delete service postgis -n aprs --ignore-not-found=true + kubectl delete deployment pgbouncer -n aprs --ignore-not-found=true + kubectl delete service pgbouncer -n aprs --ignore-not-found=true + kubectl delete cronjob postgres-cleanup -n aprs --ignore-not-found=true + kubectl delete job -l app=postgres-cleanup -n aprs --ignore-not-found=true + + echo "" + echo "PostgreSQL PVC will be retained for backup purposes" + echo "To delete it later: kubectl delete pvc postgis-pvc-ceph -n aprs" +fi + +echo "" +echo "Migration complete!" +echo "" +echo "Summary:" +echo "- APRS is now using external PostgreSQL at 10.0.19.220" +echo "- Connection is via PgBouncer on port 6432" +echo "- Local PostgreSQL resources have been removed" +echo "" +echo "To verify:" +echo "kubectl logs -n aprs -l app=aprs --tail=50" \ No newline at end of file diff --git a/clusters/aprs/monitoring/MONITORING-GUIDE.md b/clusters/aprs/monitoring/MONITORING-GUIDE.md new file mode 100644 index 0000000..190498a --- /dev/null +++ b/clusters/aprs/monitoring/MONITORING-GUIDE.md @@ -0,0 +1,244 @@ +# Monitoring and Observability Guide + +This guide explains the comprehensive monitoring stack for the APRS k3s cluster. + +## Overview + +The monitoring stack consists of: +- **Prometheus**: Metrics collection and storage +- **Grafana**: Visualization and dashboards +- **AlertManager**: Alert routing and notifications +- **Exporters**: PostgreSQL, PgBouncer, Redis, and application metrics + +## Architecture + +``` +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ APRS App │ │ PostgreSQL │ │ PgBouncer │ +│ /metrics │ │ Exporter │ │ Exporter │ +└──────┬──────┘ └──────┬──────┘ └──────┬──────┘ + │ │ │ + └───────────────────┴───────────────────┘ + │ + ┌──────▼──────┐ + │ Prometheus │ + │ (scraping) │ + └──────┬──────┘ + │ + ┌──────────┴──────────┐ + │ │ + ┌──────▼──────┐ ┌──────▼──────┐ + │ Grafana │ │ AlertManager│ + │ (dashboards)│ │ (alerts) │ + └─────────────┘ └─────────────┘ +``` + +## Installation + +1. **Install the monitoring stack**: +```bash +cd /path/to/infra/clusters/aprs/monitoring +./install-monitoring.sh +``` + +2. **Apply the enhanced deployments** (if not already done): +```bash +cd /path/to/infra/clusters/aprs +./apply-auto-healing.sh +``` + +3. **Update Redis deployment** to include metrics: +```bash +kubectl apply -f redis-deployment-with-metrics.yaml +``` + +## Access + +### Grafana +```bash +kubectl port-forward -n monitoring svc/kube-prometheus-stack-grafana 3000:80 +``` +Access at: http://localhost:3000 +- Username: `admin` +- Password: `changeme` (change this!) + +### Prometheus +```bash +kubectl port-forward -n monitoring svc/kube-prometheus-stack-prometheus 9090:9090 +``` +Access at: http://localhost:9090 + +### AlertManager +```bash +kubectl port-forward -n monitoring svc/kube-prometheus-stack-alertmanager 9093:9093 +``` +Access at: http://localhost:9093 + +## Metrics Available + +### PostgreSQL Metrics +- `pg_connection_stats_total_connections` - Current total connections +- `pg_connection_stats_max_connections` - Maximum allowed connections +- `pg_connections_count` - Connections by state (active, idle, etc.) +- `pg_long_running_queries_count` - Queries running > 5 minutes +- `pg_database_size_bytes` - Database size +- Standard `pg_*` metrics from postgres_exporter + +### PgBouncer Metrics +- `pgbouncer_pools_server_active_connections` - Active server connections +- `pgbouncer_pools_client_active_connections` - Active client connections +- `pgbouncer_pools_pool_size` - Configured pool size +- `pgbouncer_stats_*` - Various PgBouncer statistics + +### Application Metrics +- `phoenix_endpoint_stop_duration_millisecond` - HTTP request duration +- `aprsme_repo_query_*` - Database query metrics +- `aprsme_packet_pipeline_*` - Packet processing metrics +- `aprsme_spatial_pubsub_*` - Real-time broadcast metrics +- `vm_memory_total` - Erlang VM memory usage + +### Kubernetes Metrics +- `container_memory_working_set_bytes` - Container memory usage +- `container_cpu_usage_seconds_total` - Container CPU usage +- `kube_pod_container_status_restarts_total` - Pod restart count + +## Dashboards + +### Pre-installed Dashboards +1. **PostgreSQL Dashboard** (ID: 9628) - Comprehensive PostgreSQL metrics +2. **PgBouncer Dashboard** (ID: 15984) - Connection pooling metrics +3. **Kubernetes Cluster** (ID: 7249) - Cluster overview + +### Custom Dashboards +1. **APRS Application Overview** - Application-specific metrics +2. **PostgreSQL Connection Analysis** - Detailed connection monitoring + +## Alerts + +### Critical Alerts +- **PostgreSQLConnectionsCritical**: > 95% connections used +- **PostgreSQLDown**: PostgreSQL not responding +- **PgBouncerDown**: PgBouncer not responding +- **APRSPodCrashLooping**: Pod restarting frequently + +### Warning Alerts +- **PostgreSQLConnectionsHigh**: > 80% connections used +- **PostgreSQLIdleTransactionsHigh**: > 10 idle transactions +- **PostgreSQLLongRunningQueries**: Queries > 5 minutes +- **APRSHighMemoryUsage**: > 90% memory limit +- **APRSHighDatabaseConnectionQueueTime**: Queue time > 1 second + +## Troubleshooting + +### Check Metrics Collection +```bash +# Verify all exporters are running +kubectl get pods -n aprs | grep exporter + +# Check ServiceMonitors +kubectl get servicemonitor -n aprs + +# Verify Prometheus targets +kubectl port-forward -n monitoring svc/kube-prometheus-stack-prometheus 9090:9090 +# Visit http://localhost:9090/targets +``` + +### Common Issues + +1. **Metrics not appearing**: + - Check if exporter pods are running + - Verify ServiceMonitor labels match Prometheus selector + - Check Prometheus targets page for errors + +2. **Dashboards empty**: + - Ensure data source is set to "Prometheus" + - Check metric names in queries + - Verify time range selection + +3. **Alerts not firing**: + - Check PrometheusRule is loaded + - Verify alert expressions in Prometheus + - Check AlertManager configuration + +## Adding New Metrics + +### Application Metrics +Add to `/lib/aprsme_web/telemetry.ex`: +```elixir +counter("your_metric_name", + description: "Description", + tags: [:tag1, :tag2] +) +``` + +### Custom Queries +Add to postgres-exporter ConfigMap: +```yaml +your_custom_metric: + query: "SELECT ..." + metrics: + - column_name: + usage: "GAUGE" + description: "Description" +``` + +## Performance Tuning + +### Prometheus Storage +Current settings: +- Retention: 30 days +- Storage: 20Gi + +Adjust in `kube-prometheus-stack-values.yaml` if needed. + +### Scrape Intervals +Default: 30 seconds +Adjust in ServiceMonitors for specific endpoints. + +### Resource Usage +Monitor Prometheus and Grafana resource usage: +```bash +kubectl top pods -n monitoring +``` + +## Backup and Recovery + +### Grafana Dashboards +Export dashboards as JSON from Grafana UI for backup. + +### Prometheus Data +Data is stored in PVC. For backup: +```bash +kubectl exec -n monitoring prometheus-kube-prometheus-stack-prometheus-0 -- \ + tar czf /tmp/prometheus-backup.tar.gz /prometheus +kubectl cp monitoring/prometheus-kube-prometheus-stack-prometheus-0:/tmp/prometheus-backup.tar.gz \ + ./prometheus-backup.tar.gz +``` + +## Integration with CI/CD + +Add to your deployment pipeline: +```bash +# Check if metrics endpoint is healthy +curl -f http://your-app/metrics || exit 1 + +# Query Prometheus for deployment validation +curl -G http://prometheus:9090/api/v1/query \ + --data-urlencode 'query=up{job="aprs-app"}' | jq '.data.result[0].value[1]' +``` + +## Security Considerations + +1. **Change default passwords** in Grafana +2. **Configure network policies** to restrict access +3. **Enable TLS** for external access +4. **Set up authentication** for Prometheus and AlertManager +5. **Limit metric cardinality** to prevent resource exhaustion + +## Next Steps + +1. Configure AlertManager to send notifications (email, Slack, etc.) +2. Set up long-term storage with Thanos or Cortex +3. Implement SLO/SLI dashboards +4. Add distributed tracing with Jaeger +5. Set up log aggregation with Loki \ No newline at end of file diff --git a/clusters/aprs/monitoring/aprs-dashboard-configmap.yaml b/clusters/aprs/monitoring/aprs-dashboard-configmap.yaml new file mode 100644 index 0000000..22de8da --- /dev/null +++ b/clusters/aprs/monitoring/aprs-dashboard-configmap.yaml @@ -0,0 +1,114 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aprs-dashboard + namespace: monitoring + labels: + grafana_dashboard: "1" + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: kube-prometheus-stack +data: + aprs-dashboard.json: | + { + "dashboard": { + "id": null, + "title": "APRS.me Application Metrics", + "tags": ["aprs", "amateur-radio", "real-time"], + "style": "dark", + "timezone": "browser", + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "APRS-IS Connection Status", + "type": "stat", + "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0}, + "targets": [ + { + "expr": "up{job=\"aprs-metrics\"}", + "legendFormat": "Service Up" + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "steps": [ + {"color": "red", "value": 0}, + {"color": "green", "value": 1} + ] + } + } + } + }, + { + "id": 2, + "title": "Packet Processing Rate", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 6, "y": 0}, + "targets": [ + { + "expr": "rate(aprsme_packet_pipeline_batch_count_total[5m])", + "legendFormat": "Packets/sec" + }, + { + "expr": "rate(aprsme_packet_pipeline_batch_success_total[5m])", + "legendFormat": "Successful/sec" + }, + { + "expr": "rate(aprsme_packet_pipeline_batch_error_total[5m])", + "legendFormat": "Errors/sec" + } + ] + }, + { + "id": 3, + "title": "Connected Clients", + "type": "stat", + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0}, + "targets": [ + { + "expr": "aprsme_spatial_pubsub_clients_count", + "legendFormat": "Active Clients" + } + ] + }, + { + "id": 4, + "title": "Database Performance", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "targets": [ + { + "expr": "rate(aprsme_repo_query_total_time_sum[5m]) / rate(aprsme_repo_query_total_time_count[5m])", + "legendFormat": "Avg Query Time (ms)" + }, + { + "expr": "rate(aprsme_repo_query_queue_time_sum[5m]) / rate(aprsme_repo_query_queue_time_count[5m])", + "legendFormat": "Avg Queue Time (ms)" + } + ] + }, + { + "id": 5, + "title": "Spatial Broadcast Metrics", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "targets": [ + { + "expr": "rate(aprsme_spatial_pubsub_broadcasts_total[5m])", + "legendFormat": "Total Broadcasts/sec" + }, + { + "expr": "rate(aprsme_spatial_pubsub_broadcasts_filtered[5m])", + "legendFormat": "Filtered Broadcasts/sec" + } + ] + } + ] + } + } \ No newline at end of file diff --git a/clusters/aprs/monitoring/aprs-dashboard-enhanced.yaml b/clusters/aprs/monitoring/aprs-dashboard-enhanced.yaml new file mode 100644 index 0000000..7a5f5d3 --- /dev/null +++ b/clusters/aprs/monitoring/aprs-dashboard-enhanced.yaml @@ -0,0 +1,332 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aprs-dashboard-enhanced + namespace: monitoring + labels: + grafana_dashboard: "1" + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: kube-prometheus-stack +data: + aprs-dashboard-enhanced.json: | + { + "dashboard": { + "id": null, + "title": "APRS.me Complete Metrics Dashboard", + "tags": ["aprs", "postgresql", "database", "real-time"], + "style": "dark", + "timezone": "browser", + "refresh": "10s", + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "Database Connection Pool", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, + "targets": [ + { + "expr": "aprsme_repo_pool_size{job=\"aprs-metrics\"}", + "legendFormat": "Pool Size" + }, + { + "expr": "aprsme_repo_pool_busy{job=\"aprs-metrics\"}", + "legendFormat": "Busy Connections" + }, + { + "expr": "aprsme_repo_pool_idle{job=\"aprs-metrics\"}", + "legendFormat": "Idle Connections" + }, + { + "expr": "aprsme_repo_pool_queue_length{job=\"aprs-metrics\"}", + "legendFormat": "Queue Length" + } + ], + "yaxes": [ + { + "format": "short", + "label": "Connections" + }, + { + "format": "short" + } + ] + }, + { + "id": 2, + "title": "PostgreSQL Connections", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, + "targets": [ + { + "expr": "aprsme_postgres_connections_total{job=\"aprs-metrics\"}", + "legendFormat": "Total" + }, + { + "expr": "aprsme_postgres_connections_active{job=\"aprs-metrics\"}", + "legendFormat": "Active" + }, + { + "expr": "aprsme_postgres_connections_idle{job=\"aprs-metrics\"}", + "legendFormat": "Idle" + }, + { + "expr": "aprsme_postgres_connections_idle_in_transaction{job=\"aprs-metrics\"}", + "legendFormat": "Idle in Transaction" + }, + { + "expr": "aprsme_postgres_connections_waiting{job=\"aprs-metrics\"}", + "legendFormat": "Waiting" + } + ], + "yaxes": [ + { + "format": "short", + "label": "Connections" + }, + { + "format": "short" + } + ] + }, + { + "id": 3, + "title": "Database Size", + "type": "stat", + "gridPos": {"h": 4, "w": 6, "x": 0, "y": 8}, + "targets": [ + { + "expr": "aprsme_postgres_database_size_bytes{job=\"aprs-metrics\"} / 1024 / 1024 / 1024", + "legendFormat": "Database Size" + } + ], + "fieldConfig": { + "defaults": { + "unit": "GB", + "decimals": 2 + } + } + }, + { + "id": 4, + "title": "Packets Table Size", + "type": "stat", + "gridPos": {"h": 4, "w": 6, "x": 6, "y": 8}, + "targets": [ + { + "expr": "aprsme_postgres_packets_table_table_size_bytes{job=\"aprs-metrics\"} / 1024 / 1024", + "legendFormat": "Table Size" + } + ], + "fieldConfig": { + "defaults": { + "unit": "MB", + "decimals": 2 + } + } + }, + { + "id": 5, + "title": "Packets Table Rows", + "type": "stat", + "gridPos": {"h": 4, "w": 6, "x": 12, "y": 8}, + "targets": [ + { + "expr": "aprsme_postgres_packets_table_live_tuples{job=\"aprs-metrics\"}", + "legendFormat": "Live Rows" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0 + } + } + }, + { + "id": 6, + "title": "Dead Tuples Ratio", + "type": "stat", + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 8}, + "targets": [ + { + "expr": "(aprsme_postgres_packets_table_dead_tuples{job=\"aprs-metrics\"} / (aprsme_postgres_packets_table_live_tuples{job=\"aprs-metrics\"} + aprsme_postgres_packets_table_dead_tuples{job=\"aprs-metrics\"})) * 100", + "legendFormat": "Dead Tuple %" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 2, + "color": {"mode": "thresholds"}, + "thresholds": { + "steps": [ + {"color": "green", "value": 0}, + {"color": "yellow", "value": 10}, + {"color": "red", "value": 20} + ] + } + } + } + }, + { + "id": 7, + "title": "Query Performance", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12}, + "targets": [ + { + "expr": "aprsme_postgres_query_stats_avg_time_ms{job=\"aprs-metrics\"}", + "legendFormat": "Average Query Time" + }, + { + "expr": "aprsme_postgres_query_stats_max_time_ms{job=\"aprs-metrics\"}", + "legendFormat": "Max Query Time" + } + ], + "yaxes": [ + { + "format": "ms", + "label": "Query Time" + }, + { + "format": "short" + } + ] + }, + { + "id": 8, + "title": "Database Operations", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 12}, + "targets": [ + { + "expr": "rate(aprsme_postgres_packets_table_total_inserts{job=\"aprs-metrics\"}[5m])", + "legendFormat": "Insert Rate" + }, + { + "expr": "rate(aprsme_postgres_packets_table_total_updates{job=\"aprs-metrics\"}[5m])", + "legendFormat": "Update Rate" + }, + { + "expr": "rate(aprsme_postgres_packets_table_total_deletes{job=\"aprs-metrics\"}[5m])", + "legendFormat": "Delete Rate" + } + ], + "yaxes": [ + { + "format": "ops", + "label": "Operations/sec" + }, + { + "format": "short" + } + ] + }, + { + "id": 9, + "title": "Ecto Query Times", + "type": "heatmap", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 20}, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(aprsme_repo_query_total_time_bucket{job=\"aprs-metrics\"}[5m])) by (le))", + "format": "heatmap", + "legendFormat": "{{le}}" + } + ], + "dataFormat": "tsbuckets", + "yAxis": { + "format": "ms", + "decimals": 0 + } + }, + { + "id": 10, + "title": "Connection Pool Queue Time", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 20}, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(aprsme_repo_query_queue_time_bucket{job=\"aprs-metrics\"}[5m])) by (le))", + "legendFormat": "95th percentile" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(aprsme_repo_query_queue_time_bucket{job=\"aprs-metrics\"}[5m])) by (le))", + "legendFormat": "99th percentile" + } + ], + "yaxes": [ + { + "format": "ms", + "label": "Queue Time" + }, + { + "format": "short" + } + ] + }, + { + "id": 11, + "title": "APRS Packet Processing", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 28}, + "targets": [ + { + "expr": "rate(aprsme_packet_pipeline_batch_count{job=\"aprs-metrics\"}[5m])", + "legendFormat": "Packets/sec" + }, + { + "expr": "rate(aprsme_packet_pipeline_batch_success{job=\"aprs-metrics\"}[5m])", + "legendFormat": "Success/sec" + }, + { + "expr": "rate(aprsme_packet_pipeline_batch_error{job=\"aprs-metrics\"}[5m])", + "legendFormat": "Errors/sec" + } + ], + "yaxes": [ + { + "format": "short", + "label": "Packets/sec" + }, + { + "format": "short" + } + ] + }, + { + "id": 12, + "title": "Spatial Broadcasting Efficiency", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 28}, + "targets": [ + { + "expr": "aprsme_spatial_pubsub_efficiency_ratio{job=\"aprs-metrics\"}", + "legendFormat": "Efficiency %" + }, + { + "expr": "rate(aprsme_spatial_pubsub_efficiency_saved_broadcasts{job=\"aprs-metrics\"}[5m])", + "legendFormat": "Saved Broadcasts/sec", + "yaxis": 2 + } + ], + "yaxes": [ + { + "format": "percent", + "label": "Efficiency" + }, + { + "format": "short", + "label": "Saved/sec" + } + ] + } + ], + "schemaVersion": 27, + "version": 1 + } + } \ No newline at end of file diff --git a/clusters/aprs/monitoring/aprs-dashboard.json b/clusters/aprs/monitoring/aprs-dashboard.json new file mode 100644 index 0000000..4f26fec --- /dev/null +++ b/clusters/aprs/monitoring/aprs-dashboard.json @@ -0,0 +1,164 @@ +{ + "dashboard": { + "id": null, + "title": "APRS.me Application Metrics", + "tags": ["aprs", "amateur-radio", "real-time"], + "style": "dark", + "timezone": "browser", + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "APRS-IS Connection Status", + "type": "stat", + "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0}, + "targets": [ + { + "expr": "up{job=\"aprs-metrics\"}", + "legendFormat": "Service Up" + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "steps": [ + {"color": "red", "value": 0}, + {"color": "green", "value": 1} + ] + } + } + } + }, + { + "id": 2, + "title": "Packet Processing Rate", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 6, "y": 0}, + "targets": [ + { + "expr": "rate(aprsme_packet_pipeline_batch_count_total[5m])", + "legendFormat": "Packets/sec" + }, + { + "expr": "rate(aprsme_packet_pipeline_batch_success_total[5m])", + "legendFormat": "Successful/sec" + }, + { + "expr": "rate(aprsme_packet_pipeline_batch_error_total[5m])", + "legendFormat": "Errors/sec" + } + ], + "yAxes": [ + {"label": "Packets/second", "min": 0} + ] + }, + { + "id": 3, + "title": "Connected Clients", + "type": "stat", + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0}, + "targets": [ + { + "expr": "aprsme_spatial_pubsub_clients_count", + "legendFormat": "Active Clients" + } + ] + }, + { + "id": 4, + "title": "Database Performance", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "targets": [ + { + "expr": "rate(aprsme_repo_query_total_time_sum[5m]) / rate(aprsme_repo_query_total_time_count[5m])", + "legendFormat": "Avg Query Time" + }, + { + "expr": "rate(aprsme_repo_query_queue_time_sum[5m]) / rate(aprsme_repo_query_queue_time_count[5m])", + "legendFormat": "Avg Queue Time" + } + ], + "yAxes": [ + {"label": "Milliseconds", "min": 0} + ] + }, + { + "id": 5, + "title": "Spatial Broadcast Efficiency", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "targets": [ + { + "expr": "aprsme_spatial_pubsub_efficiency_ratio", + "legendFormat": "Efficiency %" + }, + { + "expr": "rate(aprsme_spatial_pubsub_broadcasts_total[5m])", + "legendFormat": "Total Broadcasts/sec" + }, + { + "expr": "rate(aprsme_spatial_pubsub_broadcasts_filtered[5m])", + "legendFormat": "Filtered Broadcasts/sec" + } + ] + }, + { + "id": 6, + "title": "HTTP Request Performance", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16}, + "targets": [ + { + "expr": "rate(phoenix_router_dispatch_stop_duration_sum[5m]) / rate(phoenix_router_dispatch_stop_duration_count[5m])", + "legendFormat": "Avg Response Time" + }, + { + "expr": "rate(phoenix_router_dispatch_stop_duration_count[5m])", + "legendFormat": "Requests/sec" + } + ] + }, + { + "id": 7, + "title": "Memory Usage", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16}, + "targets": [ + { + "expr": "vm_memory_total", + "legendFormat": "Total Memory" + } + ], + "yAxes": [ + {"label": "Bytes", "min": 0} + ] + }, + { + "id": 8, + "title": "Packet Pipeline Batch Duration", + "type": "graph", + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 24}, + "targets": [ + { + "expr": "rate(aprsme_packet_pipeline_batch_duration_ms_sum[5m]) / rate(aprsme_packet_pipeline_batch_duration_ms_count[5m])", + "legendFormat": "Avg Batch Duration (ms)" + }, + { + "expr": "histogram_quantile(0.95, rate(aprsme_packet_pipeline_batch_duration_ms_bucket[5m]))", + "legendFormat": "95th Percentile" + }, + { + "expr": "histogram_quantile(0.99, rate(aprsme_packet_pipeline_batch_duration_ms_bucket[5m]))", + "legendFormat": "99th Percentile" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/clusters/aprs/monitoring/aprs-metrics-service.yaml b/clusters/aprs/monitoring/aprs-metrics-service.yaml new file mode 100644 index 0000000..3ec4d5f --- /dev/null +++ b/clusters/aprs/monitoring/aprs-metrics-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: aprs-prometheus-metrics + namespace: aprs + labels: + app: aprs + component: metrics +spec: + selector: + app: aprs + ports: + - name: metrics + port: 9568 + targetPort: 9568 + protocol: TCP + type: ClusterIP \ No newline at end of file diff --git a/clusters/aprs/monitoring/aprs-servicemonitor.yaml b/clusters/aprs/monitoring/aprs-servicemonitor.yaml new file mode 100644 index 0000000..5d10ba8 --- /dev/null +++ b/clusters/aprs/monitoring/aprs-servicemonitor.yaml @@ -0,0 +1,37 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: aprs-metrics + namespace: monitoring + labels: + app: aprs + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app: aprs + namespaceSelector: + matchNames: + - aprs + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: v1 +kind: Service +metadata: + name: aprs-metrics-service + namespace: aprs + labels: + app: aprs +spec: + selector: + app: aprs + ports: + - name: http + port: 9568 + targetPort: 9568 + protocol: TCP + type: ClusterIP \ No newline at end of file diff --git a/clusters/aprs/monitoring/check-monitoring-status.sh b/clusters/aprs/monitoring/check-monitoring-status.sh new file mode 100755 index 0000000..c369410 --- /dev/null +++ b/clusters/aprs/monitoring/check-monitoring-status.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +echo "=== Monitoring Stack Status ===" +echo "" + +# Check Prometheus +echo "Prometheus:" +kubectl get pods -n monitoring -l app.kubernetes.io/name=prometheus -o wide +echo "" + +# Check Grafana +echo "Grafana:" +kubectl get pods -n monitoring -l app.kubernetes.io/name=grafana -o wide +echo "" + +# Check AlertManager +echo "AlertManager:" +kubectl get pods -n monitoring -l app.kubernetes.io/name=alertmanager -o wide +echo "" + +# Check Exporters +echo "Exporters in APRS namespace:" +kubectl get pods -n aprs | grep -E "(exporter|NAME)" +echo "" + +# Check ServiceMonitors +echo "ServiceMonitors:" +kubectl get servicemonitor -A | grep -E "(aprs|NAME)" +echo "" + +# Check PrometheusRules +echo "PrometheusRules:" +kubectl get prometheusrule -A | grep -E "(aprs|NAME)" +echo "" + +# Check if metrics endpoints are accessible +echo "Checking metrics endpoints:" +echo -n "PostgreSQL Exporter: " +kubectl exec -n aprs deployment/postgres-exporter -- wget -q -O- http://localhost:9187/metrics | head -1 2>/dev/null && echo "OK" || echo "FAILED" + +echo -n "PgBouncer Exporter: " +kubectl exec -n aprs deployment/pgbouncer-exporter -- wget -q -O- http://localhost:9127/metrics | head -1 2>/dev/null && echo "OK" || echo "FAILED" + +echo -n "APRS Application: " +kubectl exec -n aprs aprs-0 -- wget -q -O- http://localhost:4000/metrics | head -1 2>/dev/null && echo "OK" || echo "FAILED" + +echo "" +echo "To access Grafana: kubectl port-forward -n monitoring svc/kube-prometheus-stack-grafana 3000:80" +echo "To access Prometheus: kubectl port-forward -n monitoring svc/kube-prometheus-stack-prometheus 9090:9090" \ No newline at end of file diff --git a/clusters/aprs/monitoring/grafana-dashboards.yaml b/clusters/aprs/monitoring/grafana-dashboards.yaml new file mode 100644 index 0000000..214473f --- /dev/null +++ b/clusters/aprs/monitoring/grafana-dashboards.yaml @@ -0,0 +1,181 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aprs-custom-dashboards + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + aprs-overview.json: | + { + "dashboard": { + "title": "APRS Application Overview", + "uid": "aprs-overview", + "timezone": "browser", + "panels": [ + { + "title": "Database Connection Usage", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, + "type": "graph", + "targets": [ + { + "expr": "pg_connection_stats_total_connections", + "legendFormat": "Current Connections" + }, + { + "expr": "pg_connection_stats_max_connections", + "legendFormat": "Max Connections" + } + ] + }, + { + "title": "Connection States", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, + "type": "piechart", + "targets": [ + { + "expr": "pg_connections_count", + "legendFormat": "{{ state }}" + } + ] + }, + { + "title": "Packet Processing Rate", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "type": "graph", + "targets": [ + { + "expr": "rate(aprsme_packet_pipeline_batch_success[5m])", + "legendFormat": "Successful Batches/sec" + }, + { + "expr": "rate(aprsme_packet_pipeline_batch_error[5m])", + "legendFormat": "Failed Batches/sec" + } + ] + }, + { + "title": "Memory Usage", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "type": "graph", + "targets": [ + { + "expr": "container_memory_working_set_bytes{namespace=\"aprs\", pod=~\"aprs-.*\", container=\"aprs\"}", + "legendFormat": "{{ pod }}" + } + ] + }, + { + "title": "HTTP Request Duration (95th percentile)", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16}, + "type": "graph", + "targets": [ + { + "expr": "histogram_quantile(0.95, phoenix_endpoint_stop_duration_millisecond)", + "legendFormat": "95th percentile" + } + ] + }, + { + "title": "Database Query Time", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16}, + "type": "graph", + "targets": [ + { + "expr": "histogram_quantile(0.95, aprsme_repo_query_queue_time_millisecond)", + "legendFormat": "Queue Time (95th)" + }, + { + "expr": "histogram_quantile(0.95, aprsme_repo_query_query_time_millisecond)", + "legendFormat": "Query Time (95th)" + } + ] + } + ] + } + } + + postgres-connections.json: | + { + "dashboard": { + "title": "PostgreSQL Connection Analysis", + "uid": "postgres-connections", + "timezone": "browser", + "panels": [ + { + "title": "Connection Usage Over Time", + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 0}, + "type": "graph", + "targets": [ + { + "expr": "(pg_connection_stats_total_connections / pg_connection_stats_max_connections) * 100", + "legendFormat": "Connection Usage %" + } + ], + "alert": { + "conditions": [ + { + "evaluator": { + "params": [80], + "type": "gt" + }, + "query": { + "params": ["A", "5m", "now"] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ] + } + }, + { + "title": "Connection States Breakdown", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "type": "bargauge", + "targets": [ + { + "expr": "pg_connection_stats_active_connections", + "legendFormat": "Active" + }, + { + "expr": "pg_connection_stats_idle_connections", + "legendFormat": "Idle" + }, + { + "expr": "pg_connection_stats_idle_in_transaction", + "legendFormat": "Idle in Transaction" + } + ] + }, + { + "title": "Long Running Queries", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "type": "stat", + "targets": [ + { + "expr": "pg_long_running_queries_count", + "legendFormat": "Queries > 5 min" + } + ] + }, + { + "title": "PgBouncer Pool Usage", + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 16}, + "type": "graph", + "targets": [ + { + "expr": "pgbouncer_pools_server_active_connections", + "legendFormat": "Active Server Connections - {{ database }}" + }, + { + "expr": "pgbouncer_pools_client_active_connections", + "legendFormat": "Active Client Connections - {{ database }}" + } + ] + } + ] + } + } \ No newline at end of file diff --git a/clusters/aprs/monitoring/grafana-loadbalancer.yaml b/clusters/aprs/monitoring/grafana-loadbalancer.yaml new file mode 100644 index 0000000..76d164d --- /dev/null +++ b/clusters/aprs/monitoring/grafana-loadbalancer.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana-loadbalancer + namespace: monitoring + annotations: + metallb.universe.tf/loadBalancerIPs: 10.0.19.223 +spec: + type: LoadBalancer + ports: + - port: 80 + targetPort: 3000 + protocol: TCP + name: http + selector: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: kube-prometheus-stack \ No newline at end of file diff --git a/clusters/aprs/monitoring/install-monitoring.sh b/clusters/aprs/monitoring/install-monitoring.sh new file mode 100755 index 0000000..4a8d8af --- /dev/null +++ b/clusters/aprs/monitoring/install-monitoring.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -e + +echo "Installing monitoring stack for k3s..." + +# Create namespace +echo "Creating monitoring namespace..." +kubectl apply -f namespace.yaml + +# Add prometheus-community helm repo +echo "Adding Helm repositories..." +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo add stable https://charts.helm.sh/stable +helm repo update + +# Install kube-prometheus-stack +echo "Installing kube-prometheus-stack..." +helm upgrade --install \ + kube-prometheus-stack \ + prometheus-community/kube-prometheus-stack \ + --namespace monitoring \ + --values kube-prometheus-stack-values.yaml \ + --wait \ + --timeout 10m + +# Apply additional configurations +echo "Applying additional monitoring configurations..." +kubectl apply -f postgres-exporter.yaml +kubectl apply -f pgbouncer-exporter.yaml +kubectl apply -f servicemonitors.yaml +kubectl apply -f prometheusrules.yaml + +echo "Monitoring stack installed successfully!" +echo "" +echo "Access Grafana at: http://localhost:3000 (after port-forward)" +echo " kubectl port-forward -n monitoring svc/kube-prometheus-stack-grafana 3000:80" +echo "" +echo "Access Prometheus at: http://localhost:9090 (after port-forward)" +echo " kubectl port-forward -n monitoring svc/kube-prometheus-stack-prometheus 9090:9090" +echo "" +echo "Default Grafana credentials: admin / changeme" \ No newline at end of file diff --git a/clusters/aprs/monitoring/kube-prometheus-stack-values.yaml b/clusters/aprs/monitoring/kube-prometheus-stack-values.yaml new file mode 100644 index 0000000..fa6f15d --- /dev/null +++ b/clusters/aprs/monitoring/kube-prometheus-stack-values.yaml @@ -0,0 +1,150 @@ +# Optimized values for k3s deployment +prometheus: + prometheusSpec: + # Resource limits for edge deployment + resources: + requests: + memory: 400Mi + cpu: 100m + limits: + memory: 2Gi + cpu: 1000m + + # Retention and storage + retention: 30d + retentionSize: "10GB" + + # Storage configuration + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: ceph-rbd + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 200Gi + + # Scrape configs + additionalScrapeConfigs: + - job_name: 'postgres-exporter' + static_configs: + - targets: ['postgres-exporter.aprs:9187'] + + - job_name: 'pgbouncer-exporter' + static_configs: + - targets: ['pgbouncer-exporter.aprs:9127'] + +alertmanager: + enabled: true + alertmanagerSpec: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 100m + + storage: + volumeClaimTemplate: + spec: + storageClassName: ceph-rbd + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + +grafana: + enabled: true + adminPassword: "changeme" # Change this! + + resources: + requests: + memory: 100Mi + cpu: 100m + limits: + memory: 200Mi + cpu: 200m + + persistence: + enabled: true + size: 1Gi + storageClassName: ceph-rbd + + # Pre-install dashboards + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + + dashboards: + default: + postgresql: + gnetId: 9628 + revision: 7 + datasource: Prometheus + pgbouncer: + gnetId: 15984 + revision: 1 + datasource: Prometheus + kubernetes-cluster: + gnetId: 7249 + revision: 1 + datasource: Prometheus + aprs-application: + file: dashboards/aprs-dashboard.json + datasource: Prometheus + +# Reduce resource usage for k3s +kubeStateMetrics: + resources: + requests: + memory: 50Mi + cpu: 10m + limits: + memory: 150Mi + cpu: 100m + +nodeExporter: + resources: + requests: + memory: 30Mi + cpu: 10m + limits: + memory: 100Mi + cpu: 100m + +prometheusOperator: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 200m + +# Disable components not needed for k3s +kubeApiServer: + enabled: false +kubeControllerManager: + enabled: false +kubeScheduler: + enabled: false +kubeProxy: + enabled: false +kubeEtcd: + enabled: false + +# Enable ServiceMonitor for k3s components +kubelet: + enabled: true + serviceMonitor: + https: true \ No newline at end of file diff --git a/clusters/aprs/monitoring/namespace.yaml b/clusters/aprs/monitoring/namespace.yaml new file mode 100644 index 0000000..3335b6a --- /dev/null +++ b/clusters/aprs/monitoring/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring \ No newline at end of file diff --git a/clusters/aprs/monitoring/pgbouncer-exporter.yaml b/clusters/aprs/monitoring/pgbouncer-exporter.yaml new file mode 100644 index 0000000..3862d62 --- /dev/null +++ b/clusters/aprs/monitoring/pgbouncer-exporter.yaml @@ -0,0 +1,110 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pgbouncer-config-exporter + namespace: aprs +data: + pgbouncer.ini: | + [databases] + pgbouncer = host=127.0.0.1 port=6432 dbname=pgbouncer auth_user=stats + + [pgbouncer] + listen_port = 6433 + listen_addr = 127.0.0.1 + auth_type = trust + auth_file = /etc/pgbouncer/userlist.txt + admin_users = stats + stats_users = stats + pool_mode = session + max_client_conn = 10 + + userlist.txt: | + "stats" "trusted" +--- +apiVersion: v1 +kind: Service +metadata: + name: pgbouncer-exporter + namespace: aprs + labels: + app: pgbouncer-exporter +spec: + ports: + - port: 9127 + targetPort: 9127 + name: metrics + selector: + app: pgbouncer-exporter +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pgbouncer-exporter + namespace: aprs + labels: + app: pgbouncer-exporter +spec: + replicas: 1 + selector: + matchLabels: + app: pgbouncer-exporter + template: + metadata: + labels: + app: pgbouncer-exporter + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9127" + spec: + containers: + - name: pgbouncer-stats-proxy + image: bitnami/pgbouncer:1.22.1 + ports: + - containerPort: 6433 + env: + - name: PGBOUNCER_INI_FILE + value: /etc/pgbouncer/pgbouncer.ini + volumeMounts: + - name: config + mountPath: /etc/pgbouncer + resources: + requests: + memory: 32Mi + cpu: 10m + limits: + memory: 64Mi + cpu: 50m + + - name: pgbouncer-exporter + image: prometheuscommunity/pgbouncer-exporter:v0.7.0 + ports: + - containerPort: 9127 + name: metrics + env: + - name: PGBOUNCER_EXPORTER_CONNECTION_STRING + value: "postgres://stats:trusted@pgbouncer:5432/pgbouncer?sslmode=disable" + - name: PGBOUNCER_EXPORTER_PORT + value: "9127" + resources: + requests: + memory: 50Mi + cpu: 50m + limits: + memory: 100Mi + cpu: 100m + livenessProbe: + httpGet: + path: /metrics + port: 9127 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /metrics + port: 9127 + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: config + configMap: + name: pgbouncer-config-exporter \ No newline at end of file diff --git a/clusters/aprs/monitoring/postgres-exporter.yaml b/clusters/aprs/monitoring/postgres-exporter.yaml new file mode 100644 index 0000000..0e22eea --- /dev/null +++ b/clusters/aprs/monitoring/postgres-exporter.yaml @@ -0,0 +1,160 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres-exporter + namespace: aprs + labels: + app: postgres-exporter +spec: + ports: + - port: 9187 + targetPort: 9187 + name: metrics + selector: + app: postgres-exporter +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-exporter + namespace: aprs + labels: + app: postgres-exporter +spec: + replicas: 1 + selector: + matchLabels: + app: postgres-exporter + template: + metadata: + labels: + app: postgres-exporter + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9187" + spec: + containers: + - name: postgres-exporter + image: prometheuscommunity/postgres-exporter:v0.15.0 + ports: + - containerPort: 9187 + name: metrics + env: + - name: DATA_SOURCE_NAME + value: "postgresql://aprs:$(DATABASE_PASSWORD)@postgis:5432/aprs_prod?sslmode=disable" + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: aprs-secret + key: database-password + - name: PG_EXPORTER_AUTO_DISCOVER_DATABASES + value: "true" + - name: PG_EXPORTER_INCLUDE_DATABASES + value: "aprs_prod" + - name: PG_EXPORTER_EXTEND_QUERY_PATH + value: "/etc/postgres_exporter/queries.yaml" + volumeMounts: + - name: queries + mountPath: /etc/postgres_exporter + resources: + requests: + memory: 50Mi + cpu: 50m + limits: + memory: 150Mi + cpu: 200m + livenessProbe: + httpGet: + path: /metrics + port: 9187 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /metrics + port: 9187 + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: queries + configMap: + name: postgres-exporter-queries +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-exporter-queries + namespace: aprs +data: + queries.yaml: | + pg_connections: + query: | + SELECT + datname, + state, + COUNT(*) as count + FROM pg_stat_activity + WHERE datname IS NOT NULL + GROUP BY datname, state + metrics: + - datname: + usage: "LABEL" + description: "Database name" + - state: + usage: "LABEL" + description: "Connection state" + - count: + usage: "GAUGE" + description: "Number of connections" + + pg_connection_stats: + query: | + SELECT + (SELECT COUNT(*) FROM pg_stat_activity) as total_connections, + (SELECT setting::int FROM pg_settings WHERE name = 'max_connections') as max_connections, + (SELECT COUNT(*) FROM pg_stat_activity WHERE state = 'active') as active_connections, + (SELECT COUNT(*) FROM pg_stat_activity WHERE state = 'idle') as idle_connections, + (SELECT COUNT(*) FROM pg_stat_activity WHERE state = 'idle in transaction') as idle_in_transaction + metrics: + - total_connections: + usage: "GAUGE" + description: "Total number of connections" + - max_connections: + usage: "GAUGE" + description: "Maximum allowed connections" + - active_connections: + usage: "GAUGE" + description: "Number of active connections" + - idle_connections: + usage: "GAUGE" + description: "Number of idle connections" + - idle_in_transaction: + usage: "GAUGE" + description: "Number of idle in transaction connections" + + pg_long_running_queries: + query: | + SELECT + COUNT(*) as count + FROM pg_stat_activity + WHERE state = 'active' + AND query_start < NOW() - INTERVAL '5 minutes' + metrics: + - count: + usage: "GAUGE" + description: "Number of queries running longer than 5 minutes" + + pg_database_size: + query: | + SELECT + datname, + pg_database_size(datname) as size_bytes + FROM pg_database + WHERE datname NOT IN ('template0', 'template1', 'postgres') + metrics: + - datname: + usage: "LABEL" + description: "Database name" + - size_bytes: + usage: "GAUGE" + description: "Database size in bytes" \ No newline at end of file diff --git a/clusters/aprs/monitoring/prometheusrules.yaml b/clusters/aprs/monitoring/prometheusrules.yaml new file mode 100644 index 0000000..cae8c73 --- /dev/null +++ b/clusters/aprs/monitoring/prometheusrules.yaml @@ -0,0 +1,149 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: postgres-alerts + namespace: aprs + labels: + prometheus: kube-prometheus + role: alert-rules +spec: + groups: + - name: postgresql + interval: 30s + rules: + - alert: PostgreSQLConnectionsHigh + expr: pg_connection_stats_total_connections / pg_connection_stats_max_connections > 0.8 + for: 5m + labels: + severity: warning + annotations: + summary: "PostgreSQL connection usage is high ({{ $value | humanizePercentage }})" + description: "PostgreSQL on {{ $labels.instance }} is using {{ $value | humanizePercentage }} of available connections" + + - alert: PostgreSQLConnectionsCritical + expr: pg_connection_stats_total_connections / pg_connection_stats_max_connections > 0.95 + for: 2m + labels: + severity: critical + annotations: + summary: "PostgreSQL is almost out of connections ({{ $value | humanizePercentage }})" + description: "PostgreSQL on {{ $labels.instance }} is using {{ $value | humanizePercentage }} of available connections. Connection exhaustion imminent!" + + - alert: PostgreSQLIdleTransactionsHigh + expr: pg_connection_stats_idle_in_transaction > 10 + for: 5m + labels: + severity: warning + annotations: + summary: "High number of idle transactions ({{ $value }})" + description: "PostgreSQL has {{ $value }} idle transactions, which may be blocking other queries" + + - alert: PostgreSQLLongRunningQueries + expr: pg_long_running_queries_count > 5 + for: 5m + labels: + severity: warning + annotations: + summary: "Long running queries detected ({{ $value }})" + description: "PostgreSQL has {{ $value }} queries running longer than 5 minutes" + + - alert: PostgreSQLDown + expr: up{job="postgres-exporter"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "PostgreSQL is down" + description: "PostgreSQL exporter on {{ $labels.instance }} is not responding" +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: pgbouncer-alerts + namespace: aprs + labels: + prometheus: kube-prometheus + role: alert-rules +spec: + groups: + - name: pgbouncer + interval: 30s + rules: + - alert: PgBouncerDown + expr: up{job="pgbouncer-exporter"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "PgBouncer is down" + description: "PgBouncer exporter on {{ $labels.instance }} is not responding" + + - alert: PgBouncerPoolExhausted + expr: pgbouncer_pools_server_active_connections / pgbouncer_pools_pool_size > 0.9 + for: 5m + labels: + severity: warning + annotations: + summary: "PgBouncer pool almost exhausted" + description: "PgBouncer pool {{ $labels.database }} is using {{ $value | humanizePercentage }} of available connections" +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: aprs-app-alerts + namespace: aprs + labels: + prometheus: kube-prometheus + role: alert-rules +spec: + groups: + - name: aprs-application + interval: 30s + rules: + - alert: APRSHighMemoryUsage + expr: | + (container_memory_working_set_bytes{namespace="aprs", pod=~"aprs-.*", container="aprs"} + / container_spec_memory_limit_bytes{namespace="aprs", pod=~"aprs-.*", container="aprs"}) > 0.9 + for: 5m + labels: + severity: warning + annotations: + summary: "APRS pod memory usage is high" + description: "APRS pod {{ $labels.pod }} is using {{ $value | humanizePercentage }} of memory limit" + + - alert: APRSHighCPUUsage + expr: | + (rate(container_cpu_usage_seconds_total{namespace="aprs", pod=~"aprs-.*", container="aprs"}[5m]) + / container_spec_cpu_quota{namespace="aprs", pod=~"aprs-.*", container="aprs"} * 100000) > 80 + for: 5m + labels: + severity: warning + annotations: + summary: "APRS pod CPU usage is high" + description: "APRS pod {{ $labels.pod }} is using {{ $value }}% of CPU limit" + + - alert: APRSPodCrashLooping + expr: rate(kube_pod_container_status_restarts_total{namespace="aprs", pod=~"aprs-.*"}[15m]) > 0 + labels: + severity: critical + annotations: + summary: "APRS pod is crash looping" + description: "APRS pod {{ $labels.pod }} has restarted {{ $value }} times in the last 15 minutes" + + - alert: APRSHighDatabaseConnectionQueueTime + expr: histogram_quantile(0.95, aprsme_repo_query_queue_time_millisecond) > 1000 + for: 5m + labels: + severity: warning + annotations: + summary: "High database connection queue time" + description: "95th percentile database queue time is {{ $value }}ms, indicating connection pool exhaustion" + + - alert: APRSPacketProcessingBacklog + expr: rate(aprsme_packet_pipeline_batch_error[5m]) > 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "Packet processing errors detected" + description: "Packet pipeline is experiencing {{ $value }} errors per second" \ No newline at end of file diff --git a/clusters/aprs/monitoring/servicemonitors.yaml b/clusters/aprs/monitoring/servicemonitors.yaml new file mode 100644 index 0000000..48d581d --- /dev/null +++ b/clusters/aprs/monitoring/servicemonitors.yaml @@ -0,0 +1,67 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: postgres-exporter + namespace: aprs + labels: + app: postgres-exporter + prometheus: kube-prometheus +spec: + selector: + matchLabels: + app: postgres-exporter + endpoints: + - port: metrics + interval: 30s + path: /metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: pgbouncer-exporter + namespace: aprs + labels: + app: pgbouncer-exporter + prometheus: kube-prometheus +spec: + selector: + matchLabels: + app: pgbouncer-exporter + endpoints: + - port: metrics + interval: 30s + path: /metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: aprs-app + namespace: aprs + labels: + app: aprs + prometheus: kube-prometheus +spec: + selector: + matchLabels: + app: aprs + endpoints: + - port: http + interval: 30s + path: /metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: redis + namespace: aprs + labels: + app: redis + prometheus: kube-prometheus +spec: + selector: + matchLabels: + app: redis + endpoints: + - port: metrics + interval: 30s + path: /metrics \ No newline at end of file diff --git a/clusters/aprs/pgbouncer-configmap.yaml b/clusters/aprs/pgbouncer-configmap.yaml deleted file mode 100644 index 5107d2e..0000000 --- a/clusters/aprs/pgbouncer-configmap.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: pgbouncer-config - namespace: aprs -data: - pgbouncer.ini: | - [databases] - aprs = host=postgis port=5432 dbname=aprs - - [pgbouncer] - listen_addr = 0.0.0.0 - listen_port = 6432 - auth_type = md5 - auth_file = /etc/pgbouncer/userlist.txt - pool_mode = transaction - max_client_conn = 1000 - default_pool_size = 25 - reserve_pool_size = 5 - reserve_pool_timeout = 3 - server_lifetime = 3600 - server_idle_timeout = 600 - log_connections = 0 - log_disconnections = 0 - stats_period = 60 - - # Important for prepared statements - server_reset_query = DISCARD ALL - server_reset_query_always = 1 - - # Connection limits - min_pool_size = 10 - max_db_connections = 100 - - # Timeouts - query_timeout = 0 - query_wait_timeout = 120 - client_idle_timeout = 0 - client_login_timeout = 60 - - # For better performance - pkt_buf = 8192 - sbuf_loopcnt = 2 - tcp_keepalive = 1 - tcp_keepidle = 1 - tcp_keepintvl = 1 - tcp_keepcnt = 1 - - # Ignore certain PostgreSQL parameters - ignore_startup_parameters = extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout \ No newline at end of file diff --git a/clusters/aprs/pgbouncer-deployment-enhanced.yaml b/clusters/aprs/pgbouncer-deployment-enhanced.yaml deleted file mode 100644 index 5dcded6..0000000 --- a/clusters/aprs/pgbouncer-deployment-enhanced.yaml +++ /dev/null @@ -1,131 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: pgbouncer - namespace: aprs - labels: - app: pgbouncer -spec: - ports: - - port: 5432 - targetPort: 6432 - name: pgbouncer - selector: - app: pgbouncer - type: ClusterIP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pgbouncer - namespace: aprs - labels: - app: pgbouncer -spec: - replicas: 2 - selector: - matchLabels: - app: pgbouncer - template: - metadata: - labels: - app: pgbouncer - spec: - containers: - - name: pgbouncer - image: bitnami/pgbouncer:1.22.1 - ports: - - containerPort: 6432 - name: pgbouncer - env: - - name: POSTGRESQL_HOST - value: "postgis" - - name: POSTGRESQL_PORT - value: "5432" - - name: POSTGRESQL_DATABASE - value: "aprs" - - name: POSTGRESQL_USERNAME - value: "aprs" - - name: POSTGRESQL_PASSWORD - valueFrom: - secretKeyRef: - name: aprs-secret - key: database-password - - name: PGBOUNCER_DATABASE - value: "aprs" - - name: PGBOUNCER_PORT - value: "6432" - - name: PGBOUNCER_POOL_MODE - value: "transaction" - - name: PGBOUNCER_MAX_CLIENT_CONN - value: "1000" - - name: PGBOUNCER_DEFAULT_POOL_SIZE - value: "25" - - name: PGBOUNCER_MIN_POOL_SIZE - value: "5" - - name: PGBOUNCER_RESERVE_POOL_SIZE - value: "5" - - name: PGBOUNCER_SERVER_RESET_QUERY - value: "DISCARD ALL" - - name: PGBOUNCER_STATS_USERS - value: "aprs" - - name: PGBOUNCER_AUTH_TYPE - value: "md5" - - name: PGBOUNCER_IGNORE_STARTUP_PARAMETERS - value: "extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout" - # Enhanced connection handling - - name: PGBOUNCER_SERVER_IDLE_TIMEOUT - value: "600" # 10 minutes - - name: PGBOUNCER_SERVER_LIFETIME - value: "3600" # 1 hour - - name: PGBOUNCER_QUERY_TIMEOUT - value: "300" # 5 minutes - - name: PGBOUNCER_QUERY_WAIT_TIMEOUT - value: "120" # 2 minutes - - name: PGBOUNCER_CLIENT_IDLE_TIMEOUT - value: "300" # 5 minutes - - name: PGBOUNCER_CLIENT_LOGIN_TIMEOUT - value: "60" # 1 minute - # Connection retry settings - - name: PGBOUNCER_SERVER_CONNECT_TIMEOUT - value: "15" - - name: PGBOUNCER_SERVER_LOGIN_RETRY - value: "15" # Retry for 15 seconds before failing - - name: PGBOUNCER_SERVER_ROUND_ROBIN - value: "1" # Enable round-robin - - name: PGBOUNCER_LOG_CONNECTIONS - value: "1" - - name: PGBOUNCER_LOG_DISCONNECTIONS - value: "1" - - name: PGBOUNCER_LOG_POOLER_ERRORS - value: "1" - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "256Mi" - cpu: "200m" - livenessProbe: - tcpSocket: - port: 6432 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - exec: - command: - - /bin/sh - - -c - - | - nc -z localhost 6432 && \ - echo "SHOW POOLS;" | psql -h localhost -p 6432 -U pgbouncer pgbouncer || exit 0 - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 5 - failureThreshold: 2 - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 10"] \ No newline at end of file diff --git a/clusters/aprs/pgbouncer-deployment.yaml b/clusters/aprs/pgbouncer-deployment.yaml deleted file mode 100644 index 3bee875..0000000 --- a/clusters/aprs/pgbouncer-deployment.yaml +++ /dev/null @@ -1,96 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: pgbouncer - namespace: aprs - labels: - app: pgbouncer -spec: - ports: - - port: 5432 - targetPort: 6432 - name: pgbouncer - selector: - app: pgbouncer - type: ClusterIP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pgbouncer - namespace: aprs - labels: - app: pgbouncer -spec: - replicas: 2 - selector: - matchLabels: - app: pgbouncer - template: - metadata: - labels: - app: pgbouncer - spec: - containers: - - name: pgbouncer - image: bitnami/pgbouncer:1.22.1 - ports: - - containerPort: 6432 - name: pgbouncer - env: - - name: POSTGRESQL_HOST - value: "postgis" - - name: POSTGRESQL_PORT - value: "5432" - - name: POSTGRESQL_DATABASE - value: "aprs" - - name: POSTGRESQL_USERNAME - value: "aprs" - - name: POSTGRESQL_PASSWORD - valueFrom: - secretKeyRef: - name: aprs-secret - key: database-password - - name: PGBOUNCER_DATABASE - value: "aprs" - - name: PGBOUNCER_PORT - value: "6432" - - name: PGBOUNCER_POOL_MODE - value: "transaction" - - name: PGBOUNCER_MAX_CLIENT_CONN - value: "1000" - - name: PGBOUNCER_DEFAULT_POOL_SIZE - value: "25" - - name: PGBOUNCER_MIN_POOL_SIZE - value: "10" - - name: PGBOUNCER_RESERVE_POOL_SIZE - value: "5" - - name: PGBOUNCER_SERVER_RESET_QUERY - value: "DISCARD ALL" - - name: PGBOUNCER_STATS_USERS - value: "aprs" - - name: PGBOUNCER_AUTH_TYPE - value: "md5" - - name: PGBOUNCER_IGNORE_STARTUP_PARAMETERS - value: "extra_float_digits,synchronous_commit,work_mem,statement_timeout,idle_in_transaction_session_timeout" - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "128Mi" - cpu: "100m" - livenessProbe: - tcpSocket: - port: 6432 - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - tcpSocket: - port: 6432 - initialDelaySeconds: 5 - periodSeconds: 5 - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 10"] \ No newline at end of file diff --git a/clusters/aprs/postgis-deployment-ceph.yaml b/clusters/aprs/postgis-deployment-ceph.yaml deleted file mode 100644 index f30b17a..0000000 --- a/clusters/aprs/postgis-deployment-ceph.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgis - namespace: aprs -spec: - replicas: 1 - selector: - matchLabels: - app: postgis - template: - metadata: - labels: - app: postgis - spec: - initContainers: - - name: volume-permissions - image: busybox:1.36.1 - command: ['sh', '-c', 'chmod -R 777 /var/lib/postgresql/data || true'] - volumeMounts: - - name: postgis-storage - mountPath: /var/lib/postgresql/data - containers: - - name: postgis - image: postgis/postgis:17-3.4 - ports: - - containerPort: 5432 - env: - - name: POSTGRES_DB - value: aprs_prod - - name: POSTGRES_USER - value: aprs - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgis-secret - key: postgres-password - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - volumeMounts: - - name: postgis-storage - mountPath: /var/lib/postgresql/data - resources: - limits: - memory: "1Gi" - cpu: "500m" - requests: - memory: "512Mi" - cpu: "250m" - volumes: - - name: postgis-storage - persistentVolumeClaim: - claimName: postgis-pvc-ceph \ No newline at end of file diff --git a/clusters/aprs/postgis-deployment-enhanced.yaml b/clusters/aprs/postgis-deployment-enhanced.yaml deleted file mode 100644 index b435cb9..0000000 --- a/clusters/aprs/postgis-deployment-enhanced.yaml +++ /dev/null @@ -1,110 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgis - namespace: aprs -spec: - replicas: 1 - selector: - matchLabels: - app: postgis - template: - metadata: - labels: - app: postgis - spec: - initContainers: - - name: volume-permissions - image: busybox:1.36.1 - command: ['sh', '-c', 'chmod -R 777 /var/lib/postgresql/data || true'] - volumeMounts: - - name: postgis-storage - mountPath: /var/lib/postgresql/data - containers: - - name: postgis - image: postgis/postgis:17-3.4 - ports: - - containerPort: 5432 - env: - - name: POSTGRES_DB - value: aprs_prod - - name: POSTGRES_USER - value: aprs - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgis-secret - key: postgres-password - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - # PostgreSQL configuration for better connection handling - - name: POSTGRES_INITDB_ARGS - value: "--encoding=UTF8 --locale=en_US.utf8" - args: - - -c - - max_connections=200 # Increased from default 100 - - -c - - shared_buffers=256MB - - -c - - effective_cache_size=1GB - - -c - - maintenance_work_mem=64MB - - -c - - checkpoint_completion_target=0.7 - - -c - - wal_buffers=16MB - - -c - - default_statistics_target=100 - - -c - - random_page_cost=1.1 - - -c - - effective_io_concurrency=200 - - -c - - work_mem=4MB - - -c - - min_wal_size=1GB - - -c - - max_wal_size=2GB - - -c - - idle_in_transaction_session_timeout=30s # Kill idle transactions - - -c - - statement_timeout=300s # 5 minute statement timeout - volumeMounts: - - name: postgis-storage - mountPath: /var/lib/postgresql/data - resources: - limits: - memory: "2Gi" # Increased from 1Gi - cpu: "1000m" # Increased from 500m - requests: - memory: "1Gi" # Increased from 512Mi - cpu: "500m" # Increased from 250m - # Health checks to detect connection issues - livenessProbe: - exec: - command: - - /bin/sh - - -c - - pg_isready -U $POSTGRES_USER -d $POSTGRES_DB - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - exec: - command: - - /bin/sh - - -c - - | - pg_isready -U $POSTGRES_USER -d $POSTGRES_DB && \ - CONN_COUNT=$(psql -U $POSTGRES_USER -d $POSTGRES_DB -t -c "SELECT count(*) FROM pg_stat_activity WHERE state != 'idle';") && \ - [ $CONN_COUNT -lt 180 ] # Fail readiness if too many connections - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 2 - volumes: - - name: postgis-storage - hostPath: - path: /var/lib/rancher/k3s/storage/postgis - type: DirectoryOrCreate \ No newline at end of file diff --git a/clusters/aprs/postgis-deployment.yaml b/clusters/aprs/postgis-deployment.yaml deleted file mode 100644 index 7e8a11a..0000000 --- a/clusters/aprs/postgis-deployment.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgis - namespace: aprs -spec: - replicas: 1 - selector: - matchLabels: - app: postgis - template: - metadata: - labels: - app: postgis - spec: - initContainers: - - name: volume-permissions - image: busybox:1.36.1 - command: ['sh', '-c', 'chmod -R 777 /var/lib/postgresql/data || true'] - volumeMounts: - - name: postgis-storage - mountPath: /var/lib/postgresql/data - containers: - - name: postgis - image: postgis/postgis:17-3.4 - ports: - - containerPort: 5432 - env: - - name: POSTGRES_DB - value: aprs_prod - - name: POSTGRES_USER - value: aprs - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgis-secret - key: postgres-password - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - volumeMounts: - - name: postgis-storage - mountPath: /var/lib/postgresql/data - resources: - limits: - memory: "1Gi" - cpu: "500m" - requests: - memory: "512Mi" - cpu: "250m" - volumes: - - name: postgis-storage - hostPath: - path: /var/lib/rancher/k3s/storage/postgis - type: DirectoryOrCreate diff --git a/clusters/aprs/postgis-pdb.yaml b/clusters/aprs/postgis-pdb.yaml deleted file mode 100644 index 6bb06da..0000000 --- a/clusters/aprs/postgis-pdb.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: postgis-pdb - namespace: aprs -spec: - minAvailable: 1 - selector: - matchLabels: - app: postgis ---- -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: pgbouncer-pdb - namespace: aprs -spec: - minAvailable: 1 - selector: - matchLabels: - app: pgbouncer \ No newline at end of file diff --git a/clusters/aprs/postgis-pvc-ceph.yaml b/clusters/aprs/postgis-pvc-ceph.yaml deleted file mode 100644 index 57f81af..0000000 --- a/clusters/aprs/postgis-pvc-ceph.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: postgis-pvc-ceph - namespace: aprs -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 300Gi - storageClassName: ceph-rbd diff --git a/clusters/aprs/postgis-service.yaml b/clusters/aprs/postgis-service.yaml deleted file mode 100644 index 49d0f8b..0000000 --- a/clusters/aprs/postgis-service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: postgis - namespace: aprs -spec: - selector: - app: postgis - ports: - - port: 5432 - targetPort: 5432 - type: ClusterIP \ No newline at end of file diff --git a/clusters/aprs/postgres-cleanup-cronjob.yaml b/clusters/aprs/postgres-cleanup-cronjob.yaml deleted file mode 100644 index 7ec5505..0000000 --- a/clusters/aprs/postgres-cleanup-cronjob.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: batch/v1 -kind: CronJob -metadata: - name: postgres-cleanup - namespace: aprs -spec: - schedule: "*/15 * * * *" # Every 15 minutes - concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 3 - jobTemplate: - spec: - template: - spec: - restartPolicy: OnFailure - containers: - - name: postgres-cleanup - image: postgres:17-alpine - env: - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: postgis-secret - key: postgres-password - - name: PGHOST - value: postgis - - name: PGUSER - value: aprs - - name: PGDATABASE - value: aprs_prod - command: - - /bin/sh - - -c - - | - echo "Starting PostgreSQL connection cleanup..." - - # Kill idle connections older than 30 minutes - psql -c " - SELECT pg_terminate_backend(pid) - FROM pg_stat_activity - WHERE datname = 'aprs_prod' - AND pid <> pg_backend_pid() - AND state = 'idle' - AND state_change < NOW() - INTERVAL '30 minutes';" - - # Kill idle in transaction connections older than 5 minutes - psql -c " - SELECT pg_terminate_backend(pid) - FROM pg_stat_activity - WHERE datname = 'aprs_prod' - AND pid <> pg_backend_pid() - AND state = 'idle in transaction' - AND state_change < NOW() - INTERVAL '5 minutes';" - - # Log connection stats - psql -c " - SELECT - state, - COUNT(*) as count, - MAX(NOW() - state_change) as max_duration - FROM pg_stat_activity - WHERE datname = 'aprs_prod' - GROUP BY state - ORDER BY count DESC;" - - # Check total connections vs max - psql -c " - SELECT - (SELECT COUNT(*) FROM pg_stat_activity) as current_connections, - (SELECT setting::int FROM pg_settings WHERE name = 'max_connections') as max_connections, - ROUND(100.0 * COUNT(*) / (SELECT setting::int FROM pg_settings WHERE name = 'max_connections'), 2) as percentage_used - FROM pg_stat_activity;" - - echo "Cleanup completed successfully" ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: postgres-cleanup - namespace: aprs ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: postgres-cleanup - namespace: aprs -rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list"] -- apiGroups: [""] - resources: ["pods/exec"] - verbs: ["create"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: postgres-cleanup - namespace: aprs -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: postgres-cleanup -subjects: -- kind: ServiceAccount - name: postgres-cleanup - namespace: aprs \ No newline at end of file diff --git a/clusters/aprs/redis-deployment-with-metrics.yaml b/clusters/aprs/redis-deployment-with-metrics.yaml new file mode 100644 index 0000000..e66df8e --- /dev/null +++ b/clusters/aprs/redis-deployment-with-metrics.yaml @@ -0,0 +1,96 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: aprs + labels: + app: redis +spec: + ports: + - port: 6379 + targetPort: 6379 + name: redis + - port: 9121 + targetPort: 9121 + name: metrics + selector: + app: redis + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: aprs + labels: + app: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9121" + spec: + containers: + - name: redis + image: redis:7-alpine + ports: + - containerPort: 6379 + name: redis + command: + - redis-server + - --maxmemory + - "256mb" + - --maxmemory-policy + - allkeys-lru + - --save + - "" + - --appendonly + - "no" + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + tcpSocket: + port: 6379 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + + - name: redis-exporter + image: oliver006/redis_exporter:v1.55.0 + ports: + - containerPort: 9121 + name: metrics + env: + - name: REDIS_ADDR + value: "redis://localhost:6379" + resources: + requests: + memory: 32Mi + cpu: 10m + limits: + memory: 64Mi + cpu: 50m + livenessProbe: + httpGet: + path: /metrics + port: 9121 + initialDelaySeconds: 30 + periodSeconds: 10 \ No newline at end of file diff --git a/clusters/aprs/redis-deployment.yaml b/clusters/aprs/redis-deployment.yaml deleted file mode 100644 index ec11d74..0000000 --- a/clusters/aprs/redis-deployment.yaml +++ /dev/null @@ -1,66 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: redis - namespace: aprs - labels: - app: redis -spec: - ports: - - port: 6379 - targetPort: 6379 - name: redis - selector: - app: redis - type: ClusterIP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: redis - namespace: aprs - labels: - app: redis -spec: - replicas: 1 - selector: - matchLabels: - app: redis - template: - metadata: - labels: - app: redis - spec: - containers: - - name: redis - image: redis:7-alpine - ports: - - containerPort: 6379 - name: redis - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "256Mi" - cpu: "200m" - livenessProbe: - exec: - command: - - redis-cli - - ping - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - exec: - command: - - redis-cli - - ping - initialDelaySeconds: 5 - periodSeconds: 5 - volumeMounts: - - name: redis-data - mountPath: /data - volumes: - - name: redis-data - emptyDir: {} \ No newline at end of file diff --git a/clusters/aprs/secrets.yaml b/clusters/aprs/secrets.yaml deleted file mode 100644 index d8b02d6..0000000 --- a/clusters/aprs/secrets.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: postgis-secret - namespace: aprs -type: Opaque -stringData: - postgres-password: "mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21" ---- -apiVersion: v1 -kind: Secret -metadata: - name: aprs-secret - namespace: aprs -type: Opaque -stringData: - database-url: "ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@postgis:5432/aprs" - database-url-pgbouncer: "ecto://aprs:mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21@pgbouncer:5432/aprs" - database-password: "mjBipckSZfNOoSOWLMRsfZEqJ9I5Of21" - secret-key-base: "GJxzOUW3YXBAF7xB4mH5VcXY6BN1NYLnoJYadlZyiCxcRUGBw65w4Sco7+sNiW2t" - erlang-cookie: "aprs-cluster-cookie-ksdf8934jfklsdjf983jfklsdf" ---- -apiVersion: v1 -kind: Secret -metadata: - name: ghcr-pull-secret - namespace: aprs -type: kubernetes.io/dockerconfigjson -stringData: - .dockerconfigjson: | - { - "auths": { - "ghcr.io": { - "username": "gmcintire", - "password": "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", - "auth": "Z21jaW50aXJlOmdocF9rOFhBZkJMSmhHdzExMDZiZkZXQVhkYjdYN1E4OUkxdjlhQWk=" - } - } - } diff --git a/clusters/aprs/tailscale/00-namespace.yaml b/clusters/aprs/tailscale/00-namespace.yaml new file mode 100644 index 0000000..ca25fe5 --- /dev/null +++ b/clusters/aprs/tailscale/00-namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: tailscale \ No newline at end of file diff --git a/clusters/aprs/tailscale/01-rbac.yaml b/clusters/aprs/tailscale/01-rbac.yaml new file mode 100644 index 0000000..750be96 --- /dev/null +++ b/clusters/aprs/tailscale/01-rbac.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator + namespace: tailscale +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tailscale-operator +rules: +- apiGroups: [""] + resources: ["events", "services", "services/status"] + verbs: ["*"] +- apiGroups: ["networking.k8s.io"] + resources: ["ingresses", "ingresses/status"] + verbs: ["*"] +- apiGroups: ["networking.k8s.io"] + resources: ["ingressclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: ["tailscale.com"] + resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["*"] +- apiGroups: [""] + resources: ["serviceaccounts", "configmaps"] + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: ["statefulsets", "deployments"] + verbs: ["*"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: tailscale-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tailscale-operator +subjects: +- kind: ServiceAccount + name: operator + namespace: tailscale +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operator + namespace: tailscale +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["*"] +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: operator + namespace: tailscale +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: operator +subjects: +- kind: ServiceAccount + name: operator + namespace: tailscale \ No newline at end of file diff --git a/clusters/aprs/tailscale/02-crds.yaml b/clusters/aprs/tailscale/02-crds.yaml new file mode 100644 index 0000000..dfe7d75 --- /dev/null +++ b/clusters/aprs/tailscale/02-crds.yaml @@ -0,0 +1,296 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: connectors.tailscale.com +spec: + group: tailscale.com + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + tags: + type: array + items: + type: string + hostname: + type: string + proxyClass: + type: string + exitNode: + type: boolean + subnetRouter: + type: object + properties: + routes: + type: array + items: + type: string + status: + type: object + properties: + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastTransitionTime: + type: string + subnetRouter: + type: object + properties: + routes: + type: array + items: + type: string + isExitNode: + type: boolean + subresources: + status: {} + scope: Cluster + names: + plural: connectors + singular: connector + kind: Connector +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: proxyclasses.tailscale.com +spec: + group: tailscale.com + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + metrics: + type: object + properties: + enable: + type: boolean + statefulSet: + type: object + properties: + labels: + type: object + additionalProperties: + type: string + annotations: + type: object + additionalProperties: + type: string + pod: + type: object + properties: + labels: + type: object + additionalProperties: + type: string + annotations: + type: object + additionalProperties: + type: string + tailscaleContainer: + type: object + properties: + env: + type: array + items: + type: object + properties: + name: + type: string + value: + type: string + imagePullPolicy: + type: string + enum: ["Always", "Never", "IfNotPresent"] + resources: + type: object + properties: + requests: + type: object + additionalProperties: + type: string + limits: + type: object + additionalProperties: + type: string + securityContext: + type: object + properties: + privileged: + type: boolean + allowPrivilegeEscalation: + type: boolean + readOnlyRootFilesystem: + type: boolean + runAsNonRoot: + type: boolean + runAsUser: + type: integer + runAsGroup: + type: integer + capabilities: + type: object + properties: + add: + type: array + items: + type: string + drop: + type: array + items: + type: string + tailscaleInitContainer: + type: object + properties: + env: + type: array + items: + type: object + properties: + name: + type: string + value: + type: string + imagePullPolicy: + type: string + enum: ["Always", "Never", "IfNotPresent"] + resources: + type: object + properties: + requests: + type: object + additionalProperties: + type: string + limits: + type: object + additionalProperties: + type: string + securityContext: + type: object + properties: + privileged: + type: boolean + allowPrivilegeEscalation: + type: boolean + readOnlyRootFilesystem: + type: boolean + runAsNonRoot: + type: boolean + runAsUser: + type: integer + runAsGroup: + type: integer + capabilities: + type: object + properties: + add: + type: array + items: + type: string + drop: + type: array + items: + type: string + imagePullSecrets: + type: array + items: + type: object + properties: + name: + type: string + nodeName: + type: string + nodeSelector: + type: object + additionalProperties: + type: string + affinity: + type: object + tolerations: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + value: + type: string + effect: + type: string + tolerationSeconds: + type: integer + hostNetwork: + type: boolean + serviceAccountName: + type: string + securityContext: + type: object + properties: + runAsNonRoot: + type: boolean + runAsUser: + type: integer + runAsGroup: + type: integer + fsGroup: + type: integer + service: + type: object + properties: + type: + type: string + enum: ["ClusterIP", "LoadBalancer", "NodePort"] + status: + type: object + properties: + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + observedGeneration: + type: integer + lastTransitionTime: + type: string + subresources: + status: {} + scope: Cluster + names: + plural: proxyclasses + singular: proxyclass + kind: ProxyClass \ No newline at end of file diff --git a/clusters/aprs/tailscale/03-auth-secret.yaml.example b/clusters/aprs/tailscale/03-auth-secret.yaml.example new file mode 100644 index 0000000..ac6c3e6 --- /dev/null +++ b/clusters/aprs/tailscale/03-auth-secret.yaml.example @@ -0,0 +1,29 @@ +# IMPORTANT: Create your actual secret file as 03-auth-secret.yaml +# Do not commit the actual secret to git! +# +# To create a Tailscale auth key: +# 1. Go to https://login.tailscale.com/admin/settings/keys +# 2. Generate an auth key (reusable, with appropriate expiry) +# 3. Create the secret: +# +# kubectl create secret generic operator-oauth \ +# --namespace=tailscale \ +# --from-literal=client_id="" \ +# --from-literal=client_secret="" +# +# OR for auth key: +# +# kubectl create secret generic operator \ +# --namespace=tailscale \ +# --from-literal=authkey="" +# +# Example YAML format: +--- +apiVersion: v1 +kind: Secret +metadata: + name: operator + namespace: tailscale +type: Opaque +stringData: + authkey: "tskey-auth-YOUR-KEY-HERE" \ No newline at end of file diff --git a/clusters/aprs/tailscale/04-operator-authkey.yaml b/clusters/aprs/tailscale/04-operator-authkey.yaml new file mode 100644 index 0000000..a99208a --- /dev/null +++ b/clusters/aprs/tailscale/04-operator-authkey.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: operator + namespace: tailscale +spec: + replicas: 1 + selector: + matchLabels: + app: operator + template: + metadata: + labels: + app: operator + spec: + serviceAccountName: operator + containers: + - name: operator + image: tailscale/k8s-operator:stable + env: + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_SECRET + value: operator + - name: PROXY_IMAGE + value: tailscale/tailscale:stable + - name: PROXY_TAGS + value: tag:k8s + - name: APISERVER_PROXY + value: "false" # Set to true if you want to expose the K8s API + - name: PROXY_FIREWALL_MODE + value: auto + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + nodeSelector: + kubernetes.io/os: linux \ No newline at end of file diff --git a/clusters/aprs/tailscale/04-operator.yaml b/clusters/aprs/tailscale/04-operator.yaml new file mode 100644 index 0000000..bf1e6e8 --- /dev/null +++ b/clusters/aprs/tailscale/04-operator.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: operator + namespace: tailscale +spec: + replicas: 1 + selector: + matchLabels: + app: operator + template: + metadata: + labels: + app: operator + spec: + serviceAccountName: operator + containers: + - name: operator + image: tailscale/k8s-operator:stable + env: + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLIENT_ID_FILE + value: /oauth/client_id + - name: CLIENT_SECRET_FILE + value: /oauth/client_secret + - name: PROXY_IMAGE + value: tailscale/tailscale:stable + - name: PROXY_TAGS + value: tag:k8s + - name: APISERVER_PROXY + value: "true" + - name: PROXY_FIREWALL_MODE + value: auto + volumeMounts: + - name: oauth + mountPath: /oauth + readOnly: true + volumes: + - name: oauth + secret: + secretName: operator-oauth + optional: true + nodeSelector: + kubernetes.io/os: linux \ No newline at end of file diff --git a/clusters/aprs/tailscale/05-proxy-example.yaml b/clusters/aprs/tailscale/05-proxy-example.yaml new file mode 100644 index 0000000..56c0679 --- /dev/null +++ b/clusters/aprs/tailscale/05-proxy-example.yaml @@ -0,0 +1,45 @@ +# Example: Expose a service via Tailscale proxy +# This creates a Tailscale device that forwards traffic to a Kubernetes service +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana-tailscale + namespace: monitoring + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "aprs-grafana" + tailscale.com/tags: "tag:k8s,tag:monitoring" +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: grafana + ports: + - port: 80 + targetPort: 3000 + protocol: TCP +--- +# Alternative: Using Ingress +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana-tailscale + namespace: monitoring + annotations: + tailscale.com/ingress-class: "tailscale" +spec: + ingressClassName: tailscale + rules: + - host: grafana + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana + port: + number: 80 + tls: + - hosts: + - grafana \ No newline at end of file diff --git a/clusters/aprs/tailscale/README.md b/clusters/aprs/tailscale/README.md new file mode 100644 index 0000000..3b9d688 --- /dev/null +++ b/clusters/aprs/tailscale/README.md @@ -0,0 +1,128 @@ +# Tailscale Operator for K3s + +This directory contains the Tailscale operator configuration for the k3s cluster. + +## Setup Instructions + +1. **Apply the base configuration:** + ```bash + kubectl apply -f 00-namespace.yaml + kubectl apply -f 01-rbac.yaml + kubectl apply -f 02-crds.yaml + ``` + +2. **Create the authentication secret:** + + Option A - Using OAuth (recommended for production): + ```bash + # Get OAuth credentials from https://login.tailscale.com/admin/settings/oauth + kubectl create secret generic operator-oauth \ + --namespace=tailscale \ + --from-literal=client_id="" \ + --from-literal=client_secret="" + ``` + + Option B - Using auth key (simpler, good for testing): + ```bash + # Get auth key from https://login.tailscale.com/admin/settings/keys + kubectl create secret generic operator \ + --namespace=tailscale \ + --from-literal=authkey="" + ``` + +3. **Deploy the operator:** + ```bash + kubectl apply -f 04-operator.yaml + ``` + +4. **Verify the operator is running:** + ```bash + kubectl get pods -n tailscale + kubectl logs -n tailscale deployment/operator + ``` + +## Exposing Services + +There are several ways to expose services via Tailscale: + +### Method 1: Service Annotations +```yaml +apiVersion: v1 +kind: Service +metadata: + name: my-service + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "my-app" + tailscale.com/tags: "tag:k8s" +``` + +### Method 2: Ingress +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: my-ingress +spec: + ingressClassName: tailscale + rules: + - host: my-app + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: my-service + port: + number: 80 +``` + +### Method 3: Connector (Subnet Router/Exit Node) +```yaml +apiVersion: tailscale.com/v1alpha1 +kind: Connector +metadata: + name: subnet-router +spec: + hostname: k3s-subnet + exitNode: false + subnetRouter: + routes: + - "10.0.19.0/24" # Your cluster subnet + tags: + - "tag:k8s" + - "tag:subnet-router" +``` + +## Accessing the API Server + +The operator can expose the Kubernetes API server on your tailnet: +```bash +# This is enabled by default with APISERVER_PROXY=true +# Access via: https://tailscale-operator.tailnet-name.ts.net:443 +``` + +## Troubleshooting + +1. Check operator logs: + ```bash + kubectl logs -n tailscale deployment/operator + ``` + +2. Check proxy pods: + ```bash + kubectl get pods -A | grep ts- + ``` + +3. Verify CRDs are installed: + ```bash + kubectl get crds | grep tailscale + ``` + +## Security Notes + +- Never commit auth keys or OAuth credentials to git +- Use appropriate ACL tags in your Tailscale admin panel +- Consider using OAuth for production deployments +- Auth keys should be reusable and have appropriate expiry times \ No newline at end of file diff --git a/clusters/aprs/tailscale/grafana-tailscale.yaml b/clusters/aprs/tailscale/grafana-tailscale.yaml new file mode 100644 index 0000000..dc4f480 --- /dev/null +++ b/clusters/aprs/tailscale/grafana-tailscale.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana-tailscale + namespace: monitoring + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "aprs-grafana" + tailscale.com/tags: "tag:k8s" +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: kube-prometheus + ports: + - port: 80 + targetPort: 3000 + protocol: TCP + name: http \ No newline at end of file diff --git a/clusters/aprs/tailscale/setup.sh b/clusters/aprs/tailscale/setup.sh new file mode 100755 index 0000000..d4f299f --- /dev/null +++ b/clusters/aprs/tailscale/setup.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -e + +echo "Tailscale Operator Setup for K3s" +echo "================================" +echo "" + +# Check if kubectl is available +if ! command -v kubectl &> /dev/null; then + echo "Error: kubectl is not installed or not in PATH" + exit 1 +fi + +# Check if we can connect to the cluster +if ! kubectl cluster-info &> /dev/null; then + echo "Error: Cannot connect to Kubernetes cluster" + exit 1 +fi + +echo "Step 1: Applying base configuration..." +kubectl apply -f 00-namespace.yaml +kubectl apply -f 01-rbac.yaml +kubectl apply -f 02-crds.yaml + +echo "" +echo "Step 2: Authentication setup" +echo "You need to create an auth key from: https://login.tailscale.com/admin/settings/keys" +echo "" +echo "Choose one option:" +echo "1) I have an auth key ready" +echo "2) I'll set it up later" +echo "" +read -p "Enter your choice (1 or 2): " choice + +if [ "$choice" = "1" ]; then + read -p "Enter your Tailscale auth key: " authkey + kubectl create secret generic operator \ + --namespace=tailscale \ + --from-literal=authkey="$authkey" \ + --dry-run=client -o yaml | kubectl apply -f - + echo "Auth key secret created/updated" + + echo "" + echo "Step 3: Deploying operator..." + kubectl apply -f 04-operator-authkey.yaml + + echo "" + echo "Waiting for operator to start..." + sleep 5 + kubectl wait --for=condition=ready pod -l app=operator -n tailscale --timeout=60s + + echo "" + echo "Operator status:" + kubectl get pods -n tailscale + + echo "" + echo "Step 4: Would you like to expose Grafana via Tailscale? (y/n)" + read -p "Choice: " expose_grafana + + if [ "$expose_grafana" = "y" ]; then + kubectl apply -f grafana-tailscale.yaml + echo "Grafana will be available at: https://aprs-grafana..ts.net" + fi +else + echo "" + echo "Skipping auth key setup. To complete the setup later:" + echo "1. Create an auth key at https://login.tailscale.com/admin/settings/keys" + echo "2. Run: kubectl create secret generic operator --namespace=tailscale --from-literal=authkey=''" + echo "3. Apply the operator: kubectl apply -f 04-operator-authkey.yaml" +fi + +echo "" +echo "Setup complete!" +echo "" +echo "To check the operator status: kubectl logs -n tailscale deployment/operator" +echo "To see Tailscale proxies: kubectl get pods -A | grep ts-" \ No newline at end of file diff --git a/terraform/dns_w5isp.tf b/terraform/dns_w5isp.tf index 495a82e..5420286 100644 --- a/terraform/dns_w5isp.tf +++ b/terraform/dns_w5isp.tf @@ -1,7 +1,7 @@ resource "dnsimple_zone_record" "w5isp_root" { zone_name = "w5isp.com" name = "" - value = "204.110.191.210" + value = "204.110.191.8" type = "A" ttl = 3600 } @@ -54,6 +54,14 @@ resource "dnsimple_zone_record" "w5isp_skippy" { ttl = 3600 } +resource "dnsimple_zone_record" "w5isp_ntfy" { + zone_name = "w5isp.com" + name = "ntfy" + value = "204.110.191.8" + type = "A" + ttl = 3600 +} + resource "dnsimple_zone_record" "w5isp_jellyfin" { zone_name = "w5isp.com" name = "jellyfin"