From a3bf64521bf2650acf1086cd3b5f6bc423390a27 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Wed, 20 May 2026 12:06:54 -0500 Subject: [PATCH] updates --- ansible/group_vars/all/vault.yml.example | 13 +++- .../{db.towerops.net.yml => db.w5isp.com.yml} | 31 ++++++++- ansible/hosts | 1 + ansible/playbook.yml | 14 ++++ ansible/roles/pgbouncer/README.md | 38 +++++++++++ ansible/roles/pgbouncer/defaults/main.yml | 68 +++++++++++++++++++ ansible/roles/pgbouncer/handlers/main.yml | 14 ++++ .../roles/pgbouncer/tasks/backend_auth.yml | 41 +++++++++++ ansible/roles/pgbouncer/tasks/main.yml | 45 ++++++++++++ .../pgbouncer/templates/pgbouncer.ini.j2 | 41 +++++++++++ .../roles/pgbouncer/templates/userlist.txt.j2 | 4 ++ 11 files changed, 307 insertions(+), 3 deletions(-) rename ansible/host_vars/{db.towerops.net.yml => db.w5isp.com.yml} (78%) create mode 100644 ansible/roles/pgbouncer/README.md create mode 100644 ansible/roles/pgbouncer/defaults/main.yml create mode 100644 ansible/roles/pgbouncer/handlers/main.yml create mode 100644 ansible/roles/pgbouncer/tasks/backend_auth.yml create mode 100644 ansible/roles/pgbouncer/tasks/main.yml create mode 100644 ansible/roles/pgbouncer/templates/pgbouncer.ini.j2 create mode 100644 ansible/roles/pgbouncer/templates/userlist.txt.j2 diff --git a/ansible/group_vars/all/vault.yml.example b/ansible/group_vars/all/vault.yml.example index fa14a4a..c4b445b 100644 --- a/ansible/group_vars/all/vault.yml.example +++ b/ansible/group_vars/all/vault.yml.example @@ -10,4 +10,15 @@ vault_netbox_secret_key: "change-me-to-very-long-random-string-at-least-50-chars vault_netbox_admin_password: "change-me-to-secure-admin-password" vault_netbox_redis_password: "optional-redis-password" vault_netbox_redis_cache_password: "optional-redis-cache-password" -vault_netbox_superuser_api_token: "optional-api-token-for-automation" \ No newline at end of file +vault_netbox_superuser_api_token: "optional-api-token-for-automation" + +# PostgreSQL role passwords (db.towerops.net). Omit to leave existing +# passwords untouched; set to (re)provision a role's password. +vault_postgresql_towerops_password: "change-me" +vault_postgresql_aprsme_password: "change-me" +vault_postgresql_gridmap_password: "change-me" +vault_postgresql_prop_password: "change-me" + +# PgBouncer dedicated auth_query lookup role. REQUIRED before applying the +# pgbouncer role. Generate with: openssl rand -base64 24 +vault_pgbouncer_auth_password: "change-me-random" \ No newline at end of file diff --git a/ansible/host_vars/db.towerops.net.yml b/ansible/host_vars/db.w5isp.com.yml similarity index 78% rename from ansible/host_vars/db.towerops.net.yml rename to ansible/host_vars/db.w5isp.com.yml index 6c31d5d..62c2f30 100644 --- a/ansible/host_vars/db.towerops.net.yml +++ b/ansible/host_vars/db.w5isp.com.yml @@ -1,6 +1,6 @@ --- ansible_user: graham -ansible_host: db-towerops +ansible_host: 10.0.15.30 ansible_python_interpreter: /usr/bin/python3 ansible_ssh_private_key_file: /Users/graham/.ssh/id_ed25519 ansible_ssh_common_args: '-o StrictHostKeyChecking=accept-new' @@ -15,6 +15,9 @@ firewall_allow_rules: - port: 5432 proto: tcp comment: PostgreSQL + - port: 6432 + proto: tcp + comment: PgBouncer # Override role defaults - this server uses default Debian paths, not RAID postgresql_version: "17" @@ -33,7 +36,10 @@ timescaledb_apt_repo: "deb https://packagecloud.io/timescale/timescaledb/debian/ postgresql_config: # Connection listen_addresses: "'*'" - max_connections: 100 + # Matches the live server (was raised from 100). With PgBouncer fronting the + # backend, real backend connections stay well under this; the headroom covers + # apps still connecting directly during/after cutover. + max_connections: 300 port: 5432 # Memory (tuned for 8GB ARM server) @@ -142,6 +148,8 @@ postgresql_databases: owner: gridmap - name: towerops owner: towerops + - name: prop + owner: prop postgresql_users: - name: aprsme @@ -150,6 +158,8 @@ postgresql_users: password: "{{ vault_postgresql_gridmap_password | default(omit) }}" - name: towerops password: "{{ vault_postgresql_towerops_password | default(omit) }}" + - name: prop + password: "{{ vault_postgresql_prop_password | default(omit) }}" postgresql_extensions: aprsme_prod: @@ -157,3 +167,20 @@ postgresql_extensions: - pg_stat_statements - pg_trgm - postgis + +# --- PgBouncer --- +# Per-database backend connection pools. pool_size sums to ~165, leaving ample +# headroom under max_connections (300) for TimescaleDB workers and any direct +# connections. All start in session mode (safe for Ecto prepared statements and +# Oban LISTEN/NOTIFY); switch an app to pool_mode: transaction only after +# verifying it tolerates transaction pooling. +pgbouncer_max_db_connections: 100 +pgbouncer_databases: + - name: towerops + pool_size: 25 + - name: aprsme_prod + pool_size: 40 + - name: prop + pool_size: 80 + - name: gridmap + pool_size: 20 diff --git a/ansible/hosts b/ansible/hosts index 5a9aa82..278a2f9 100755 --- a/ansible/hosts +++ b/ansible/hosts @@ -66,6 +66,7 @@ node2 ansible_host=node2 node3 ansible_host=node3 [postgresql_servers] +db.w5isp.com [home_cluster:children] proxmox_servers diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 759b004..de2282b 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -61,6 +61,20 @@ roles: - caddy +- name: Configure PostgreSQL servers and PgBouncer + hosts: postgresql_servers + become: true + gather_facts: true + tags: + - postgresql + roles: + - role: base + tags: base + - role: geerlingguy.postgresql + tags: postgres + - role: pgbouncer + tags: pgbouncer + - name: Provision BIND9 VM on Proxmox hosts: localhost diff --git a/ansible/roles/pgbouncer/README.md b/ansible/roles/pgbouncer/README.md new file mode 100644 index 0000000..979205f --- /dev/null +++ b/ansible/roles/pgbouncer/README.md @@ -0,0 +1,38 @@ +# pgbouncer + +Installs and configures [PgBouncer](https://www.pgbouncer.org/) on a PostgreSQL +host. PgBouncer fronts the local backend and bounds the number of real backend +connections per database, so one app can't exhaust the server's +`max_connections` and starve its neighbours. + +## How it works + +- Listens on `:6432` (apps point `DATABASE_URL` there instead of `:5432`). +- Clients authenticate to PgBouncer; PgBouncer resolves each role's verifier at + connect time via `auth_query` against a `SECURITY DEFINER` lookup function, so + no per-app passwords live in this repo. +- Default `pool_mode` is `session` — safe for Ecto/Postgrex prepared statements + and Oban's `LISTEN/NOTIFY`. Override to `transaction` per database only for + apps verified to tolerate it. + +## Required variables + +Set in the DB host's `host_vars` and vault: + +```yaml +# vault +vault_pgbouncer_auth_password: "" + +# host_vars +pgbouncer_databases: + - { name: towerops } # session mode (default) + - { name: aprsme_prod, pool_size: 30 } + - { name: prop, pool_mode: transaction } # opt-in transaction pooling +``` + +## Cutover + +1. Apply the role, then verify: `psql -h -p 6432 -U -c 'select 1'`. +2. Repoint each app's `DATABASE_URL` host/port to the PgBouncer listener. +3. Inspect pools from the admin console: + `psql -h -p 6432 -U pgbouncer pgbouncer -c 'SHOW POOLS'`. diff --git a/ansible/roles/pgbouncer/defaults/main.yml b/ansible/roles/pgbouncer/defaults/main.yml new file mode 100644 index 0000000..17d50ca --- /dev/null +++ b/ansible/roles/pgbouncer/defaults/main.yml @@ -0,0 +1,68 @@ +--- +# PgBouncer connection pooler — defaults. +# Runs on the PostgreSQL host itself and proxies clients to the local backend, +# bounding the number of real backend connections so no single app can exhaust +# the server's max_connections (the noisy-neighbor problem). + +pgbouncer_package: pgbouncer +pgbouncer_service: pgbouncer + +# Listener. 6432 is the PgBouncer convention; backend Postgres stays on 5432. +pgbouncer_listen_addr: "*" +pgbouncer_listen_port: 6432 + +# Where the backend Postgres lives (same host). +pgbouncer_backend_host: 127.0.0.1 +pgbouncer_backend_port: 5432 + +# Pooling. session is the safe default: it preserves LISTEN/NOTIFY, session +# GUCs and server-side prepared statements (needed by Ecto/Postgrex + Oban). +# Override per-database to "transaction" only for apps verified to tolerate it. +pgbouncer_default_pool_mode: session + +# Client-side capacity (cheap) vs backend connections (the scarce resource). +pgbouncer_max_client_conn: 2000 +pgbouncer_default_pool_size: 20 +pgbouncer_min_pool_size: 0 +pgbouncer_reserve_pool_size: 5 +pgbouncer_reserve_pool_timeout: 3 +# Hard cap on real backend connections per database. Sum across databases must +# stay comfortably under the backend's max_connections. +pgbouncer_max_db_connections: 50 + +# Auth. Clients authenticate to PgBouncer; PgBouncer fetches each role's stored +# verifier from the backend via auth_query (no per-app secrets duplicated here). +pgbouncer_auth_type: scram-sha-256 +pgbouncer_auth_user: pgbouncer +# Dedicated lookup role's password (set in vault). Required. +pgbouncer_auth_password: "{{ vault_pgbouncer_auth_password | default('') }}" +pgbouncer_auth_query: "SELECT username, password FROM pgbouncer.get_auth($1)" + +# Roles allowed to use the PgBouncer admin console (SHOW POOLS, etc). +pgbouncer_admin_users: + - "{{ pgbouncer_auth_user }}" +pgbouncer_stats_users: + - "{{ pgbouncer_auth_user }}" + +# PgBouncer -> backend TLS. Disabled because it connects over loopback. +pgbouncer_server_tls_sslmode: disable + +# Startup parameters PgBouncer should accept rather than reject. Postgrex/Ecto +# send statement_timeout (and extra_float_digits) in the startup packet; without +# this PgBouncer aborts the connection with "unsupported startup parameter". +# Where a value must persist (e.g. statement_timeout), set it as a role default +# (ALTER ROLE ... SET ...) since ignored startup params are not applied. +pgbouncer_ignore_startup_parameters: "extra_float_digits,options,IntervalStyle,statement_timeout" + +# Pooled databases. Each item: +# name: client-facing db name (what apps connect to) +# dbname: actual backend db name (defaults to name) +# pool_mode: optional override of pgbouncer_default_pool_mode +# pool_size: optional override of pgbouncer_default_pool_size +# Define the real list in host_vars for the DB server. +pgbouncer_databases: [] + +# Manage the backend auth role + lookup function via community.postgresql. +# Requires local peer access as the postgres OS user (the postgres role's model). +pgbouncer_manage_backend_auth: true +pgbouncer_backend_postgres_user: postgres diff --git a/ansible/roles/pgbouncer/handlers/main.yml b/ansible/roles/pgbouncer/handlers/main.yml new file mode 100644 index 0000000..6d346fc --- /dev/null +++ b/ansible/roles/pgbouncer/handlers/main.yml @@ -0,0 +1,14 @@ +--- +# Reload picks up [databases] and userlist changes without dropping +# established client connections. +- name: Reload pgbouncer + ansible.builtin.systemd: + name: "{{ pgbouncer_service }}" + state: reloaded + +# Restart is required for listener changes (listen_addr/listen_port), which +# SIGHUP/reload does not apply. +- name: Restart pgbouncer + ansible.builtin.systemd: + name: "{{ pgbouncer_service }}" + state: restarted diff --git a/ansible/roles/pgbouncer/tasks/backend_auth.yml b/ansible/roles/pgbouncer/tasks/backend_auth.yml new file mode 100644 index 0000000..a623249 --- /dev/null +++ b/ansible/roles/pgbouncer/tasks/backend_auth.yml @@ -0,0 +1,41 @@ +--- +# Create the dedicated lookup role and SECURITY DEFINER function PgBouncer's +# auth_query relies on. Runs as the postgres OS user over the local socket +# (peer auth), matching how the postgres role manages users. + +- name: Install psycopg2 (required by community.postgresql modules) + ansible.builtin.apt: + name: python3-psycopg2 + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Create pgbouncer lookup role + become: true + become_user: "{{ pgbouncer_backend_postgres_user }}" + community.postgresql.postgresql_user: + name: "{{ pgbouncer_auth_user }}" + password: "{{ pgbouncer_auth_password }}" + role_attr_flags: LOGIN,NOSUPERUSER,NOCREATEDB,NOCREATEROLE + login_unix_socket: /var/run/postgresql + +# auth_query executes in whichever database the client targets, so the lookup +# function must exist in every pooled database. +- name: Install auth lookup function in each pooled database + become: true + become_user: "{{ pgbouncer_backend_postgres_user }}" + community.postgresql.postgresql_query: + login_db: "{{ item.dbname | default(item.name) }}" + login_unix_socket: /var/run/postgresql + query: | + CREATE SCHEMA IF NOT EXISTS pgbouncer AUTHORIZATION {{ pgbouncer_backend_postgres_user }}; + CREATE OR REPLACE FUNCTION pgbouncer.get_auth(p_usename TEXT) + RETURNS TABLE(username TEXT, password TEXT) + LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog AS + $$ SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow WHERE usename = p_usename; $$; + REVOKE ALL ON FUNCTION pgbouncer.get_auth(TEXT) FROM PUBLIC; + GRANT USAGE ON SCHEMA pgbouncer TO {{ pgbouncer_auth_user }}; + GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(TEXT) TO {{ pgbouncer_auth_user }}; + loop: "{{ pgbouncer_databases }}" + loop_control: + label: "{{ item.dbname | default(item.name) }}" diff --git a/ansible/roles/pgbouncer/tasks/main.yml b/ansible/roles/pgbouncer/tasks/main.yml new file mode 100644 index 0000000..8050a76 --- /dev/null +++ b/ansible/roles/pgbouncer/tasks/main.yml @@ -0,0 +1,45 @@ +--- +# Install and configure PgBouncer in front of the local PostgreSQL backend. + +- name: Assert PgBouncer auth password is set + ansible.builtin.assert: + that: + - pgbouncer_auth_password | length > 0 + fail_msg: >- + pgbouncer_auth_password is empty. Set vault_pgbouncer_auth_password in the + vault before applying the pgbouncer role. + +- name: Install PgBouncer + ansible.builtin.apt: + name: "{{ pgbouncer_package }}" + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Configure backend auth role and lookup function + ansible.builtin.include_tasks: backend_auth.yml + when: pgbouncer_manage_backend_auth | bool + +- name: Deploy pgbouncer.ini + ansible.builtin.template: + src: pgbouncer.ini.j2 + dest: /etc/pgbouncer/pgbouncer.ini + owner: postgres + group: postgres + mode: "0640" + notify: Restart pgbouncer + +- name: Deploy userlist.txt + ansible.builtin.template: + src: userlist.txt.j2 + dest: /etc/pgbouncer/userlist.txt + owner: postgres + group: postgres + mode: "0640" + notify: Reload pgbouncer + +- name: Enable and start PgBouncer + ansible.builtin.systemd: + name: "{{ pgbouncer_service }}" + enabled: true + state: started diff --git a/ansible/roles/pgbouncer/templates/pgbouncer.ini.j2 b/ansible/roles/pgbouncer/templates/pgbouncer.ini.j2 new file mode 100644 index 0000000..f2151b7 --- /dev/null +++ b/ansible/roles/pgbouncer/templates/pgbouncer.ini.j2 @@ -0,0 +1,41 @@ +; {{ ansible_managed }} +; PgBouncer connection pooler for the local PostgreSQL backend. + +[databases] +{% for db in pgbouncer_databases %} +{{ db.name }} = host={{ pgbouncer_backend_host }} port={{ pgbouncer_backend_port }} dbname={{ db.dbname | default(db.name) }}{% if db.pool_mode is defined %} pool_mode={{ db.pool_mode }}{% endif %}{% if db.pool_size is defined %} pool_size={{ db.pool_size }}{% endif %} + +{% endfor %} + +[pgbouncer] +listen_addr = {{ pgbouncer_listen_addr }} +listen_port = {{ pgbouncer_listen_port }} +unix_socket_dir = /var/run/postgresql + +; --- authentication --- +auth_type = {{ pgbouncer_auth_type }} +auth_file = /etc/pgbouncer/userlist.txt +auth_user = {{ pgbouncer_auth_user }} +auth_query = {{ pgbouncer_auth_query }} + +; --- pooling --- +pool_mode = {{ pgbouncer_default_pool_mode }} +max_client_conn = {{ pgbouncer_max_client_conn }} +default_pool_size = {{ pgbouncer_default_pool_size }} +min_pool_size = {{ pgbouncer_min_pool_size }} +reserve_pool_size = {{ pgbouncer_reserve_pool_size }} +reserve_pool_timeout = {{ pgbouncer_reserve_pool_timeout }} +max_db_connections = {{ pgbouncer_max_db_connections }} + +; --- backend connection --- +server_tls_sslmode = {{ pgbouncer_server_tls_sslmode }} +ignore_startup_parameters = {{ pgbouncer_ignore_startup_parameters }} + +; --- admin / monitoring --- +admin_users = {{ pgbouncer_admin_users | join(', ') }} +stats_users = {{ pgbouncer_stats_users | join(', ') }} + +; --- logging --- +logfile = /var/log/postgresql/pgbouncer.log +pidfile = /var/run/postgresql/pgbouncer.pid +syslog = 0 diff --git a/ansible/roles/pgbouncer/templates/userlist.txt.j2 b/ansible/roles/pgbouncer/templates/userlist.txt.j2 new file mode 100644 index 0000000..179eb7e --- /dev/null +++ b/ansible/roles/pgbouncer/templates/userlist.txt.j2 @@ -0,0 +1,4 @@ +; {{ ansible_managed }} +; Only the auth_user needs an entry here; every other role is resolved at +; connect time via auth_query against the backend's pg_shadow. +"{{ pgbouncer_auth_user }}" "{{ pgbouncer_auth_password }}"