updates

ansible/group_vars/all/vault.yml.example

@@ -10,4 +10,15 @@ vault_netbox_secret_key: "change-me-to-very-long-random-string-at-least-50-chars
 vault_netbox_admin_password: "change-me-to-secure-admin-password"
 vault_netbox_redis_password: "optional-redis-password"
 vault_netbox_redis_cache_password: "optional-redis-cache-password"
-vault_netbox_superuser_api_token: "optional-api-token-for-automation"
+vault_netbox_superuser_api_token: "optional-api-token-for-automation"
+
+# PostgreSQL role passwords (db.towerops.net). Omit to leave existing
+# passwords untouched; set to (re)provision a role's password.
+vault_postgresql_towerops_password: "change-me"
+vault_postgresql_aprsme_password: "change-me"
+vault_postgresql_gridmap_password: "change-me"
+vault_postgresql_prop_password: "change-me"
+
+# PgBouncer dedicated auth_query lookup role. REQUIRED before applying the
+# pgbouncer role. Generate with: openssl rand -base64 24
+vault_pgbouncer_auth_password: "change-me-random"

ansible/host_vars/db.w5isp.com.yml

@@ -1,6 +1,6 @@
 ---
 ansible_user: graham
-ansible_host: db-towerops
+ansible_host: 10.0.15.30
 ansible_python_interpreter: /usr/bin/python3
 ansible_ssh_private_key_file: /Users/graham/.ssh/id_ed25519
 ansible_ssh_common_args: '-o StrictHostKeyChecking=accept-new'
@@ -15,6 +15,9 @@ firewall_allow_rules:
   - port: 5432
     proto: tcp
     comment: PostgreSQL
+  - port: 6432
+    proto: tcp
+    comment: PgBouncer
 
 # Override role defaults - this server uses default Debian paths, not RAID
 postgresql_version: "17"
@@ -33,7 +36,10 @@ timescaledb_apt_repo: "deb https://packagecloud.io/timescale/timescaledb/debian/
 postgresql_config:
   # Connection
   listen_addresses: "'*'"
-  max_connections: 100
+  # Matches the live server (was raised from 100). With PgBouncer fronting the
+  # backend, real backend connections stay well under this; the headroom covers
+  # apps still connecting directly during/after cutover.
+  max_connections: 300
   port: 5432
 
   # Memory (tuned for 8GB ARM server)
@@ -142,6 +148,8 @@ postgresql_databases:
     owner: gridmap
   - name: towerops
     owner: towerops
+  - name: prop
+    owner: prop
 
 postgresql_users:
   - name: aprsme
@@ -150,6 +158,8 @@ postgresql_users:
     password: "{{ vault_postgresql_gridmap_password | default(omit) }}"
   - name: towerops
     password: "{{ vault_postgresql_towerops_password | default(omit) }}"
+  - name: prop
+    password: "{{ vault_postgresql_prop_password | default(omit) }}"
 
 postgresql_extensions:
   aprsme_prod:
@@ -157,3 +167,20 @@ postgresql_extensions:
     - pg_stat_statements
     - pg_trgm
     - postgis
+
+# --- PgBouncer ---
+# Per-database backend connection pools. pool_size sums to ~165, leaving ample
+# headroom under max_connections (300) for TimescaleDB workers and any direct
+# connections. All start in session mode (safe for Ecto prepared statements and
+# Oban LISTEN/NOTIFY); switch an app to pool_mode: transaction only after
+# verifying it tolerates transaction pooling.
+pgbouncer_max_db_connections: 100
+pgbouncer_databases:
+  - name: towerops
+    pool_size: 25
+  - name: aprsme_prod
+    pool_size: 40
+  - name: prop
+    pool_size: 80
+  - name: gridmap
+    pool_size: 20

ansible/hosts

@@ -66,6 +66,7 @@ node2 ansible_host=node2
 node3 ansible_host=node3
 
 [postgresql_servers]
+db.w5isp.com
 
 [home_cluster:children]
 proxmox_servers

ansible/playbook.yml

@@ -61,6 +61,20 @@
   roles:
     - caddy
 
+- name: Configure PostgreSQL servers and PgBouncer
+  hosts: postgresql_servers
+  become: true
+  gather_facts: true
+  tags:
+    - postgresql
+  roles:
+    - role: base
+      tags: base
+    - role: geerlingguy.postgresql
+      tags: postgres
+    - role: pgbouncer
+      tags: pgbouncer
+
 
 - name: Provision BIND9 VM on Proxmox
   hosts: localhost

ansible/roles/pgbouncer/README.md

@@ -0,0 +1,38 @@
+# pgbouncer
+
+Installs and configures [PgBouncer](https://www.pgbouncer.org/) on a PostgreSQL
+host. PgBouncer fronts the local backend and bounds the number of real backend
+connections per database, so one app can't exhaust the server's
+`max_connections` and starve its neighbours.
+
+## How it works
+
+- Listens on `:6432` (apps point `DATABASE_URL` there instead of `:5432`).
+- Clients authenticate to PgBouncer; PgBouncer resolves each role's verifier at
+  connect time via `auth_query` against a `SECURITY DEFINER` lookup function, so
+  no per-app passwords live in this repo.
+- Default `pool_mode` is `session` — safe for Ecto/Postgrex prepared statements
+  and Oban's `LISTEN/NOTIFY`. Override to `transaction` per database only for
+  apps verified to tolerate it.
+
+## Required variables
+
+Set in the DB host's `host_vars` and vault:
+
+```yaml
+# vault
+vault_pgbouncer_auth_password: "<random>"
+
+# host_vars
+pgbouncer_databases:
+  - { name: towerops }                       # session mode (default)
+  - { name: aprsme_prod, pool_size: 30 }
+  - { name: prop, pool_mode: transaction }   # opt-in transaction pooling
+```
+
+## Cutover
+
+1. Apply the role, then verify: `psql -h <host> -p 6432 -U <user> <db> -c 'select 1'`.
+2. Repoint each app's `DATABASE_URL` host/port to the PgBouncer listener.
+3. Inspect pools from the admin console:
+   `psql -h <host> -p 6432 -U pgbouncer pgbouncer -c 'SHOW POOLS'`.

ansible/roles/pgbouncer/defaults/main.yml

@@ -0,0 +1,68 @@
+---
+# PgBouncer connection pooler — defaults.
+# Runs on the PostgreSQL host itself and proxies clients to the local backend,
+# bounding the number of real backend connections so no single app can exhaust
+# the server's max_connections (the noisy-neighbor problem).
+
+pgbouncer_package: pgbouncer
+pgbouncer_service: pgbouncer
+
+# Listener. 6432 is the PgBouncer convention; backend Postgres stays on 5432.
+pgbouncer_listen_addr: "*"
+pgbouncer_listen_port: 6432
+
+# Where the backend Postgres lives (same host).
+pgbouncer_backend_host: 127.0.0.1
+pgbouncer_backend_port: 5432
+
+# Pooling. session is the safe default: it preserves LISTEN/NOTIFY, session
+# GUCs and server-side prepared statements (needed by Ecto/Postgrex + Oban).
+# Override per-database to "transaction" only for apps verified to tolerate it.
+pgbouncer_default_pool_mode: session
+
+# Client-side capacity (cheap) vs backend connections (the scarce resource).
+pgbouncer_max_client_conn: 2000
+pgbouncer_default_pool_size: 20
+pgbouncer_min_pool_size: 0
+pgbouncer_reserve_pool_size: 5
+pgbouncer_reserve_pool_timeout: 3
+# Hard cap on real backend connections per database. Sum across databases must
+# stay comfortably under the backend's max_connections.
+pgbouncer_max_db_connections: 50
+
+# Auth. Clients authenticate to PgBouncer; PgBouncer fetches each role's stored
+# verifier from the backend via auth_query (no per-app secrets duplicated here).
+pgbouncer_auth_type: scram-sha-256
+pgbouncer_auth_user: pgbouncer
+# Dedicated lookup role's password (set in vault). Required.
+pgbouncer_auth_password: "{{ vault_pgbouncer_auth_password | default('') }}"
+pgbouncer_auth_query: "SELECT username, password FROM pgbouncer.get_auth($1)"
+
+# Roles allowed to use the PgBouncer admin console (SHOW POOLS, etc).
+pgbouncer_admin_users:
+  - "{{ pgbouncer_auth_user }}"
+pgbouncer_stats_users:
+  - "{{ pgbouncer_auth_user }}"
+
+# PgBouncer -> backend TLS. Disabled because it connects over loopback.
+pgbouncer_server_tls_sslmode: disable
+
+# Startup parameters PgBouncer should accept rather than reject. Postgrex/Ecto
+# send statement_timeout (and extra_float_digits) in the startup packet; without
+# this PgBouncer aborts the connection with "unsupported startup parameter".
+# Where a value must persist (e.g. statement_timeout), set it as a role default
+# (ALTER ROLE ... SET ...) since ignored startup params are not applied.
+pgbouncer_ignore_startup_parameters: "extra_float_digits,options,IntervalStyle,statement_timeout"
+
+# Pooled databases. Each item:
+#   name:      client-facing db name (what apps connect to)
+#   dbname:    actual backend db name (defaults to name)
+#   pool_mode: optional override of pgbouncer_default_pool_mode
+#   pool_size: optional override of pgbouncer_default_pool_size
+# Define the real list in host_vars for the DB server.
+pgbouncer_databases: []
+
+# Manage the backend auth role + lookup function via community.postgresql.
+# Requires local peer access as the postgres OS user (the postgres role's model).
+pgbouncer_manage_backend_auth: true
+pgbouncer_backend_postgres_user: postgres

ansible/roles/pgbouncer/handlers/main.yml

@@ -0,0 +1,14 @@
+---
+# Reload picks up [databases] and userlist changes without dropping
+# established client connections.
+- name: Reload pgbouncer
+  ansible.builtin.systemd:
+    name: "{{ pgbouncer_service }}"
+    state: reloaded
+
+# Restart is required for listener changes (listen_addr/listen_port), which
+# SIGHUP/reload does not apply.
+- name: Restart pgbouncer
+  ansible.builtin.systemd:
+    name: "{{ pgbouncer_service }}"
+    state: restarted

ansible/roles/pgbouncer/tasks/backend_auth.yml

@@ -0,0 +1,41 @@
+---
+# Create the dedicated lookup role and SECURITY DEFINER function PgBouncer's
+# auth_query relies on. Runs as the postgres OS user over the local socket
+# (peer auth), matching how the postgres role manages users.
+
+- name: Install psycopg2 (required by community.postgresql modules)
+  ansible.builtin.apt:
+    name: python3-psycopg2
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Create pgbouncer lookup role
+  become: true
+  become_user: "{{ pgbouncer_backend_postgres_user }}"
+  community.postgresql.postgresql_user:
+    name: "{{ pgbouncer_auth_user }}"
+    password: "{{ pgbouncer_auth_password }}"
+    role_attr_flags: LOGIN,NOSUPERUSER,NOCREATEDB,NOCREATEROLE
+    login_unix_socket: /var/run/postgresql
+
+# auth_query executes in whichever database the client targets, so the lookup
+# function must exist in every pooled database.
+- name: Install auth lookup function in each pooled database
+  become: true
+  become_user: "{{ pgbouncer_backend_postgres_user }}"
+  community.postgresql.postgresql_query:
+    login_db: "{{ item.dbname | default(item.name) }}"
+    login_unix_socket: /var/run/postgresql
+    query: |
+      CREATE SCHEMA IF NOT EXISTS pgbouncer AUTHORIZATION {{ pgbouncer_backend_postgres_user }};
+      CREATE OR REPLACE FUNCTION pgbouncer.get_auth(p_usename TEXT)
+        RETURNS TABLE(username TEXT, password TEXT)
+        LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog AS
+      $$ SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow WHERE usename = p_usename; $$;
+      REVOKE ALL ON FUNCTION pgbouncer.get_auth(TEXT) FROM PUBLIC;
+      GRANT USAGE ON SCHEMA pgbouncer TO {{ pgbouncer_auth_user }};
+      GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(TEXT) TO {{ pgbouncer_auth_user }};
+  loop: "{{ pgbouncer_databases }}"
+  loop_control:
+    label: "{{ item.dbname | default(item.name) }}"

ansible/roles/pgbouncer/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+# Install and configure PgBouncer in front of the local PostgreSQL backend.
+
+- name: Assert PgBouncer auth password is set
+  ansible.builtin.assert:
+    that:
+      - pgbouncer_auth_password | length > 0
+    fail_msg: >-
+      pgbouncer_auth_password is empty. Set vault_pgbouncer_auth_password in the
+      vault before applying the pgbouncer role.
+
+- name: Install PgBouncer
+  ansible.builtin.apt:
+    name: "{{ pgbouncer_package }}"
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Configure backend auth role and lookup function
+  ansible.builtin.include_tasks: backend_auth.yml
+  when: pgbouncer_manage_backend_auth | bool
+
+- name: Deploy pgbouncer.ini
+  ansible.builtin.template:
+    src: pgbouncer.ini.j2
+    dest: /etc/pgbouncer/pgbouncer.ini
+    owner: postgres
+    group: postgres
+    mode: "0640"
+  notify: Restart pgbouncer
+
+- name: Deploy userlist.txt
+  ansible.builtin.template:
+    src: userlist.txt.j2
+    dest: /etc/pgbouncer/userlist.txt
+    owner: postgres
+    group: postgres
+    mode: "0640"
+  notify: Reload pgbouncer
+
+- name: Enable and start PgBouncer
+  ansible.builtin.systemd:
+    name: "{{ pgbouncer_service }}"
+    enabled: true
+    state: started

ansible/roles/pgbouncer/templates/pgbouncer.ini.j2

@@ -0,0 +1,41 @@
+; {{ ansible_managed }}
+; PgBouncer connection pooler for the local PostgreSQL backend.
+
+[databases]
+{% for db in pgbouncer_databases %}
+{{ db.name }} = host={{ pgbouncer_backend_host }} port={{ pgbouncer_backend_port }} dbname={{ db.dbname | default(db.name) }}{% if db.pool_mode is defined %} pool_mode={{ db.pool_mode }}{% endif %}{% if db.pool_size is defined %} pool_size={{ db.pool_size }}{% endif %}
+
+{% endfor %}
+
+[pgbouncer]
+listen_addr = {{ pgbouncer_listen_addr }}
+listen_port = {{ pgbouncer_listen_port }}
+unix_socket_dir = /var/run/postgresql
+
+; --- authentication ---
+auth_type = {{ pgbouncer_auth_type }}
+auth_file = /etc/pgbouncer/userlist.txt
+auth_user = {{ pgbouncer_auth_user }}
+auth_query = {{ pgbouncer_auth_query }}
+
+; --- pooling ---
+pool_mode = {{ pgbouncer_default_pool_mode }}
+max_client_conn = {{ pgbouncer_max_client_conn }}
+default_pool_size = {{ pgbouncer_default_pool_size }}
+min_pool_size = {{ pgbouncer_min_pool_size }}
+reserve_pool_size = {{ pgbouncer_reserve_pool_size }}
+reserve_pool_timeout = {{ pgbouncer_reserve_pool_timeout }}
+max_db_connections = {{ pgbouncer_max_db_connections }}
+
+; --- backend connection ---
+server_tls_sslmode = {{ pgbouncer_server_tls_sslmode }}
+ignore_startup_parameters = {{ pgbouncer_ignore_startup_parameters }}
+
+; --- admin / monitoring ---
+admin_users = {{ pgbouncer_admin_users | join(', ') }}
+stats_users = {{ pgbouncer_stats_users | join(', ') }}
+
+; --- logging ---
+logfile = /var/log/postgresql/pgbouncer.log
+pidfile = /var/run/postgresql/pgbouncer.pid
+syslog = 0

ansible/roles/pgbouncer/templates/userlist.txt.j2

@@ -0,0 +1,4 @@
+; {{ ansible_managed }}
+; Only the auth_user needs an entry here; every other role is resolved at
+; connect time via auth_query against the backend's pg_shadow.
+"{{ pgbouncer_auth_user }}" "{{ pgbouncer_auth_password }}"