Graham McIntire
8a54f9a866
Pods were running at 500m/500m (100% of limit) with 44-47% of scheduling periods being CPU-throttled. SSL decryption is CPU-intensive — when the BEAM gets throttled mid ssl_gen_statem:call/2, the SSL recv stalls until the cgroup allows CPU again, causing the 15s DBConnection timeout. This was the actual root cause of "client timed out because it queued and checked out the connection for longer than 15000ms" errors. Previous fixes (TCP keepalive, pool tuning, oban pruner) were treating symptoms. - CPU limit: 500m → 2000m (4 cores burst capacity) - CPU request: 100m → 500m (guarantee adequate baseline) Reviewed-on: graham/towerops-web#96
242 lines
7.6 KiB
YAML
242 lines
7.6 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: towerops
|
|
namespace: towerops
|
|
spec:
|
|
replicas: 2 # Run 3 replicas for high availability
|
|
strategy:
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
maxSurge: 1 # Limit to 3 pods during rollout to avoid exceeding PG max_connections
|
|
maxUnavailable: 0 # Keep all pods running during rollout
|
|
minReadySeconds: 10 # Wait 10s after pod is ready before considering it available
|
|
selector:
|
|
matchLabels:
|
|
app: towerops
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: towerops
|
|
spec:
|
|
# Use system-cluster-critical priority to ensure towerops starts after CNI is ready
|
|
priorityClassName: system-cluster-critical
|
|
terminationGracePeriodSeconds: 90 # Allow 90s for graceful shutdown and connection draining
|
|
imagePullSecrets:
|
|
- name: forgejo-registry
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
fsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
initContainers:
|
|
- name: migrate
|
|
image: git.mcintire.me/graham/towerops-web:production-1773753909-becd2ef
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/app/bin/migrate"]
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
env:
|
|
- name: MIX_ENV
|
|
value: "prod"
|
|
- name: DATABASE_SSL
|
|
value: "true"
|
|
- name: DATABASE_SSL_VERIFY
|
|
value: "false"
|
|
- name: RELEASE_COOKIE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: RELEASE_COOKIE
|
|
- name: SECRET_KEY_BASE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: SECRET_KEY_BASE
|
|
- name: CLOAK_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: CLOAK_KEY
|
|
- name: STRIPE_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_SECRET_KEY
|
|
optional: true
|
|
- name: STRIPE_WEBHOOK_SECRET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_WEBHOOK_SECRET
|
|
optional: true
|
|
- name: STRIPE_PRICE_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_PRICE_ID
|
|
optional: true
|
|
- name: STRIPE_METER_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_METER_ID
|
|
optional: true
|
|
envFrom:
|
|
- secretRef:
|
|
name: towerops-db
|
|
- secretRef:
|
|
name: towerops-aws
|
|
resources:
|
|
requests:
|
|
memory: "512Mi"
|
|
cpu: "100m"
|
|
limits:
|
|
memory: "1Gi"
|
|
cpu: "500m"
|
|
containers:
|
|
- name: towerops
|
|
image: git.mcintire.me/graham/towerops-web:production-1773753909-becd2ef
|
|
imagePullPolicy: IfNotPresent
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
ports:
|
|
- containerPort: 4000
|
|
name: http
|
|
- containerPort: 4369
|
|
name: epmd
|
|
- containerPort: 9000
|
|
name: dist
|
|
lifecycle:
|
|
preStop:
|
|
exec:
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
# Stop accepting new connections by failing readiness probe
|
|
# Give load balancer time to remove pod from rotation
|
|
# Allow existing connections to complete
|
|
sleep 20
|
|
env:
|
|
- name: POD_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.podIP
|
|
- name: POD_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.name
|
|
- name: POD_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
- name: ETCD_ENABLED
|
|
value: "true"
|
|
- name: RELEASE_DISTRIBUTION
|
|
value: "name"
|
|
- name: RELEASE_NODE
|
|
value: "towerops@$(POD_IP)"
|
|
- name: PORT
|
|
value: "4000"
|
|
- name: PHX_HOST
|
|
value: "towerops.net"
|
|
- name: MIX_ENV
|
|
value: "prod"
|
|
- name: RELEASE_COOKIE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: RELEASE_COOKIE
|
|
- name: SECRET_KEY_BASE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: SECRET_KEY_BASE
|
|
- name: CLOAK_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: CLOAK_KEY
|
|
- name: DATABASE_SSL
|
|
value: "true"
|
|
- name: DATABASE_SSL_VERIFY
|
|
value: "false"
|
|
- name: STRIPE_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_SECRET_KEY
|
|
optional: true
|
|
- name: STRIPE_WEBHOOK_SECRET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_WEBHOOK_SECRET
|
|
optional: true
|
|
- name: STRIPE_PRICE_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_PRICE_ID
|
|
optional: true
|
|
- name: STRIPE_METER_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_METER_ID
|
|
optional: true
|
|
envFrom:
|
|
# Redis connection configured via towerops-redis secret
|
|
- secretRef:
|
|
name: towerops-redis
|
|
- secretRef:
|
|
name: towerops-db
|
|
- secretRef:
|
|
name: towerops-aws
|
|
resources:
|
|
requests:
|
|
memory: "1.5Gi"
|
|
cpu: "500m"
|
|
limits:
|
|
memory: "3Gi"
|
|
cpu: "2000m"
|
|
startupProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 4000
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 5
|
|
timeoutSeconds: 3
|
|
failureThreshold: 12 # 10s + (12 * 5s) = 70s max startup time
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 4000
|
|
periodSeconds: 10
|
|
timeoutSeconds: 3
|
|
failureThreshold: 3
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 4000
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 5
|
|
timeoutSeconds: 2
|
|
successThreshold: 1 # Mark ready after first successful check
|
|
failureThreshold: 2 # Allow 1 failure before marking not ready
|