Graham McIntire
b5e3c1f977
Three changes targeting the root cause of production up/down alerting: 1. Skip BruteForceProtection for /health and /health/live paths (removes unnecessary Repo.get_by(IpBlock) DB query from every k8s probe, matching the existing /socket/agent exemption). 2. Relax k8s readiness probe: periodSeconds 5→10, timeoutSeconds 2→5, failureThreshold 2→3, successThreshold 3→2 so transient DB/Redis blips don't cascade into full service unavailability. 3. Use the persistent Towerops.Redix connection for health checks instead of opening a brand-new TCP connection (handshake→AUTH→ PING→close) on every probe. Towerops.Redix.Fake gains a set_ping_response/2 toggle for testing failure paths.
307 lines
10 KiB
YAML
307 lines
10 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: towerops
|
|
namespace: towerops
|
|
spec:
|
|
replicas: 2
|
|
strategy:
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
maxSurge: 1 # Allow 3rd pod during rollout to avoid downtime
|
|
maxUnavailable: 0 # Keep all pods running during rollout
|
|
minReadySeconds: 30 # Wait 30s after pod is ready before considering it available (endpoint propagation buffer)
|
|
selector:
|
|
matchLabels:
|
|
app: towerops
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: towerops
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "9568"
|
|
prometheus.io/path: "/metrics"
|
|
prometheus.io/job: "towerops"
|
|
spec:
|
|
# Use system-cluster-critical priority to ensure towerops starts after CNI is ready
|
|
priorityClassName: system-cluster-critical
|
|
terminationGracePeriodSeconds: 90 # Allow 90s for graceful shutdown and connection draining
|
|
imagePullSecrets:
|
|
- name: forgejo-registry
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
fsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
initContainers:
|
|
- name: migrate
|
|
image: codeberg.org/gmcintire/towerops:main-1777403311-23304e1
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/app/bin/migrate"]
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
env:
|
|
- name: MIX_ENV
|
|
value: "prod"
|
|
- name: RELEASE_COOKIE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: RELEASE_COOKIE
|
|
- name: SECRET_KEY_BASE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: SECRET_KEY_BASE
|
|
- name: CLOAK_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: CLOAK_KEY
|
|
- name: SESSION_SIGNING_SALT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: SESSION_SIGNING_SALT
|
|
- name: SESSION_ENCRYPTION_SALT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: SESSION_ENCRYPTION_SALT
|
|
- name: LIVE_VIEW_SIGNING_SALT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: LIVE_VIEW_SIGNING_SALT
|
|
- name: STRIPE_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_SECRET_KEY
|
|
optional: true
|
|
- name: STRIPE_WEBHOOK_SECRET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_WEBHOOK_SECRET
|
|
optional: true
|
|
- name: STRIPE_PRICE_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_PRICE_ID
|
|
optional: true
|
|
- name: STRIPE_METER_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_METER_ID
|
|
optional: true
|
|
- name: DEEPSEEK_API_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-llm
|
|
key: DEEPSEEK_API_KEY
|
|
optional: true
|
|
- name: DEEPSEEK_MODEL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-llm
|
|
key: DEEPSEEK_MODEL
|
|
optional: true
|
|
envFrom:
|
|
- secretRef:
|
|
name: towerops-db
|
|
- secretRef:
|
|
name: towerops-aws
|
|
resources:
|
|
requests:
|
|
memory: "512Mi"
|
|
cpu: "100m"
|
|
limits:
|
|
memory: "1Gi"
|
|
cpu: "500m"
|
|
containers:
|
|
- name: towerops
|
|
image: codeberg.org/gmcintire/towerops:main-1777403311-23304e1
|
|
imagePullPolicy: IfNotPresent
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
ports:
|
|
- containerPort: 4000
|
|
name: http
|
|
- containerPort: 4369
|
|
name: epmd
|
|
- containerPort: 9000
|
|
name: dist
|
|
- containerPort: 9568
|
|
name: metrics
|
|
lifecycle:
|
|
preStop:
|
|
exec:
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
# Stop accepting new connections by failing readiness probe
|
|
# Give load balancer time to remove pod from rotation
|
|
# Allow existing connections to complete
|
|
sleep 20
|
|
env:
|
|
- name: POD_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.podIP
|
|
- name: POD_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.name
|
|
- name: POD_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
- name: ETCD_ENABLED
|
|
value: "true"
|
|
- name: RELEASE_DISTRIBUTION
|
|
value: "name"
|
|
- name: RELEASE_NODE
|
|
value: "towerops@$(POD_IP)"
|
|
- name: PORT
|
|
value: "4000"
|
|
- name: PHX_HOST
|
|
value: "towerops.net"
|
|
- name: MIX_ENV
|
|
value: "prod"
|
|
- name: RELEASE_COOKIE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: RELEASE_COOKIE
|
|
- name: SECRET_KEY_BASE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: SECRET_KEY_BASE
|
|
- name: CLOAK_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: CLOAK_KEY
|
|
- name: SESSION_SIGNING_SALT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: SESSION_SIGNING_SALT
|
|
- name: SESSION_ENCRYPTION_SALT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: SESSION_ENCRYPTION_SALT
|
|
- name: LIVE_VIEW_SIGNING_SALT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-secrets
|
|
key: LIVE_VIEW_SIGNING_SALT
|
|
- name: STRIPE_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_SECRET_KEY
|
|
optional: true
|
|
- name: STRIPE_WEBHOOK_SECRET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_WEBHOOK_SECRET
|
|
optional: true
|
|
- name: STRIPE_PRICE_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_PRICE_ID
|
|
optional: true
|
|
- name: STRIPE_METER_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-billing
|
|
key: STRIPE_METER_ID
|
|
optional: true
|
|
- name: DEEPSEEK_API_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-llm
|
|
key: DEEPSEEK_API_KEY
|
|
optional: true
|
|
- name: DEEPSEEK_MODEL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: towerops-llm
|
|
key: DEEPSEEK_MODEL
|
|
optional: true
|
|
envFrom:
|
|
# Redis connection configured via towerops-redis secret
|
|
- secretRef:
|
|
name: towerops-redis
|
|
- secretRef:
|
|
name: towerops-db
|
|
- secretRef:
|
|
name: towerops-aws
|
|
resources:
|
|
requests:
|
|
memory: "1.5Gi"
|
|
cpu: "500m"
|
|
limits:
|
|
memory: "3Gi"
|
|
cpu: "2000m"
|
|
# Startup and liveness use the shallow /health/live (no db/redis) so a
|
|
# transient dependency outage can't prevent boot or kill a running
|
|
# pod. Readiness uses the deep /health so the pod is pulled from the
|
|
# load balancer (but NOT restarted) while db/redis are unreachable.
|
|
startupProbe:
|
|
httpGet:
|
|
path: /health/live
|
|
port: 4000
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 5
|
|
timeoutSeconds: 3
|
|
failureThreshold: 24 # 10s + (24 * 5s) = 130s max startup time
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /health/live
|
|
port: 4000
|
|
periodSeconds: 10
|
|
timeoutSeconds: 3
|
|
failureThreshold: 3
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 4000
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10 # Check every 10s instead of 5s (halves probe pressure)
|
|
timeoutSeconds: 5 # Give DB+Redis checks enough time to respond
|
|
successThreshold: 2 # 2 consecutive successes (20s) before marking ready
|
|
failureThreshold: 3 # Allow 2 consecutive failures before marking not ready
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /data
|
|
volumes:
|
|
- name: data
|
|
nfs:
|
|
server: 10.0.15.103
|
|
path: /data
|