diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index f6fce47d..9c84567a 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -3,12 +3,14 @@ kind: Kustomization namespace: towerops resources: - namespace.yaml - - mib-pvc.yaml - deployment.yaml - service.yaml - service-headless.yaml + - valkey-configmap.yaml - valkey-statefulset.yaml - valkey-service.yaml + - valkey-sentinel-statefulset.yaml + - valkey-sentinel-service.yaml - etcd-statefulset.yaml - certificate.yaml - ingressroute.yaml diff --git a/k8s/valkey-configmap.yaml b/k8s/valkey-configmap.yaml new file mode 100644 index 00000000..c6a9edfb --- /dev/null +++ b/k8s/valkey-configmap.yaml @@ -0,0 +1,82 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: valkey-config + namespace: towerops +data: + master.conf: | + # Valkey master configuration + port 6379 + + # Network settings + tcp-backlog 511 + timeout 300 + tcp-keepalive 60 + + # Connection limits + maxclients 10000 + + # Memory management + maxmemory 256mb + maxmemory-policy allkeys-lru + + # Persistence (disable for cache-only mode, faster recovery) + save "" + appendonly no + + # Logging + loglevel notice + + # Security + protected-mode no + + # Replication settings + repl-diskless-sync yes + repl-diskless-sync-delay 5 + + replica.conf: | + # Valkey replica configuration + port 6379 + + # Network settings + tcp-backlog 511 + timeout 300 + tcp-keepalive 60 + + # Connection limits + maxclients 10000 + + # Memory management + maxmemory 256mb + maxmemory-policy allkeys-lru + + # Persistence (disable for cache-only mode) + save "" + appendonly no + + # Logging + loglevel notice + + # Security + protected-mode no + + # Replication settings + replicaof valkey-0.valkey.towerops.svc.cluster.local 6379 + replica-read-only yes + + sentinel.conf: | + # Valkey Sentinel configuration + port 26379 + + # Monitor the master + sentinel monitor mymaster valkey-0.valkey.towerops.svc.cluster.local 6379 2 + sentinel down-after-milliseconds mymaster 5000 + sentinel parallel-syncs mymaster 1 + sentinel failover-timeout mymaster 10000 + + # Logging + loglevel notice + + # Security + protected-mode no diff --git a/k8s/valkey-sentinel-service.yaml b/k8s/valkey-sentinel-service.yaml new file mode 100644 index 00000000..613d16a7 --- /dev/null +++ b/k8s/valkey-sentinel-service.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: valkey-sentinel + namespace: towerops +spec: + type: ClusterIP + clusterIP: None # Headless service for StatefulSet DNS + selector: + app: valkey-sentinel + ports: + - port: 26379 + targetPort: 26379 + name: sentinel diff --git a/k8s/valkey-sentinel-statefulset.yaml b/k8s/valkey-sentinel-statefulset.yaml new file mode 100644 index 00000000..662c7f83 --- /dev/null +++ b/k8s/valkey-sentinel-statefulset.yaml @@ -0,0 +1,133 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: valkey-sentinel + namespace: towerops +spec: + serviceName: valkey-sentinel + replicas: 3 # Sentinel requires 3+ instances for quorum + selector: + matchLabels: + app: valkey-sentinel + template: + metadata: + labels: + app: valkey-sentinel + spec: + priorityClassName: system-cluster-critical + securityContext: + runAsNonRoot: true + runAsUser: 999 + fsGroup: 999 + seccompProfile: + type: RuntimeDefault + initContainers: + - name: wait-for-network + image: busybox:1.36 + command: + - sh + - -c + - | + echo "Waiting for network to be ready..." + until nslookup kubernetes.default.svc.cluster.local > /dev/null 2>&1; do + echo "Network not ready, retrying in 2s..." + sleep 2 + done + echo "Network is ready!" + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 999 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + - name: config-init + image: busybox:1.36 + command: + - sh + - -c + - | + cp /config-ro/sentinel.conf /config/sentinel.conf + # Make it writable for Sentinel (it updates the config file) + chmod 666 /config/sentinel.conf + volumeMounts: + - name: config-ro + mountPath: /config-ro + - name: config + mountPath: /config + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 999 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + containers: + - name: sentinel + image: valkey/valkey:8.0-alpine + imagePullPolicy: IfNotPresent + command: + - valkey-sentinel + - /config/sentinel.conf + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 999 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + ports: + - containerPort: 26379 + name: sentinel + volumeMounts: + - name: config + mountPath: /config + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "100m" + livenessProbe: + exec: + command: + - valkey-cli + - -p + - "26379" + - ping + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + readinessProbe: + exec: + command: + - sh + - -c + - | + # Check if Sentinel can reach the master + MASTER=$(valkey-cli -p 26379 sentinel get-master-addr-by-name mymaster 2>/dev/null | head -1) + if [ -n "$MASTER" ]; then + exit 0 + else + exit 1 + fi + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 2 + successThreshold: 1 + failureThreshold: 3 + volumes: + - name: config-ro + configMap: + name: valkey-config + - name: config + emptyDir: {} diff --git a/k8s/valkey-statefulset.yaml b/k8s/valkey-statefulset.yaml index 5328206a..c45a99f1 100644 --- a/k8s/valkey-statefulset.yaml +++ b/k8s/valkey-statefulset.yaml @@ -6,7 +6,7 @@ metadata: namespace: towerops spec: serviceName: valkey - replicas: 1 + replicas: 3 # 1 master + 2 replicas selector: matchLabels: app: valkey @@ -46,10 +46,41 @@ spec: - ALL seccompProfile: type: RuntimeDefault + - name: config-init + image: busybox:1.36 + command: + - sh + - -c + - | + # Determine if this pod should be master or replica + if [ "$(hostname)" = "valkey-0" ]; then + echo "This pod is master" + cp /config-ro/master.conf /config/valkey.conf + else + echo "This pod is replica" + cp /config-ro/replica.conf /config/valkey.conf + fi + volumeMounts: + - name: config-ro + mountPath: /config-ro + - name: config + mountPath: /config + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 999 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault containers: - name: valkey image: valkey/valkey:8.0-alpine imagePullPolicy: IfNotPresent + command: + - valkey-server + - /config/valkey.conf securityContext: allowPrivilegeEscalation: false runAsNonRoot: true @@ -62,13 +93,16 @@ spec: ports: - containerPort: 6379 name: valkey + volumeMounts: + - name: config + mountPath: /config resources: requests: - memory: "128Mi" - cpu: "50m" + memory: "256Mi" + cpu: "100m" limits: - memory: "512Mi" - cpu: "200m" + memory: "1Gi" + cpu: "500m" livenessProbe: exec: command: @@ -81,10 +115,29 @@ spec: readinessProbe: exec: command: - - valkey-cli - - ping + - sh + - -c + - | + # Master is always ready, replicas need to check sync status + if [ "$(hostname)" = "valkey-0" ]; then + valkey-cli ping + else + # Check if replica is in sync + ROLE=$(valkey-cli info replication | grep role | cut -d: -f2 | tr -d '\r') + if [ "$ROLE" = "slave" ]; then + valkey-cli ping + else + exit 1 + fi + fi initialDelaySeconds: 5 periodSeconds: 5 timeoutSeconds: 2 successThreshold: 1 failureThreshold: 2 + volumes: + - name: config-ro + configMap: + name: valkey-config + - name: config + emptyDir: {} diff --git a/lib/towerops/application.ex b/lib/towerops/application.ex index d69399d9..7e21d308 100644 --- a/lib/towerops/application.ex +++ b/lib/towerops/application.ex @@ -110,11 +110,32 @@ defmodule Towerops.Application do defp redis_pubsub_config do redis_config = Application.get_env(:towerops, :redis, []) - [ + # Base configuration for Phoenix.PubSub.Redis + base_opts = [ host: Keyword.get(redis_config, :host, "localhost"), port: Keyword.get(redis_config, :port, 6379), node_name: node() ] + + # Add resilience options for better reconnection handling + resilience_opts = [ + # Redix connection options for better resilience + socket_opts: [ + # TCP keepalive to detect dead connections + keepalive: true + ], + # Connection timeout (5 seconds) + timeout: 5_000, + # Backoff settings for reconnection + backoff_initial: 500, + backoff_max: 30_000, + # Don't exit process on disconnection - let PubSub handle it + exit_on_disconnection: false, + # Sync connect to fail fast if Redis unavailable at startup + sync_connect: false + ] + + Keyword.merge(base_opts, resilience_opts) end @doc """ diff --git a/lib/towerops/exq_supervisor.ex b/lib/towerops/exq_supervisor.ex index ce39d421..cbf3281c 100644 --- a/lib/towerops/exq_supervisor.ex +++ b/lib/towerops/exq_supervisor.ex @@ -91,7 +91,9 @@ defmodule Towerops.ExqSupervisor do timeout: 5_000, # Backoff for reconnection attempts backoff_initial: 500, - backoff_max: 30_000 + backoff_max: 30_000, + # TCP keepalive to detect dead connections + socket_opts: [keepalive: true] ] ] end