From be818b49b8cba45414530a92878e4d155a8d4e60 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sat, 24 Jan 2026 14:12:02 -0600 Subject: [PATCH] refactor: remove Valkey from K8s, move to Proxmox hosts Removing all Valkey (Redis) resources from Kubernetes due to instability caused by Flannel CNI networking issues. Redis will now run on Proxmox hosts for better stability and performance. Changes: - Delete Valkey StatefulSet (master + 2 replicas) - Delete Valkey Sentinel StatefulSet (3 instances) - Delete Valkey services (headless and sentinel) - Delete Valkey ConfigMap - Remove Valkey resources from kustomization.yaml - Update deployment to use towerops-redis secret for connection Next Steps: - Set up Redis Sentinel on 3 Proxmox hosts/LXC containers - Create towerops-redis secret with REDIS_HOST and REDIS_PORT - Test failover and application connectivity Benefits: - Not affected by K8s networking issues (Flannel failures) - More stable (no restarts from node issues) - Better performance (no K8s overhead) - Independent lifecycle from K8s cluster --- k8s/deployment.yaml | 8 +- k8s/kustomization.yaml | 5 - k8s/valkey-configmap.yaml | 82 --------------- k8s/valkey-sentinel-service.yaml | 15 --- k8s/valkey-sentinel-statefulset.yaml | 133 ------------------------- k8s/valkey-service.yaml | 15 --- k8s/valkey-statefulset.yaml | 143 --------------------------- 7 files changed, 3 insertions(+), 398 deletions(-) delete mode 100644 k8s/valkey-configmap.yaml delete mode 100644 k8s/valkey-sentinel-service.yaml delete mode 100644 k8s/valkey-sentinel-statefulset.yaml delete mode 100644 k8s/valkey-service.yaml delete mode 100644 k8s/valkey-statefulset.yaml diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index e26d7855..d29774a9 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -99,12 +99,10 @@ spec: secretKeyRef: name: towerops-secrets key: SECRET_KEY_BASE - # Redis connection to standalone Valkey StatefulSet - - name: REDIS_HOST - value: "valkey" - - name: REDIS_PORT - value: "6379" envFrom: + # Redis connection configured via towerops-redis secret + - secretRef: + name: towerops-redis - secretRef: name: towerops-db - secretRef: diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 9c84567a..c91c930e 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -6,11 +6,6 @@ resources: - deployment.yaml - service.yaml - service-headless.yaml - - valkey-configmap.yaml - - valkey-statefulset.yaml - - valkey-service.yaml - - valkey-sentinel-statefulset.yaml - - valkey-sentinel-service.yaml - etcd-statefulset.yaml - certificate.yaml - ingressroute.yaml diff --git a/k8s/valkey-configmap.yaml b/k8s/valkey-configmap.yaml deleted file mode 100644 index c6a9edfb..00000000 --- a/k8s/valkey-configmap.yaml +++ /dev/null @@ -1,82 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: valkey-config - namespace: towerops -data: - master.conf: | - # Valkey master configuration - port 6379 - - # Network settings - tcp-backlog 511 - timeout 300 - tcp-keepalive 60 - - # Connection limits - maxclients 10000 - - # Memory management - maxmemory 256mb - maxmemory-policy allkeys-lru - - # Persistence (disable for cache-only mode, faster recovery) - save "" - appendonly no - - # Logging - loglevel notice - - # Security - protected-mode no - - # Replication settings - repl-diskless-sync yes - repl-diskless-sync-delay 5 - - replica.conf: | - # Valkey replica configuration - port 6379 - - # Network settings - tcp-backlog 511 - timeout 300 - tcp-keepalive 60 - - # Connection limits - maxclients 10000 - - # Memory management - maxmemory 256mb - maxmemory-policy allkeys-lru - - # Persistence (disable for cache-only mode) - save "" - appendonly no - - # Logging - loglevel notice - - # Security - protected-mode no - - # Replication settings - replicaof valkey-0.valkey.towerops.svc.cluster.local 6379 - replica-read-only yes - - sentinel.conf: | - # Valkey Sentinel configuration - port 26379 - - # Monitor the master - sentinel monitor mymaster valkey-0.valkey.towerops.svc.cluster.local 6379 2 - sentinel down-after-milliseconds mymaster 5000 - sentinel parallel-syncs mymaster 1 - sentinel failover-timeout mymaster 10000 - - # Logging - loglevel notice - - # Security - protected-mode no diff --git a/k8s/valkey-sentinel-service.yaml b/k8s/valkey-sentinel-service.yaml deleted file mode 100644 index 613d16a7..00000000 --- a/k8s/valkey-sentinel-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: valkey-sentinel - namespace: towerops -spec: - type: ClusterIP - clusterIP: None # Headless service for StatefulSet DNS - selector: - app: valkey-sentinel - ports: - - port: 26379 - targetPort: 26379 - name: sentinel diff --git a/k8s/valkey-sentinel-statefulset.yaml b/k8s/valkey-sentinel-statefulset.yaml deleted file mode 100644 index 662c7f83..00000000 --- a/k8s/valkey-sentinel-statefulset.yaml +++ /dev/null @@ -1,133 +0,0 @@ ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: valkey-sentinel - namespace: towerops -spec: - serviceName: valkey-sentinel - replicas: 3 # Sentinel requires 3+ instances for quorum - selector: - matchLabels: - app: valkey-sentinel - template: - metadata: - labels: - app: valkey-sentinel - spec: - priorityClassName: system-cluster-critical - securityContext: - runAsNonRoot: true - runAsUser: 999 - fsGroup: 999 - seccompProfile: - type: RuntimeDefault - initContainers: - - name: wait-for-network - image: busybox:1.36 - command: - - sh - - -c - - | - echo "Waiting for network to be ready..." - until nslookup kubernetes.default.svc.cluster.local > /dev/null 2>&1; do - echo "Network not ready, retrying in 2s..." - sleep 2 - done - echo "Network is ready!" - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 999 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - - name: config-init - image: busybox:1.36 - command: - - sh - - -c - - | - cp /config-ro/sentinel.conf /config/sentinel.conf - # Make it writable for Sentinel (it updates the config file) - chmod 666 /config/sentinel.conf - volumeMounts: - - name: config-ro - mountPath: /config-ro - - name: config - mountPath: /config - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 999 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - containers: - - name: sentinel - image: valkey/valkey:8.0-alpine - imagePullPolicy: IfNotPresent - command: - - valkey-sentinel - - /config/sentinel.conf - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 999 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - ports: - - containerPort: 26379 - name: sentinel - volumeMounts: - - name: config - mountPath: /config - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "128Mi" - cpu: "100m" - livenessProbe: - exec: - command: - - valkey-cli - - -p - - "26379" - - ping - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 2 - failureThreshold: 3 - readinessProbe: - exec: - command: - - sh - - -c - - | - # Check if Sentinel can reach the master - MASTER=$(valkey-cli -p 26379 sentinel get-master-addr-by-name mymaster 2>/dev/null | head -1) - if [ -n "$MASTER" ]; then - exit 0 - else - exit 1 - fi - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 2 - successThreshold: 1 - failureThreshold: 3 - volumes: - - name: config-ro - configMap: - name: valkey-config - - name: config - emptyDir: {} diff --git a/k8s/valkey-service.yaml b/k8s/valkey-service.yaml deleted file mode 100644 index c8cb5b3f..00000000 --- a/k8s/valkey-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: valkey - namespace: towerops -spec: - type: ClusterIP - clusterIP: None # Headless service for StatefulSet DNS - selector: - app: valkey - ports: - - port: 6379 - targetPort: 6379 - name: valkey diff --git a/k8s/valkey-statefulset.yaml b/k8s/valkey-statefulset.yaml deleted file mode 100644 index c45a99f1..00000000 --- a/k8s/valkey-statefulset.yaml +++ /dev/null @@ -1,143 +0,0 @@ ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: valkey - namespace: towerops -spec: - serviceName: valkey - replicas: 3 # 1 master + 2 replicas - selector: - matchLabels: - app: valkey - template: - metadata: - labels: - app: valkey - spec: - # Use system-cluster-critical priority to ensure Valkey starts after CNI is ready - priorityClassName: system-cluster-critical - securityContext: - runAsNonRoot: true - runAsUser: 999 - fsGroup: 999 - seccompProfile: - type: RuntimeDefault - # Init container to wait for network to be ready (Flannel CNI) - initContainers: - - name: wait-for-network - image: busybox:1.36 - command: - - sh - - -c - - | - echo "Waiting for network to be ready..." - until nslookup kubernetes.default.svc.cluster.local > /dev/null 2>&1; do - echo "Network not ready, retrying in 2s..." - sleep 2 - done - echo "Network is ready!" - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 999 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - - name: config-init - image: busybox:1.36 - command: - - sh - - -c - - | - # Determine if this pod should be master or replica - if [ "$(hostname)" = "valkey-0" ]; then - echo "This pod is master" - cp /config-ro/master.conf /config/valkey.conf - else - echo "This pod is replica" - cp /config-ro/replica.conf /config/valkey.conf - fi - volumeMounts: - - name: config-ro - mountPath: /config-ro - - name: config - mountPath: /config - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 999 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - containers: - - name: valkey - image: valkey/valkey:8.0-alpine - imagePullPolicy: IfNotPresent - command: - - valkey-server - - /config/valkey.conf - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 999 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - ports: - - containerPort: 6379 - name: valkey - volumeMounts: - - name: config - mountPath: /config - resources: - requests: - memory: "256Mi" - cpu: "100m" - limits: - memory: "1Gi" - cpu: "500m" - livenessProbe: - exec: - command: - - valkey-cli - - ping - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 2 - failureThreshold: 3 - readinessProbe: - exec: - command: - - sh - - -c - - | - # Master is always ready, replicas need to check sync status - if [ "$(hostname)" = "valkey-0" ]; then - valkey-cli ping - else - # Check if replica is in sync - ROLE=$(valkey-cli info replication | grep role | cut -d: -f2 | tr -d '\r') - if [ "$ROLE" = "slave" ]; then - valkey-cli ping - else - exit 1 - fi - fi - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 2 - successThreshold: 1 - failureThreshold: 2 - volumes: - - name: config-ro - configMap: - name: valkey-config - - name: config - emptyDir: {}