refactor: remove Valkey from K8s, move to Proxmox hosts
Removing all Valkey (Redis) resources from Kubernetes due to instability caused by Flannel CNI networking issues. Redis will now run on Proxmox hosts for better stability and performance. Changes: - Delete Valkey StatefulSet (master + 2 replicas) - Delete Valkey Sentinel StatefulSet (3 instances) - Delete Valkey services (headless and sentinel) - Delete Valkey ConfigMap - Remove Valkey resources from kustomization.yaml - Update deployment to use towerops-redis secret for connection Next Steps: - Set up Redis Sentinel on 3 Proxmox hosts/LXC containers - Create towerops-redis secret with REDIS_HOST and REDIS_PORT - Test failover and application connectivity Benefits: - Not affected by K8s networking issues (Flannel failures) - More stable (no restarts from node issues) - Better performance (no K8s overhead) - Independent lifecycle from K8s cluster
This commit is contained in:
parent
3156eb19ac
commit
be818b49b8
7 changed files with 3 additions and 398 deletions
|
|
@ -99,12 +99,10 @@ spec:
|
|||
secretKeyRef:
|
||||
name: towerops-secrets
|
||||
key: SECRET_KEY_BASE
|
||||
# Redis connection to standalone Valkey StatefulSet
|
||||
- name: REDIS_HOST
|
||||
value: "valkey"
|
||||
- name: REDIS_PORT
|
||||
value: "6379"
|
||||
envFrom:
|
||||
# Redis connection configured via towerops-redis secret
|
||||
- secretRef:
|
||||
name: towerops-redis
|
||||
- secretRef:
|
||||
name: towerops-db
|
||||
- secretRef:
|
||||
|
|
|
|||
|
|
@ -6,11 +6,6 @@ resources:
|
|||
- deployment.yaml
|
||||
- service.yaml
|
||||
- service-headless.yaml
|
||||
- valkey-configmap.yaml
|
||||
- valkey-statefulset.yaml
|
||||
- valkey-service.yaml
|
||||
- valkey-sentinel-statefulset.yaml
|
||||
- valkey-sentinel-service.yaml
|
||||
- etcd-statefulset.yaml
|
||||
- certificate.yaml
|
||||
- ingressroute.yaml
|
||||
|
|
|
|||
|
|
@ -1,82 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: valkey-config
|
||||
namespace: towerops
|
||||
data:
|
||||
master.conf: |
|
||||
# Valkey master configuration
|
||||
port 6379
|
||||
|
||||
# Network settings
|
||||
tcp-backlog 511
|
||||
timeout 300
|
||||
tcp-keepalive 60
|
||||
|
||||
# Connection limits
|
||||
maxclients 10000
|
||||
|
||||
# Memory management
|
||||
maxmemory 256mb
|
||||
maxmemory-policy allkeys-lru
|
||||
|
||||
# Persistence (disable for cache-only mode, faster recovery)
|
||||
save ""
|
||||
appendonly no
|
||||
|
||||
# Logging
|
||||
loglevel notice
|
||||
|
||||
# Security
|
||||
protected-mode no
|
||||
|
||||
# Replication settings
|
||||
repl-diskless-sync yes
|
||||
repl-diskless-sync-delay 5
|
||||
|
||||
replica.conf: |
|
||||
# Valkey replica configuration
|
||||
port 6379
|
||||
|
||||
# Network settings
|
||||
tcp-backlog 511
|
||||
timeout 300
|
||||
tcp-keepalive 60
|
||||
|
||||
# Connection limits
|
||||
maxclients 10000
|
||||
|
||||
# Memory management
|
||||
maxmemory 256mb
|
||||
maxmemory-policy allkeys-lru
|
||||
|
||||
# Persistence (disable for cache-only mode)
|
||||
save ""
|
||||
appendonly no
|
||||
|
||||
# Logging
|
||||
loglevel notice
|
||||
|
||||
# Security
|
||||
protected-mode no
|
||||
|
||||
# Replication settings
|
||||
replicaof valkey-0.valkey.towerops.svc.cluster.local 6379
|
||||
replica-read-only yes
|
||||
|
||||
sentinel.conf: |
|
||||
# Valkey Sentinel configuration
|
||||
port 26379
|
||||
|
||||
# Monitor the master
|
||||
sentinel monitor mymaster valkey-0.valkey.towerops.svc.cluster.local 6379 2
|
||||
sentinel down-after-milliseconds mymaster 5000
|
||||
sentinel parallel-syncs mymaster 1
|
||||
sentinel failover-timeout mymaster 10000
|
||||
|
||||
# Logging
|
||||
loglevel notice
|
||||
|
||||
# Security
|
||||
protected-mode no
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: valkey-sentinel
|
||||
namespace: towerops
|
||||
spec:
|
||||
type: ClusterIP
|
||||
clusterIP: None # Headless service for StatefulSet DNS
|
||||
selector:
|
||||
app: valkey-sentinel
|
||||
ports:
|
||||
- port: 26379
|
||||
targetPort: 26379
|
||||
name: sentinel
|
||||
|
|
@ -1,133 +0,0 @@
|
|||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: valkey-sentinel
|
||||
namespace: towerops
|
||||
spec:
|
||||
serviceName: valkey-sentinel
|
||||
replicas: 3 # Sentinel requires 3+ instances for quorum
|
||||
selector:
|
||||
matchLabels:
|
||||
app: valkey-sentinel
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: valkey-sentinel
|
||||
spec:
|
||||
priorityClassName: system-cluster-critical
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
fsGroup: 999
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
- name: wait-for-network
|
||||
image: busybox:1.36
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
echo "Waiting for network to be ready..."
|
||||
until nslookup kubernetes.default.svc.cluster.local > /dev/null 2>&1; do
|
||||
echo "Network not ready, retrying in 2s..."
|
||||
sleep 2
|
||||
done
|
||||
echo "Network is ready!"
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
- name: config-init
|
||||
image: busybox:1.36
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
cp /config-ro/sentinel.conf /config/sentinel.conf
|
||||
# Make it writable for Sentinel (it updates the config file)
|
||||
chmod 666 /config/sentinel.conf
|
||||
volumeMounts:
|
||||
- name: config-ro
|
||||
mountPath: /config-ro
|
||||
- name: config
|
||||
mountPath: /config
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: sentinel
|
||||
image: valkey/valkey:8.0-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- valkey-sentinel
|
||||
- /config/sentinel.conf
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
ports:
|
||||
- containerPort: 26379
|
||||
name: sentinel
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
resources:
|
||||
requests:
|
||||
memory: "64Mi"
|
||||
cpu: "50m"
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- valkey-cli
|
||||
- -p
|
||||
- "26379"
|
||||
- ping
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
# Check if Sentinel can reach the master
|
||||
MASTER=$(valkey-cli -p 26379 sentinel get-master-addr-by-name mymaster 2>/dev/null | head -1)
|
||||
if [ -n "$MASTER" ]; then
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 2
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
volumes:
|
||||
- name: config-ro
|
||||
configMap:
|
||||
name: valkey-config
|
||||
- name: config
|
||||
emptyDir: {}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: valkey
|
||||
namespace: towerops
|
||||
spec:
|
||||
type: ClusterIP
|
||||
clusterIP: None # Headless service for StatefulSet DNS
|
||||
selector:
|
||||
app: valkey
|
||||
ports:
|
||||
- port: 6379
|
||||
targetPort: 6379
|
||||
name: valkey
|
||||
|
|
@ -1,143 +0,0 @@
|
|||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: valkey
|
||||
namespace: towerops
|
||||
spec:
|
||||
serviceName: valkey
|
||||
replicas: 3 # 1 master + 2 replicas
|
||||
selector:
|
||||
matchLabels:
|
||||
app: valkey
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: valkey
|
||||
spec:
|
||||
# Use system-cluster-critical priority to ensure Valkey starts after CNI is ready
|
||||
priorityClassName: system-cluster-critical
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
fsGroup: 999
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
# Init container to wait for network to be ready (Flannel CNI)
|
||||
initContainers:
|
||||
- name: wait-for-network
|
||||
image: busybox:1.36
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
echo "Waiting for network to be ready..."
|
||||
until nslookup kubernetes.default.svc.cluster.local > /dev/null 2>&1; do
|
||||
echo "Network not ready, retrying in 2s..."
|
||||
sleep 2
|
||||
done
|
||||
echo "Network is ready!"
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
- name: config-init
|
||||
image: busybox:1.36
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
# Determine if this pod should be master or replica
|
||||
if [ "$(hostname)" = "valkey-0" ]; then
|
||||
echo "This pod is master"
|
||||
cp /config-ro/master.conf /config/valkey.conf
|
||||
else
|
||||
echo "This pod is replica"
|
||||
cp /config-ro/replica.conf /config/valkey.conf
|
||||
fi
|
||||
volumeMounts:
|
||||
- name: config-ro
|
||||
mountPath: /config-ro
|
||||
- name: config
|
||||
mountPath: /config
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: valkey
|
||||
image: valkey/valkey:8.0-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- valkey-server
|
||||
- /config/valkey.conf
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: valkey
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- valkey-cli
|
||||
- ping
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
# Master is always ready, replicas need to check sync status
|
||||
if [ "$(hostname)" = "valkey-0" ]; then
|
||||
valkey-cli ping
|
||||
else
|
||||
# Check if replica is in sync
|
||||
ROLE=$(valkey-cli info replication | grep role | cut -d: -f2 | tr -d '\r')
|
||||
if [ "$ROLE" = "slave" ]; then
|
||||
valkey-cli ping
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 2
|
||||
successThreshold: 1
|
||||
failureThreshold: 2
|
||||
volumes:
|
||||
- name: config-ro
|
||||
configMap:
|
||||
name: valkey-config
|
||||
- name: config
|
||||
emptyDir: {}
|
||||
Loading…
Add table
Reference in a new issue