155 lines
No EOL
4.4 KiB
YAML
155 lines
No EOL
4.4 KiB
YAML
---
|
|
# This playbook sets up direct Ceph RBD access for Kubernetes
|
|
# It replaces NFS-based access with native Ceph RBD CSI
|
|
|
|
- name: Ensure Ceph kubernetes pool exists
|
|
hosts: proxmox[0]
|
|
become: true
|
|
tasks:
|
|
- name: Check if kubernetes pool exists
|
|
command: ceph osd pool ls
|
|
register: ceph_pools
|
|
changed_when: false
|
|
|
|
- name: Create kubernetes pool if not exists
|
|
command: ceph osd pool create kubernetes 128 128
|
|
when: "'kubernetes' not in ceph_pools.stdout"
|
|
|
|
- name: Initialize pool for RBD use
|
|
command: rbd pool init kubernetes
|
|
when: "'kubernetes' not in ceph_pools.stdout"
|
|
|
|
- name: Check if kubernetes user exists
|
|
command: ceph auth list
|
|
register: ceph_users
|
|
changed_when: false
|
|
|
|
- name: Create kubernetes user if not exists
|
|
shell: |
|
|
ceph auth get-or-create client.kubernetes \
|
|
mon 'profile rbd' \
|
|
osd 'profile rbd pool=kubernetes' \
|
|
mgr 'profile rbd pool=kubernetes'
|
|
when: "'client.kubernetes' not in ceph_users.stdout"
|
|
|
|
- name: Get kubernetes user key
|
|
command: ceph auth print-key client.kubernetes
|
|
register: k8s_key
|
|
changed_when: false
|
|
|
|
- name: Display kubernetes key
|
|
debug:
|
|
msg: "Kubernetes key: {{ k8s_key.stdout }}"
|
|
|
|
- name: Set fact for kubernetes key
|
|
set_fact:
|
|
ceph_kubernetes_key: "{{ k8s_key.stdout }}"
|
|
|
|
- name: Setup Ceph client on K3s nodes
|
|
import_playbook: setup-ceph-clients-k3s.yml
|
|
|
|
- name: Deploy Ceph CSI
|
|
import_playbook: deploy-ceph-csi.yml
|
|
|
|
- name: Verify Ceph RBD setup
|
|
hosts: localhost
|
|
connection: local
|
|
vars:
|
|
kubeconfig: "{{ playbook_dir }}/kubeconfig"
|
|
|
|
tasks:
|
|
- name: Check CSI driver status
|
|
shell: |
|
|
export KUBECONFIG={{ kubeconfig }}
|
|
echo "=== CSI Drivers ==="
|
|
kubectl get csidrivers
|
|
echo -e "\n=== CSI Nodes ==="
|
|
kubectl get csinodes
|
|
echo -e "\n=== Storage Classes ==="
|
|
kubectl get storageclass
|
|
echo -e "\n=== CSI Pods Status ==="
|
|
kubectl get pods -n ceph-csi
|
|
register: csi_status
|
|
|
|
- name: Display CSI status
|
|
debug:
|
|
var: csi_status.stdout_lines
|
|
|
|
- name: Create test application with Ceph RBD
|
|
shell: |
|
|
export KUBECONFIG={{ kubeconfig }}
|
|
cat <<EOF | kubectl apply -f -
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: ceph-test
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: test-rbd-pvc
|
|
namespace: ceph-test
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: ceph-rbd
|
|
resources:
|
|
requests:
|
|
storage: 5Gi
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: test-rbd-pod
|
|
namespace: ceph-test
|
|
spec:
|
|
containers:
|
|
- name: test
|
|
image: nginx:alpine
|
|
volumeMounts:
|
|
- name: test-volume
|
|
mountPath: /data
|
|
volumes:
|
|
- name: test-volume
|
|
persistentVolumeClaim:
|
|
claimName: test-rbd-pvc
|
|
EOF
|
|
register: test_app
|
|
|
|
- name: Wait for test pod to be ready
|
|
shell: |
|
|
export KUBECONFIG={{ kubeconfig }}
|
|
kubectl wait --for=condition=ready pod -n ceph-test test-rbd-pod --timeout=120s
|
|
register: wait_result
|
|
ignore_errors: yes
|
|
|
|
- name: Check test application status
|
|
shell: |
|
|
export KUBECONFIG={{ kubeconfig }}
|
|
echo "=== Test PVC Status ==="
|
|
kubectl get pvc -n ceph-test
|
|
echo -e "\n=== Test Pod Status ==="
|
|
kubectl get pod -n ceph-test
|
|
echo -e "\n=== RBD Image List ==="
|
|
kubectl exec -n ceph-test test-rbd-pod -- df -h /data
|
|
register: test_status
|
|
when: wait_result is succeeded
|
|
|
|
- name: Display test results
|
|
debug:
|
|
var: test_status.stdout_lines
|
|
when: wait_result is succeeded
|
|
|
|
- name: Cleanup test resources
|
|
shell: |
|
|
export KUBECONFIG={{ kubeconfig }}
|
|
kubectl delete namespace ceph-test --ignore-not-found
|
|
when: test_app is succeeded
|
|
|
|
- name: Summary
|
|
debug:
|
|
msg:
|
|
- "Ceph RBD CSI setup complete!"
|
|
- "Storage class 'ceph-rbd' is now available for direct Ceph RBD access"
|
|
- "You can now create PVCs with storageClassName: ceph-rbd"
|
|
- "This provides better performance than NFS-based access" |