diff --git a/clusters/aprs/aprs-service.yaml b/clusters/aprs/aprs-service.yaml index ccb1fc0..baf8fe9 100644 --- a/clusters/aprs/aprs-service.yaml +++ b/clusters/aprs/aprs-service.yaml @@ -3,10 +3,13 @@ kind: Service metadata: name: aprs namespace: aprs + annotations: + metallb.universe.tf/address-pool: default-pool spec: selector: app: aprs ports: - - port: 4000 + - port: 80 targetPort: 4000 - type: ClusterIP \ No newline at end of file + type: LoadBalancer + loadBalancerIP: 10.0.19.221 \ No newline at end of file diff --git a/clusters/aprs/ceph-csi/ceph-config.yaml b/clusters/aprs/ceph-csi/ceph-config.yaml new file mode 100644 index 0000000..b6c2a45 --- /dev/null +++ b/clusters/aprs/ceph-csi/ceph-config.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-config + namespace: ceph-csi +data: + ceph.conf: | + [global] + fsid = 6e5e8932-3613-471d-afa0-410fade4d4ff + mon_initial_members = lab02,lab03,lab04 + mon_host = 10.0.16.231,10.0.16.232,10.0.16.233 + auth_cluster_required = cephx + auth_service_required = cephx + auth_client_required = cephx +--- +apiVersion: v1 +kind: Secret +metadata: + name: ceph-secret + namespace: ceph-csi +type: kubernetes.io/rbd +stringData: + userID: kubernetes + userKey: AQAq14Jo/g4ZORAAwxpPOmqj1PYgmih6s6b4vA== +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-csi-config + namespace: ceph-csi +data: + config.json: | + [ + { + "clusterID": "6e5e8932-3613-471d-afa0-410fade4d4ff", + "monitors": [ + "10.0.16.231:6789", + "10.0.16.232:6789", + "10.0.16.233:6789" + ] + } + ] \ No newline at end of file diff --git a/clusters/aprs/ceph-csi/ceph-csi-configmap.yaml b/clusters/aprs/ceph-csi/ceph-csi-configmap.yaml new file mode 100644 index 0000000..659140d --- /dev/null +++ b/clusters/aprs/ceph-csi/ceph-csi-configmap.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-csi-config + namespace: ceph-csi +data: + config.json: | + [ + { + "clusterID": "6e5e8932-3613-471d-afa0-410fade4d4ff", + "monitors": [ + "10.0.16.231:6789", + "10.0.16.232:6789", + "10.0.16.233:6789" + ] + } + ] +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-csi-encryption-kms-config + namespace: ceph-csi +data: + config.json: | + {} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-config + namespace: ceph-csi +data: + ceph.conf: | + [global] + fsid = 6e5e8932-3613-471d-afa0-410fade4d4ff + mon_initial_members = lab02,lab03,lab04 + mon_host = 10.0.16.231,10.0.16.232,10.0.16.233 + auth_cluster_required = cephx + auth_service_required = cephx + auth_client_required = cephx + keyring: | \ No newline at end of file diff --git a/clusters/aprs/ceph-csi/ceph-csi-deployment.yaml b/clusters/aprs/ceph-csi/ceph-csi-deployment.yaml new file mode 100644 index 0000000..e5c642b --- /dev/null +++ b/clusters/aprs/ceph-csi/ceph-csi-deployment.yaml @@ -0,0 +1,223 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: csi-rbdplugin-provisioner + namespace: ceph-csi +spec: + replicas: 1 + selector: + matchLabels: + app: csi-rbdplugin-provisioner + template: + metadata: + labels: + app: csi-rbdplugin-provisioner + spec: + serviceAccountName: rbd-csi-provisioner + containers: + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0 + args: + - "--csi-address=/csi/csi.sock" + - "--v=5" + - "--timeout=150s" + - "--retry-interval-start=500ms" + - "--feature-gates=Topology=false" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.0 + args: + - "--csi-address=/csi/csi.sock" + - "--v=5" + - "--timeout=150s" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.4.0 + args: + - "--csi-address=/csi/csi.sock" + - "--v=5" + - "--timeout=150s" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.9.0 + args: + - "--csi-address=/csi/csi.sock" + - "--v=5" + - "--timeout=150s" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-rbdplugin + image: quay.io/cephcsi/cephcsi:v3.9.0 + args: + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--type=rbd" + - "--drivername=rbd.csi.ceph.com" + - "--configmap=ceph-csi-config" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CEPH_CONF + value: /etc/ceph/ceph.conf + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: host-sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-dev + mountPath: /dev + - name: ceph-config + mountPath: /etc/ceph/ + - name: ceph-config-dir + mountPath: /etc/ceph-csi + volumes: + - name: socket-dir + emptyDir: {} + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev + - name: ceph-config + configMap: + name: ceph-config + - name: ceph-config-dir + emptyDir: {} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: csi-rbdplugin + namespace: ceph-csi +spec: + selector: + matchLabels: + app: csi-rbdplugin + template: + metadata: + labels: + app: csi-rbdplugin + spec: + serviceAccountName: rbd-csi-nodeplugin + hostNetwork: true + hostPID: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0 + args: + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-rbdplugin + image: quay.io/cephcsi/cephcsi:v3.9.0 + args: + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--type=rbd" + - "--drivername=rbd.csi.ceph.com" + - "--configmap=ceph-csi-config" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CEPH_CONF + value: /etc/ceph/ceph.conf + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins + mountPropagation: "Bidirectional" + - name: mountpoint-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: host-sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-dev + mountPath: /dev + - name: ceph-config + mountPath: /etc/ceph/ + - name: ceph-config-dir + mountPath: /etc/ceph-csi + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins + type: Directory + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + - name: mountpoint-dir + hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev + - name: ceph-config + configMap: + name: ceph-config + - name: ceph-config-dir + emptyDir: {} \ No newline at end of file diff --git a/clusters/aprs/ceph-csi/ceph-csi-rbac-fix.yaml b/clusters/aprs/ceph-csi/ceph-csi-rbac-fix.yaml new file mode 100644 index 0000000..2197a56 --- /dev/null +++ b/clusters/aprs/ceph-csi/ceph-csi-rbac-fix.yaml @@ -0,0 +1,23 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rbd-csi-provisioner-leader-election + namespace: ceph-csi +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rbd-csi-provisioner-leader-election + namespace: ceph-csi +subjects: + - kind: ServiceAccount + name: rbd-csi-provisioner + namespace: ceph-csi +roleRef: + kind: Role + name: rbd-csi-provisioner-leader-election + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/clusters/aprs/ceph-csi/ceph-csi-rbd-driver.yaml b/clusters/aprs/ceph-csi/ceph-csi-rbd-driver.yaml new file mode 100644 index 0000000..f09d8b2 --- /dev/null +++ b/clusters/aprs/ceph-csi/ceph-csi-rbd-driver.yaml @@ -0,0 +1,99 @@ +# This deploys the Ceph CSI RBD driver +# Source: https://github.com/ceph/ceph-csi +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rbd-csi-provisioner + namespace: ceph-csi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rbd-csi-nodeplugin + namespace: ceph-csi +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "create", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner +subjects: + - kind: ServiceAccount + name: rbd-csi-provisioner + namespace: ceph-csi +roleRef: + kind: ClusterRole + name: rbd-csi-provisioner + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rbd-csi-nodeplugin + namespace: ceph-csi +roleRef: + kind: ClusterRole + name: rbd-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/clusters/aprs/ceph-csi/ceph-csi-secret.yaml b/clusters/aprs/ceph-csi/ceph-csi-secret.yaml new file mode 100644 index 0000000..de5b2aa --- /dev/null +++ b/clusters/aprs/ceph-csi/ceph-csi-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: csi-rbd-secret + namespace: ceph-csi +type: kubernetes.io/rbd +stringData: + userID: kubernetes + userKey: AQAq14Jo/g4ZORAAwxpPOmqj1PYgmih6s6b4vA== \ No newline at end of file diff --git a/clusters/aprs/ceph-csi/ceph-storageclass.yaml b/clusters/aprs/ceph-csi/ceph-storageclass.yaml new file mode 100644 index 0000000..01e81d0 --- /dev/null +++ b/clusters/aprs/ceph-csi/ceph-storageclass.yaml @@ -0,0 +1,31 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ceph-rbd + annotations: + storageclass.kubernetes.io/is-default-class: "false" +provisioner: rbd.csi.ceph.com +parameters: + # Ceph cluster ID + clusterID: "6e5e8932-3613-471d-afa0-410fade4d4ff" + + # Ceph pool name - usually 'rbd' or create a dedicated pool for K8s + pool: "kubernetes" + + # Image features - using only layering for compatibility + imageFeatures: "layering" + + # CSI secrets for provisioner + csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret + csi.storage.k8s.io/provisioner-secret-namespace: ceph-csi + csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret + csi.storage.k8s.io/controller-expand-secret-namespace: ceph-csi + csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret + csi.storage.k8s.io/node-stage-secret-namespace: ceph-csi + + # File system type + csi.storage.k8s.io/fstype: ext4 + +reclaimPolicy: Delete +allowVolumeExpansion: true +volumeBindingMode: Immediate \ No newline at end of file diff --git a/clusters/aprs/ceph-csi/csi-config-map-template.yaml b/clusters/aprs/ceph-csi/csi-config-map-template.yaml new file mode 100644 index 0000000..44ef712 --- /dev/null +++ b/clusters/aprs/ceph-csi/csi-config-map-template.yaml @@ -0,0 +1,15 @@ +# +# /!\ DO NOT MODIFY THIS FILE +# +# This file has been automatically generated by Ceph-CSI yamlgen. +# The source for the contents can be found in the api/deploy directory, make +# your modifications there. +# +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: "ceph-csi-config" +data: + config.json: |- + [] diff --git a/clusters/aprs/ceph-csi/csi-driver.yaml b/clusters/aprs/ceph-csi/csi-driver.yaml new file mode 100644 index 0000000..be582b0 --- /dev/null +++ b/clusters/aprs/ceph-csi/csi-driver.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: rbd.csi.ceph.com +spec: + attachRequired: true + podInfoOnMount: false + volumeLifecycleModes: + - Persistent \ No newline at end of file diff --git a/clusters/aprs/ceph-csi/csi-rbdplugin-provisioner.yaml b/clusters/aprs/ceph-csi/csi-rbdplugin-provisioner.yaml new file mode 100644 index 0000000..876e9bd --- /dev/null +++ b/clusters/aprs/ceph-csi/csi-rbdplugin-provisioner.yaml @@ -0,0 +1,248 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: csi-rbdplugin-provisioner + # replace with non-default namespace name + namespace: default + labels: + app: csi-metrics +spec: + selector: + app: csi-rbdplugin-provisioner + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8680 + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-rbdplugin-provisioner + # replace with non-default namespace name + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + app: csi-rbdplugin-provisioner + template: + metadata: + labels: + app: csi-rbdplugin-provisioner + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-rbdplugin-provisioner + topologyKey: "kubernetes.io/hostname" + serviceAccountName: rbd-csi-provisioner + priorityClassName: system-cluster-critical + containers: + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + args: + - "--csi-address=$(ADDRESS)" + - "--v=1" + - "--timeout=150s" + - "--retry-interval-start=500ms" + - "--leader-election=true" + # set it to true to use topology based provisioning + - "--feature-gates=Topology=false" + - "--feature-gates=HonorPVReclaimPolicy=true" + - "--prevent-volume-mode-conversion=true" + # if fstype is not specified in storageclass, ext4 is default + - "--default-fstype=ext4" + - "--extra-create-metadata=true" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + args: + - "--csi-address=$(ADDRESS)" + - "--v=1" + - "--timeout=150s" + - "--leader-election=true" + - "--extra-create-metadata=true" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 + args: + - "--v=1" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--retry-interval-start=500ms" + - "--default-fstype=ext4" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + args: + - "--csi-address=$(ADDRESS)" + - "--v=1" + - "--timeout=150s" + - "--leader-election" + - "--retry-interval-start=500ms" + - "--handle-volume-inuse-error=false" + - "--feature-gates=RecoverVolumeExpansionFailure=true" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-rbdplugin + image: quay.io/cephcsi/cephcsi:v3.9.0 + args: + - "--nodeid=$(NODE_ID)" + - "--type=rbd" + - "--controllerserver=true" + - "--endpoint=$(CSI_ENDPOINT)" + - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)" + - "--v=5" + - "--drivername=rbd.csi.ceph.com" + - "--pidlimit=-1" + - "--rbdhardmaxclonedepth=8" + - "--rbdsoftmaxclonedepth=4" + - "--enableprofiling=false" + - "--setmetadata=true" + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # - name: KMS_CONFIGMAP_NAME + # value: encryptionConfig + - name: CSI_ENDPOINT + value: unix:///csi/csi-provisioner.sock + - name: CSI_ADDONS_ENDPOINT + value: unix:///csi/csi-addons.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: ceph-csi-encryption-kms-config + mountPath: /etc/ceph-csi-encryption-kms-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + - name: ceph-config + mountPath: /etc/ceph/ + - name: oidc-token + mountPath: /run/secrets/tokens + readOnly: true + - name: csi-rbdplugin-controller + image: quay.io/cephcsi/cephcsi:v3.9.0 + args: + - "--type=controller" + - "--v=5" + - "--drivername=rbd.csi.ceph.com" + - "--drivernamespace=$(DRIVER_NAMESPACE)" + - "--setmetadata=true" + env: + - name: DRIVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + - name: ceph-config + mountPath: /etc/ceph/ + - name: liveness-prometheus + image: quay.io/cephcsi/cephcsi:v3.9.0 + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metricsport=8680" + - "--metricspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" + volumes: + - name: host-dev + hostPath: + path: /dev + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: socket-dir + emptyDir: { + medium: "Memory" + } + - name: ceph-config + configMap: + name: ceph-config + - name: ceph-csi-config + configMap: + name: ceph-csi-config + - name: ceph-csi-encryption-kms-config + configMap: + name: ceph-csi-encryption-kms-config + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } + - name: oidc-token + projected: + sources: + - serviceAccountToken: + path: oidc-token + expirationSeconds: 3600 + audience: ceph-csi-kms diff --git a/clusters/aprs/ceph-csi/csi-rbdplugin.yaml b/clusters/aprs/ceph-csi/csi-rbdplugin.yaml new file mode 100644 index 0000000..0107079 --- /dev/null +++ b/clusters/aprs/ceph-csi/csi-rbdplugin.yaml @@ -0,0 +1,229 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-rbdplugin + # replace with non-default namespace name + namespace: default +spec: + selector: + matchLabels: + app: csi-rbdplugin + template: + metadata: + labels: + app: csi-rbdplugin + spec: + serviceAccountName: rbd-csi-nodeplugin + hostNetwork: true + hostPID: true + priorityClassName: system-node-critical + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + securityContext: + privileged: true + allowPrivilegeEscalation: true + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + args: + - "--v=1" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: quay.io/cephcsi/cephcsi:v3.9.0 + args: + - "--nodeid=$(NODE_ID)" + - "--pluginpath=/var/lib/kubelet/plugins" + - "--stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/" + - "--type=rbd" + - "--nodeserver=true" + - "--endpoint=$(CSI_ENDPOINT)" + - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)" + - "--v=5" + - "--drivername=rbd.csi.ceph.com" + - "--enableprofiling=false" + # If topology based provisioning is desired, configure required + # node labels representing the nodes topology domain + # and pass the label names below, for CSI to consume and advertise + # its equivalent topology domain + # - "--domainlabels=failure-domain/region,failure-domain/zone" + # + # Options to enable read affinity. + # If enabled Ceph CSI will fetch labels from kubernetes node and + # pass `read_from_replica=localize,crush_location=type:value` during + # rbd map command. refer: + # https://docs.ceph.com/en/latest/man/8/rbd/#kernel-rbd-krbd-options + # for more details. + # - "--enable-read-affinity=true" + # - "--crush-location-labels=topology.io/zone,topology.io/rack" + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # - name: KMS_CONFIGMAP_NAME + # value: encryptionConfig + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: CSI_ADDONS_ENDPOINT + value: unix:///csi/csi-addons.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /run/mount + name: host-mount + - mountPath: /etc/selinux + name: etc-selinux + readOnly: true + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: ceph-csi-encryption-kms-config + mountPath: /etc/ceph-csi-encryption-kms-config/ + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins + mountPropagation: "Bidirectional" + - name: mountpoint-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + - name: ceph-logdir + mountPath: /var/log/ceph + - name: ceph-config + mountPath: /etc/ceph/ + - name: oidc-token + mountPath: /run/secrets/tokens + readOnly: true + - name: liveness-prometheus + securityContext: + privileged: true + allowPrivilegeEscalation: true + image: quay.io/cephcsi/cephcsi:v3.9.0 + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metricsport=8680" + - "--metricspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins + type: Directory + - name: mountpoint-dir + hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + - name: ceph-logdir + hostPath: + path: /var/log/ceph + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: host-dev + hostPath: + path: /dev + - name: host-sys + hostPath: + path: /sys + - name: etc-selinux + hostPath: + path: /etc/selinux + - name: host-mount + hostPath: + path: /run/mount + - name: lib-modules + hostPath: + path: /lib/modules + - name: ceph-config + configMap: + name: ceph-config + - name: ceph-csi-config + configMap: + name: ceph-csi-config + - name: ceph-csi-encryption-kms-config + configMap: + name: ceph-csi-encryption-kms-config + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } + - name: oidc-token + projected: + sources: + - serviceAccountToken: + path: oidc-token + expirationSeconds: 3600 + audience: ceph-csi-kms +--- +# This is a service to expose the liveness metrics +apiVersion: v1 +kind: Service +metadata: + name: csi-metrics-rbdplugin + # replace with non-default namespace name + namespace: default + labels: + app: csi-metrics +spec: + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8680 + selector: + app: csi-rbdplugin diff --git a/clusters/aprs/ceph-csi/kustomization.yaml b/clusters/aprs/ceph-csi/kustomization.yaml new file mode 100644 index 0000000..3e5322c --- /dev/null +++ b/clusters/aprs/ceph-csi/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: ceph-csi + +resources: + - namespace.yaml + - ceph-csi-configmap.yaml + - ceph-csi-secret.yaml + - ceph-csi-rbd-driver.yaml + - csi-driver.yaml + - csi-rbdplugin-provisioner.yaml + - csi-rbdplugin.yaml + - ceph-storageclass.yaml \ No newline at end of file diff --git a/clusters/aprs/ceph-csi/namespace.yaml b/clusters/aprs/ceph-csi/namespace.yaml new file mode 100644 index 0000000..5554155 --- /dev/null +++ b/clusters/aprs/ceph-csi/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ceph-csi \ No newline at end of file diff --git a/clusters/aprs/ceph-setup-commands.sh b/clusters/aprs/ceph-setup-commands.sh new file mode 100755 index 0000000..152f38e --- /dev/null +++ b/clusters/aprs/ceph-setup-commands.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Run these commands on your Proxmox node (e.g., lab02) + +# Create a Ceph pool for Kubernetes with 128 placement groups +echo "Creating Kubernetes pool..." +ceph osd pool create kubernetes 128 +ceph osd pool application enable kubernetes rbd + +# Create a Ceph user for Kubernetes with appropriate permissions +echo "Creating Kubernetes user..." +ceph auth get-or-create client.kubernetes \ + mon 'profile rbd' \ + osd 'profile rbd pool=kubernetes' \ + mgr 'profile rbd pool=kubernetes' + +# Display the key for configuration +echo -e "\n\nCopy the key value below for your configuration:" +ceph auth get client.kubernetes | grep key \ No newline at end of file diff --git a/clusters/aprs/ceph-setup-guide.md b/clusters/aprs/ceph-setup-guide.md new file mode 100644 index 0000000..fd8e162 --- /dev/null +++ b/clusters/aprs/ceph-setup-guide.md @@ -0,0 +1,105 @@ +# Setting up Ceph Storage for PostgreSQL in K3s + +## Prerequisites + +Before applying these manifests, you need to gather information from your Proxmox Ceph cluster: + +1. **Ceph Monitor IPs**: Get the monitor IPs from Proxmox + ```bash + # On Proxmox node: + ceph mon dump + ``` + +2. **Ceph Cluster ID**: Get your Ceph cluster ID + ```bash + # On Proxmox node: + ceph fsid + ``` + +3. **Create a Ceph pool for Kubernetes** (if not already exists): + ```bash + # On Proxmox node: + ceph osd pool create kubernetes 128 + ceph osd pool application enable kubernetes rbd + ``` + +4. **Create a Ceph user for Kubernetes**: + ```bash + # On Proxmox node: + ceph auth get-or-create client.kubernetes mon 'profile rbd' osd 'profile rbd pool=kubernetes' mgr 'profile rbd pool=kubernetes' + ``` + +## Configuration Steps + +1. **Update the Ceph configuration** in `ceph-csi/ceph-config.yaml`: + - Replace `mon_host` with your actual Ceph monitor IPs + - Replace the `userKey` with the key from step 4 above + +2. **Update the StorageClass** in `ceph-csi/ceph-storageclass.yaml`: + - Replace `clusterID` with your Ceph cluster ID from step 2 + - Ensure the `pool` name matches what you created in step 3 + +3. **Apply the Ceph CSI driver**: + ```bash + kubectl apply -k ceph-csi/ + ``` + +4. **Verify the installation**: + ```bash + # Check if CSI pods are running + kubectl get pods -n ceph-csi + + # Check if StorageClass is created + kubectl get storageclass ceph-rbd + ``` + +## Migrating PostgreSQL to Ceph Storage + +1. **Backup existing data** (if any): + ```bash + kubectl exec -n aprs deployment/postgis -- pg_dump -U aprs aprs_prod > backup.sql + ``` + +2. **Scale down the current deployment**: + ```bash + kubectl scale deployment postgis -n aprs --replicas=0 + ``` + +3. **Create the new PVC with Ceph**: + ```bash + kubectl apply -f postgis-pvc-ceph.yaml + ``` + +4. **Update the deployment to use Ceph PVC**: + ```bash + kubectl apply -f postgis-deployment-ceph.yaml + ``` + +5. **Restore data** (if needed): + ```bash + kubectl exec -i -n aprs deployment/postgis -- psql -U aprs aprs_prod < backup.sql + ``` + +## Troubleshooting + +If pods fail to mount Ceph volumes: + +1. Check CSI driver logs: + ```bash + kubectl logs -n ceph-csi deployment/csi-rbdplugin-provisioner + kubectl logs -n ceph-csi daemonset/csi-rbdplugin + ``` + +2. Verify Ceph connectivity from K3s nodes: + ```bash + # Install ceph-common on K3s nodes + apt-get install ceph-common + + # Test connection + ceph -c /etc/ceph/ceph.conf -n client.kubernetes --keyring /etc/ceph/keyring status + ``` + +3. Check PVC events: + ```bash + kubectl describe pvc postgis-pvc-ceph -n aprs + ``` \ No newline at end of file diff --git a/clusters/aprs/create-ghcr-secret.sh b/clusters/aprs/create-ghcr-secret.sh new file mode 100755 index 0000000..23debf9 --- /dev/null +++ b/clusters/aprs/create-ghcr-secret.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# This script creates a Kubernetes secret for pulling images from ghcr.io + +echo "Enter your GitHub username:" +read GITHUB_USERNAME + +echo "Enter your GitHub Personal Access Token (with read:packages scope):" +read -s GITHUB_TOKEN + +# Create the secret +kubectl create secret docker-registry ghcr-pull-secret \ + --docker-server=ghcr.io \ + --docker-username="$GITHUB_USERNAME" \ + --docker-password="$GITHUB_TOKEN" \ + --namespace=aprs \ + --dry-run=client -o yaml > ghcr-pull-secret.yaml + +echo "Secret YAML created in ghcr-pull-secret.yaml" +echo "Apply it with: kubectl apply -f ghcr-pull-secret.yaml" \ No newline at end of file diff --git a/clusters/aprs/ghcr-pull-secret-template.yaml b/clusters/aprs/ghcr-pull-secret-template.yaml new file mode 100644 index 0000000..8c6abe5 --- /dev/null +++ b/clusters/aprs/ghcr-pull-secret-template.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ghcr-pull-secret + namespace: aprs +type: kubernetes.io/dockerconfigjson +data: + # This needs to be base64 encoded JSON in the format: + # {"auths":{"ghcr.io":{"username":"YOUR_GITHUB_USERNAME","password":"YOUR_GITHUB_PAT","auth":"BASE64_ENCODED_USERNAME:PAT"}}} + # + # You can generate this by running: + # echo -n '{"auths":{"ghcr.io":{"username":"YOUR_GITHUB_USERNAME","password":"YOUR_GITHUB_PAT","auth":"'$(echo -n "YOUR_GITHUB_USERNAME:YOUR_GITHUB_PAT" | base64)'"}}}' | base64 + .dockerconfigjson: REPLACE_THIS_WITH_BASE64_ENCODED_DOCKER_CONFIG \ No newline at end of file diff --git a/clusters/aprs/ghcr-secret.yaml b/clusters/aprs/ghcr-secret.yaml new file mode 100644 index 0000000..99f13b0 --- /dev/null +++ b/clusters/aprs/ghcr-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ghcr-secret + namespace: aprs +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: eyJhdXRocyI6eyJnaGNyLmlvIjp7ImF1dGgiOiIifX19 \ No newline at end of file diff --git a/clusters/aprs/metallb-ippool.yaml b/clusters/aprs/metallb-ippool.yaml new file mode 100644 index 0000000..782dc3f --- /dev/null +++ b/clusters/aprs/metallb-ippool.yaml @@ -0,0 +1,17 @@ +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: default-pool + namespace: metallb-system +spec: + addresses: + - 10.0.19.220-10.0.19.230 +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: default + namespace: metallb-system +spec: + ipAddressPools: + - default-pool \ No newline at end of file diff --git a/clusters/aprs/metallb-namespace.yaml b/clusters/aprs/metallb-namespace.yaml new file mode 100644 index 0000000..6059071 --- /dev/null +++ b/clusters/aprs/metallb-namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: metallb-system + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged \ No newline at end of file diff --git a/clusters/aprs/postgis-deployment-ceph.yaml b/clusters/aprs/postgis-deployment-ceph.yaml new file mode 100644 index 0000000..f30b17a --- /dev/null +++ b/clusters/aprs/postgis-deployment-ceph.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgis + namespace: aprs +spec: + replicas: 1 + selector: + matchLabels: + app: postgis + template: + metadata: + labels: + app: postgis + spec: + initContainers: + - name: volume-permissions + image: busybox:1.36.1 + command: ['sh', '-c', 'chmod -R 777 /var/lib/postgresql/data || true'] + volumeMounts: + - name: postgis-storage + mountPath: /var/lib/postgresql/data + containers: + - name: postgis + image: postgis/postgis:17-3.4 + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: aprs_prod + - name: POSTGRES_USER + value: aprs + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgis-secret + key: postgres-password + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + volumeMounts: + - name: postgis-storage + mountPath: /var/lib/postgresql/data + resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" + volumes: + - name: postgis-storage + persistentVolumeClaim: + claimName: postgis-pvc-ceph \ No newline at end of file diff --git a/clusters/aprs/postgis-deployment.yaml b/clusters/aprs/postgis-deployment.yaml index b114287..7e8a11a 100644 --- a/clusters/aprs/postgis-deployment.yaml +++ b/clusters/aprs/postgis-deployment.yaml @@ -13,6 +13,13 @@ spec: labels: app: postgis spec: + initContainers: + - name: volume-permissions + image: busybox:1.36.1 + command: ['sh', '-c', 'chmod -R 777 /var/lib/postgresql/data || true'] + volumeMounts: + - name: postgis-storage + mountPath: /var/lib/postgresql/data containers: - name: postgis image: postgis/postgis:17-3.4 @@ -42,5 +49,6 @@ spec: cpu: "250m" volumes: - name: postgis-storage - persistentVolumeClaim: - claimName: postgis-pvc \ No newline at end of file + hostPath: + path: /var/lib/rancher/k3s/storage/postgis + type: DirectoryOrCreate diff --git a/clusters/aprs/postgis-pvc.yaml b/clusters/aprs/postgis-pvc-ceph.yaml similarity index 64% rename from clusters/aprs/postgis-pvc.yaml rename to clusters/aprs/postgis-pvc-ceph.yaml index 162d10a..57f81af 100644 --- a/clusters/aprs/postgis-pvc.yaml +++ b/clusters/aprs/postgis-pvc-ceph.yaml @@ -1,12 +1,12 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: postgis-pvc + name: postgis-pvc-ceph namespace: aprs spec: accessModes: - ReadWriteOnce resources: requests: - storage: 10Gi - storageClassName: local-path \ No newline at end of file + storage: 300Gi + storageClassName: ceph-rbd diff --git a/clusters/aprs/setup-ghcr-auth.md b/clusters/aprs/setup-ghcr-auth.md new file mode 100644 index 0000000..fc15528 --- /dev/null +++ b/clusters/aprs/setup-ghcr-auth.md @@ -0,0 +1,32 @@ +# Setting up GitHub Container Registry Authentication + +## Step 1: Create GitHub Personal Access Token + +1. Go to https://github.com/settings/tokens/new +2. Name: "k3s-ghcr-pull" (or any name you prefer) +3. Expiration: Choose based on your security requirements +4. Scopes: Select `read:packages` +5. Click "Generate token" +6. Copy the token (starts with `ghp_`) + +## Step 2: Create Kubernetes Secret + +Replace `YOUR_GITHUB_USERNAME` and `YOUR_GITHUB_PAT` in the command below: + +```bash +kubectl create secret docker-registry ghcr-pull-secret \ + --docker-server=ghcr.io \ + --docker-username=YOUR_GITHUB_USERNAME \ + --docker-password=YOUR_GITHUB_PAT \ + --namespace=aprs +``` + +## Step 3: Verify Secret + +```bash +kubectl get secret ghcr-pull-secret -n aprs +``` + +## Step 4: The deployment will be automatically updated to use this secret + +The aprs-deployment.yaml already includes the imagePullSecrets configuration. \ No newline at end of file