From 11c06b13df0874584208d1a48ef127536d88d101 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 19 Feb 2026 13:24:35 -0600 Subject: [PATCH] Add Longhorn storage to Talos worker nodes Configure passthrough NVMe drives (/dev/sdb) on each worker with XFS partition mounted at /var/mnt/longhorn. Switch worker installer image to Talos Image Factory build with iscsi-tools and util-linux-tools extensions required by Longhorn. Remove conflicting grubUseUKICmdline setting. --- CLAUDE.md | 19 ++++++++++++++++--- talos/configs/talos-worker1.yaml | 26 ++++++++++++++------------ talos/configs/talos-worker2.yaml | 26 ++++++++++++++------------ talos/configs/talos-worker3.yaml | 26 ++++++++++++++------------ talos/worker.yaml | 26 ++++++++++++++------------ 5 files changed, 72 insertions(+), 51 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 874e1a1..54dd115 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -183,16 +183,29 @@ tofu plan - **Worker Nodes**: 3 (10.0.15.4-6) - **API Endpoint**: https://10.0.15.1:6443 - **Proxmox Cluster**: "home" (node1, node2, node3 at 10.0.15.101-103) -- **Storage**: Proxmox Ceph for VM disks +- **VM Disks**: Proxmox Ceph for VM boot disks +- **Persistent Storage**: Longhorn on passthrough NVMe drives (`/dev/sdb` on each worker, mounted at `/var/mnt/longhorn`) + - Worker1: 2TB, Worker2: 2TB, Worker3: 1TB + - `longhorn` StorageClass is the default + - `longhorn-system` namespace requires privileged PodSecurity policy - **CNI**: Flannel +- **Talos Extensions** (workers only): + - `iscsi-tools` v0.2.0 — required by Longhorn + - `util-linux-tools` 2.41.2 — required by Longhorn + - Installer image: `factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:v1.12.4` + - Control plane nodes use the stock `ghcr.io/siderolabs/installer:v1.12.4` - **Deployment**: Managed via OpenTofu in `/tofu/` - **Configuration**: `/talos/` directory - `talosconfig` - Talosctl client config - `secrets.yaml` - Cluster secrets - - `controlplane.yaml` / `worker.yaml` - Node configs + - `controlplane.yaml` / `worker.yaml` - Base node configs + - `configs/talos-worker{1,2,3}.yaml` - Per-node worker configs (with disk/extension settings) + - `configs/talos-cp{1,2,3}.yaml` - Per-node control plane configs - **Applications**: `/cluster/` directory contains Kubernetes manifests - - `ceph-csi/` - Ceph CSI RBD driver for persistent storage (default StorageClass) - **Documentation**: See `/talos/README.md` and `/cluster/README.md` for detailed operations +- **Config Notes**: + - `install.extraKernelArgs` and `install.grubUseUKICmdline` cannot be used together + - Worker disks with existing data (e.g. old Ceph LVM) must be wiped before Talos will partition them: `talosctl reset --user-disks-to-wipe /dev/sdb --wipe-mode user-disks --reboot` ### K3s Cluster at 204.110.191.2 - **Purpose**: Home lab Kubernetes cluster diff --git a/talos/configs/talos-worker1.yaml b/talos/configs/talos-worker1.yaml index 087f18d..0d5a265 100644 --- a/talos/configs/talos-worker1.yaml +++ b/talos/configs/talos-worker1.yaml @@ -31,16 +31,14 @@ machine: # extraArgs: # key: value - # # The `extraMounts` field is used to add additional mounts to the kubelet container. - # extraMounts: - # - destination: /var/lib/example # Destination is the absolute path where the mount will be placed in the container. - # type: bind # Type specifies the mount kind. - # source: /var/lib/example # Source specifies the source path of the mount. - # # Options are fstab style mount options. - # options: - # - bind - # - rshared - # - rw + extraMounts: + - destination: /var/mnt/longhorn + type: bind + source: /var/mnt/longhorn + options: + - bind + - rshared + - rw # # The `extraConfig` field is used to provide kubelet configuration overrides. # extraConfig: @@ -77,9 +75,8 @@ machine: # Used to provide instructions for installations. install: disk: /dev/sda # The disk used for installations. - image: ghcr.io/siderolabs/installer:v1.12.4 # Allows for supplying the image used to perform the installation. + image: factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:v1.12.4 # Talos installer with iscsi-tools and util-linux-tools extensions for Longhorn. wipe: false # Indicates if the installation disk should be wiped at installation time. - grubUseUKICmdline: true # Indicates if legacy GRUB bootloader should use kernel cmdline from the UKI instead of building it on the host. extraKernelArgs: - console=tty0 @@ -88,6 +85,11 @@ machine: # size: 4GB # Disk size. # model: WDC* # Disk model `/sys/block//device/model`. # busPath: /pci0000:00/0000:00:17.0/ata1/host0/target0:0:0/0:0:0:0 # Disk bus path. + # Extra disks to configure (Longhorn storage). + disks: + - device: /dev/sdb + partitions: + - mountpoint: /var/mnt/longhorn registries: {} # Features describe individual Talos features that can be switched on or off. features: diff --git a/talos/configs/talos-worker2.yaml b/talos/configs/talos-worker2.yaml index dbfaed5..f86809a 100644 --- a/talos/configs/talos-worker2.yaml +++ b/talos/configs/talos-worker2.yaml @@ -31,16 +31,14 @@ machine: # extraArgs: # key: value - # # The `extraMounts` field is used to add additional mounts to the kubelet container. - # extraMounts: - # - destination: /var/lib/example # Destination is the absolute path where the mount will be placed in the container. - # type: bind # Type specifies the mount kind. - # source: /var/lib/example # Source specifies the source path of the mount. - # # Options are fstab style mount options. - # options: - # - bind - # - rshared - # - rw + extraMounts: + - destination: /var/mnt/longhorn + type: bind + source: /var/mnt/longhorn + options: + - bind + - rshared + - rw # # The `extraConfig` field is used to provide kubelet configuration overrides. # extraConfig: @@ -77,9 +75,8 @@ machine: # Used to provide instructions for installations. install: disk: /dev/sda # The disk used for installations. - image: ghcr.io/siderolabs/installer:v1.12.4 # Allows for supplying the image used to perform the installation. + image: factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:v1.12.4 # Talos installer with iscsi-tools and util-linux-tools extensions for Longhorn. wipe: false # Indicates if the installation disk should be wiped at installation time. - grubUseUKICmdline: true # Indicates if legacy GRUB bootloader should use kernel cmdline from the UKI instead of building it on the host. extraKernelArgs: - console=tty0 @@ -88,6 +85,11 @@ machine: # size: 4GB # Disk size. # model: WDC* # Disk model `/sys/block//device/model`. # busPath: /pci0000:00/0000:00:17.0/ata1/host0/target0:0:0/0:0:0:0 # Disk bus path. + # Extra disks to configure (Longhorn storage). + disks: + - device: /dev/sdb + partitions: + - mountpoint: /var/mnt/longhorn registries: {} # Features describe individual Talos features that can be switched on or off. features: diff --git a/talos/configs/talos-worker3.yaml b/talos/configs/talos-worker3.yaml index 13d472b..afbab23 100644 --- a/talos/configs/talos-worker3.yaml +++ b/talos/configs/talos-worker3.yaml @@ -31,16 +31,14 @@ machine: # extraArgs: # key: value - # # The `extraMounts` field is used to add additional mounts to the kubelet container. - # extraMounts: - # - destination: /var/lib/example # Destination is the absolute path where the mount will be placed in the container. - # type: bind # Type specifies the mount kind. - # source: /var/lib/example # Source specifies the source path of the mount. - # # Options are fstab style mount options. - # options: - # - bind - # - rshared - # - rw + extraMounts: + - destination: /var/mnt/longhorn + type: bind + source: /var/mnt/longhorn + options: + - bind + - rshared + - rw # # The `extraConfig` field is used to provide kubelet configuration overrides. # extraConfig: @@ -77,9 +75,8 @@ machine: # Used to provide instructions for installations. install: disk: /dev/sda # The disk used for installations. - image: ghcr.io/siderolabs/installer:v1.12.4 # Allows for supplying the image used to perform the installation. + image: factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:v1.12.4 # Talos installer with iscsi-tools and util-linux-tools extensions for Longhorn. wipe: false # Indicates if the installation disk should be wiped at installation time. - grubUseUKICmdline: true # Indicates if legacy GRUB bootloader should use kernel cmdline from the UKI instead of building it on the host. extraKernelArgs: - console=tty0 @@ -88,6 +85,11 @@ machine: # size: 4GB # Disk size. # model: WDC* # Disk model `/sys/block//device/model`. # busPath: /pci0000:00/0000:00:17.0/ata1/host0/target0:0:0/0:0:0:0 # Disk bus path. + # Extra disks to configure (Longhorn storage). + disks: + - device: /dev/sdb + partitions: + - mountpoint: /var/mnt/longhorn registries: {} # Features describe individual Talos features that can be switched on or off. features: diff --git a/talos/worker.yaml b/talos/worker.yaml index 9ced197..dca05cb 100644 --- a/talos/worker.yaml +++ b/talos/worker.yaml @@ -31,16 +31,14 @@ machine: # extraArgs: # key: value - # # The `extraMounts` field is used to add additional mounts to the kubelet container. - # extraMounts: - # - destination: /var/lib/example # Destination is the absolute path where the mount will be placed in the container. - # type: bind # Type specifies the mount kind. - # source: /var/lib/example # Source specifies the source path of the mount. - # # Options are fstab style mount options. - # options: - # - bind - # - rshared - # - rw + extraMounts: + - destination: /var/mnt/longhorn + type: bind + source: /var/mnt/longhorn + options: + - bind + - rshared + - rw # # The `extraConfig` field is used to provide kubelet configuration overrides. # extraConfig: @@ -81,15 +79,19 @@ machine: # Used to provide instructions for installations. install: disk: /dev/sda # The disk used for installations. - image: ghcr.io/siderolabs/installer:v1.12.4 # Allows for supplying the image used to perform the installation. + image: factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:v1.12.4 # Talos installer with iscsi-tools and util-linux-tools extensions for Longhorn. wipe: false # Indicates if the installation disk should be wiped at installation time. - grubUseUKICmdline: true # Indicates if legacy GRUB bootloader should use kernel cmdline from the UKI instead of building it on the host. # # Look up disk using disk attributes like model, size, serial and others. # diskSelector: # size: 4GB # Disk size. # model: WDC* # Disk model `/sys/block//device/model`. # busPath: /pci0000:00/0000:00:17.0/ata1/host0/target0:0:0/0:0:0:0 # Disk bus path. + # Extra disks to configure (Longhorn storage). + disks: + - device: /dev/sdb + partitions: + - mountpoint: /var/mnt/longhorn registries: {} # Features describe individual Talos features that can be switched on or off. features: