diff --git a/ansible/host_vars/skippy.w5isp.com.yml b/ansible/host_vars/skippy.w5isp.com.yml index 6843cde..1202266 100644 --- a/ansible/host_vars/skippy.w5isp.com.yml +++ b/ansible/host_vars/skippy.w5isp.com.yml @@ -8,6 +8,14 @@ ansible_ssh_common_args: '-o StrictHostKeyChecking=accept-new' network: skip: true +nfs_exports: + - path: /data/srtm + clients: + - host: 10.0.0.0/16 + options: rw,sync,no_subtree_check,all_squash + - host: 204.110.191.1 + options: ro,sync,no_subtree_check,all_squash + firewall_allow_rules: - port: 80 proto: tcp @@ -15,3 +23,15 @@ firewall_allow_rules: - port: 443 proto: tcp comment: HTTPS + - port: 2049 + proto: tcp + from_ip: 10.0.0.0/16 + comment: NFS from internal + - port: 2049 + proto: tcp + from_ip: 204.110.188.0/22 + comment: NFS from VNTX + - port: 2049 + proto: tcp + from_ip: 100.64.0.0/10 + comment: NFS from Tailscale diff --git a/ansible/roles/firewall/tasks/main.yml b/ansible/roles/firewall/tasks/main.yml index d0a0a0c..1fc15db 100644 --- a/ansible/roles/firewall/tasks/main.yml +++ b/ansible/roles/firewall/tasks/main.yml @@ -130,6 +130,7 @@ rule: allow port: "{{ item.port | string }}" proto: "{{ item.proto }}" + from_ip: "{{ item.from_ip | default('any') }}" comment: "{{ item.comment }}" loop: "{{ firewall_allow_rules }}" when: diff --git a/ansible/roles/general/handlers/main.yml b/ansible/roles/general/handlers/main.yml new file mode 100644 index 0000000..2704cdf --- /dev/null +++ b/ansible/roles/general/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Re-export NFS shares + command: exportfs -ra diff --git a/ansible/roles/general/tasks/debian/nfs.yml b/ansible/roles/general/tasks/debian/nfs.yml new file mode 100644 index 0000000..88b45d2 --- /dev/null +++ b/ansible/roles/general/tasks/debian/nfs.yml @@ -0,0 +1,25 @@ +--- +- name: Install NFS server packages + apt: + name: nfs-kernel-server + state: present + update_cache: true + cache_valid_time: 3600 + when: nfs_exports is defined + +- name: Configure /etc/exports + template: + src: templates/etc/exports.j2 + dest: /etc/exports + owner: root + group: root + mode: "0644" + when: nfs_exports is defined + notify: Re-export NFS shares + +- name: Enable and start nfs-server + systemd: + name: nfs-server + enabled: true + state: started + when: nfs_exports is defined diff --git a/ansible/roles/general/tasks/main.yml b/ansible/roles/general/tasks/main.yml index 8544bb8..327503c 100644 --- a/ansible/roles/general/tasks/main.yml +++ b/ansible/roles/general/tasks/main.yml @@ -32,4 +32,9 @@ when: ansible_os_family == "Debian" tags: - icinga2 - - monitoring \ No newline at end of file + - monitoring + +- import_tasks: debian/nfs.yml + when: ansible_os_family == "Debian" + tags: + - nfs \ No newline at end of file diff --git a/ansible/roles/general/templates/etc/exports.j2 b/ansible/roles/general/templates/etc/exports.j2 new file mode 100644 index 0000000..468a8f4 --- /dev/null +++ b/ansible/roles/general/templates/etc/exports.j2 @@ -0,0 +1,6 @@ +# /etc/exports - managed by Ansible +# Do not edit manually; changes will be overwritten. +{% for export in nfs_exports %} +{{ export.path }} {% for client in export.clients %}{{ client.host }}({{ client.options }}) {% endfor %} + +{% endfor %} diff --git a/talos/CLAUDE.md b/talos/CLAUDE.md index 3f84e94..8b7f82f 100644 --- a/talos/CLAUDE.md +++ b/talos/CLAUDE.md @@ -6,7 +6,7 @@ This document contains notes on how the Talos home cluster was created and how t - **Cluster Name**: home-cluster - **Kubernetes Version**: v1.35.0 -- **Talos Version**: v1.12.1 +- **Talos Version**: v1.12.6 - **Control Plane Nodes**: 3 (10.0.15.1-3) - **Worker Nodes**: 3 (10.0.15.4-6) - **API Endpoint**: https://10.0.15.1:6443 @@ -245,6 +245,23 @@ kubectl get nodes 5. **Node Naming**: Kubernetes node names come from the Talos hostname configuration, not VM names in Proxmox. +6. **Recreating Individual VMs from Template**: The OpenTofu VM definitions have been removed. To recreate a single VM directly on Proxmox: + ```bash + ssh root@ "qm clone 9001 --name --full true --storage local-lvm" + ssh root@ "qm set --cpu host --balloon 0 --numa 0 --sockets 1 --net0 virtio=,bridge=vmbr0,firewall=0 --onboot 1 --agent enabled=1,fstrim_cloned_disks=0,type=virtio --vga std,memory=16" + ssh root@ "qm resize scsi0 64G" + ssh root@ "qm start " + # Wait for DHCP, then apply config: + talosctl --talosconfig talosconfig apply-config --nodes --file configs/.yaml --insecure + # Then upgrade to the factory image with extensions: + talosctl --talosconfig talosconfig --nodes upgrade --image factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:v1.12.6 + ``` + **Critical VM settings**: `cpu: host` is required — without it, Talos boots but never gets a network address. Also set `balloon: 0`, `firewall=0` on NIC. + +7. **Disk Dependencies Block Boot**: If a worker config references `/dev/sdb` for Longhorn but the Proxmox node has no passthrough NVMe disk, the `UserDiskConfigController` errors will prevent CRI and kubelet from starting. Comment out the `disks:` and `extraMounts:` sections for nodes without the extra disk. + +8. **Node1 Has No NVMe**: node1 (10.0.15.101) only has a single 238GB SSD. No NVMe passthrough disk is available for Longhorn storage on talos-worker1. + ## Troubleshooting ### Nodes won't accept new config @@ -277,9 +294,9 @@ talosctl --talosconfig talosconfig service --nodes 10.0.15.1 ## Storage -Nodes use Proxmox Ceph for VM disks. For Kubernetes persistent storage, deploy: -- Ceph CSI driver (to use Proxmox Ceph directly) -- Or Longhorn for cluster-managed storage +- **VM boot disks**: local-lvm on each Proxmox node (no ceph storage available) +- **Longhorn**: Workers 2 and 3 have passthrough NVMe disks (`/dev/sdb`) for Longhorn persistent storage. Worker 1 (node1) has no NVMe disk. +- **Longhorn StorageClass**: `longhorn` (default) ## Maintenance diff --git a/talos/configs/talos-worker1.yaml b/talos/configs/talos-worker1.yaml index 0d5a265..bd7e6f4 100644 --- a/talos/configs/talos-worker1.yaml +++ b/talos/configs/talos-worker1.yaml @@ -31,14 +31,14 @@ machine: # extraArgs: # key: value - extraMounts: - - destination: /var/mnt/longhorn - type: bind - source: /var/mnt/longhorn - options: - - bind - - rshared - - rw + # extraMounts: + # - destination: /var/mnt/longhorn + # type: bind + # source: /var/mnt/longhorn + # options: + # - bind + # - rshared + # - rw # # The `extraConfig` field is used to provide kubelet configuration overrides. # extraConfig: @@ -86,10 +86,10 @@ machine: # model: WDC* # Disk model `/sys/block//device/model`. # busPath: /pci0000:00/0000:00:17.0/ata1/host0/target0:0:0/0:0:0:0 # Disk bus path. # Extra disks to configure (Longhorn storage). - disks: - - device: /dev/sdb - partitions: - - mountpoint: /var/mnt/longhorn + # disks: + # - device: /dev/sdb + # partitions: + # - mountpoint: /var/mnt/longhorn registries: {} # Features describe individual Talos features that can be switched on or off. features: