graham/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

k3s deployment working

Browse source
This commit is contained in:
Graham McIntire 2025-07-24 16:29:33 -05:00
parent 718f1d2f90
commit 103810701f
No known key found for this signature in database
12 changed files with 588 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
ansible/K8S-DEPLOYMENT.md
View file

@ -1,10 +1,10 @@
# Kubernetes Cluster Deployment on Proxmox
# Kubernetes/K3s Cluster Deployment on Proxmox
This guide helps you deploy a 3-node Kubernetes cluster on Proxmox using Debian 12.
This guide helps you deploy either a full Kubernetes cluster or a lightweight K3s cluster on Proxmox using Debian 12.
## Prerequisites
1. Three Proxmox hosts (lab01, lab02, lab03)
1. Three Proxmox hosts (lab02, lab03, lab04)
2. Proxmox API access with root credentials
3. Network configured with:
- Gateway: 10.0.19.254
@ -158,12 +158,76 @@ Replace Flannel with Calico:
- Check node resources: `kubectl describe nodes`
- Verify CNI is working: `kubectl get pods -n kube-system`
## K3s Deployment (Recommended for Lightweight Clusters)
K3s is a certified Kubernetes distribution designed for production workloads in resource-constrained environments.
### K3s Architecture
- **k3s-server** (10.0.19.210): Control plane + etcd
- **k3s-agent1** (10.0.19.211): Worker node
- **k3s-agent2** (10.0.19.212): Worker node
All nodes run Debian 12 with:
- 2 CPU cores
- 2GB RAM (vs 4GB for full K8s)
- 20GB disk (vs 40GB for full K8s)
- Single binary (~100MB)
- Built-in SQLite (no external etcd)
### Deploy K3s Cluster
```bash
export PROXMOX_PASSWORD='your-proxmox-password'
export K3S_VM_PASSWORD='secure-password' # Optional, defaults to 'changeme'
./deploy-k3s-cluster.sh
```
Or manually:
```bash
ansible-playbook -i inventories/k3s/hosts k3s-cluster.yml \
-e proxmox_password='your-password' \
-e vm_password='secure-password'
```
### K3s Features
Out of the box, K3s includes:
- Flannel CNI for pod networking
- CoreDNS for cluster DNS
- Traefik ingress controller (disabled in our setup)
- Local-path storage provisioner
- Metrics server
- Helm controller
### Access K3s Cluster
```bash
# SSH to server
ssh debian@10.0.19.210
kubectl get nodes
# Copy kubeconfig locally
scp debian@10.0.19.210:~/.kube/config ~/.kube/config-k3s
export KUBECONFIG=~/.kube/config-k3s
```
## Cleanup
To remove the cluster:
### K8s Cluster
```bash
# Delete VMs from Proxmox
# Delete K8s VMs from Proxmox
for vmid in 200 201 202; do
ssh root@lab01.w5isp.com "qm stop $vmid && qm destroy $vmid"
ssh root@lab02.w5isp.com "qm stop $vmid && qm destroy $vmid"
done
```
### K3s Cluster
```bash
# Delete K3s VMs from Proxmox
for vmid in 300 301 302; do
ssh root@lab02.w5isp.com "qm stop $vmid && qm destroy $vmid"
done
```

39
ansible/create-vm-template.yml
View file

@ -9,9 +9,20 @@
template_vmid: 9000
template_name: debian12-cloud-init
debian_image_url: https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2
storage: local-lvm
storage: ceph
tasks:
- name: Check if template already exists
shell: |
qm status {{ template_vmid }} >/dev/null 2>&1 && echo "exists" || echo "missing"
register: template_check
changed_when: false
- name: Display template status
debug:
msg: "Template VM {{ template_vmid }} already exists. Skipping creation."
when: template_check.stdout == "exists"
- name: Install required packages
apt:
name:
@ -19,56 +30,78 @@
- libguestfs-tools
state: present
update_cache: yes
when: template_check.stdout == "missing"
- name: Download Debian cloud image
get_url:
url: "{{ debian_image_url }}"
dest: /var/lib/vz/template/iso/debian-12-genericcloud-amd64.qcow2
mode: '0644'
when: template_check.stdout == "missing"
- name: Create VM for template
shell: |
qm create {{ template_vmid }} --name {{ template_name }} --memory 2048 --cores 2 --net0 virtio,bridge=vmbr0
args:
creates: /etc/pve/qemu-server/{{ template_vmid }}.conf
when: template_check.stdout == "missing"
- name: Import disk image to VM
shell: |
qm importdisk {{ template_vmid }} /var/lib/vz/template/iso/debian-12-genericcloud-amd64.qcow2 {{ storage }}
register: import_result
changed_when: "'successfully imported' in import_result.stdout"
when: template_check.stdout == "missing"
- name: Get imported disk name
shell: |
pvesm list {{ storage }} | grep "vm-{{ template_vmid }}-disk-" | awk '{print $1}' | head -1
register: disk_name
when: template_check.stdout == "missing"
- name: Attach imported disk to VM
shell: |
qm set {{ template_vmid }} --scsihw virtio-scsi-pci --scsi0 {{ disk_name.stdout }}
when: disk_name.stdout != ""
when:
- template_check.stdout == "missing"
- disk_name.stdout != ""
- name: Check if cloud-init drive exists
shell: |
qm config {{ template_vmid }} | grep -q 'ide2:.*cloudinit' && echo "exists" || echo "missing"
register: cloudinit_check
changed_when: false
when: template_check.stdout == "missing"
- name: Add cloud-init drive
shell: |
qm set {{ template_vmid }} --ide2 {{ storage }}:cloudinit
when:
- template_check.stdout == "missing"
- cloudinit_check.stdout == "missing"
- name: Set boot disk
shell: |
qm set {{ template_vmid }} --boot c --bootdisk scsi0
when: template_check.stdout == "missing"
- name: Add serial console
shell: |
qm set {{ template_vmid }} --serial0 socket --vga serial0
when: template_check.stdout == "missing"
- name: Enable QEMU guest agent
shell: |
qm set {{ template_vmid }} --agent enabled=1
when: template_check.stdout == "missing"
- name: Convert VM to template
shell: |
qm template {{ template_vmid }}
when: template_check.stdout == "missing"
- name: Clean up downloaded image
file:
path: /var/lib/vz/template/iso/debian-12-genericcloud-amd64.qcow2
state: absent
state: absent
when: template_check.stdout == "missing"

75
ansible/deploy-k3s-cluster.sh Executable file
View file

@ -0,0 +1,75 @@
#!/bin/bash
# Deploy a 3-node K3s cluster on Proxmox using Debian 12
set -e
echo "=== K3s Cluster Deployment Script ==="
echo "This will create a lightweight 3-node K3s cluster on your Proxmox servers"
echo
echo "VMs to be created:"
echo " - k3s-server (10.0.19.210) - 2 CPU, 2GB RAM, 20GB disk"
echo " - k3s-agent1 (10.0.19.211) - 2 CPU, 2GB RAM, 20GB disk"
echo " - k3s-agent2 (10.0.19.212) - 2 CPU, 2GB RAM, 20GB disk"
echo
# Check if proxmox password is set
if [ -z "$PROXMOX_PASSWORD" ]; then
echo "Please set PROXMOX_PASSWORD environment variable"
echo "export PROXMOX_PASSWORD='your-proxmox-password'"
exit 1
fi
# Install required collections if needed
if ! ansible-galaxy collection list | grep -q community.general; then
echo "Installing required Ansible collections..."
ansible-galaxy collection install community.general
fi
# Check if template exists
echo "Checking if Debian 12 template exists..."
TEMPLATE_EXISTS=$(ansible -i inventories/k3s/hosts proxmox -m shell -a "qm list | grep -q debian12-cloud-init && echo yes || echo no" -e proxmox_password="$PROXMOX_PASSWORD" | grep -o "yes\|no" | head -1)
if [ "$TEMPLATE_EXISTS" != "yes" ]; then
echo "Debian 12 template not found. Please run create-vm-template.yml first."
read -p "Create template now? (y/n) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
ansible-playbook create-vm-template.yml -e proxmox_password="$PROXMOX_PASSWORD"
else
exit 1
fi
fi
# Deploy K3s cluster
echo
echo "Ready to deploy K3s cluster"
read -p "Continue? (y/n) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
echo "Creating VMs and installing K3s..."
ansible-playbook -i inventories/k3s/hosts k3s-cluster.yml \
-e proxmox_password="$PROXMOX_PASSWORD" \
-e vm_password="${K3S_VM_PASSWORD:-changeme}"
fi
echo
echo "=== Deployment Complete ==="
echo "To access your K3s cluster:"
echo " ssh debian@10.0.19.210"
echo " kubectl get nodes"
echo
echo "To copy kubeconfig to your local machine:"
echo " mkdir -p ~/.kube"
echo " scp debian@10.0.19.210:~/.kube/config ~/.kube/config-k3s"
echo " export KUBECONFIG=~/.kube/config-k3s"
echo
echo "K3s includes:"
echo " - Embedded SQLite (no external datastore needed)"
echo " - Flannel CNI for networking"
echo " - CoreDNS for cluster DNS"
echo " - Local-path provisioner for storage"
echo " - Metrics server"
echo
echo "To install additional components:"
echo " - Kubernetes Dashboard: kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml"
echo " - Cert-Manager: kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cert-manager.yaml"

3
ansible/host_vars/lab01.w5isp.com.yml
View file

@ -1,3 +0,0 @@
---
# Host-specific variables for lab01.w5isp.com
ansible_host: 10.0.16.231

2
ansible/host_vars/lab02.w5isp.com.yml
View file

@ -1,3 +1,3 @@
---
# Host-specific variables for lab02.w5isp.com
ansible_host: 10.0.16.232
ansible_host: 10.0.16.231

2
ansible/host_vars/lab03.w5isp.com.yml
View file

@ -1,3 +1,3 @@
---
# Host-specific variables for lab03.w5isp.com
ansible_host: 10.0.16.233
ansible_host: 10.0.16.232

3
ansible/host_vars/lab04.w5isp.com.yml Normal file
View file

@ -0,0 +1,3 @@
---
# Host-specific variables for lab04.w5isp.com
ansible_host: 10.0.16.233

4
ansible/hosts
View file

@ -20,14 +20,14 @@ skippy.w5isp.com
g.w5isp.com
[proxmox]
lab01.w5isp.com
lab02.w5isp.com
lab03.w5isp.com
lab04.w5isp.com
[tailscale_home]
lab01.w5isp.com
lab02.w5isp.com
lab03.w5isp.com
lab04.w5isp.com
g.w5isp.com
[caddy_servers]

23
ansible/inventories/k3s/hosts Normal file
View file

@ -0,0 +1,23 @@
# K3s cluster inventory
[proxmox]
lab02.w5isp.com
lab03.w5isp.com
lab04.w5isp.com
[proxmox:vars]
ansible_python_interpreter=/usr/bin/python3
[k3s_server]
k3s-server ansible_host=10.0.19.210 ansible_user=ansible
[k3s_agents]
k3s-agent1 ansible_host=10.0.19.211 ansible_user=ansible
k3s-agent2 ansible_host=10.0.19.212 ansible_user=ansible
[k3s_cluster:children]
k3s_server
k3s_agents
[k3s_cluster:vars]
ansible_python_interpreter=/usr/bin/python3

2
ansible/inventories/k8s-cluster/hosts
View file

@ -1,9 +1,9 @@
# Kubernetes cluster inventory
[proxmox]
lab01.w5isp.com
lab02.w5isp.com
lab03.w5isp.com
lab04.w5isp.com
[k8s_master]
k8s-master ansible_host=10.0.19.200 ansible_user=debian

351
ansible/k3s-cluster.yml Normal file
View file

@ -0,0 +1,351 @@
---
# Deploy K3s cluster on Proxmox
# This playbook creates VMs and installs K3s
- name: Create K3s VMs on Proxmox
hosts: proxmox
vars:
k3s_vms:
- name: k3s-server
vmid: 300
node: lab02
cores: 4
memory: 8192
disk: 64
ip: 10.0.19.210/22
gateway: 10.0.19.254
- name: k3s-agent1
vmid: 301
node: lab03
cores: 4
memory: 8192
disk: 64
ip: 10.0.19.211/22
gateway: 10.0.19.254
- name: k3s-agent2
vmid: 302
node: lab04
cores: 4
memory: 8192
disk: 64
ip: 10.0.19.212/22
gateway: 10.0.19.254
template_vmid: 9000
proxmox_api_host: "{{ ansible_host | default(inventory_hostname) }}"
proxmox_api_user: root@pam
proxmox_api_password: "{{ proxmox_password }}"
vm_user: ansible
vm_password: "{{ lookup('password', '/dev/null length=16 chars=ascii_letters,digits') }}"
ssh_key_path: "~/.ssh/id_ed25519.pub"
tasks:
- name: Clone VMs from template on lab02
community.general.proxmox_kvm:
api_host: "{{ proxmox_api_host }}"
api_user: "{{ proxmox_api_user }}"
api_password: "{{ proxmox_api_password }}"
validate_certs: false
clone: "debian12-cloud-init"
newid: "{{ item.vmid }}"
name: "{{ item.name }}"
node: "lab02"
storage: "ceph"
full: true
timeout: 600
loop: "{{ k3s_vms }}"
when: inventory_hostname == 'lab02.w5isp.com'
delegate_to: localhost
- name: Wait for clones to settle
ansible.builtin.pause:
seconds: 10
when: inventory_hostname == 'lab02.w5isp.com'
- name: Check current VM location
shell: |
qm config {{ item.vmid }} >/dev/null 2>&1 && echo "local" || echo "remote"
register: vm_location
loop: "{{ k3s_vms }}"
when: inventory_hostname == 'lab02.w5isp.com'
changed_when: false
- name: Migrate VMs to target nodes
shell: |
pvesh create /nodes/lab02/qemu/{{ item.0.vmid }}/migrate \
-target {{ item.0.node }} \
-online 0
loop: "{{ k3s_vms | zip(vm_location.results) | list }}"
when:
- inventory_hostname == 'lab02.w5isp.com'
- item.0.node != 'lab02'
- item.1.stdout == 'local'
- name: Wait for migrations to complete
ansible.builtin.pause:
seconds: 30
when: inventory_hostname == 'lab02.w5isp.com'
- name: Resize VM disk
community.general.proxmox_disk:
api_host: "{{ proxmox_api_host }}"
api_user: "{{ proxmox_api_user }}"
api_password: "{{ proxmox_api_password }}"
validate_certs: false
vmid: "{{ item.vmid }}"
disk: scsi0
size: "{{ item.disk }}G"
state: resized
loop: "{{ k3s_vms }}"
when: inventory_hostname == 'lab02.w5isp.com'
delegate_to: localhost
- name: Configure VM resources
community.general.proxmox_kvm:
api_host: "{{ proxmox_api_host }}"
api_user: "{{ proxmox_api_user }}"
api_password: "{{ proxmox_api_password }}"
validate_certs: false
vmid: "{{ item.vmid }}"
node: "{{ item.node }}"
cores: "{{ item.cores }}"
memory: "{{ item.memory }}"
onboot: true
update: true
loop: "{{ k3s_vms }}"
when: inventory_hostname == 'lab02.w5isp.com'
delegate_to: localhost
- name: Configure cloud-init
community.general.proxmox_kvm:
api_host: "{{ proxmox_api_host }}"
api_user: "{{ proxmox_api_user }}"
api_password: "{{ proxmox_api_password }}"
validate_certs: false
vmid: "{{ item.vmid }}"
node: "{{ item.node }}"
ipconfig:
ipconfig0: "ip={{ item.ip }},gw={{ item.gateway }}"
nameservers: "9.9.9.9"
ciuser: "{{ vm_user }}"
cipassword: "{{ vm_password }}"
sshkeys: "{{ lookup('file', ssh_key_path) }}"
update: true
loop: "{{ k3s_vms }}"
when: inventory_hostname == 'lab02.w5isp.com'
delegate_to: localhost
- name: Start VMs
community.general.proxmox_kvm:
api_host: "{{ proxmox_api_host }}"
api_user: "{{ proxmox_api_user }}"
api_password: "{{ proxmox_api_password }}"
validate_certs: false
vmid: "{{ item.vmid }}"
node: "{{ item.node }}"
state: started
loop: "{{ k3s_vms }}"
when: inventory_hostname == 'lab02.w5isp.com'
delegate_to: localhost
- name: Wait for VMs to be reachable
ansible.builtin.wait_for:
host: "{{ item.ip.split('/')[0] }}"
port: 22
delay: 30
timeout: 300
loop: "{{ k3s_vms }}"
when: inventory_hostname == 'lab02.w5isp.com'
delegate_to: localhost
- name: Bootstrap K3s nodes
hosts: k3s_cluster
become: true
gather_facts: false
tasks:
- name: Wait for system to be ready
ansible.builtin.wait_for_connection:
delay: 10
timeout: 300
- name: Gather facts
ansible.builtin.setup:
- name: Update apt cache
ansible.builtin.apt:
update_cache: true
cache_valid_time: 3600
- name: Install required packages
ansible.builtin.apt:
name:
- curl
- ca-certificates
- apt-transport-https
- sudo
state: present
- name: Create graham user
ansible.builtin.user:
name: graham
groups: sudo
shell: /bin/bash
create_home: true
state: present
- name: Set up SSH key for graham user
ansible.posix.authorized_key:
user: graham
key: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}"
state: present
- name: Configure sudo for graham user
ansible.builtin.lineinfile:
path: /etc/sudoers.d/graham
line: 'graham ALL=(ALL) NOPASSWD:ALL'
create: true
validate: 'visudo -cf %s'
- name: Configure DNS to use Quad9
ansible.builtin.copy:
content: |
nameserver 9.9.9.9
nameserver 149.112.112.112
dest: /etc/resolv.conf
owner: root
group: root
mode: '0644'
- name: Prevent resolv.conf from being overwritten
ansible.builtin.file:
path: /etc/resolv.conf
attr: +i
- name: Check if Tailscale service exists
ansible.builtin.stat:
path: /etc/systemd/system/tailscaled.service
register: tailscale_service
- name: Stop and disable Tailscale if present
ansible.builtin.systemd:
name: tailscaled
state: stopped
enabled: false
when: tailscale_service.stat.exists
- name: Install K3s on server
hosts: k3s_server
become: true
vars:
k3s_version: v1.28.5+k3s1
k3s_server_args: "--disable traefik --write-kubeconfig-mode 644"
tasks:
- name: Download K3s installer
ansible.builtin.get_url:
url: https://get.k3s.io
dest: /tmp/k3s-install.sh
mode: '0755'
- name: Install K3s server
ansible.builtin.shell: |
INSTALL_K3S_VERSION="{{ k3s_version }}" \
INSTALL_K3S_EXEC="{{ k3s_server_args }}" \
sh /tmp/k3s-install.sh
args:
creates: /usr/local/bin/k3s
- name: Wait for K3s to be ready
ansible.builtin.wait_for:
path: /var/lib/rancher/k3s/server/node-token
state: present
timeout: 60
- name: Get node token
ansible.builtin.slurp:
src: /var/lib/rancher/k3s/server/node-token
register: node_token
- name: Store token for agents
ansible.builtin.set_fact:
k3s_token: "{{ node_token.content | b64decode | trim }}"
- name: Create .kube directory
ansible.builtin.file:
path: /home/{{ ansible_user }}/.kube
state: directory
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: '0755'
- name: Copy kubeconfig
ansible.builtin.copy:
src: /etc/rancher/k3s/k3s.yaml
dest: /home/{{ ansible_user }}/.kube/config
remote_src: true
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: '0600'
- name: Update kubeconfig server address
ansible.builtin.replace:
path: /home/{{ ansible_user }}/.kube/config
regexp: 'https://127.0.0.1:6443'
replace: 'https://{{ ansible_host }}:6443'
- name: Install K3s on agents
hosts: k3s_agents
become: true
vars:
k3s_version: v1.28.5+k3s1
k3s_server_host: "{{ hostvars[groups['k3s_server'][0]]['ansible_host'] }}"
k3s_token: "{{ hostvars[groups['k3s_server'][0]]['k3s_token'] }}"
tasks:
- name: Download K3s installer
ansible.builtin.get_url:
url: https://get.k3s.io
dest: /tmp/k3s-install.sh
mode: '0755'
- name: Install K3s agent
ansible.builtin.shell: |
INSTALL_K3S_VERSION="{{ k3s_version }}" \
K3S_URL="https://{{ k3s_server_host }}:6443" \
K3S_TOKEN="{{ k3s_token }}" \
sh /tmp/k3s-install.sh
args:
creates: /usr/local/bin/k3s
- name: Configure K3s cluster
hosts: k3s_server
become: true
tasks:
- name: Install Helm
ansible.builtin.shell: |
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
args:
creates: /usr/local/bin/helm
- name: Wait for all nodes to be ready
ansible.builtin.shell: |
kubectl get nodes --no-headers | grep -v Ready || true
register: not_ready_nodes
until: not_ready_nodes.stdout == ""
retries: 30
delay: 10
become_user: "{{ ansible_user }}"
- name: Label worker nodes
ansible.builtin.shell: |
kubectl label node {{ item }} node-role.kubernetes.io/worker=worker --overwrite
loop:
- k3s-agent1
- k3s-agent2
become_user: "{{ ansible_user }}"
- name: Show cluster status
ansible.builtin.shell: kubectl get nodes
register: nodes_status
become_user: "{{ ansible_user }}"
- name: Display cluster status
ansible.builtin.debug:
msg: "{{ nodes_status.stdout_lines }}"

26
ansible/migrate-template-to-ceph.yml Normal file
View file

@ -0,0 +1,26 @@
---
# Migrate template from local-lvm to ceph storage
- name: Migrate template to Ceph storage
hosts: proxmox[0]
become: yes
vars:
template_vmid: 9000
tasks:
- name: Check if template exists
shell: |
qm status {{ template_vmid }} >/dev/null 2>&1 && echo "exists" || echo "missing"
register: template_check
changed_when: false
- name: Stop and destroy existing template
shell: |
qm destroy {{ template_vmid }} --purge
when: template_check.stdout == "exists"
- name: Recreate template on Ceph
ansible.builtin.include_tasks:
file: create-vm-template.yml
apply:
delegate_to: "{{ inventory_hostname }}"