From 63968031db98694b6996ce1725994a12a3fe1f1b Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 19 Oct 2025 14:22:39 -0500 Subject: [PATCH] update for latest k8s deployment --- CLAUDE.md | 31 ++- .../aprs-statefulset-with-pullsecret.yaml | 22 +- home/cluster/aprs/postgres-deployment.yaml | 19 +- home/cluster/controlplane.yaml | 9 + home/cluster/deploy.md | 195 ++++++++++++++++++ home/cluster/high-availability-setup.md | 44 ---- home/cluster/longhorn-uservolume.yaml | 8 + home/cluster/metallb-config.yaml | 9 +- home/cluster/n8n/n8n-deployment.yaml | 2 +- home/cluster/nodered/AUTH.md | 53 ----- home/cluster/nodered/deploy.sh | 3 + home/cluster/nodered/deployment-local.yaml | 38 ---- home/cluster/nodered/deployment-simple.yaml | 30 --- home/cluster/nodered/deployment.yaml | 20 +- home/cluster/nodered/generate-password.yaml | 17 -- home/cluster/nodered/ingress.yaml | 72 +++++-- home/cluster/nodered/settings-configmap.yaml | 2 +- home/cluster/update-k3s-cluster.md | 63 ------ home/cluster/update-k3s-node.sh | 64 ------ home/terraform/dns_w5isp.tf | 8 + 20 files changed, 359 insertions(+), 350 deletions(-) delete mode 100644 home/cluster/high-availability-setup.md create mode 100644 home/cluster/longhorn-uservolume.yaml delete mode 100644 home/cluster/nodered/AUTH.md delete mode 100644 home/cluster/nodered/deployment-local.yaml delete mode 100644 home/cluster/nodered/deployment-simple.yaml delete mode 100644 home/cluster/nodered/generate-password.yaml delete mode 100644 home/cluster/update-k3s-cluster.md delete mode 100755 home/cluster/update-k3s-node.sh diff --git a/CLAUDE.md b/CLAUDE.md index 9f2593f..73c2b44 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -190,4 +190,33 @@ tofu plan - As you learn new things, keep claude.md updated ## Clusters -- k3s cluster at 204.110.191.2 \ No newline at end of file +- k3s cluster at 204.110.191.2 + +## Application Deployment Patterns + +### Standard App Deployment Process +1. **Check for existing manifests** in `/home/cluster/[app-name]/` +2. **Set namespace security policy**: `kubectl label namespace [app] pod-security.kubernetes.io/enforce=privileged --overwrite` +3. **Use ClusterIP services** (internal only) with Traefik for external access +4. **Deploy via existing deploy.sh** scripts when available +5. **Use Longhorn storage** for persistent volumes +6. **Configure Traefik IngressRoute** for SSL and routing + +### Deployed Applications +- **APRS.me**: https://aprs.me (PostgreSQL + Redis, 2-replica StatefulSet) +- **Node-RED**: https://nodered.w5isp.com (automation platform) +- **n8n**: https://n8n.w5isp.com (workflow automation, PostgreSQL backend) +- **Wavelog**: https://log.w5isp.com (amateur radio logging, MariaDB backend) + +### Common Issues & Solutions +- **PodSecurity violations**: Set namespace to privileged security policy +- **Database connectivity**: Use short service names (`postgres` not `postgres.namespace.svc.cluster.local`) +- **Init container permissions**: Required for volume ownership fixes +- **SSL certificates**: cert-manager with Let's Encrypt, handled automatically via Traefik +- **Service DNS**: Services accessible via `service-name` within same namespace + +### Traefik Configuration +- **Host-based routing**: Multiple apps on same IP (204.110.191.2) +- **Automatic SSL**: cert-manager integration with Let's Encrypt +- **HTTP to HTTPS redirect**: Configured in IngressRoute manifests +- **ClusterIP backend**: All apps use ClusterIP, Traefik provides external access \ No newline at end of file diff --git a/home/cluster/aprs/aprs-statefulset-with-pullsecret.yaml b/home/cluster/aprs/aprs-statefulset-with-pullsecret.yaml index 4aaa945..c0d80cc 100644 --- a/home/cluster/aprs/aprs-statefulset-with-pullsecret.yaml +++ b/home/cluster/aprs/aprs-statefulset-with-pullsecret.yaml @@ -19,7 +19,7 @@ metadata: name: aprs namespace: aprs spec: - type: LoadBalancer + type: ClusterIP selector: app: aprs ports: @@ -65,6 +65,11 @@ spec: - ALL seccompProfile: type: RuntimeDefault + volumes: + - name: tmp-volume + emptyDir: {} + - name: app-volume + emptyDir: {} containers: - name: aprs image: ghcr.io/aprsme/aprs.me:latest @@ -108,13 +113,20 @@ spec: limits: memory: "1Gi" cpu: "500m" + volumeMounts: + - name: tmp-volume + mountPath: /tmp + - name: app-volume + mountPath: /app/tmp securityContext: allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 capabilities: drop: - ALL seccompProfile: - type: RuntimeDefault \ No newline at end of file + type: RuntimeDefault + volumes: + - name: tmp-volume + emptyDir: {} + - name: app-volume + emptyDir: {} \ No newline at end of file diff --git a/home/cluster/aprs/postgres-deployment.yaml b/home/cluster/aprs/postgres-deployment.yaml index ada12ba..ecb6134 100644 --- a/home/cluster/aprs/postgres-deployment.yaml +++ b/home/cluster/aprs/postgres-deployment.yaml @@ -10,7 +10,7 @@ spec: ports: - port: 5432 targetPort: 5432 - clusterIP: None + type: ClusterIP --- apiVersion: apps/v1 kind: StatefulSet @@ -29,6 +29,9 @@ spec: app: postgres-aprs spec: terminationGracePeriodSeconds: 600 # 10 minutes to shutdown gracefully + securityContext: + fsGroup: 999 + fsGroupChangePolicy: OnRootMismatch containers: - name: postgres image: postgis/postgis:17-3.4 @@ -59,20 +62,20 @@ spec: - /bin/sh - -c - pg_isready -U $POSTGRES_USER -d $POSTGRES_DB - initialDelaySeconds: 300 # 5 minutes to start + initialDelaySeconds: 60 # 1 minute to start periodSeconds: 30 # Check every 30s timeoutSeconds: 10 # 10s timeout for probe - failureThreshold: 20 # Allow 20 failures (10 minutes) + failureThreshold: 3 # Allow 3 failures readinessProbe: exec: command: - /bin/sh - -c - pg_isready -U $POSTGRES_USER -d $POSTGRES_DB - initialDelaySeconds: 300 # 5 minutes to start - periodSeconds: 30 # Check every 30s - timeoutSeconds: 10 # 10s timeout for probe - failureThreshold: 20 # Allow 20 failures (10 minutes) + initialDelaySeconds: 30 # 30 seconds to start + periodSeconds: 10 # Check every 10s + timeoutSeconds: 5 # 5s timeout for probe + failureThreshold: 3 # Allow 3 failures volumeClaimTemplates: - metadata: name: postgres-storage @@ -81,4 +84,4 @@ spec: storageClassName: longhorn resources: requests: - storage: 100Gi \ No newline at end of file + storage: 256Gi \ No newline at end of file diff --git a/home/cluster/controlplane.yaml b/home/cluster/controlplane.yaml index 58e641f..3de067f 100644 --- a/home/cluster/controlplane.yaml +++ b/home/cluster/controlplane.yaml @@ -21,6 +21,15 @@ machine: image: ghcr.io/siderolabs/kubelet:v1.34.0 # The `image` field is an optional reference to an alternative kubelet image. defaultRuntimeSeccompProfileEnabled: true # Enable container runtime default Seccomp profile. disableManifestsDirectory: true # The `disableManifestsDirectory` field configures the kubelet to get static pod manifests from the /etc/kubernetes/manifests directory. + # Extra mounts for Longhorn storage + extraMounts: + - destination: /var/mnt/longhorn + type: bind + source: /var/mnt/longhorn + options: + - bind + - rshared + - rw # # The `ClusterDNS` field is an optional reference to an alternative kubelet clusterDNS ip list. # clusterDNS: diff --git a/home/cluster/deploy.md b/home/cluster/deploy.md index bcab384..88fcbad 100644 --- a/home/cluster/deploy.md +++ b/home/cluster/deploy.md @@ -8,6 +8,176 @@ Fresh Talos cluster has been set up. Document deployment commands below as they ## Deployment Commands Log +### Latest APRS Deployment (2025-10-18) + +```bash +# Generate kubeconfig from Talos +talosctl --talosconfig=./talosconfig -e 10.0.101.21 --nodes 10.0.101.21 kubeconfig + +# Force re-deploy APRS.me with new GitHub pull token +cd /Users/graham/dev/infra/home/cluster/aprs +./deploy.sh + +# Check pod status +kubectl get pods -n aprs -o wide + +# Force restart of APRS StatefulSet to pull new image with GitHub token +kubectl delete pod aprs-0 -n aprs +kubectl rollout restart statefulset/aprs -n aprs + +# Check persistent volume claim status +kubectl get pvc -n aprs +kubectl get storageclass + +# Check cluster nodes +kubectl get nodes -o wide +``` + +**Status**: Image pull now working with GitHub token, but PostgreSQL stuck due to missing storage class + +### Storage Configuration (2025-10-18) + +```bash +# Initially tried Longhorn but had iscsi-tools issues in Talos +kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.7.2/deploy/longhorn.yaml +kubectl label namespace longhorn-system pod-security.kubernetes.io/enforce=privileged --overwrite +kubectl rollout restart ds/longhorn-manager -n longhorn-system + +# Installed local-path-provisioner as alternative +kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.28/deploy/local-path-storage.yaml +kubectl label namespace local-path-storage pod-security.kubernetes.io/enforce=privileged --overwrite +kubectl rollout restart deployment/local-path-provisioner -n local-path-storage + +# Updated PostgreSQL to use local-path with 256GB storage +# Modified postgres-deployment.yaml: storageClassName: local-path, storage: 256Gi +kubectl delete pvc postgres-storage-postgres-aprs-0 -n aprs +kubectl delete statefulset postgres-aprs -n aprs +kubectl apply -f /Users/graham/dev/infra/home/cluster/aprs/postgres-deployment.yaml + +# Verify storage configuration +kubectl get pvc -n aprs +kubectl get storageclass +``` + +**Status**: PostgreSQL with PostGIS now configured with 256GB local-path storage, APRS.me deploying with GitHub token + +### Longhorn Storage Final Configuration (2025-10-18) + +```bash +# Updated controlplane.yaml to add kubelet extraMounts for Longhorn +# Added: +# extraMounts: +# - destination: /var/mnt/longhorn +# type: bind +# source: /var/mnt/longhorn +# options: [bind, rshared, rw] + +# Applied updated configuration to both nodes +talosctl apply-config --talosconfig=./talosconfig -e 10.0.101.21 --nodes 10.0.101.21 --file controlplane.yaml --config-patch @node1.patch.yaml --config-patch @tailscale.patch.yaml +talosctl apply-config --talosconfig=./talosconfig -e 10.0.101.22 --nodes 10.0.101.22 --file controlplane.yaml --config-patch @node2.patch.yaml --config-patch @tailscale.patch.yaml + +# Generated custom Talos image with iscsi-tools extension +curl -X POST --data-binary @schematic.yaml https://factory.talos.dev/schematics +# Returned schematic ID: 077514df2c1b6436460bc60faabc976687b16193b8a1290fda4366c69024fec2 + +# Upgraded both nodes with custom image including iscsi-tools +talosctl upgrade --talosconfig=./talosconfig -e 10.0.101.21 --nodes 10.0.101.21 --image factory.talos.dev/metal-installer/708747e350d604ae9e57227d8dcf274091453ddb1097b765d4ea8884f1992c1f:v1.11.3 --preserve --force +talosctl upgrade --talosconfig=./talosconfig -e 10.0.101.22 --nodes 10.0.101.22 --image factory.talos.dev/metal-installer/708747e350d604ae9e57227d8dcf274091453ddb1097b765d4ea8884f1992c1f:v1.11.3 --preserve --force + +# Reinstalled Longhorn after node upgrade +kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.7.2/deploy/longhorn.yaml +kubectl label namespace longhorn-system pod-security.kubernetes.io/enforce=privileged --overwrite + +# Configured PostgreSQL to use Longhorn storage with 256GB +# Modified postgres-deployment.yaml: storageClassName: longhorn, storage: 256Gi +kubectl delete pvc postgres-storage-postgres-aprs-0 -n aprs +kubectl delete statefulset postgres-aprs -n aprs +kubectl apply -f /Users/graham/dev/infra/home/cluster/aprs/postgres-deployment.yaml + +# Verify Longhorn storage +kubectl get storageclass +kubectl get nodes.longhorn.io -n longhorn-system +kubectl get pvc -n aprs +``` + +**Status**: Longhorn storage successfully configured on both nodes, PostgreSQL with PostGIS using 256GB Longhorn storage + +### APRS.me Application Deployment Success (2025-10-19) + +```bash +# Fixed PostgreSQL service to use ClusterIP instead of headless service +# Modified postgres-deployment.yaml: type: ClusterIP +kubectl delete svc postgres-aprs -n aprs +kubectl apply -f /Users/graham/dev/infra/home/cluster/aprs/postgres-deployment.yaml + +# Fixed PostgreSQL permissions with fsGroup +# Added to postgres-deployment.yaml: +# securityContext: +# fsGroup: 999 +# fsGroupChangePolicy: OnRootMismatch + +# Shortened PostgreSQL readiness probe delays +# Updated readiness probe: initialDelaySeconds: 30, periodSeconds: 10, failureThreshold: 3 + +# Fixed APRS container permissions +# Removed strict runAsUser constraints from aprs-statefulset-with-pullsecret.yaml +# Added writable volumes for tmp directories + +# Set APRS namespace to privileged pod security +kubectl label namespace aprs pod-security.kubernetes.io/enforce=privileged --overwrite + +# Final deployment verification +kubectl get pods -n aprs +kubectl get svc -n aprs +kubectl logs aprs-0 -n aprs -c aprs +``` + +**Status**: ✅ APRS.me application fully deployed and operational +- PostgreSQL 17 with PostGIS running on 256GB Longhorn storage +- Redis cache operational +- APRS.me web application responding on port 4000 +- APRS-IS packet receiver active and processing +- Multi-replica deployment with Erlang clustering +- All services healthy and ready + +### MetalLB LoadBalancer Configuration (2025-10-19) + +```bash +# Install MetalLB for LoadBalancer services +kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.14.8/config/manifests/metallb-native.yaml +kubectl wait --for=condition=ready pod -l app=metallb -n metallb-system --timeout=60s + +# Create MetalLB configuration with public IP +cat > /Users/graham/dev/infra/home/cluster/metallb-config.yaml << 'EOF' +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: public-ip-pool + namespace: metallb-system +spec: + addresses: + - 204.110.191.2/32 +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: public-ip-advertisement + namespace: metallb-system +spec: + ipAddressPools: + - public-ip-pool +EOF + +# Apply MetalLB configuration +kubectl apply -f /Users/graham/dev/infra/home/cluster/metallb-config.yaml +``` + +**Status**: ✅ LoadBalancer services now working with MetalLB +- Traefik assigned external IP 204.110.191.2 +- HTTPS access to aprs.me working correctly +- HTTP redirects to HTTPS automatically + ### Attempted APRS Deployment ```bash @@ -118,6 +288,31 @@ tofu -chdir=/Users/graham/dev/infra/home/terraform state show cloudflare_record. # Confirmed: Both aprs.me and www.aprs.me point to 204.110.191.2 ``` +### Node-RED Deployment (2025-10-19) + +```bash +# Added 204.110.191.3 to node1 configuration +talosctl apply-config --talosconfig=./talosconfig -e 10.0.101.21 --nodes 10.0.101.21 --file controlplane.yaml --config-patch @node1.patch.yaml --config-patch @tailscale.patch.yaml + +# Deployed nginx proxy for multiple services on 204.110.191.3:8080/8443 +cd /Users/graham/dev/infra/home/cluster/nginx-proxy +./deploy.sh + +# Deployed Node-RED to cluster +cd /Users/graham/dev/infra/home/cluster/nodered +./deploy.sh + +# Updated DNS for nodered.w5isp.com to point to 204.110.191.3 +tofu -chdir=/Users/graham/dev/infra/home/terraform apply -target=dnsimple_zone_record.w5isp_nodered -auto-approve +``` + +**Status**: ✅ Node-RED deployed and accessible +- Traefik handles routing and SSL termination via host-based routing +- APRS: https://aprs.me (204.110.191.2 via Traefik) +- Node-RED: https://nodered.w5isp.com (204.110.191.2 via Traefik) +- Both services use ClusterIP (internal only) +- Uses Longhorn for persistent storage + ## Key Configuration Files - `controlplane.yaml` - Talos control plane configuration diff --git a/home/cluster/high-availability-setup.md b/home/cluster/high-availability-setup.md deleted file mode 100644 index 5fa3614..0000000 --- a/home/cluster/high-availability-setup.md +++ /dev/null @@ -1,44 +0,0 @@ -# High Availability Setup for K3s Cluster - -## Current Situation -- **node1** (10.0.101.21): Has public IP 204.110.191.2, currently being updated -- **node2** (10.0.101.22): No public IP -- **node3** (10.0.101.23): No public IP - -## Solution Options - -### Option 1: Floating IP (Recommended if your provider supports it) -Use a floating/virtual IP that can be moved between nodes: - -1. Get a floating IP from your provider -2. Use Keepalived to manage automatic failover -3. Install MetalLB or Kube-VIP for in-cluster load balancing - -### Option 2: External Load Balancer (Best for production) -Use your hosting provider's load balancer or set up HAProxy/Nginx: - -1. Set up an external load balancer pointing to all 3 nodes -2. Configure health checks on port 6443 (K3s API) -3. For services, use NodePort and configure LB to forward to all nodes - -### Option 3: DNS Round-Robin with Health Checks (Simple but less reliable) -Configure multiple A records with health checking: - -1. Add A records for all node IPs to your domain -2. Use a DNS provider that supports health checks (like Route53) -3. Automatically remove unhealthy nodes from rotation - -### Option 4: Tailscale + Caddy on Separate Host (Quick solution) -Since you have Tailscale, use a separate small VPS as a reverse proxy: - -1. Get a small VPS with public IP -2. Install Tailscale and join your network -3. Install Caddy to reverse proxy to cluster services -4. Point DNS to this VPS instead of directly to cluster - -## Immediate Solution: Configure Services for High Availability - -While node1 is down, ensure your apps remain accessible: - -### 1. Configure Pod Anti-Affinity -Ensure critical pods run on different nodes: \ No newline at end of file diff --git a/home/cluster/longhorn-uservolume.yaml b/home/cluster/longhorn-uservolume.yaml new file mode 100644 index 0000000..ae8ab77 --- /dev/null +++ b/home/cluster/longhorn-uservolume.yaml @@ -0,0 +1,8 @@ +apiVersion: v1alpha1 +kind: UserVolumeConfig +name: longhorn +provisioning: + diskSelector: + match: disk.transport == "nvme" + grow: false + maxSize: 1700GB \ No newline at end of file diff --git a/home/cluster/metallb-config.yaml b/home/cluster/metallb-config.yaml index ccd59ed..4af92be 100644 --- a/home/cluster/metallb-config.yaml +++ b/home/cluster/metallb-config.yaml @@ -1,17 +1,18 @@ +--- apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: - name: default-pool + name: public-ip-pool namespace: metallb-system spec: addresses: - - 10.0.101.30-10.0.101.50 + - 204.110.191.2/32 --- apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: - name: default + name: public-ip-advertisement namespace: metallb-system spec: ipAddressPools: - - default-pool \ No newline at end of file + - public-ip-pool \ No newline at end of file diff --git a/home/cluster/n8n/n8n-deployment.yaml b/home/cluster/n8n/n8n-deployment.yaml index 63e8cc9..beb7548 100644 --- a/home/cluster/n8n/n8n-deployment.yaml +++ b/home/cluster/n8n/n8n-deployment.yaml @@ -12,7 +12,7 @@ data: N8N_METRICS: "true" N8N_BASIC_AUTH_ACTIVE: "false" DB_TYPE: "postgresdb" - DB_POSTGRESDB_HOST: "postgres.n8n.svc.cluster.local" + DB_POSTGRESDB_HOST: "postgres" DB_POSTGRESDB_PORT: "5432" DB_POSTGRESDB_DATABASE: "n8n" DB_POSTGRESDB_USER: "n8n" diff --git a/home/cluster/nodered/AUTH.md b/home/cluster/nodered/AUTH.md deleted file mode 100644 index 6ac2951..0000000 --- a/home/cluster/nodered/AUTH.md +++ /dev/null @@ -1,53 +0,0 @@ -# Node-RED Authentication - -## Current Configuration - -Node-RED is configured with authentication enabled: - -- **Username**: `admin` -- **Password**: `changeme` - -## Changing the Password - -1. Edit the `generate-password.yaml` file and change the PASSWORD value -2. Run the password generator: - ```bash - kubectl delete job nodered-password-generator -n nodered - kubectl apply -f generate-password.yaml - kubectl logs job/nodered-password-generator -n nodered - ``` -3. Copy the generated hash -4. Edit `settings-configmap.yaml` and replace the password hash -5. Apply the changes: - ```bash - kubectl apply -f settings-configmap.yaml - kubectl rollout restart deployment/nodered -n nodered - ``` - -## Adding More Users - -Edit the `settings-configmap.yaml` file and add more users to the `users` array: - -```javascript -users: [{ - username: "admin", - password: "$2b$08$...", - permissions: "*" -}, { - username: "viewer", - password: "$2b$08$...", - permissions: "read" -}] -``` - -Permissions can be: -- `*` - Full access -- `read` - Read-only access - -## Securing HTTP Nodes (Optional) - -To also secure HTTP endpoints created in Node-RED flows, uncomment the `httpNodeAuth` line in `settings-configmap.yaml`. - -## Disabling Authentication - -To disable authentication, remove or comment out the `adminAuth` section in `settings-configmap.yaml`. \ No newline at end of file diff --git a/home/cluster/nodered/deploy.sh b/home/cluster/nodered/deploy.sh index b0543c7..28a9fde 100755 --- a/home/cluster/nodered/deploy.sh +++ b/home/cluster/nodered/deploy.sh @@ -8,6 +8,9 @@ kubectl apply -f namespace.yaml # Create PVC kubectl apply -f pvc.yaml +# Create settings ConfigMap +kubectl apply -f settings-configmap.yaml + # Deploy Node-RED kubectl apply -f deployment.yaml diff --git a/home/cluster/nodered/deployment-local.yaml b/home/cluster/nodered/deployment-local.yaml deleted file mode 100644 index f07a77d..0000000 --- a/home/cluster/nodered/deployment-local.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nodered - namespace: nodered -spec: - replicas: 1 - selector: - matchLabels: - app: nodered - template: - metadata: - labels: - app: nodered - spec: - containers: - - name: nodered - image: nodered/node-red:latest - ports: - - containerPort: 1880 - env: - - name: TZ - value: "UTC" - volumeMounts: - - name: node-red-data - mountPath: /data - resources: - requests: - memory: "256Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" - volumes: - - name: node-red-data - hostPath: - path: /var/lib/nodered - type: DirectoryOrCreate \ No newline at end of file diff --git a/home/cluster/nodered/deployment-simple.yaml b/home/cluster/nodered/deployment-simple.yaml deleted file mode 100644 index a80f711..0000000 --- a/home/cluster/nodered/deployment-simple.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nodered - namespace: nodered -spec: - replicas: 1 - selector: - matchLabels: - app: nodered - template: - metadata: - labels: - app: nodered - spec: - containers: - - name: nodered - image: nodered/node-red:latest - ports: - - containerPort: 1880 - env: - - name: TZ - value: "UTC" - resources: - requests: - memory: "256Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" \ No newline at end of file diff --git a/home/cluster/nodered/deployment.yaml b/home/cluster/nodered/deployment.yaml index e270860..15bb3dd 100644 --- a/home/cluster/nodered/deployment.yaml +++ b/home/cluster/nodered/deployment.yaml @@ -25,7 +25,15 @@ spec: - name: node-red-data mountPath: /data securityContext: - runAsUser: 0 + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault containers: - name: nodered image: nodered/node-red:latest @@ -47,6 +55,16 @@ spec: limits: memory: "512Mi" cpu: "500m" + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumes: - name: node-red-data persistentVolumeClaim: diff --git a/home/cluster/nodered/generate-password.yaml b/home/cluster/nodered/generate-password.yaml deleted file mode 100644 index bd9a80d..0000000 --- a/home/cluster/nodered/generate-password.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: nodered-password-generator - namespace: nodered -spec: - template: - spec: - restartPolicy: Never - containers: - - name: generate-hash - image: nodered/node-red:latest - command: ["node"] - args: ["-e", "console.log(require('bcryptjs').hashSync(process.env.PASSWORD || 'changeme', 8));"] - env: - - name: PASSWORD - value: "changeme" # Change this to your desired password \ No newline at end of file diff --git a/home/cluster/nodered/ingress.yaml b/home/cluster/nodered/ingress.yaml index ad4a816..e058fc4 100644 --- a/home/cluster/nodered/ingress.yaml +++ b/home/cluster/nodered/ingress.yaml @@ -1,25 +1,57 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress +--- +apiVersion: cert-manager.io/v1 +kind: Certificate metadata: - name: nodered-ingress + name: nodered-tls namespace: nodered - annotations: - cert-manager.io/cluster-issuer: "letsencrypt-prod" - traefik.ingress.kubernetes.io/router.entrypoints: "websecure" - traefik.ingress.kubernetes.io/router.tls: "true" spec: - tls: - - hosts: + secretName: nodered-tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: - nodered.w5isp.com +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: nodered-https + namespace: nodered +spec: + entryPoints: + - websecure + routes: + - match: Host(`nodered.w5isp.com`) + kind: Rule + services: + - name: nodered + port: 80 + tls: secretName: nodered-tls - rules: - - host: nodered.w5isp.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: nodered - port: - number: 80 \ No newline at end of file +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: nodered-http + namespace: nodered +spec: + entryPoints: + - web + routes: + - match: Host(`nodered.w5isp.com`) + kind: Rule + services: + - name: nodered + port: 80 + middlewares: + - name: redirect-to-https +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: redirect-to-https + namespace: nodered +spec: + redirectScheme: + scheme: https + permanent: true \ No newline at end of file diff --git a/home/cluster/nodered/settings-configmap.yaml b/home/cluster/nodered/settings-configmap.yaml index 9b0ae71..a65316c 100644 --- a/home/cluster/nodered/settings-configmap.yaml +++ b/home/cluster/nodered/settings-configmap.yaml @@ -25,7 +25,7 @@ data: type: "credentials", users: [{ username: "admin", - password: "$2b$08$eqrvB6AnIA68TG8yIJGDfumTlW0tSgUAcrnSxs.sy3mdNYUFZca/a", // changeme + password: "$2b$08$dKYDBSTCOTgDVQqPNXCq/uzopu6XMbayUNtlvHhoLF0vNVbtscH.a", permissions: "*" }] }, diff --git a/home/cluster/update-k3s-cluster.md b/home/cluster/update-k3s-cluster.md deleted file mode 100644 index 3c512ec..0000000 --- a/home/cluster/update-k3s-cluster.md +++ /dev/null @@ -1,63 +0,0 @@ -# K3s Cluster Update Instructions - -## Update Order for Multi-Master Cluster - -When updating a multi-master K3s cluster with embedded etcd, follow this order: - -1. **Update node2 first** (secondary master) -2. **Update node1 second** (primary master) -3. **Update node3 last** (if it's successfully added to etcd cluster) - -## Steps for Each Node - -1. Copy the update script to the node: - ```bash - scp update-k3s-node.sh node2:~/ - ``` - -2. SSH to the node and run the script: - ```bash - ssh node2 - chmod +x update-k3s-node.sh - ./update-k3s-node.sh - ``` - -3. When prompted about draining: - - Answer **'y'** for production safety (moves pods to other nodes) - - Answer **'n'** if you want faster update and can tolerate brief downtime - -4. Monitor the update progress and verify the node comes back online - -## Verification Commands - -From your local machine: -```bash -# Check all nodes are ready and on correct version -kubectl get nodes -o wide - -# Check etcd cluster health (run on any master node) -ssh node1 'sudo k3s etcdctl endpoint health' - -# Check for any pods having issues -kubectl get pods -A | grep -v Running - -# Check cluster info -kubectl cluster-info -``` - -## Rollback (if needed) - -If something goes wrong, you can rollback to the previous version: -```bash -# On the affected node -sudo systemctl stop k3s -curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="v1.33.4+k3s1" INSTALL_K3S_SKIP_START=true sh - -sudo systemctl start k3s -``` - -## Important Notes - -- The update process will cause a brief disruption to pods on that node -- Embedded etcd cluster requires majority of nodes to be online -- Always update secondary masters before primary master -- Monitor etcd logs during update: `sudo journalctl -u k3s -f | grep etcd` \ No newline at end of file diff --git a/home/cluster/update-k3s-node.sh b/home/cluster/update-k3s-node.sh deleted file mode 100755 index 40eff78..0000000 --- a/home/cluster/update-k3s-node.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Script to update K3s on a node to version 1.33.5 -# Run this script ON each node you want to update - -set -e - -NODE_NAME=$(hostname) -K3S_VERSION="v1.33.5+k3s1" - -echo "=== Updating K3s on ${NODE_NAME} to ${K3S_VERSION} ===" - -# Step 1: Check current version -echo "Current K3s version:" -k3s --version - -# Step 2: Drain the node (make workloads move to other nodes) -echo "" -echo "Do you want to drain this node first? (recommended for safety)" -echo "This will move pods to other nodes during the update." -read -p "Drain node? (y/n): " DRAIN_NODE - -if [[ "$DRAIN_NODE" == "y" ]]; then - echo "Draining node ${NODE_NAME}..." - kubectl drain ${NODE_NAME} --ignore-daemonsets --delete-emptydir-data --force --grace-period=60 -fi - -# Step 3: Stop K3s service -echo "" -echo "Stopping K3s service..." -sudo systemctl stop k3s - -# Step 4: Download and install new K3s version -echo "Installing K3s ${K3S_VERSION}..." -curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="${K3S_VERSION}" INSTALL_K3S_SKIP_START=true sh - - -# Step 5: Start K3s with new version -echo "Starting K3s..." -sudo systemctl start k3s - -# Step 6: Wait for node to be ready -echo "Waiting for K3s to be ready..." -sleep 20 - -# Step 7: Check new version -echo "" -echo "New K3s version:" -k3s --version - -# Step 8: Check node status -kubectl get node ${NODE_NAME} - -# Step 9: Uncordon the node if it was drained -if [[ "$DRAIN_NODE" == "y" ]]; then - echo "" - echo "Uncordoning node ${NODE_NAME}..." - kubectl uncordon ${NODE_NAME} -fi - -echo "" -echo "=== Update complete on ${NODE_NAME} ===" -echo "" -echo "Verify cluster health with:" -echo " kubectl get nodes" -echo " kubectl get pods -A | grep -v Running" \ No newline at end of file diff --git a/home/terraform/dns_w5isp.tf b/home/terraform/dns_w5isp.tf index 02cb18a..767f26e 100644 --- a/home/terraform/dns_w5isp.tf +++ b/home/terraform/dns_w5isp.tf @@ -178,3 +178,11 @@ resource "dnsimple_zone_record" "w5isp_n8n" { type = "A" ttl = 3600 } + +resource "dnsimple_zone_record" "w5isp_nodered" { + zone_name = "w5isp.com" + name = "nodered" + value = "204.110.191.2" # Node-RED via Traefik on k3s cluster + type = "A" + ttl = 3600 +}