home-kubernetes/experiments/rke2/step-by-step.md

VMS creation

# hypervisor: beelink (192.168.0.6)

vms_path=/srv/vms/images
isos_path=/srv/vms/isos
cd $isos_path
# Grab Ubuntu 24.04 cloud image
wget https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img

# Create a base disk from the cloud image (repeat per node)
for NODE in rke2-server rke2-agent1 rke2-agent2; do
  DISK_SIZE="30G"
  [[ "$NODE" == rke2-agent* ]] && DISK_SIZE="50G"

  qemu-img create -f qcow2 -F qcow2 -b $isos_path/noble-server-cloudimg-amd64.img $vms_path/${NODE}.qcow2
  qemu-img resize $vms_path/${NODE}.qcow2 ${DISK_SIZE}
done

# prepare cloud-init/user-data
apt install cloud-image-utils

# prepare cloud-init and launch VMs
declare -A nodes=(
  [rke2-server]="192.168.0.51"
  [rke2-agent1]="192.168.0.52"
  [rke2-agent2]="192.168.0.53"
)

for node in "${!nodes[@]}"; do
  ip="${nodes[$node]}"

  # user-data
  cat <<EOF > user-data-${node}
#cloud-config
hostname: ${node}
manage_etc_hosts: false
users:
  - name: sre
    sudo: ALL=(ALL) NOPASSWD:ALL
    shell: /bin/bash
    ssh_authorized_keys:
      - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQxxkqmvtVI+8c5BkTaJ5c2HfBFRXJWMmEcevvfP9tV jan.novak@Jans-MacBook-Air.local
write_files:
  - path: /etc/hosts
    append: true
    content: |
      192.168.0.51 rke2-server
      192.168.0.52 rke2-agent1
      192.168.0.53 rke2-agent2
  - path: /etc/modules-load.d/rke2.conf
    content: |
      br_netfilter
      overlay
  - path: /etc/sysctl.d/99-rke2.conf
    content: |
      net.bridge.bridge-nf-call-iptables  = 1
      net.bridge.bridge-nf-call-ip6tables = 1
      net.ipv4.ip_forward                 = 1
swap:
  filename: /swap.img
  size: 0
  maxsize: 0
runcmd:
  - swapoff -a
  - sed -i '/swap/d' /etc/fstab
  - modprobe br_netfilter
  - modprobe overlay
  - sysctl --system
package_update: true
packages:
  - qemu-guest-agent
  - nfs-common
  - open-iscsi
power_state:
  mode: reboot
EOF

  # network config
  cat <<EOF > network-config-${node}
network:
  version: 2
  ethernets:
    eth0:
      match:
        driver: "virtio_net"
      addresses:
      - "${ip}/24"
      nameservers:
        addresses:
        - 8.8.8.8
      routes:
      - to: "default"
        via: "192.168.0.4"
EOF

  cloud-localds --network-config=./network-config-${node} \
    $vms_path/${node}-seed.iso ./user-data-${node}
done

# Launch VMs
for node in "${!nodes[@]}"; do
  virt-install \
    --name ${node} \
    --ram 4096 --vcpus 2 \
    --os-variant ubuntu24.04 \
    --disk $vms_path/${node}.qcow2,bus=virtio \
    --disk $vms_path/${node}-seed.iso,device=cdrom \
    --network bridge=br0,model=virtio \
    --graphics none \
    --console pty,target_type=serial \
    --noautoconsole \
    --import
done

RKE2 installation

# there are no .deb packages - only rpm or tarball
# "magic" install script can handle that
curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=v1.32 sudo sh -

# Create config directory
sudo mkdir -p /etc/rancher/rke2

# Server configuration
cat <<EOF | sudo tee /etc/rancher/rke2/config.yaml
# Bind the API to the node's IP (not 127.0.0.1)
tls-san:
  - rke2-server
  - 192.168.0.51
# Write kubeconfig readable by non-root
write-kubeconfig-mode: "0644"
# CNI - canal is default and fine for home lab
# Alternatives: cilium, calico, multus
cni:
  - canal
# Disable servicelb if you plan to use metallb
disable:
  - rke2-service-lb
EOF

# Enable and start
sudo systemctl enable rke2-server.service
sudo systemctl start rke2-server.service

# Watch bootstrap (takes 2-3 min on first run)
sudo journalctl -u rke2-server -f

Grab the join token and kubeconfig

# Token for agents to join
sudo cat /var/lib/rancher/rke2/server/node-token
# Save this somewhere - you'll need it on every agent

# Set up kubectl on the server node
echo 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml' >> ~/.bashrc
echo 'export PATH=$PATH:/var/lib/rancher/rke2/bin' >> ~/.bashrc
source ~/.bashrc

# Verify
kubectl get nodes

Install RKE2 Agents (Workers)

# Install RKE2 agent - same channel as server
curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=v1.32 INSTALL_RKE2_TYPE=agent sudo sh -

# Create config
sudo mkdir -p /etc/rancher/rke2

cat <<EOF | sudo tee /etc/rancher/rke2/config.yaml
server: https://192.168.0.51:9345
token: K10dba0bfff01d610ffed41c6b82a1b8861ee19e5af34a3bdd21970936823a846da::server:1f02fb36a288dcd06770cab28b015bd7
EOF

# Enable and start
sudo systemctl enable rke2-agent.service
sudo systemctl start rke2-agent.service

# Watch join
sudo journalctl -u rke2-agent -f

Copy Kubeconfig to Your Workstation

# On your workstation (not the VMs)
scp sre@192.168.0.51:/etc/rancher/rke2/rke2.yaml ~/.kube/rke2-homelab.yaml

# Fix the server address (it'll say 127.0.0.1)
gsed -i 's/127.0.0.1/192.168.0.51/' ~/.kube/rke2-homelab.yaml

export KUBECONFIG=~/.kube/rke2-homelab.yaml
kubectl get nodes