misc: zot registry, k8s OIDC, server configs, sandbox experiments, and notes
- docker-30/zot: add Zot OCI registry with on-demand sync to docker.io, registry.k8s.io, ghcr.io, quay.io - kubernetes-kvm-terraform: wire Kanidm OIDC via structured AuthenticationConfiguration; add reference apiserver manifest and join-node-02 helper - servers: reorganize shadow/ under servers/, add saint vhost config and utility-101 VM definition, add shadow hrajfrisbee.cz vhost and storage-23 notes - experiments: add notes and configs for e2b dev VM, kata + firecracker on kube, microsandbox, orb-stack k3s (terraform + cloud-init), rke2 - vms/docker: document tailscale + node-exporter setup - blog: stub post on Gateway API - chore: gitignore tmp/, smtp_password, and the two local-only credential caches; add per-project .claude/settings.json Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
215
experiments/rke2/step-by-step.md
Normal file
215
experiments/rke2/step-by-step.md
Normal file
@@ -0,0 +1,215 @@
|
||||
## VMS creation
|
||||
|
||||
|
||||
|
||||
```bash
|
||||
# hypervisor: beelink (192.168.0.6)
|
||||
|
||||
vms_path=/srv/vms/images
|
||||
isos_path=/srv/vms/isos
|
||||
cd $isos_path
|
||||
# Grab Ubuntu 24.04 cloud image
|
||||
wget https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
|
||||
|
||||
# Create a base disk from the cloud image (repeat per node)
|
||||
for NODE in rke2-server rke2-agent1 rke2-agent2; do
|
||||
DISK_SIZE="30G"
|
||||
[[ "$NODE" == rke2-agent* ]] && DISK_SIZE="50G"
|
||||
|
||||
qemu-img create -f qcow2 -F qcow2 -b $isos_path/noble-server-cloudimg-amd64.img $vms_path/${NODE}.qcow2
|
||||
qemu-img resize $vms_path/${NODE}.qcow2 ${DISK_SIZE}
|
||||
done
|
||||
```
|
||||
|
||||
|
||||
```bash
|
||||
# prepare cloud-init/user-data
|
||||
apt install cloud-image-utils
|
||||
|
||||
# prepare cloud-init and launch VMs
|
||||
declare -A nodes=(
|
||||
[rke2-server]="192.168.0.51"
|
||||
[rke2-agent1]="192.168.0.52"
|
||||
[rke2-agent2]="192.168.0.53"
|
||||
)
|
||||
|
||||
for node in "${!nodes[@]}"; do
|
||||
ip="${nodes[$node]}"
|
||||
|
||||
# user-data
|
||||
cat <<EOF > user-data-${node}
|
||||
#cloud-config
|
||||
hostname: ${node}
|
||||
manage_etc_hosts: false
|
||||
users:
|
||||
- name: sre
|
||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
||||
shell: /bin/bash
|
||||
ssh_authorized_keys:
|
||||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQxxkqmvtVI+8c5BkTaJ5c2HfBFRXJWMmEcevvfP9tV jan.novak@Jans-MacBook-Air.local
|
||||
write_files:
|
||||
- path: /etc/hosts
|
||||
append: true
|
||||
content: |
|
||||
192.168.0.51 rke2-server
|
||||
192.168.0.52 rke2-agent1
|
||||
192.168.0.53 rke2-agent2
|
||||
- path: /etc/modules-load.d/rke2.conf
|
||||
content: |
|
||||
br_netfilter
|
||||
overlay
|
||||
- path: /etc/sysctl.d/99-rke2.conf
|
||||
content: |
|
||||
net.bridge.bridge-nf-call-iptables = 1
|
||||
net.bridge.bridge-nf-call-ip6tables = 1
|
||||
net.ipv4.ip_forward = 1
|
||||
swap:
|
||||
filename: /swap.img
|
||||
size: 0
|
||||
maxsize: 0
|
||||
runcmd:
|
||||
- swapoff -a
|
||||
- sed -i '/swap/d' /etc/fstab
|
||||
- modprobe br_netfilter
|
||||
- modprobe overlay
|
||||
- sysctl --system
|
||||
package_update: true
|
||||
packages:
|
||||
- qemu-guest-agent
|
||||
- nfs-common
|
||||
- open-iscsi
|
||||
power_state:
|
||||
mode: reboot
|
||||
EOF
|
||||
|
||||
# network config
|
||||
cat <<EOF > network-config-${node}
|
||||
network:
|
||||
version: 2
|
||||
ethernets:
|
||||
eth0:
|
||||
match:
|
||||
driver: "virtio_net"
|
||||
addresses:
|
||||
- "${ip}/24"
|
||||
nameservers:
|
||||
addresses:
|
||||
- 8.8.8.8
|
||||
routes:
|
||||
- to: "default"
|
||||
via: "192.168.0.4"
|
||||
EOF
|
||||
|
||||
cloud-localds --network-config=./network-config-${node} \
|
||||
$vms_path/${node}-seed.iso ./user-data-${node}
|
||||
done
|
||||
|
||||
# Launch VMs
|
||||
for node in "${!nodes[@]}"; do
|
||||
virt-install \
|
||||
--name ${node} \
|
||||
--ram 4096 --vcpus 2 \
|
||||
--os-variant ubuntu24.04 \
|
||||
--disk $vms_path/${node}.qcow2,bus=virtio \
|
||||
--disk $vms_path/${node}-seed.iso,device=cdrom \
|
||||
--network bridge=br0,model=virtio \
|
||||
--graphics none \
|
||||
--console pty,target_type=serial \
|
||||
--noautoconsole \
|
||||
--import
|
||||
done
|
||||
|
||||
```
|
||||
|
||||
|
||||
## RKE2 installation
|
||||
|
||||
```bash
|
||||
# there are no .deb packages - only rpm or tarball
|
||||
# "magic" install script can handle that
|
||||
curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=v1.32 sudo sh -
|
||||
|
||||
# Create config directory
|
||||
sudo mkdir -p /etc/rancher/rke2
|
||||
|
||||
# Server configuration
|
||||
cat <<EOF | sudo tee /etc/rancher/rke2/config.yaml
|
||||
# Bind the API to the node's IP (not 127.0.0.1)
|
||||
tls-san:
|
||||
- rke2-server
|
||||
- 192.168.0.51
|
||||
# Write kubeconfig readable by non-root
|
||||
write-kubeconfig-mode: "0644"
|
||||
# CNI - canal is default and fine for home lab
|
||||
# Alternatives: cilium, calico, multus
|
||||
cni:
|
||||
- canal
|
||||
# Disable servicelb if you plan to use metallb
|
||||
disable:
|
||||
- rke2-service-lb
|
||||
EOF
|
||||
|
||||
# Enable and start
|
||||
sudo systemctl enable rke2-server.service
|
||||
sudo systemctl start rke2-server.service
|
||||
|
||||
# Watch bootstrap (takes 2-3 min on first run)
|
||||
sudo journalctl -u rke2-server -f
|
||||
|
||||
```
|
||||
|
||||
|
||||
## Grab the join token and kubeconfig
|
||||
|
||||
```bash
|
||||
|
||||
# Token for agents to join
|
||||
sudo cat /var/lib/rancher/rke2/server/node-token
|
||||
# Save this somewhere - you'll need it on every agent
|
||||
|
||||
# Set up kubectl on the server node
|
||||
echo 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml' >> ~/.bashrc
|
||||
echo 'export PATH=$PATH:/var/lib/rancher/rke2/bin' >> ~/.bashrc
|
||||
source ~/.bashrc
|
||||
|
||||
# Verify
|
||||
kubectl get nodes
|
||||
```
|
||||
|
||||
|
||||
## Install RKE2 Agents (Workers)
|
||||
|
||||
```bash
|
||||
# Install RKE2 agent - same channel as server
|
||||
curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=v1.32 INSTALL_RKE2_TYPE=agent sudo sh -
|
||||
|
||||
# Create config
|
||||
sudo mkdir -p /etc/rancher/rke2
|
||||
|
||||
cat <<EOF | sudo tee /etc/rancher/rke2/config.yaml
|
||||
server: https://192.168.0.51:9345
|
||||
token: K10dba0bfff01d610ffed41c6b82a1b8861ee19e5af34a3bdd21970936823a846da::server:1f02fb36a288dcd06770cab28b015bd7
|
||||
EOF
|
||||
|
||||
# Enable and start
|
||||
sudo systemctl enable rke2-agent.service
|
||||
sudo systemctl start rke2-agent.service
|
||||
|
||||
# Watch join
|
||||
sudo journalctl -u rke2-agent -f
|
||||
|
||||
```
|
||||
|
||||
|
||||
## Copy Kubeconfig to Your Workstation
|
||||
|
||||
```bash
|
||||
# On your workstation (not the VMs)
|
||||
scp sre@192.168.0.51:/etc/rancher/rke2/rke2.yaml ~/.kube/rke2-homelab.yaml
|
||||
|
||||
# Fix the server address (it'll say 127.0.0.1)
|
||||
gsed -i 's/127.0.0.1/192.168.0.51/' ~/.kube/rke2-homelab.yaml
|
||||
|
||||
export KUBECONFIG=~/.kube/rke2-homelab.yaml
|
||||
kubectl get nodes
|
||||
```
|
||||
Reference in New Issue
Block a user