misc: zot registry, k8s OIDC, server configs, sandbox experiments, and notes

- docker-30/zot: add Zot OCI registry with on-demand sync to docker.io, registry.k8s.io, ghcr.io, quay.io - kubernetes-kvm-terraform: wire Kanidm OIDC via structured AuthenticationConfiguration; add reference apiserver manifest and join-node-02 helper - servers: reorganize shadow/ under servers/, add saint vhost config and utility-101 VM definition, add shadow hrajfrisbee.cz vhost and storage-23 notes - experiments: add notes and configs for e2b dev VM, kata + firecracker on kube, microsandbox, orb-stack k3s (terraform + cloud-init), rke2 - vms/docker: document tailscale + node-exporter setup - blog: stub post on Gateway API - chore: gitignore tmp/, smtp_password, and the two local-only credential caches; add per-project .claude/settings.json Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-01 18:12:38 +02:00
parent 5ca27a832b
commit 80d0cc1168
34 changed files with 2814 additions and 1 deletions
--- a/experiments/rke2/rke2-basics.md
+++ b/experiments/rke2/rke2-basics.md
@@ -0,0 +1,219 @@
+## install
+
+```bash
+# master node
+curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=v1.32.12+rke2r1 sh -
+systemctl enable rke2-server.service
+systemctl start rke2-server.service
+journalctl -u rke2-server -f
+
+# open firewalld
+sudo firewall-cmd --permanent --add-port=9345/tcp
+sudo firewall-cmd --permanent --add-port=6443/tcp
+sudo firewall-cmd --permanent --add-port=10250/tcp # Kubelet
+sudo firewall-cmd --reload
+
+# install nerdctl
+# Set the version
+VERSION="2.2.1" # Check GitHub for the latest version
+
+# Download the tarball
+wget https://github.com/containerd/nerdctl/releases/download/v${VERSION}/nerdctl-${VERSION}-linux-arm64.tar.gz
+
+# Extract to your path
+sudo tar -C /usr/local/bin -xzvf nerdctl-${VERSION}-linux-arm64.tar.gz nerdctl
+
+# configure nerdctl
+sudo mkdir -p /etc/nerdctl
+sudo tee /etc/nerdctl/nerdctl.toml <<EOF
+address   = "unix:///run/k3s/containerd/containerd.sock"
+namespace = "k8s.io"
+EOF
+
+# install buildkit
+# Set current stable version
+BK_VER="0.28.0"
+
+# Download arm64 binary
+wget https://github.com/moby/buildkit/releases/download/v${BK_VER}/buildkit-v${BK_VER}.linux-arm64.tar.gz
+
+# Extract only the binaries to /usr/local/bin
+sudo tar -C /usr/local/bin -xzvf buildkit-v${BK_VER}.linux-arm64.tar.gz --strip-components=1 bin/
+
+# Create the service file
+sudo tee /etc/systemd/system/buildkit.service <<EOF
+[Unit]
+Description=BuildKit
+Documentation=https://github.com/moby/buildkit
+
+[Service]
+ExecStart=/usr/local/bin/buildkitd --addr unix:///run/buildkit/buildkitd.sock
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# Reload and Start
+sudo systemctl daemon-reload
+sudo systemctl enable --now buildkit
+
+# ---------------------------------------------
+
+
+# agent/worker node
+curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" INSTALL_RKE2_VERSION=v1.32.12+rke2r1 sh -
+systemctl enable rke2-agent.service
+mkdir -p /etc/rancher/rke2/
+# token from master node
+# cat /var/lib/rancher/rke2/server/node-token
+cat <<EOF | sudo tee /etc/rancher/rke2/config.yaml
+server: https://192.168.64.3:9345
+token: K107618960f87b9efb3a3255ce00a9743d29f1db9376820c9144cb85fa3c554dc69::server:06b2effdf0c9ce3952efc8a5d80bf084
+EOF
+systemctl start rke2-agent.service
+journalctl -u rke2-agent -f
+
+
+# Set up kubectl on the server node
+echo 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml' >> ~/.bashrc
+echo 'export PATH=$PATH:/var/lib/rancher/rke2/bin' >> ~/.bashrc
+source ~/.bashrc
+```
+
+
+
+## build and deploy application
+
+```bash
+# build container with nerdctl
+nerdctl --namespace k8s.io build --tag hello-world:latest .
+
+# export image as tar on master node
+nerdctl save hello-world:latest -o hello-world.tar
+# copy it over to worker node
+scp hello-world.tar novakj@192.168.64.4:~/
+# import image on the agent node
+sudo /var/lib/rancher/rke2/bin/ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images import hello-world.tar
+
+kubectl create namespace rke2-apps
+
+cat <<EOF > deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hello-world-deployment
+  namespace: rke2-apps
+  labels:
+    type: staticwebapp
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      type: staticwebapp
+  template:
+    metadata:
+      labels:
+        type: staticwebapp
+    spec:
+      containers:
+      - name: staticwebapp
+        image: hello-world:latest
+        imagePullPolicy: Never
+        ports:
+        - containerPort: 80
+        resources:
+          requests:
+            memory: "32Mi"
+            cpu: "200m"
+          limits:
+            memory: "64Mi"
+            cpu: "300m"
+EOF
+
+kubectl create -f deployment.yaml
+
+# expose deployment
+kubectl expose deployment hello-world-deployment --name hello-world-service --port=8080 --target-port=80 -n rke2-apps
+
+# install ingress-nginx (even though i thought that there is ingress controller already deployed)
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.2/deploy/static/provider/cloud/deploy.yaml
+
+# create ingress with "localhost" as host
+kubectl create ingress hello-world-ingress --class=nginx --rule="test-host/*=hello-world-service:8080" -n rke2-apps
+kubectl port-forward -n ingress-nginx service/ingress-nginx-controller 8081:80
+
+
+
+# incomplete completion configuration ;-)
+dnf install bash-completion -y
+alias 'k=kubectl'
+
+# ~/.bashrc
+
+# 1. Load the main bash-completion package first
+# On Rocky/RHEL, it's usually at this path:
+[[ -r "/usr/share/bash-completion/bash_completion" ]] && . "/usr/share/bash-completion/bash_completion"
+
+# Enable kubectl bash completion
+source <(kubectl completion bash)
+
+# Set up the alias
+alias k=kubectl
+
+# Link the kubectl completion logic to the 'k' alias
+complete -o default -F __start_kubectl k
+```
+
+
+
+## upgrading RKE2
+
+```bash
+# install upgrade controller
+kubectl apply -f https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/system-upgrade-controller.yaml
+
+# server upgrade
+cat <<EOF | kubectl apply -f -
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: rke2-server-upgrade
+  namespace: system-upgrade
+spec:
+  concurrency: 1
+  cordon: true
+  nodeSelector:
+    matchExpressions:
+      - key: node-role.kubernetes.io/control-plane
+        operator: In
+        values: ["true"]
+  serviceAccountName: system-upgrade
+  upgrade:
+    image: rancher/rke2-upgrade
+  version: v1.33.9+rke2r1
+EOF
+
+# agent upgrade
+cat <<EOF | kubectl apply -f -
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: rke2-agent-upgrade
+  namespace: system-upgrade
+spec:
+  concurrency: 1
+  cordon: true
+  nodeSelector:
+    matchExpressions:
+      - key: node-role.kubernetes.io/control-plane
+        operator: DoesNotExist
+  prepare:
+    # Logic: "Don't start workers until servers are done"
+    args: ["wait-for-plan", "rke2-server-upgrade"]
+    image: rancher/rke2-upgrade
+  serviceAccountName: system-upgrade
+  upgrade:
+    image: rancher/rke2-upgrade
+  version: v1.33.9+rke2r1
+EOF
+```
--- a/experiments/rke2/step-by-step.md
+++ b/experiments/rke2/step-by-step.md
@@ -0,0 +1,215 @@
+## VMS creation
+
+
+
+```bash
+# hypervisor: beelink (192.168.0.6)
+
+vms_path=/srv/vms/images
+isos_path=/srv/vms/isos
+cd $isos_path
+# Grab Ubuntu 24.04 cloud image
+wget https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+
+# Create a base disk from the cloud image (repeat per node)
+for NODE in rke2-server rke2-agent1 rke2-agent2; do
+  DISK_SIZE="30G"
+  [[ "$NODE" == rke2-agent* ]] && DISK_SIZE="50G"
+
+  qemu-img create -f qcow2 -F qcow2 -b $isos_path/noble-server-cloudimg-amd64.img $vms_path/${NODE}.qcow2
+  qemu-img resize $vms_path/${NODE}.qcow2 ${DISK_SIZE}
+done
+```
+
+
+```bash
+# prepare cloud-init/user-data
+apt install cloud-image-utils
+
+# prepare cloud-init and launch VMs
+declare -A nodes=(
+  [rke2-server]="192.168.0.51"
+  [rke2-agent1]="192.168.0.52"
+  [rke2-agent2]="192.168.0.53"
+)
+
+for node in "${!nodes[@]}"; do
+  ip="${nodes[$node]}"
+
+  # user-data
+  cat <<EOF > user-data-${node}
+#cloud-config
+hostname: ${node}
+manage_etc_hosts: false
+users:
+  - name: sre
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    shell: /bin/bash
+    ssh_authorized_keys:
+      - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQxxkqmvtVI+8c5BkTaJ5c2HfBFRXJWMmEcevvfP9tV jan.novak@Jans-MacBook-Air.local
+write_files:
+  - path: /etc/hosts
+    append: true
+    content: |
+      192.168.0.51 rke2-server
+      192.168.0.52 rke2-agent1
+      192.168.0.53 rke2-agent2
+  - path: /etc/modules-load.d/rke2.conf
+    content: |
+      br_netfilter
+      overlay
+  - path: /etc/sysctl.d/99-rke2.conf
+    content: |
+      net.bridge.bridge-nf-call-iptables  = 1
+      net.bridge.bridge-nf-call-ip6tables = 1
+      net.ipv4.ip_forward                 = 1
+swap:
+  filename: /swap.img
+  size: 0
+  maxsize: 0
+runcmd:
+  - swapoff -a
+  - sed -i '/swap/d' /etc/fstab
+  - modprobe br_netfilter
+  - modprobe overlay
+  - sysctl --system
+package_update: true
+packages:
+  - qemu-guest-agent
+  - nfs-common
+  - open-iscsi
+power_state:
+  mode: reboot
+EOF
+
+  # network config
+  cat <<EOF > network-config-${node}
+network:
+  version: 2
+  ethernets:
+    eth0:
+      match:
+        driver: "virtio_net"
+      addresses:
+      - "${ip}/24"
+      nameservers:
+        addresses:
+        - 8.8.8.8
+      routes:
+      - to: "default"
+        via: "192.168.0.4"
+EOF
+
+  cloud-localds --network-config=./network-config-${node} \
+    $vms_path/${node}-seed.iso ./user-data-${node}
+done
+
+# Launch VMs
+for node in "${!nodes[@]}"; do
+  virt-install \
+    --name ${node} \
+    --ram 4096 --vcpus 2 \
+    --os-variant ubuntu24.04 \
+    --disk $vms_path/${node}.qcow2,bus=virtio \
+    --disk $vms_path/${node}-seed.iso,device=cdrom \
+    --network bridge=br0,model=virtio \
+    --graphics none \
+    --console pty,target_type=serial \
+    --noautoconsole \
+    --import
+done
+
+```
+
+
+## RKE2 installation
+
+```bash
+# there are no .deb packages - only rpm or tarball
+# "magic" install script can handle that
+curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=v1.32 sudo sh -
+
+# Create config directory
+sudo mkdir -p /etc/rancher/rke2
+
+# Server configuration
+cat <<EOF | sudo tee /etc/rancher/rke2/config.yaml
+# Bind the API to the node's IP (not 127.0.0.1)
+tls-san:
+  - rke2-server
+  - 192.168.0.51
+# Write kubeconfig readable by non-root
+write-kubeconfig-mode: "0644"
+# CNI - canal is default and fine for home lab
+# Alternatives: cilium, calico, multus
+cni:
+  - canal
+# Disable servicelb if you plan to use metallb
+disable:
+  - rke2-service-lb
+EOF
+
+# Enable and start
+sudo systemctl enable rke2-server.service
+sudo systemctl start rke2-server.service
+
+# Watch bootstrap (takes 2-3 min on first run)
+sudo journalctl -u rke2-server -f
+
+```
+
+
+## Grab the join token and kubeconfig
+
+```bash
+
+# Token for agents to join
+sudo cat /var/lib/rancher/rke2/server/node-token
+# Save this somewhere - you'll need it on every agent
+
+# Set up kubectl on the server node
+echo 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml' >> ~/.bashrc
+echo 'export PATH=$PATH:/var/lib/rancher/rke2/bin' >> ~/.bashrc
+source ~/.bashrc
+
+# Verify
+kubectl get nodes
+```
+
+
+## Install RKE2 Agents (Workers)
+
+```bash
+# Install RKE2 agent - same channel as server
+curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=v1.32 INSTALL_RKE2_TYPE=agent sudo sh -
+
+# Create config
+sudo mkdir -p /etc/rancher/rke2
+
+cat <<EOF | sudo tee /etc/rancher/rke2/config.yaml
+server: https://192.168.0.51:9345
+token: K10dba0bfff01d610ffed41c6b82a1b8861ee19e5af34a3bdd21970936823a846da::server:1f02fb36a288dcd06770cab28b015bd7
+EOF
+
+# Enable and start
+sudo systemctl enable rke2-agent.service
+sudo systemctl start rke2-agent.service
+
+# Watch join
+sudo journalctl -u rke2-agent -f
+
+```
+
+
+## Copy Kubeconfig to Your Workstation
+
+```bash
+# On your workstation (not the VMs)
+scp sre@192.168.0.51:/etc/rancher/rke2/rke2.yaml ~/.kube/rke2-homelab.yaml
+
+# Fix the server address (it'll say 127.0.0.1)
+gsed -i 's/127.0.0.1/192.168.0.51/' ~/.kube/rke2-homelab.yaml
+
+export KUBECONFIG=~/.kube/rke2-homelab.yaml
+kubectl get nodes
+```