misc: zot registry, k8s OIDC, server configs, sandbox experiments, and notes
- docker-30/zot: add Zot OCI registry with on-demand sync to docker.io, registry.k8s.io, ghcr.io, quay.io - kubernetes-kvm-terraform: wire Kanidm OIDC via structured AuthenticationConfiguration; add reference apiserver manifest and join-node-02 helper - servers: reorganize shadow/ under servers/, add saint vhost config and utility-101 VM definition, add shadow hrajfrisbee.cz vhost and storage-23 notes - experiments: add notes and configs for e2b dev VM, kata + firecracker on kube, microsandbox, orb-stack k3s (terraform + cloud-init), rke2 - vms/docker: document tailscale + node-exporter setup - blog: stub post on Gateway API - chore: gitignore tmp/, smtp_password, and the two local-only credential caches; add per-project .claude/settings.json Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
198
experiments/kata_and_fc_on_kube/readme.md
Normal file
198
experiments/kata_and_fc_on_kube/readme.md
Normal file
@@ -0,0 +1,198 @@
|
||||
## Node: kube-node-34
|
||||
|
||||
```bash
|
||||
apt install cpu-checker
|
||||
|
||||
# check kvm availability
|
||||
kvm-ok
|
||||
cat /sys/module/kvm_amd/parameters/nested
|
||||
lsmod | grep kvm
|
||||
|
||||
# install kata containers
|
||||
# using kata-deploy ?
|
||||
|
||||
export VERSION=$(curl -sSL https://api.github.com/repos/kata-containers/kata-containers/releases/latest | jq .tag_name | tr -d '"')
|
||||
export CHART="oci://ghcr.io/kata-containers/kata-deploy-charts/kata-deploy"
|
||||
|
||||
helm upgrade --install kata-deploy "${CHART}" --version "${VERSION}" --values experiments/kata_and_fc_on_kube/manifests/kata-deploy-values.yaml
|
||||
|
||||
# label only specific nodes to be used with kata-fc
|
||||
kubectl label node kube-node-34 runtime=kata-fc
|
||||
# taint node with NoSchedule
|
||||
kubectl taint nodes kube-node-34 dedicated=kata-fc:NoSchedule
|
||||
|
||||
# configure runtime class
|
||||
k apply -f experiments/kata_and_fc_on_kube/manifests//runtime-class_kata-fc.yaml
|
||||
|
||||
# devmapper configuration for firecracker in containerd
|
||||
ctr plugins ls | grep devmapper
|
||||
|
||||
# create data and metadata files
|
||||
# might want to replace this with LVM !!!!
|
||||
# sudo mkdir -p /var/lib/containerd/devmapper
|
||||
# sudo truncate -s 10G /var/lib/containerd/devmapper/data
|
||||
# sudo truncate -s 1G /var/lib/containerd/devmapper/init_metadata
|
||||
|
||||
# # Associate the files with Loop Devices
|
||||
# sudo losetup /dev/loop10 /var/lib/containerd/devmapper/data
|
||||
# sudo losetup /dev/loop11 /var/lib/containerd/devmapper/init_metadata
|
||||
|
||||
# # Use dmsetup to create the pool:
|
||||
# # This command creates the mapping.
|
||||
# # The numbers '0 20971520' represent the size in 512-byte sectors (for a 10GB file).
|
||||
# sudo dmsetup create containerd-pool --table "0 20971520 thin-pool /dev/loop11 /dev/loop10 128 32768 1"
|
||||
|
||||
lvcreate -L 10G -T kata-vg/kata-pool
|
||||
|
||||
# Add the configuration to /etc/containerd/config.toml:
|
||||
# Find the [plugins."io.containerd.snapshotter.v1.devmapper"] section and update it:
|
||||
|
||||
[plugins."io.containerd.snapshotter.v1.devmapper"]
|
||||
# LVM uses a specific naming convention in /dev/mapper/
|
||||
# It is VolumeGroupName-LogicalVolumeName
|
||||
pool_name = "kata--vg-kata--pool"
|
||||
root_path = "/var/lib/containerd/devmapper"
|
||||
base_image_size = "10GB"
|
||||
discard_blocks = true
|
||||
|
||||
|
||||
# restart containerd
|
||||
systemctl restart containerd
|
||||
|
||||
|
||||
|
||||
```
|
||||
|
||||
|
||||
### kube node preparation & deployment
|
||||
|
||||
```bash
|
||||
# ---------------------------------------------------------------------------
|
||||
# Packages
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
apt-get update
|
||||
apt-get install -y \
|
||||
qemu-guest-agent \
|
||||
openssh-server \
|
||||
apt-transport-https \
|
||||
ca-certificates \
|
||||
curl \
|
||||
gnupg \
|
||||
nvme-cli
|
||||
|
||||
systemctl enable --now qemu-guest-agent
|
||||
systemctl enable --now ssh
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# nvme-tcp
|
||||
# ---------------------------------------------------------------------------
|
||||
apt-get install -y linux-modules-extra-$(uname -r)
|
||||
modprobe nvme-tcp
|
||||
echo "nvme-tcp" >> /etc/modules-load.d/nvme-tcp.conf
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kernel modules for Kubernetes
|
||||
# ---------------------------------------------------------------------------
|
||||
cat > /etc/modules-load.d/k8s.conf <<'EOF'
|
||||
overlay
|
||||
br_netfilter
|
||||
EOF
|
||||
|
||||
modprobe overlay
|
||||
modprobe br_netfilter
|
||||
|
||||
cat > /etc/sysctl.d/k8s.conf <<'EOF'
|
||||
net.bridge.bridge-nf-call-iptables = 1
|
||||
net.bridge.bridge-nf-call-ip6tables = 1
|
||||
net.ipv4.ip_forward = 1
|
||||
EOF
|
||||
|
||||
sysctl --system
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# containerd
|
||||
# ---------------------------------------------------------------------------
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
|
||||
echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
|
||||
> /etc/apt/sources.list.d/docker.list
|
||||
apt-get update && apt-get install -y containerd.io
|
||||
|
||||
cat > /etc/containerd/config.toml <<'EOF'
|
||||
version = 2
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
|
||||
runtime_type = "io.containerd.runc.v2"
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
|
||||
SystemdCgroup = true
|
||||
[plugins."io.containerd.grpc.v1.cri".registry]
|
||||
config_path = "/etc/containerd/certs.d"
|
||||
EOF
|
||||
|
||||
# Registry mirrors pointing to Zot at 192.168.0.30:5000
|
||||
mkdir -p \
|
||||
/etc/containerd/certs.d/docker.io \
|
||||
/etc/containerd/certs.d/registry.k8s.io \
|
||||
/etc/containerd/certs.d/ghcr.io \
|
||||
/etc/containerd/certs.d/quay.io
|
||||
|
||||
cat > /etc/containerd/certs.d/docker.io/hosts.toml <<'EOF'
|
||||
server = "https://registry-1.docker.io"
|
||||
[host."http://192.168.0.30:5000/v2/docker.io"]
|
||||
capabilities = ["pull", "resolve"]
|
||||
skip_verify = true
|
||||
override_path = true
|
||||
EOF
|
||||
|
||||
cat > /etc/containerd/certs.d/registry.k8s.io/hosts.toml <<'EOF'
|
||||
server = "https://registry.k8s.io"
|
||||
[host."http://192.168.0.30:5000/v2/registry.k8s.io"]
|
||||
capabilities = ["pull", "resolve"]
|
||||
skip_verify = true
|
||||
override_path = true
|
||||
EOF
|
||||
|
||||
cat > /etc/containerd/certs.d/ghcr.io/hosts.toml <<'EOF'
|
||||
server = "https://ghcr.io"
|
||||
[host."http://192.168.0.30:5000/v2/ghcr.io"]
|
||||
capabilities = ["pull", "resolve"]
|
||||
skip_verify = true
|
||||
override_path = true
|
||||
EOF
|
||||
|
||||
cat > /etc/containerd/certs.d/quay.io/hosts.toml <<'EOF'
|
||||
server = "https://quay.io"
|
||||
[host."http://192.168.0.30:5000/v2/quay.io"]
|
||||
capabilities = ["pull", "resolve"]
|
||||
skip_verify = true
|
||||
override_path = true
|
||||
EOF
|
||||
|
||||
systemctl restart containerd
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# kubelet systemd drop-in
|
||||
# ---------------------------------------------------------------------------
|
||||
mkdir -p /etc/systemd/system/kubelet.service.d
|
||||
cat > /etc/systemd/system/kubelet.service.d/10-containerd.conf <<'EOF'
|
||||
[Unit]
|
||||
After=containerd.service
|
||||
Requires=containerd.service
|
||||
|
||||
[Service]
|
||||
ExecStartPre=/bin/bash -c 'until [ -S /var/run/containerd/containerd.sock ]; do sleep 1; done'
|
||||
ExecStartPre=/usr/bin/crictl info
|
||||
EOF
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# kubeadm / kubelet / kubectl v1.32
|
||||
# ---------------------------------------------------------------------------
|
||||
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key \
|
||||
| gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
|
||||
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /" \
|
||||
> /etc/apt/sources.list.d/kubernetes.list
|
||||
apt-get update && apt-get install -y kubelet kubeadm kubectl
|
||||
apt-mark hold kubelet kubeadm kubectl
|
||||
|
||||
|
||||
# join kube cluster
|
||||
JOIN_COMMAND="kubeadm join 192.168.0.31:6443 --token <token> --discovery-token-ca-cert-hash sha256:<hash>"
|
||||
Reference in New Issue
Block a user