home-kubernetes/experiments/kata_and_fc_on_kube

Node: kube-node-34

apt install cpu-checker

# check kvm availability
kvm-ok
cat /sys/module/kvm_amd/parameters/nested
lsmod | grep kvm

# install kata containers
# using kata-deploy ? 

export VERSION=$(curl -sSL https://api.github.com/repos/kata-containers/kata-containers/releases/latest | jq .tag_name | tr -d '"')
export CHART="oci://ghcr.io/kata-containers/kata-deploy-charts/kata-deploy"

helm upgrade --install  kata-deploy "${CHART}" --version "${VERSION}" --values experiments/kata_and_fc_on_kube/manifests/kata-deploy-values.yaml

# label only specific nodes to be used with kata-fc
kubectl label node kube-node-34 runtime=kata-fc
# taint node with NoSchedule
kubectl taint nodes kube-node-34 dedicated=kata-fc:NoSchedule

# configure runtime class
k apply -f experiments/kata_and_fc_on_kube/manifests//runtime-class_kata-fc.yaml

# devmapper configuration for firecracker in containerd
ctr plugins ls | grep devmapper

# create data and metadata files
# might want to replace this with LVM !!!!
# sudo mkdir -p /var/lib/containerd/devmapper
# sudo truncate -s 10G /var/lib/containerd/devmapper/data
# sudo truncate -s 1G /var/lib/containerd/devmapper/init_metadata

# # Associate the files with Loop Devices
# sudo losetup /dev/loop10 /var/lib/containerd/devmapper/data
# sudo losetup /dev/loop11 /var/lib/containerd/devmapper/init_metadata

# # Use dmsetup to create the pool:
# # This command creates the mapping. 
# # The numbers '0 20971520' represent the size in 512-byte sectors (for a 10GB file).
# sudo dmsetup create containerd-pool --table "0 20971520 thin-pool /dev/loop11 /dev/loop10 128 32768 1"

lvcreate -L 10G -T kata-vg/kata-pool

# Add the configuration to /etc/containerd/config.toml:
# Find the [plugins."io.containerd.snapshotter.v1.devmapper"] section and update it:

[plugins."io.containerd.snapshotter.v1.devmapper"]
  # LVM uses a specific naming convention in /dev/mapper/
  # It is VolumeGroupName-LogicalVolumeName
  pool_name = "kata--vg-kata--pool"
  root_path = "/var/lib/containerd/devmapper"
  base_image_size = "10GB"
  discard_blocks = true

  
# restart containerd
systemctl restart containerd

kube node preparation & deployment

# ---------------------------------------------------------------------------
# Packages
# ---------------------------------------------------------------------------

apt-get update
apt-get install -y \
  qemu-guest-agent \
  openssh-server \
  apt-transport-https \
  ca-certificates \
  curl \
  gnupg \
  nvme-cli

systemctl enable --now qemu-guest-agent
systemctl enable --now ssh

# ---------------------------------------------------------------------------
# nvme-tcp
# ---------------------------------------------------------------------------
apt-get install -y linux-modules-extra-$(uname -r)
modprobe nvme-tcp
echo "nvme-tcp" >> /etc/modules-load.d/nvme-tcp.conf

# ---------------------------------------------------------------------------
# Kernel modules for Kubernetes
# ---------------------------------------------------------------------------
cat > /etc/modules-load.d/k8s.conf <<'EOF'
overlay
br_netfilter
EOF

modprobe overlay
modprobe br_netfilter

cat > /etc/sysctl.d/k8s.conf <<'EOF'
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sysctl --system

# ---------------------------------------------------------------------------
# containerd
# ---------------------------------------------------------------------------
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
  > /etc/apt/sources.list.d/docker.list
apt-get update && apt-get install -y containerd.io

cat > /etc/containerd/config.toml <<'EOF'
version = 2
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
  runtime_type = "io.containerd.runc.v2"
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".registry]
  config_path = "/etc/containerd/certs.d"
EOF

# Registry mirrors pointing to Zot at 192.168.0.30:5000
mkdir -p \
  /etc/containerd/certs.d/docker.io \
  /etc/containerd/certs.d/registry.k8s.io \
  /etc/containerd/certs.d/ghcr.io \
  /etc/containerd/certs.d/quay.io

cat > /etc/containerd/certs.d/docker.io/hosts.toml <<'EOF'
server = "https://registry-1.docker.io"
[host."http://192.168.0.30:5000/v2/docker.io"]
  capabilities = ["pull", "resolve"]
  skip_verify = true
  override_path = true
EOF

cat > /etc/containerd/certs.d/registry.k8s.io/hosts.toml <<'EOF'
server = "https://registry.k8s.io"
[host."http://192.168.0.30:5000/v2/registry.k8s.io"]
  capabilities = ["pull", "resolve"]
  skip_verify = true
  override_path = true
EOF

cat > /etc/containerd/certs.d/ghcr.io/hosts.toml <<'EOF'
server = "https://ghcr.io"
[host."http://192.168.0.30:5000/v2/ghcr.io"]
  capabilities = ["pull", "resolve"]
  skip_verify = true
  override_path = true
EOF

cat > /etc/containerd/certs.d/quay.io/hosts.toml <<'EOF'
server = "https://quay.io"
[host."http://192.168.0.30:5000/v2/quay.io"]
  capabilities = ["pull", "resolve"]
  skip_verify = true
  override_path = true
EOF

systemctl restart containerd

# ---------------------------------------------------------------------------
# kubelet systemd drop-in
# ---------------------------------------------------------------------------
mkdir -p /etc/systemd/system/kubelet.service.d
cat > /etc/systemd/system/kubelet.service.d/10-containerd.conf <<'EOF'
[Unit]
After=containerd.service
Requires=containerd.service

[Service]
ExecStartPre=/bin/bash -c 'until [ -S /var/run/containerd/containerd.sock ]; do sleep 1; done'
ExecStartPre=/usr/bin/crictl info
EOF

# ---------------------------------------------------------------------------
# kubeadm / kubelet / kubectl v1.32
# ---------------------------------------------------------------------------
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key \
  | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /" \
  > /etc/apt/sources.list.d/kubernetes.list
apt-get update && apt-get install -y kubelet kubeadm kubectl
apt-mark hold kubelet kubeadm kubectl


# join kube cluster
JOIN_COMMAND="kubeadm join 192.168.0.31:6443 --token <token> --discovery-token-ca-cert-hash sha256:<hash>"