terraform: extend kubernetes a little bit
This commit is contained in:
Jan Novak
314
kubernetes-kvm-terraform/nodes-on-homer.tf
Normal file
314
kubernetes-kvm-terraform/nodes-on-homer.tf
Normal file
@@ -0,0 +1,314 @@
|
||||
locals {
|
||||
vm_node_01_name = "kube-node-32"
|
||||
vm_node_01_ip_address = "192.168.0.32"
|
||||
}
|
||||
|
||||
|
||||
resource "libvirt_volume" "node_01_disk" {
|
||||
provider = libvirt.kvm-homer
|
||||
name = "${local.vm_node_01_name}.qcow2"
|
||||
pool = "default"
|
||||
backing_store = {
|
||||
path = libvirt_volume.ubuntu_base_homer.path
|
||||
format = {
|
||||
type = "qcow2"
|
||||
}
|
||||
}
|
||||
target = {
|
||||
format = {
|
||||
type = "qcow2"
|
||||
}
|
||||
}
|
||||
capacity = 21474836480
|
||||
}
|
||||
|
||||
locals {
|
||||
user_data_node_01 = <<-EOF
|
||||
#cloud-config
|
||||
hostname: ${local.vm_node_01_name}
|
||||
users:
|
||||
- name: ubuntu
|
||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
||||
shell: /bin/bash
|
||||
ssh_authorized_keys:
|
||||
- ${file("~/.ssh/id_rsa.pub")}
|
||||
chpasswd:
|
||||
list: |
|
||||
ubuntu:yourpassword
|
||||
expire: false
|
||||
ssh_pwauth: true
|
||||
package_update: true
|
||||
packages:
|
||||
- qemu-guest-agent
|
||||
- openssh-server
|
||||
- apt-transport-https
|
||||
- ca-certificates
|
||||
- curl
|
||||
- gnupg
|
||||
- nvme-cli
|
||||
|
||||
write_files:
|
||||
- path: /etc/modules-load.d/k8s.conf
|
||||
content: |
|
||||
overlay
|
||||
br_netfilter
|
||||
|
||||
- path: /etc/sysctl.d/k8s.conf
|
||||
content: |
|
||||
net.bridge.bridge-nf-call-iptables = 1
|
||||
net.bridge.bridge-nf-call-ip6tables = 1
|
||||
net.ipv4.ip_forward = 1
|
||||
|
||||
- path: /etc/containerd/config.toml
|
||||
content: |
|
||||
version = 2
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
|
||||
runtime_type = "io.containerd.runc.v2"
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
|
||||
SystemdCgroup = true
|
||||
|
||||
# Update existing containerd config to enable registry config_path
|
||||
- path: /etc/containerd/config.toml
|
||||
content: |
|
||||
version = 2
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
|
||||
runtime_type = "io.containerd.runc.v2"
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
|
||||
SystemdCgroup = true
|
||||
[plugins."io.containerd.grpc.v1.cri".registry]
|
||||
config_path = "/etc/containerd/certs.d"
|
||||
|
||||
# Mirror configs for each upstream registry
|
||||
- path: /etc/containerd/certs.d/docker.io/hosts.toml
|
||||
content: |
|
||||
server = "https://registry-1.docker.io"
|
||||
[host."http://${local.zot_registry_ip}:5000/v2/docker.io"]
|
||||
capabilities = ["pull", "resolve"]
|
||||
skip_verify = true
|
||||
override_path = true
|
||||
|
||||
- path: /etc/containerd/certs.d/registry.k8s.io/hosts.toml
|
||||
content: |
|
||||
server = "https://registry.k8s.io"
|
||||
[host."http://${local.zot_registry_ip}:5000/v2/registry.k8s.io"]
|
||||
capabilities = ["pull", "resolve"]
|
||||
skip_verify = true
|
||||
override_path = true
|
||||
|
||||
- path: /etc/containerd/certs.d/ghcr.io/hosts.toml
|
||||
content: |
|
||||
server = "https://ghcr.io"
|
||||
[host."http://${local.zot_registry_ip}:5000/v2/ghcr.io"]
|
||||
capabilities = ["pull", "resolve"]
|
||||
skip_verify = true
|
||||
override_path = true
|
||||
|
||||
- path: /etc/containerd/certs.d/quay.io/hosts.toml
|
||||
content: |
|
||||
server = "https://quay.io"
|
||||
[host."http://${local.zot_registry_ip}:5000/v2/quay.io"]
|
||||
capabilities = ["pull", "resolve"]
|
||||
skip_verify = true
|
||||
override_path = true
|
||||
|
||||
- path: /root/kubeadm-config.yaml
|
||||
content: |
|
||||
apiVersion: kubeadm.k8s.io/v1beta3
|
||||
kind: InitConfiguration
|
||||
nodeRegistration:
|
||||
criSocket: unix:///run/containerd/containerd.sock
|
||||
---
|
||||
apiVersion: kubeadm.k8s.io/v1beta3
|
||||
kind: ClusterConfiguration
|
||||
networking:
|
||||
podSubnet: "10.244.0.0/16"
|
||||
---
|
||||
apiVersion: kubelet.config.k8s.io/v1beta1
|
||||
kind: KubeletConfiguration
|
||||
cgroupDriver: systemd
|
||||
|
||||
- path: /etc/profile.d/kubectl.sh
|
||||
content: |
|
||||
alias k='kubectl'
|
||||
source <(kubectl completion bash)
|
||||
complete -o default -F __start_kubectl k
|
||||
|
||||
runcmd:
|
||||
- systemctl enable --now qemu-guest-agent
|
||||
- systemctl enable --now ssh
|
||||
|
||||
# needed for nvme-tcp module
|
||||
- apt install linux-modules-extra-$(uname -r)
|
||||
- modprobe nvme-tcp
|
||||
- echo "nvme-tcp" >> /etc/modules-load.d/nvme-tcp.conf
|
||||
|
||||
# relevant to kubernetes
|
||||
- modprobe overlay
|
||||
- modprobe br_netfilter
|
||||
- sysctl --system
|
||||
|
||||
# containerd
|
||||
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
|
||||
- echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list
|
||||
- apt-get update && apt-get install -y containerd.io
|
||||
- systemctl restart containerd
|
||||
|
||||
# kubeadm/kubelet/kubectl v1.32
|
||||
- curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
|
||||
- echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
|
||||
- apt-get update && apt-get install -y kubelet kubeadm kubectl
|
||||
- apt-mark hold kubelet kubeadm kubectl
|
||||
|
||||
# join cluster
|
||||
- ${local.join_command}
|
||||
|
||||
EOF
|
||||
|
||||
network_config_node_01 = <<-EOF
|
||||
version: 2
|
||||
ethernets:
|
||||
eth0:
|
||||
match:
|
||||
driver: virtio_net
|
||||
addresses:
|
||||
- ${local.vm_node_01_ip_address}/${local.cidr}
|
||||
routes:
|
||||
- to: default
|
||||
via: ${local.gateway}
|
||||
nameservers:
|
||||
addresses:
|
||||
- ${local.dns}
|
||||
EOF
|
||||
}
|
||||
|
||||
resource "libvirt_cloudinit_disk" "commoninit_node_01" {
|
||||
provider = libvirt.kvm-homer
|
||||
name = "${local.vm_node_01_name}-cloudinit.iso"
|
||||
user_data = local.user_data_node_01
|
||||
meta_data = yamlencode({
|
||||
instance-id = local.vm_node_01_name
|
||||
local-hostname = local.vm_node_01_name
|
||||
})
|
||||
network_config = local.network_config_node_01
|
||||
}
|
||||
|
||||
# Create a volume from the cloud-init ISO
|
||||
resource "libvirt_volume" "cloudinit_node_01" {
|
||||
provider = libvirt.kvm-homer
|
||||
name = "${local.vm_node_01_name}-cloudinit.iso"
|
||||
pool = "default"
|
||||
create = {
|
||||
content = {
|
||||
url = libvirt_cloudinit_disk.commoninit_node_01.path
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
resource "libvirt_domain" "node_01" {
|
||||
provider = libvirt.kvm-homer
|
||||
name = local.vm_node_01_name
|
||||
memory = "8192"
|
||||
memory_unit = "MiB"
|
||||
vcpu = 4
|
||||
type = "kvm"
|
||||
autostart = true
|
||||
running = true
|
||||
|
||||
os = {
|
||||
type = "hvm"
|
||||
type_arch = "x86_64"
|
||||
type_machine = "q35"
|
||||
}
|
||||
|
||||
devices = {
|
||||
disks = [
|
||||
{
|
||||
driver = {
|
||||
name = "qemu"
|
||||
type = "qcow2"
|
||||
}
|
||||
source = {
|
||||
file = {
|
||||
file = libvirt_volume.node_01_disk.path
|
||||
}
|
||||
}
|
||||
target = {
|
||||
dev = "vda"
|
||||
bus = "virtio"
|
||||
}
|
||||
},
|
||||
{
|
||||
device = "cdrom"
|
||||
driver = {
|
||||
name = "qemu"
|
||||
type = "raw"
|
||||
}
|
||||
source = {
|
||||
file = {
|
||||
file = libvirt_volume.cloudinit_node_01.path
|
||||
}
|
||||
}
|
||||
target = {
|
||||
dev = "sda"
|
||||
bus = "sata"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
interfaces = [
|
||||
{
|
||||
type = "bridge"
|
||||
model = {
|
||||
type = "virtio"
|
||||
}
|
||||
source = {
|
||||
bridge = {
|
||||
bridge = local.bridge
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
serials = [
|
||||
{
|
||||
type = "pty"
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
xml = {
|
||||
xslt = <<-XSLT
|
||||
<?xml version="1.0"?>
|
||||
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
|
||||
<xsl:output method="xml" indent="yes"/>
|
||||
<xsl:template match="@*|node()">
|
||||
<xsl:copy>
|
||||
<xsl:apply-templates select="@*|node()"/>
|
||||
</xsl:copy>
|
||||
</xsl:template>
|
||||
<xsl:template match="devices">
|
||||
<xsl:copy>
|
||||
<xsl:apply-templates select="@*|node()"/>
|
||||
<serial type="pty">
|
||||
<target port="0"/>
|
||||
</serial>
|
||||
<console type="pty">
|
||||
<target type="serial" port="0"/>
|
||||
</console>
|
||||
<graphics type="vnc" port="-1" autoport="yes" listen="0.0.0.0">
|
||||
<listen type="address" address="0.0.0.0"/>
|
||||
</graphics>
|
||||
<channel type="unix">
|
||||
<target type="virtio" name="org.qemu.guest_agent.0"/>
|
||||
</channel>
|
||||
</xsl:copy>
|
||||
</xsl:template>
|
||||
</xsl:stylesheet>
|
||||
XSLT
|
||||
}
|
||||
}
|
||||
}
|
||||
output "node_01_ip" {
|
||||
value = local.vm_node_01_ip_address
|
||||
}
|
||||
Reference in New Issue
Block a user