home-kubernetes/kubernetes-kvm-terraform/nodes-on-homer.tf

locals {
  vm_node_01_name    = "kube-node-32"
  vm_node_01_ip_address = "192.168.0.32"
}


resource "libvirt_volume" "node_01_disk" {
  provider = libvirt.kvm-homer
  name           = "${local.vm_node_01_name}.qcow2"
  pool           = "default"
  backing_store = {
    path   = libvirt_volume.ubuntu_base_homer.path
    format = {
        type = "qcow2"
    }
  }
  target = {
    format = {
      type = "qcow2"
    }
  }  
  capacity       = 53687091200
}

locals {
  user_data_node_01 = <<-EOF
    #cloud-config
    hostname: ${local.vm_node_01_name}
    users:
      - name: ubuntu
        sudo: ALL=(ALL) NOPASSWD:ALL
        shell: /bin/bash
        ssh_authorized_keys:
          - ${file("~/.ssh/id_rsa.pub")}
    chpasswd:
      list: |
        ubuntu:yourpassword
      expire: false
    ssh_pwauth: true
    package_update: true
    packages:
      - qemu-guest-agent
      - openssh-server
      - apt-transport-https
      - ca-certificates
      - curl
      - gnupg
      - nvme-cli

    write_files:
      - path: /etc/modules-load.d/k8s.conf
        content: |
          overlay
          br_netfilter

      - path: /etc/sysctl.d/k8s.conf
        content: |
          net.bridge.bridge-nf-call-iptables = 1
          net.bridge.bridge-nf-call-ip6tables = 1
          net.ipv4.ip_forward = 1

      - path: /etc/containerd/config.toml
        content: |
          version = 2
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
            runtime_type = "io.containerd.runc.v2"
            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
              SystemdCgroup = true

      # Update existing containerd config to enable registry config_path
      - path: /etc/containerd/config.toml
        content: |
          version = 2
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
            runtime_type = "io.containerd.runc.v2"
            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
              SystemdCgroup = true
          [plugins."io.containerd.grpc.v1.cri".registry]
            config_path = "/etc/containerd/certs.d"

      # Mirror configs for each upstream registry
      - path: /etc/containerd/certs.d/docker.io/hosts.toml
        content: |
          server = "https://registry-1.docker.io"
          [host."http://${local.zot_registry_ip}:5000/v2/docker.io"]
            capabilities = ["pull", "resolve"]
            skip_verify = true
            override_path = true

      - path: /etc/containerd/certs.d/registry.k8s.io/hosts.toml
        content: |
          server = "https://registry.k8s.io"
          [host."http://${local.zot_registry_ip}:5000/v2/registry.k8s.io"]
            capabilities = ["pull", "resolve"]
            skip_verify = true
            override_path = true

      - path: /etc/containerd/certs.d/ghcr.io/hosts.toml
        content: |
          server = "https://ghcr.io"
          [host."http://${local.zot_registry_ip}:5000/v2/ghcr.io"]
            capabilities = ["pull", "resolve"]
            skip_verify = true
            override_path = true

      - path: /etc/containerd/certs.d/quay.io/hosts.toml
        content: |
          server = "https://quay.io"
          [host."http://${local.zot_registry_ip}:5000/v2/quay.io"]
            capabilities = ["pull", "resolve"]
            skip_verify = true
            override_path = true              

      - path: /root/kubeadm-config.yaml
        content: |
          apiVersion: kubeadm.k8s.io/v1beta3
          kind: InitConfiguration
          nodeRegistration:
            criSocket: unix:///run/containerd/containerd.sock
          ---
          apiVersion: kubeadm.k8s.io/v1beta3
          kind: ClusterConfiguration
          networking:
            podSubnet: "10.244.0.0/16"
          ---
          apiVersion: kubelet.config.k8s.io/v1beta1
          kind: KubeletConfiguration
          cgroupDriver: systemd

      - path: /etc/profile.d/kubectl.sh
        content: |
          alias k='kubectl'
          source <(kubectl completion bash)
          complete -o default -F __start_kubectl k

      - path: /etc/systemd/system/kubelet.service.d/10-containerd.conf
        content: |
          [Unit]
          After=containerd.service
          Requires=containerd.service

          [Service]
          ExecStartPre=/bin/bash -c 'until [ -S /var/run/containerd/containerd.sock ]; do sleep 1; done'
          ExecStartPre=/usr/bin/crictl info


    runcmd:
      - systemctl enable --now qemu-guest-agent
      - systemctl enable --now ssh

      # needed for nvme-tcp module
      - apt install linux-modules-extra-$(uname -r)
      - modprobe nvme-tcp
      - echo "nvme-tcp" >> /etc/modules-load.d/nvme-tcp.conf

      # relevant to kubernetes
      - modprobe overlay
      - modprobe br_netfilter
      - sysctl --system

      # containerd
      - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
      - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list
      - apt-get update && apt-get install -y containerd.io
      - |
        cat > /etc/containerd/config.toml <<'CONTAINERD'
        version = 2
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          runtime_type = "io.containerd.runc.v2"
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            SystemdCgroup = true
        [plugins."io.containerd.grpc.v1.cri".registry]
          config_path = "/etc/containerd/certs.d"
        CONTAINERD
      - systemctl restart containerd

      # kubeadm/kubelet/kubectl v1.32
      - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
      - echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
      - apt-get update && apt-get install -y kubelet kubeadm kubectl
      - apt-mark hold kubelet kubeadm kubectl

      # join cluster
      - ${local.join_command}

  EOF

  network_config_node_01 = <<-EOF
    version: 2
    ethernets:
      eth0:
        match:
          driver: virtio_net
        addresses:
          - ${local.vm_node_01_ip_address}/${local.cidr}
        routes:
          - to: default
            via: ${local.gateway}
        nameservers:
          addresses:
            - ${local.dns}
  EOF
}

resource "libvirt_cloudinit_disk" "commoninit_node_01" {
  provider = libvirt.kvm-homer
  name           = "${local.vm_node_01_name}-cloudinit.iso"
  user_data      = local.user_data_node_01
  meta_data = yamlencode({
    instance-id    = local.vm_node_01_name
    local-hostname = local.vm_node_01_name
  })
  network_config = local.network_config_node_01
}

# Create a volume from the cloud-init ISO
resource "libvirt_volume" "cloudinit_node_01" {
  provider = libvirt.kvm-homer
  name = "${local.vm_node_01_name}-cloudinit.iso"
  pool = "default"
  create = {
    content = {
      url = libvirt_cloudinit_disk.commoninit_node_01.path
    }
  }
}


resource "libvirt_domain" "node_01" {
  provider = libvirt.kvm-homer
  name   = local.vm_node_01_name
  memory = "8192"
  memory_unit = "MiB" 
  vcpu   = 4
  type   = "kvm"
  autostart = true
  running   = true

  cpu = {
    mode = "host-passthrough"
  }
  os = {
    type         = "hvm"
    type_arch    = "x86_64"
    type_machine = "q35"
  }  

  devices = {
    disks = [
      {
        driver = {
          name = "qemu"
          type = "qcow2"
        }
        source = {
          file = {
            file = libvirt_volume.node_01_disk.path
          }
        }
        target = {
          dev = "vda"
          bus = "virtio"
        }
      },
      {
        device = "cdrom"
        driver = {
          name = "qemu"
          type = "raw"
        }
        source = {
          file = {
            file = libvirt_volume.cloudinit_node_01.path
          }
        }
        target = {
          dev = "sda"
          bus = "sata"
        }
      }
    ]

    interfaces = [
      {
        type = "bridge"
        model = {
          type = "virtio"
        }
        source = {
          bridge = {
            bridge = local.bridge
          }
        }
      }
    ]

    serials = [
      {
        type = "pty"
      }
    ]


    xml = {
        xslt = <<-XSLT
        <?xml version="1.0"?>
        <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
            <xsl:output method="xml" indent="yes"/>
            <xsl:template match="@*|node()">
            <xsl:copy>
                <xsl:apply-templates select="@*|node()"/>
            </xsl:copy>
            </xsl:template>
            <xsl:template match="devices">
            <xsl:copy>
                <xsl:apply-templates select="@*|node()"/>
                <serial type="pty">
                <target port="0"/>
                </serial>
                <console type="pty">
                <target type="serial" port="0"/>
                </console>
                <graphics type="vnc" port="-1" autoport="yes" listen="0.0.0.0">
                <listen type="address" address="0.0.0.0"/>
                </graphics>
                <channel type="unix">
                <target type="virtio" name="org.qemu.guest_agent.0"/>
                </channel>
            </xsl:copy>
            </xsl:template>
        </xsl:stylesheet>
        XSLT
    }
  }
}
output "node_01_ip" {
  value = local.vm_node_01_ip_address
}