diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1c99dc1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+.terraform/
diff --git a/kubernetes-kvm-terraform/.terraform.lock.hcl b/kubernetes-kvm-terraform/.terraform.lock.hcl
new file mode 100644
index 0000000..b5d9835
--- /dev/null
+++ b/kubernetes-kvm-terraform/.terraform.lock.hcl
@@ -0,0 +1,57 @@
+# This file is maintained automatically by "tofu init".
+# Manual edits may be lost in future updates.
+
+provider "registry.opentofu.org/dmacvicar/libvirt" {
+ version = "0.9.1"
+ constraints = "~> 0.8"
+ hashes = [
+ "h1:94QTfgc4WIynVQPmag+oVlrEP0po7YttSzND7u56Q0c=",
+ "zh:0480649879c36dff007d4e510ff68b473c24703c66649622739f75fa6303c9df",
+ "zh:320ebb4758739d53054e81cd3eec1bb1dfebc9cda32c6b40b9820cd0a12b21ea",
+ "zh:339365c8bd5e02129daa210138af6c9c9e8e15622798d248db46963d4c39c0dd",
+ "zh:4f280769ba65eaee764a64c8d119d96b8ac53425a5298e94e1976c7bd00569ae",
+ "zh:7381c242accfc030bab88cdf8c06afa950ed5294c28278d101c7bbc93b57fb31",
+ "zh:83ea6c39bc62024958a3830d2d940210969eb80541a0288cd53d5dcb3a0d664a",
+ "zh:8c000e8b5d0526cabbf30fab6aad1fba3f828cb59b18244379999069ed51d605",
+ "zh:8e0d66ed50aa1737e2fb03b29694ddb5dcb93bbafbb521d1b571e78709f49c7d",
+ "zh:9161fddae518f7d6e0dbf8fcf7d6686d660ec47f8fa90dbcab22e293fb3ad785",
+ "zh:b2e80faecf4ba0b361a98036856027b0c08480a59581e8f461b0637b251bab2c",
+ "zh:b2fb50332d71dce8158be69ef53d3a87436bb5103ea3939c1ce81665298f9e48",
+ "zh:cf22d7c5b42cfcfc101392735a4c7f2ee8518942a257e524acd48a0a5e02ae08",
+ "zh:f5156222e08403b1df5a9b707bacf8f1caabaf9580b69794121b252e3bd93cd5",
+ "zh:f7eac58fdccc1ae378ee1c6c0bb9185455b18dcd9b367aeed7067f681ecc372f",
+ ]
+}
+
+provider "registry.opentofu.org/hashicorp/local" {
+ version = "2.6.1"
+ hashes = [
+ "h1:+XfQ7VmNtYMp0eOnoQH6cZpSMk12IP1X6tEkMoMGQ/A=",
+ "zh:0416d7bf0b459a995cf48f202af7b7ffa252def7d23386fc05b34f67347a22ba",
+ "zh:24743d559026b59610eb3d9fa9ec7fbeb06399c0ef01272e46fe5c313eb5c6ff",
+ "zh:2561cdfbc90090fee7f844a5cb5cbed8472ce264f5d505acb18326650a5b563f",
+ "zh:3ebc3f2dc7a099bd83e5c4c2b6918e5b56ec746766c58a31a3f5d189cb837db5",
+ "zh:490e0ce925fc3848027e10017f960e9e19e7f9c3b620524f67ce54217d1c6390",
+ "zh:bf08934295877f831f2e5f17a0b3ebb51dd608b2509077f7b22afa7722e28950",
+ "zh:c298c0f72e1485588a73768cb90163863b6c3d4c71982908c219e9b87904f376",
+ "zh:cedbaed4967818903ef378675211ed541c8243c4597304161363e828c7dc3d36",
+ "zh:edda76726d7874128cf1e182640c332c5a5e6a66a053c0aa97e2a0e4267b3b92",
+ ]
+}
+
+provider "registry.opentofu.org/hashicorp/null" {
+ version = "3.2.4"
+ hashes = [
+ "h1:i+WKhUHL2REY5EGmiHjfUljJB8UKZ9QdhdM5uTeUhC4=",
+ "zh:1769783386610bed8bb1e861a119fe25058be41895e3996d9216dd6bb8a7aee3",
+ "zh:32c62a9387ad0b861b5262b41c5e9ed6e940eda729c2a0e58100e6629af27ddb",
+ "zh:339bf8c2f9733fce068eb6d5612701144c752425cebeafab36563a16be460fb2",
+ "zh:36731f23343aee12a7e078067a98644c0126714c4fe9ac930eecb0f2361788c4",
+ "zh:3d106c7e32a929e2843f732625a582e562ff09120021e510a51a6f5d01175b8d",
+ "zh:74bcb3567708171ad83b234b92c9d63ab441ef882b770b0210c2b14fdbe3b1b6",
+ "zh:90b55bdbffa35df9204282251059e62c178b0ac7035958b93a647839643c0072",
+ "zh:ae24c0e5adc692b8f94cb23a000f91a316070fdc19418578dcf2134ff57cf447",
+ "zh:b5c10d4ad860c4c21273203d1de6d2f0286845edf1c64319fa2362df526b5f58",
+ "zh:e05bbd88e82e1d6234988c85db62fd66f11502645838fff594a2ec25352ecd80",
+ ]
+}
diff --git a/kubernetes-kvm-terraform/common.tf b/kubernetes-kvm-terraform/common.tf
new file mode 100644
index 0000000..bda5a38
--- /dev/null
+++ b/kubernetes-kvm-terraform/common.tf
@@ -0,0 +1,29 @@
+terraform {
+ required_providers {
+ libvirt = {
+ source = "dmacvicar/libvirt"
+ version = "~> 0.8"
+ }
+ }
+}
+
+
+provider "libvirt" {
+ uri = "qemu+ssh://novakj@192.168.0.7/system"
+}
+
+
+resource "libvirt_volume" "ubuntu_base" {
+ name = "ubuntu-24.04-base.qcow2"
+ pool = "default"
+ create = {
+ content = {
+ url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+ }
+ }
+ target = {
+ format = {
+ type = "qcow2"
+ }
+ }
+}
diff --git a/kubernetes-kvm-terraform/join-command.txt b/kubernetes-kvm-terraform/join-command.txt
new file mode 100644
index 0000000..2bf1e31
--- /dev/null
+++ b/kubernetes-kvm-terraform/join-command.txt
@@ -0,0 +1 @@
+kubeadm join 192.168.0.31:6443 --token ps457c.7g4koxq4awqigdwy --discovery-token-ca-cert-hash sha256:bd969ebcf4c68c424a76c8fc483f194ab8b32a2fd52f1cf992e164f496907447
diff --git a/kubernetes-kvm-terraform/join_command.tf b/kubernetes-kvm-terraform/join_command.tf
new file mode 100644
index 0000000..e1de364
--- /dev/null
+++ b/kubernetes-kvm-terraform/join_command.tf
@@ -0,0 +1,58 @@
+resource "null_resource" "kubeadm_token" {
+ depends_on = [libvirt_domain.master] # or whatever your master resource is
+
+ provisioner "remote-exec" {
+ inline = [
+ "until sudo kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes; do sleep 5; done",
+ "sudo kubeadm token create --print-join-command > /tmp/join-command.txt"
+ ]
+
+ connection {
+ type = "ssh"
+ host = local.master_ip
+ user = "ubuntu"
+ private_key = file("~/.ssh/id_rsa")
+ timeout = "10m" # connection timeout
+ }
+ }
+
+ provisioner "local-exec" {
+ command = "scp -o StrictHostKeyChecking=no ubuntu@${local.master_ip}:/tmp/join-command.txt ./join-command.txt"
+ }
+}
+
+data "local_file" "join_command" {
+ depends_on = [null_resource.kubeadm_token]
+ filename = "./join-command.txt"
+}
+
+# get kubeconfig and store it locally
+resource "null_resource" "kubeconfig" {
+
+ provisioner "remote-exec" {
+ inline = [
+ "until sudo ls -la /etc/kubernetes/admin.conf; do sleep 5; done",
+ "sudo cp /etc/kubernetes/admin.conf /tmp/admin.conf",
+ "sudo chown $(id -u):$(id -g) /tmp/admin.conf",
+ # Rewrite server address from localhost/internal to accessible IP
+ "sudo sed -i 's|server: https://.*:6443|server: https://${local.master_ip}:6443|' /tmp/admin.conf"
+ ]
+
+ connection {
+ type = "ssh"
+ host = local.master_ip
+ user = "ubuntu"
+ private_key = file("~/.ssh/id_rsa")
+ }
+ }
+
+ provisioner "local-exec" {
+ command = "scp -o StrictHostKeyChecking=no ubuntu@${local.master_ip}:/tmp/admin.conf ./kubeconfig"
+ }
+}
+
+
+locals {
+ # parse: kubeadm join 192.168.1.10:6443 --token xxx --discovery-token-ca-cert-hash sha256:yyy
+ join_command = trimspace(data.local_file.join_command.content)
+}
\ No newline at end of file
diff --git a/kubernetes-kvm-terraform/kubeconfig b/kubernetes-kvm-terraform/kubeconfig
new file mode 100644
index 0000000..715e98d
--- /dev/null
+++ b/kubernetes-kvm-terraform/kubeconfig
@@ -0,0 +1,19 @@
+apiVersion: v1
+clusters:
+- cluster:
+ certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJSWh2TjVRZFNKUnd3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TlRFeU1qa3hNekUyTXpGYUZ3MHpOVEV5TWpjeE16SXhNekZhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUUR6c2NVNnpMUWd0RFVkeHV3TUdPeFZqdzVhNVNCRzJrWEtWcHJnWEsyT1Yrd0IrRFFUK3JsUjVCM1QKSTFpd3BUMG5EcG5FcCtOM0VNYkg0T0RhWERhZkNrMUIwcXFDbW9LSXViT1V5TEZ1bmNNd3BJdW9ZOXJDN3puOQpmYVFabFNYeks5b1gzcUc4WjZCWTE4L3c3WHMybVJMYWgxVDQ5cGVwOEp0dGYyKy90dkFqcVBUU3NtVDVHTHErCmNvOXlVNUNDQUMwM0ZPakpIZU9PMEsyTVYycEZPc2ZoUGMxamJsR1VjNHdiVDU0WTR1em1LNjJTM3EyYm9PenQKcHI2RHZkbmJwZHcrR1lJQXZjVWtjMnZtODl1MXVtd0RNWUhYd3A3UWM4N2x1MGZSeHByUjdqLyt0MEVrSlVTWApuV25YVzhnNnhtYm5rakVISDUwaTdXakRMMUw5QWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJUTnBEYVptMmpSUmpjRVNhbDltMC92SkpZalhUQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ1ZvT25DUnlBLwowUXBMYkt5Y2VPallFeWFBYXRwbU5lTTJRNmJOVklzNnhlaHBPSWQ1NWQzZGdTR05TSXZtcllwVHYwNHNhZVo3CjF3VTBndHVtSVN6U2V5OGNiZXp1bGNIamlmT0dhYVpESS9OSXdoVStPSW5qcG9jTFNONTliZDJrNXV3VVV6dHMKU1UrWXNHT1NnNE84YlRyWUxwSUc3SEMyWGFiQmJCeXpLV3ZNc0t3bGtORFZPSnNLMENZRklBTnEvNWZmNC96NwpPd1c2bXJTTkwxM25wV2JLNVd0REtlUVlzcXdZVGhnM2E3MjJwVHc3U2l5bjFtSkFLUzVrMSttMG43WlNUc2dHCjVwLzg3Q2NrZ01xNHJaRml3K0VJL09mbW0vRk9scXhmYjVjR1RCdy9QZTRwQ1NNUUVYV2RjUTNHbVE1bnUvMHEKLy84L2RlL244VlFBCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+ server: https://192.168.0.31:6443
+ name: kubernetes
+contexts:
+- context:
+ cluster: kubernetes
+ user: kubernetes-admin
+ name: kubernetes-admin@kubernetes
+current-context: kubernetes-admin@kubernetes
+kind: Config
+preferences: {}
+users:
+- name: kubernetes-admin
+ user:
+ client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURLVENDQWhHZ0F3SUJBZ0lJR0lTZmJsakh4akV3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TlRFeU1qa3hNekUyTXpGYUZ3MHlOakV5TWpreE16SXhNekZhTUR3eApIekFkQmdOVkJBb1RGbXQxWW1WaFpHMDZZMngxYzNSbGNpMWhaRzFwYm5NeEdUQVhCZ05WQkFNVEVHdDFZbVZ5CmJtVjBaWE10WVdSdGFXNHdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFDeGV1eHEKQkpxbmozR2R3ZWdsYVd1R0pKL0FFVTFNYTQ2V2Z3Si9FWTB1Z216VDdmREZvaS9vbUtWblUvZmF5ZHVjVmVQMApEYTd6d2lKODFZWTdMZTdwUis4MHlUNVJ6WWVsMmtIeEpQWS9OTXVEeXk2SzdZT2E4NVRSZVg2dmNjTlZncGt1CnkzSE5vbmNLcGNWay8vZlRKUXFaVmtpdmFNcS90bUszQjFvYlRLR2pzY1M4TmNpRnNFS09iU3ZWdUpXOTcySFcKUTBMNUFCNWU4SGd5TlFHeEE2VGE2by9EZFQyZmJGby91ZkRKUU4relhXQzRtTTRMajdZNDhxMnMyLzUrSUg3OQpHY2ZLcXp6bE5aZjFHSHFwczdOc1pYVUhqaFZIdHU5VnNteWpjY2JENWREYVNyRWowS0tMNWhyM3BtWDZoYlVQCnJWQVkyRS9PbU1xTUxjT1BBZ01CQUFHalZqQlVNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUsKQmdnckJnRUZCUWNEQWpBTUJnTlZIUk1CQWY4RUFqQUFNQjhHQTFVZEl3UVlNQmFBRk0ya05wbWJhTkZHTndSSgpxWDJiVCs4a2xpTmRNQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUNGMGdzUU9YM1ZYZTEwY1ZpVVplblpaQ1ZyCjkrdFhvT2pJcDlxRDU4eHhRSzZSMmhBTmhRa0s4Z2ZrbUpkaDJsNDcvaVgvQVF6MnhTdVhNVUhoRlpVYlEzb08KaXkvamZNZGJNZ24wRzdmYVQ3Vm0xYnIzTkpyMFJQbjVPRkNpZTRtclN4c0hXQVJnL3o4Q3hLalIxb1FwamU1dwpsNFdlcXhFckx2MHRDYnNneTZndldBL3I5eGVSeTBIbHE4bG9yOWdZejJlaUxJSFU3UStRRXdyU3B5akdDQjRNCnlhZGdqT0x5bzNidHhYWjRXQ2JMV2kwOFZCSWJIWThVWC83QmN6YU9wdm9oOTVDTE9VVWY0RlV3MitoUlVhWDIKbjRTbTJwdE1TSmNKdFBabDgzN3A2aGNtYkQ0Y2VnMHlMSGhZVGJSYnhkVFZTM0hxSHhFS0JXdTRwaTFwCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+ client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBc1hyc2FnU2FwNDl4bmNIb0pXbHJoaVNmd0JGTlRHdU9sbjhDZnhHTkxvSnMwKzN3CnhhSXY2SmlsWjFQMzJzbmJuRlhqOUEydTg4SWlmTldHT3kzdTZVZnZOTWsrVWMySHBkcEI4U1QyUHpUTGc4c3UKaXUyRG12T1UwWGwrcjNIRFZZS1pMc3R4emFKM0NxWEZaUC8zMHlVS21WWklyMmpLdjdaaXR3ZGFHMHlobzdIRQp2RFhJaGJCQ2ptMHIxYmlWdmU5aDFrTkMrUUFlWHZCNE1qVUJzUU9rMnVxUHczVTluMnhhUDdud3lVRGZzMTFnCnVKak9DNCsyT1BLdHJOditmaUIrL1JuSHlxczg1VFdYOVJoNnFiT3piR1YxQjQ0VlI3YnZWYkpzbzNIR3crWFEKMmtxeEk5Q2lpK1lhOTZabCtvVzFENjFRR05oUHpwaktqQzNEandJREFRQUJBb0lCQUI3SnpVTmpBeFU0VENScQo1QVBCY3ZhMVdjVXUrekxib2NPM3lzNFNWR1NTR2FEMk1RT0g3RG0wYVZVd2V0bklsTUZkSVF0UWFGNCsrazZmCm1BYWlKRVR4eHF4b2tOZ09DSThiK3I2RGcvaFpCMGtZc1QxR2tNTUxxSEJpWERqQlNucUdDbDdVeWl3N0Y5MlAKL2x0VUVFRGZ0eVA4bE9WaEQzTU1oN2xxbkJZSi9qTzlDbitsWWJiZnJhcitEZUpTQzhiUDZWb1hjVFlacHpoSworeUZmNW1vdjZDMTdHeTMrb1E1ZDJMWG5aL3Exb2k5YjJ6VFBhUUREaTJndStnT0I4U0hWam5FMGFpeHRwc2NpCklGSW1sZlh0WXltSy9KQ3kvMCtkdDF1cEhIUnc1ZXRtMmRmdXNKUS9JdkhxU2JyVVRoVllVRjZ3RXZrWk9BaGYKQVNvVldxRUNnWUVBMExHaTA5ejA3dS91aFlPenNNQVV5eVFTakN3U3hBbTFLcmhqVXNsSis0TVJRci9QdlVRRwpmSGZTWm1UeTdLbjZIOS9PL2dmNTFQb25KOHpOWHVqUFcvdGRCWTlHT2Z2aE9LS1lieWo5QWpPUXdFREhTblFpClhuejBwV3BvYUIvcWU5cGdYSk44U000b2lEbW1XdmJKTk52TXpOTHlpQzl2Rkpwc1VtTmJXVEVDZ1lFQTJiWDUKL3BIWFh0WklZVkJEc2ZrenNwK0xSZVF0SWtrNjcwaVBMNGplbVdzcUQzc2N5ZllnUWYvSytKeURwWkd5SXhWSQprYVZxZlRmeUlqWEh1SllvOG45ZTFvVnRoT3E4dEVpSnd2MXg2WWc3anpIZjB4U1hHdEtwYytJTDQ3MFRMMWdiCitqRTQva2Q2SS9ybUhja2NPRStaTzAzRDZIZk9mU3FhWFZiYnVMOENnWUVBbHBLa0RjS05nbHZndDdwWGlyNTgKQWptTG5GRXNWUDlBRkl1bk5oTC9heFdjL1pFUlhOaFk2Yk0yUGlTMTFTV3F6eGFXN1ZocDNFMkV0TVdpbzhqTAoyK3pYcCt4QTJoTXU0OE1Fd3FkTExTeHhVeFVwOVVFbmp3OFBJTTVPZUZvV3Y3Zkc4T1NFdHE3d05hR0tzblQrCnBVRzFXVThXS0VZNUdoOTA2bGkvanZFQ2dZQWlEb0dJL1M5L3VtMmdLM29yZ1J4Qk9Rd0VadWxZajdRVWNubFQKcWttSUhtZDUvSHhJYTZRb0x1ZUZkSlp6ZEJNMXdjS1VaaGYrZHAzK0ZBT21NSGMvV0FWeDhYVnNjWHdMaDcxbApMMWgrZTcwMWJVdmlMVHBtQzhaT1JuSXRzZ29xUWRJK2xTeWhPblUwNDY3VHdmNFVJUCtMYzJMT0hzL0NiTU94CkRrRlk2UUtCZ0RYLzUwV1MzSVZuejM4TFlkcnpCSC9OUXAyMzJpcmxaekwzZ0E2VVJ6SVpUU1ZKRWUrd2RmY3QKQXZPQlpEa2hYZ00rcm1XU2FjUmhCY3ZHVysrUDVTVy9jbkNFTUd4cC95L3VaeVpmS01Kd3VHZStzZWZhbVJkWgowVVpYZzRtdFhLdndCSCtCMks0dWJTSXZQYzI2TTJyVFNYVUtqTnAyUkwycjFCYmVoK1Q3Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==
diff --git a/kubernetes-kvm-terraform/master.tf b/kubernetes-kvm-terraform/master.tf
new file mode 100644
index 0000000..469ecba
--- /dev/null
+++ b/kubernetes-kvm-terraform/master.tf
@@ -0,0 +1,327 @@
+locals {
+ master_vm_name = "kube-master-31"
+ master_ip = "192.168.0.31"
+ gateway = "192.168.0.4"
+ dns = "8.8.8.8"
+ cidr = 24
+ bridge = "br0"
+ zot_registry_ip = "192.168.0.30" # wherever zot is running
+}
+
+
+resource "libvirt_volume" "ubuntu_disk" {
+ name = "${local.master_vm_name}.qcow2"
+ pool = "default"
+ backing_store = {
+ path = libvirt_volume.ubuntu_base.path
+ format = {
+ type = "qcow2"
+ }
+ }
+ target = {
+ format = {
+ type = "qcow2"
+ }
+ }
+ capacity = 21474836480
+}
+
+locals {
+ user_data = <<-EOF
+ #cloud-config
+ hostname: ${local.master_vm_name}
+ users:
+ - name: ubuntu
+ sudo: ALL=(ALL) NOPASSWD:ALL
+ shell: /bin/bash
+ ssh_authorized_keys:
+ - ${file("~/.ssh/id_rsa.pub")}
+ chpasswd:
+ list: |
+ ubuntu:yourpassword
+ expire: false
+ ssh_pwauth: true
+ package_update: true
+ packages:
+ - qemu-guest-agent
+ - openssh-server
+ - apt-transport-https
+ - ca-certificates
+ - curl
+ - gnupg
+
+ write_files:
+ - path: /etc/modules-load.d/k8s.conf
+ content: |
+ overlay
+ br_netfilter
+
+ - path: /etc/sysctl.d/k8s.conf
+ content: |
+ net.bridge.bridge-nf-call-iptables = 1
+ net.bridge.bridge-nf-call-ip6tables = 1
+ net.ipv4.ip_forward = 1
+
+ - path: /etc/containerd/config.toml
+ content: |
+ version = 2
+ [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+ runtime_type = "io.containerd.runc.v2"
+ [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+ SystemdCgroup = true
+
+ # Update existing containerd config to enable registry config_path
+ - path: /etc/containerd/config.toml
+ content: |
+ version = 2
+ [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+ runtime_type = "io.containerd.runc.v2"
+ [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+ SystemdCgroup = true
+ [plugins."io.containerd.grpc.v1.cri".registry]
+ config_path = "/etc/containerd/certs.d"
+
+ # Mirror configs for each upstream registry
+ - path: /etc/containerd/certs.d/docker.io/hosts.toml
+ content: |
+ server = "https://registry-1.docker.io"
+ [host."http://${local.zot_registry_ip}:5000/v2/docker.io"]
+ capabilities = ["pull", "resolve"]
+ skip_verify = true
+ override_path = true
+
+ - path: /etc/containerd/certs.d/registry.k8s.io/hosts.toml
+ content: |
+ server = "https://registry.k8s.io"
+ [host."http://${local.zot_registry_ip}:5000/v2/registry.k8s.io"]
+ capabilities = ["pull", "resolve"]
+ skip_verify = true
+ override_path = true
+
+ - path: /etc/containerd/certs.d/ghcr.io/hosts.toml
+ content: |
+ server = "https://ghcr.io"
+ [host."http://${local.zot_registry_ip}:5000/v2/ghcr.io"]
+ capabilities = ["pull", "resolve"]
+ skip_verify = true
+ override_path = true
+
+ - path: /etc/containerd/certs.d/quay.io/hosts.toml
+ content: |
+ server = "https://quay.io"
+ [host."http://${local.zot_registry_ip}:5000/v2/quay.io"]
+ capabilities = ["pull", "resolve"]
+ skip_verify = true
+ override_path = true
+
+ - path: /root/kubeadm-config.yaml
+ content: |
+ apiVersion: kubeadm.k8s.io/v1beta3
+ kind: InitConfiguration
+ nodeRegistration:
+ criSocket: unix:///run/containerd/containerd.sock
+ ---
+ apiVersion: kubeadm.k8s.io/v1beta3
+ kind: ClusterConfiguration
+ networking:
+ podSubnet: "10.244.0.0/16"
+ ---
+ apiVersion: kubelet.config.k8s.io/v1beta1
+ kind: KubeletConfiguration
+ cgroupDriver: systemd
+
+ - path: /etc/profile.d/kubectl.sh
+ content: |
+ alias k='kubectl'
+ source <(kubectl completion bash)
+ complete -o default -F __start_kubectl k
+
+ runcmd:
+ - systemctl enable --now qemu-guest-agent
+ - systemctl enable --now ssh
+
+ # relevant to kubernetes
+ - modprobe overlay
+ - modprobe br_netfilter
+ - sysctl --system
+
+ # containerd
+ - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+ - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list
+ - apt-get update && apt-get install -y containerd.io
+ - systemctl restart containerd
+
+ # kubeadm/kubelet/kubectl v1.32
+ - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+ - echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
+ - apt-get update && apt-get install -y kubelet kubeadm kubectl
+ - apt-mark hold kubelet kubeadm kubectl
+
+
+ # init cluster
+ - kubeadm init --config=/root/kubeadm-config.yaml --skip-phases=addon/kube-proxy
+
+ # kubeconfig for root
+ - mkdir -p /root/.kube && cp /etc/kubernetes/admin.conf /root/.kube/config
+
+ # wait for API server
+ - |
+ echo "Waiting for API server..."
+ until kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes ; do
+ echo "Waiting for API server..."
+ sleep 5
+ done
+
+ # CNI (cilium example, swap for flannel/calico as needed)
+ - |
+ CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
+ curl -L --remote-name-all https://github.com/cilium/cilium-cli/releases/download/$${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz
+ tar xzvf cilium-linux-amd64.tar.gz -C /usr/local/bin
+ cilium install --kubeconfig=/etc/kubernetes/admin.conf --set kubeProxyReplacement=true --wait
+ EOF
+
+ network_config = <<-EOF
+ version: 2
+ ethernets:
+ eth0:
+ match:
+ driver: virtio_net
+ addresses:
+ - ${local.master_ip}/${local.cidr}
+ routes:
+ - to: default
+ via: ${local.gateway}
+ nameservers:
+ addresses:
+ - ${local.dns}
+ EOF
+}
+
+resource "libvirt_cloudinit_disk" "commoninit" {
+ name = "${local.master_vm_name}-cloudinit.iso"
+ user_data = local.user_data
+ meta_data = yamlencode({
+ instance-id = local.master_vm_name
+ local-hostname = local.master_vm_name
+ })
+ network_config = local.network_config
+}
+
+# Create a volume from the cloud-init ISO
+resource "libvirt_volume" "cloudinit" {
+ name = "${local.master_vm_name}-cloudinit.iso"
+ pool = "default"
+ create = {
+ content = {
+ url = libvirt_cloudinit_disk.commoninit.path
+ }
+ }
+}
+
+
+resource "libvirt_domain" "master" {
+ name = local.master_vm_name
+ memory = "2048"
+ memory_unit = "MiB"
+ vcpu = 2
+ type = "kvm"
+ autostart = true
+ running = true
+
+ os = {
+ type = "hvm"
+ type_arch = "x86_64"
+ type_machine = "q35"
+ }
+
+ devices = {
+ disks = [
+ {
+ driver = {
+ name = "qemu"
+ type = "qcow2"
+ }
+ source = {
+ file = {
+ file = libvirt_volume.ubuntu_disk.path
+ }
+ }
+ target = {
+ dev = "vda"
+ bus = "virtio"
+ }
+ },
+ {
+ device = "cdrom"
+ driver = {
+ name = "qemu"
+ type = "raw"
+ }
+ source = {
+ file = {
+ file = libvirt_volume.cloudinit.path
+ }
+ }
+ target = {
+ dev = "sda"
+ bus = "sata"
+ }
+ }
+ ]
+
+ interfaces = [
+ {
+ type = "bridge"
+ model = {
+ type = "virtio"
+ }
+ source = {
+ bridge = {
+ bridge = local.bridge
+ }
+ }
+ }
+ ]
+
+ serials = [
+ {
+ type = "pty"
+ }
+ ]
+
+
+ xml = {
+ xslt = <<-XSLT
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ XSLT
+ }
+ }
+}
+output "master_ip" {
+ value = local.master_ip
+}
\ No newline at end of file
diff --git a/kubernetes-kvm-terraform/nodes.tf b/kubernetes-kvm-terraform/nodes.tf
new file mode 100644
index 0000000..674de0f
--- /dev/null
+++ b/kubernetes-kvm-terraform/nodes.tf
@@ -0,0 +1,304 @@
+locals {
+ vm_node_01_name = "kube-node-32"
+ vm_node_01_ip_address = "192.168.0.32"
+}
+
+
+resource "libvirt_volume" "node_01_disk" {
+ name = "${local.vm_node_01_name}.qcow2"
+ pool = "default"
+ backing_store = {
+ path = libvirt_volume.ubuntu_base.path
+ format = {
+ type = "qcow2"
+ }
+ }
+ target = {
+ format = {
+ type = "qcow2"
+ }
+ }
+ capacity = 21474836480
+}
+
+locals {
+ user_data_node = <<-EOF
+ #cloud-config
+ hostname: ${local.vm_node_01_name}
+ users:
+ - name: ubuntu
+ sudo: ALL=(ALL) NOPASSWD:ALL
+ shell: /bin/bash
+ ssh_authorized_keys:
+ - ${file("~/.ssh/id_rsa.pub")}
+ chpasswd:
+ list: |
+ ubuntu:yourpassword
+ expire: false
+ ssh_pwauth: true
+ package_update: true
+ packages:
+ - qemu-guest-agent
+ - openssh-server
+ - apt-transport-https
+ - ca-certificates
+ - curl
+ - gnupg
+
+ write_files:
+ - path: /etc/modules-load.d/k8s.conf
+ content: |
+ overlay
+ br_netfilter
+
+ - path: /etc/sysctl.d/k8s.conf
+ content: |
+ net.bridge.bridge-nf-call-iptables = 1
+ net.bridge.bridge-nf-call-ip6tables = 1
+ net.ipv4.ip_forward = 1
+
+ - path: /etc/containerd/config.toml
+ content: |
+ version = 2
+ [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+ runtime_type = "io.containerd.runc.v2"
+ [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+ SystemdCgroup = true
+
+ # Update existing containerd config to enable registry config_path
+ - path: /etc/containerd/config.toml
+ content: |
+ version = 2
+ [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+ runtime_type = "io.containerd.runc.v2"
+ [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+ SystemdCgroup = true
+ [plugins."io.containerd.grpc.v1.cri".registry]
+ config_path = "/etc/containerd/certs.d"
+
+ # Mirror configs for each upstream registry
+ - path: /etc/containerd/certs.d/docker.io/hosts.toml
+ content: |
+ server = "https://registry-1.docker.io"
+ [host."http://${local.zot_registry_ip}:5000/v2/docker.io"]
+ capabilities = ["pull", "resolve"]
+ skip_verify = true
+ override_path = true
+
+ - path: /etc/containerd/certs.d/registry.k8s.io/hosts.toml
+ content: |
+ server = "https://registry.k8s.io"
+ [host."http://${local.zot_registry_ip}:5000/v2/registry.k8s.io"]
+ capabilities = ["pull", "resolve"]
+ skip_verify = true
+ override_path = true
+
+ - path: /etc/containerd/certs.d/ghcr.io/hosts.toml
+ content: |
+ server = "https://ghcr.io"
+ [host."http://${local.zot_registry_ip}:5000/v2/ghcr.io"]
+ capabilities = ["pull", "resolve"]
+ skip_verify = true
+ override_path = true
+
+ - path: /etc/containerd/certs.d/quay.io/hosts.toml
+ content: |
+ server = "https://quay.io"
+ [host."http://${local.zot_registry_ip}:5000/v2/quay.io"]
+ capabilities = ["pull", "resolve"]
+ skip_verify = true
+ override_path = true
+
+ - path: /root/kubeadm-config.yaml
+ content: |
+ apiVersion: kubeadm.k8s.io/v1beta3
+ kind: InitConfiguration
+ nodeRegistration:
+ criSocket: unix:///run/containerd/containerd.sock
+ ---
+ apiVersion: kubeadm.k8s.io/v1beta3
+ kind: ClusterConfiguration
+ networking:
+ podSubnet: "10.244.0.0/16"
+ ---
+ apiVersion: kubelet.config.k8s.io/v1beta1
+ kind: KubeletConfiguration
+ cgroupDriver: systemd
+
+ - path: /etc/profile.d/kubectl.sh
+ content: |
+ alias k='kubectl'
+ source <(kubectl completion bash)
+ complete -o default -F __start_kubectl k
+
+ runcmd:
+ - systemctl enable --now qemu-guest-agent
+ - systemctl enable --now ssh
+
+ # relevant to kubernetes
+ - modprobe overlay
+ - modprobe br_netfilter
+ - sysctl --system
+
+ # containerd
+ - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+ - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list
+ - apt-get update && apt-get install -y containerd.io
+ - systemctl restart containerd
+
+ # kubeadm/kubelet/kubectl v1.32
+ - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+ - echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
+ - apt-get update && apt-get install -y kubelet kubeadm kubectl
+ - apt-mark hold kubelet kubeadm kubectl
+
+ # join cluster
+ - ${local.join_command}
+
+ EOF
+
+ network_config_node_01 = <<-EOF
+ version: 2
+ ethernets:
+ eth0:
+ match:
+ driver: virtio_net
+ addresses:
+ - ${local.vm_node_01_ip_address}/${local.cidr}
+ routes:
+ - to: default
+ via: ${local.gateway}
+ nameservers:
+ addresses:
+ - ${local.dns}
+ EOF
+}
+
+resource "libvirt_cloudinit_disk" "commoninit_node_01" {
+ name = "${local.vm_node_01_name}-cloudinit.iso"
+ user_data = local.user_data_node
+ meta_data = yamlencode({
+ instance-id = local.vm_node_01_name
+ local-hostname = local.vm_node_01_name
+ })
+ network_config = local.network_config_node_01
+}
+
+# Create a volume from the cloud-init ISO
+resource "libvirt_volume" "cloudinit_node_01" {
+ name = "${local.vm_node_01_name}-cloudinit.iso"
+ pool = "default"
+ create = {
+ content = {
+ url = libvirt_cloudinit_disk.commoninit_node_01.path
+ }
+ }
+}
+
+
+resource "libvirt_domain" "node_01" {
+ name = local.vm_node_01_name
+ memory = "4096"
+ memory_unit = "MiB"
+ vcpu = 2
+ type = "kvm"
+ autostart = true
+ running = true
+
+ os = {
+ type = "hvm"
+ type_arch = "x86_64"
+ type_machine = "q35"
+ }
+
+ devices = {
+ disks = [
+ {
+ driver = {
+ name = "qemu"
+ type = "qcow2"
+ }
+ source = {
+ file = {
+ file = libvirt_volume.node_01_disk.path
+ }
+ }
+ target = {
+ dev = "vda"
+ bus = "virtio"
+ }
+ },
+ {
+ device = "cdrom"
+ driver = {
+ name = "qemu"
+ type = "raw"
+ }
+ source = {
+ file = {
+ file = libvirt_volume.cloudinit_node_01.path
+ }
+ }
+ target = {
+ dev = "sda"
+ bus = "sata"
+ }
+ }
+ ]
+
+ interfaces = [
+ {
+ type = "bridge"
+ model = {
+ type = "virtio"
+ }
+ source = {
+ bridge = {
+ bridge = local.bridge
+ }
+ }
+ }
+ ]
+
+ serials = [
+ {
+ type = "pty"
+ }
+ ]
+
+
+ xml = {
+ xslt = <<-XSLT
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ XSLT
+ }
+ }
+}
+output "node_01_ip" {
+ value = local.vm_node_01_ip_address
+}
\ No newline at end of file
diff --git a/kubernetes-kvm-terraform/readme.md b/kubernetes-kvm-terraform/readme.md
new file mode 100644
index 0000000..e1e1c07
--- /dev/null
+++ b/kubernetes-kvm-terraform/readme.md
@@ -0,0 +1,19 @@
+# kubernetes
+
+## 1: create VMs on hypervisor
+## 2: install kubernetes with kubeadm
+## 3: bootstrap flux
+## 4: deploy the rest
+
+
+
+### Random notes
+
+```bash
+# recreate specific resources
+tofu destroy -target=libvirt_domain.ubuntu_vm -target=libvirt_volume.cloudinit
+tofu apply
+
+# taint resource to have it recreated
+tofu taint libvirt_domain.ubuntu_vm
+```
\ No newline at end of file
diff --git a/kubernetes-kvm-terraform/terraform.tfstate b/kubernetes-kvm-terraform/terraform.tfstate
new file mode 100644
index 0000000..44a3d16
--- /dev/null
+++ b/kubernetes-kvm-terraform/terraform.tfstate
@@ -0,0 +1 @@
+{"version":4,"terraform_version":"1.11.2","serial":66,"lineage":"69f81358-9017-b0fa-2e3d-404364b1f698","outputs":{"master_ip":{"value":"192.168.0.31","type":"string"},"node_01_ip":{"value":"192.168.0.32","type":"string"}},"resources":[{"mode":"data","type":"local_file","name":"join_command","provider":"provider[\"registry.opentofu.org/hashicorp/local\"]","instances":[{"schema_version":0,"attributes":{"content":"kubeadm join 192.168.0.31:6443 --token ps457c.7g4koxq4awqigdwy --discovery-token-ca-cert-hash sha256:bd969ebcf4c68c424a76c8fc483f194ab8b32a2fd52f1cf992e164f496907447 \n","content_base64":"a3ViZWFkbSBqb2luIDE5Mi4xNjguMC4zMTo2NDQzIC0tdG9rZW4gcHM0NTdjLjdnNGtveHE0YXdxaWdkd3kgLS1kaXNjb3ZlcnktdG9rZW4tY2EtY2VydC1oYXNoIHNoYTI1NjpiZDk2OWViY2Y0YzY4YzQyNGE3NmM4ZmM0ODNmMTk0YWI4YjMyYTJmZDUyZjFjZjk5MmUxNjRmNDk2OTA3NDQ3IAo=","content_base64sha256":"Nfe4cEuj9IcdF7E6B/NCohZAmoqEE1jkbv+5/fFg6Q4=","content_base64sha512":"/9NQ9J2SuaSzn2wG48VRTANdbi9tdr9AaVq9YLjYOGHRyJyrtntbjgJy636+fuhUvAcHXIad+t9ey4aEpOiWdg==","content_md5":"a50d45f29d7a87e59055f88ba97a6778","content_sha1":"431c014c1cef740ad9ae9e2e028d2ab426eb4f1d","content_sha256":"35f7b8704ba3f4871d17b13a07f342a216409a8a841358e46effb9fdf160e90e","content_sha512":"ffd350f49d92b9a4b39f6c06e3c5514c035d6e2f6d76bf40695abd60b8d83861d1c89cabb67b5b8e0272eb7ebe7ee854bc07075c869dfadf5ecb8684a4e89676","filename":"./join-command.txt","id":"431c014c1cef740ad9ae9e2e028d2ab426eb4f1d"},"sensitive_attributes":[]}]},{"mode":"managed","type":"libvirt_cloudinit_disk","name":"commoninit","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"id":"9b5efd668378a2cc","meta_data":"\"instance-id\": \"kube-master-31\"\n\"local-hostname\": \"kube-master-31\"\n","name":"kube-master-31-cloudinit.iso","network_config":"version: 2\nethernets:\n eth0:\n match:\n driver: virtio_net\n addresses:\n - 192.168.0.31/24\n routes:\n - to: default\n via: 192.168.0.4\n nameservers:\n addresses:\n - 8.8.8.8\n","path":"/var/folders/dq/h32yg2mx55b2m8hxmf1yh66r0000gn/T/terraform-provider-libvirt-cloudinit/cloudinit-9b5efd668378a2cc.iso","size":49152,"user_data":"#cloud-config\nhostname: kube-master-31\nusers:\n - name: ubuntu\n sudo: ALL=(ALL) NOPASSWD:ALL\n shell: /bin/bash\n ssh_authorized_keys:\n - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDTLxDOQMsumLpjIzMXvpN/hvoMfbVFcl7TDKXTYLBJIvTOLWO/elI3PeJkXNAlaIT1lgJQB7NwQED+DciPLjCLzkG3aSxuJdZRo0Z+vHm9CEUkWnq8icLTQs0zYadK6p24VpMqT61cGNv9L/riciMRWT6l2SdRN2ZBR6MhCXMZy/XgRRlcV9iN1n9T0Q9aZjaQ2CjgG59wTLdQ0bHRlFBSt2q6erFmCIiBHUJba4avDVRvYryvMgkopayL/eK2gxaloMjcMt4z+zIokxhQONPPU2AvqMcqQIsWGQYIzIkJ0UOO8ly/PAFCpgvukWJepmZR30vVBALFCRZCUN1zMbI3 jan.novak@Jans-MacBook-Air-9.local\n\nchpasswd:\n list: |\n ubuntu:yourpassword\n expire: false\nssh_pwauth: true\npackage_update: true\npackages:\n - qemu-guest-agent\n - openssh-server\n - apt-transport-https\n - ca-certificates\n - curl\n - gnupg \n\nwrite_files:\n - path: /etc/modules-load.d/k8s.conf\n content: |\n overlay\n br_netfilter\n\n - path: /etc/sysctl.d/k8s.conf\n content: |\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.ipv4.ip_forward = 1\n\n - path: /etc/containerd/config.toml\n content: |\n version = 2\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n runtime_type = \"io.containerd.runc.v2\"\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n SystemdCgroup = true\n\n # Update existing containerd config to enable registry config_path\n - path: /etc/containerd/config.toml\n content: |\n version = 2\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n runtime_type = \"io.containerd.runc.v2\"\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n SystemdCgroup = true\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n config_path = \"/etc/containerd/certs.d\"\n\n # Mirror configs for each upstream registry\n - path: /etc/containerd/certs.d/docker.io/hosts.toml\n content: |\n server = \"https://registry-1.docker.io\"\n [host.\"http://192.168.0.30:5000/v2/docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/registry.k8s.io/hosts.toml\n content: |\n server = \"https://registry.k8s.io\"\n [host.\"http://192.168.0.30:5000/v2/registry.k8s.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/ghcr.io/hosts.toml\n content: |\n server = \"https://ghcr.io\"\n [host.\"http://192.168.0.30:5000/v2/ghcr.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/quay.io/hosts.toml\n content: |\n server = \"https://quay.io\"\n [host.\"http://192.168.0.30:5000/v2/quay.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true \n\n - path: /root/kubeadm-config.yaml\n content: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: InitConfiguration\n nodeRegistration:\n criSocket: unix:///run/containerd/containerd.sock\n ---\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: ClusterConfiguration\n networking:\n podSubnet: \"10.244.0.0/16\"\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n cgroupDriver: systemd\n\n - path: /etc/profile.d/kubectl.sh\n content: |\n alias k='kubectl'\n source \u003c(kubectl completion bash)\n complete -o default -F __start_kubectl k \n\nruncmd:\n - systemctl enable --now qemu-guest-agent\n - systemctl enable --now ssh\n\n # relevant to kubernetes\n - modprobe overlay\n - modprobe br_netfilter\n - sysctl --system\n\n # containerd\n - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg\n - echo \"deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" \u003e /etc/apt/sources.list.d/docker.list\n - apt-get update \u0026\u0026 apt-get install -y containerd.io\n - systemctl restart containerd\n\n # kubeadm/kubelet/kubectl v1.32\n - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg\n - echo \"deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /\" \u003e /etc/apt/sources.list.d/kubernetes.list\n - apt-get update \u0026\u0026 apt-get install -y kubelet kubeadm kubectl\n - apt-mark hold kubelet kubeadm kubectl\n\n\n # init cluster\n - kubeadm init --config=/root/kubeadm-config.yaml --skip-phases=addon/kube-proxy\n \n # kubeconfig for root\n - mkdir -p /root/.kube \u0026\u0026 cp /etc/kubernetes/admin.conf /root/.kube/config\n\n # wait for API server\n - |\n echo \"Waiting for API server...\"\n until kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes ; do\n echo \"Waiting for API server...\"\n sleep 5\n done\n\n # CNI (cilium example, swap for flannel/calico as needed)\n - |\n CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)\n curl -L --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz\n tar xzvf cilium-linux-amd64.tar.gz -C /usr/local/bin\n cilium install --kubeconfig=/etc/kubernetes/admin.conf --set kubeProxyReplacement=true --wait \n"},"sensitive_attributes":[]}]},{"mode":"managed","type":"libvirt_cloudinit_disk","name":"commoninit_node_01","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"id":"2d6abdd2982994a2","meta_data":"\"instance-id\": \"kube-node-32\"\n\"local-hostname\": \"kube-node-32\"\n","name":"kube-node-32-cloudinit.iso","network_config":"version: 2\nethernets:\n eth0:\n match:\n driver: virtio_net\n addresses:\n - 192.168.0.32/24\n routes:\n - to: default\n via: 192.168.0.4\n nameservers:\n addresses:\n - 8.8.8.8\n","path":"/var/folders/dq/h32yg2mx55b2m8hxmf1yh66r0000gn/T/terraform-provider-libvirt-cloudinit/cloudinit-2d6abdd2982994a2.iso","size":49152,"user_data":"#cloud-config\nhostname: kube-node-32\nusers:\n - name: ubuntu\n sudo: ALL=(ALL) NOPASSWD:ALL\n shell: /bin/bash\n ssh_authorized_keys:\n - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDTLxDOQMsumLpjIzMXvpN/hvoMfbVFcl7TDKXTYLBJIvTOLWO/elI3PeJkXNAlaIT1lgJQB7NwQED+DciPLjCLzkG3aSxuJdZRo0Z+vHm9CEUkWnq8icLTQs0zYadK6p24VpMqT61cGNv9L/riciMRWT6l2SdRN2ZBR6MhCXMZy/XgRRlcV9iN1n9T0Q9aZjaQ2CjgG59wTLdQ0bHRlFBSt2q6erFmCIiBHUJba4avDVRvYryvMgkopayL/eK2gxaloMjcMt4z+zIokxhQONPPU2AvqMcqQIsWGQYIzIkJ0UOO8ly/PAFCpgvukWJepmZR30vVBALFCRZCUN1zMbI3 jan.novak@Jans-MacBook-Air-9.local\n\nchpasswd:\n list: |\n ubuntu:yourpassword\n expire: false\nssh_pwauth: true\npackage_update: true\npackages:\n - qemu-guest-agent\n - openssh-server\n - apt-transport-https\n - ca-certificates\n - curl\n - gnupg \n\nwrite_files:\n - path: /etc/modules-load.d/k8s.conf\n content: |\n overlay\n br_netfilter\n\n - path: /etc/sysctl.d/k8s.conf\n content: |\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.ipv4.ip_forward = 1\n\n - path: /etc/containerd/config.toml\n content: |\n version = 2\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n runtime_type = \"io.containerd.runc.v2\"\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n SystemdCgroup = true\n\n # Update existing containerd config to enable registry config_path\n - path: /etc/containerd/config.toml\n content: |\n version = 2\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n runtime_type = \"io.containerd.runc.v2\"\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n SystemdCgroup = true\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n config_path = \"/etc/containerd/certs.d\"\n\n # Mirror configs for each upstream registry\n - path: /etc/containerd/certs.d/docker.io/hosts.toml\n content: |\n server = \"https://registry-1.docker.io\"\n [host.\"http://192.168.0.30:5000/v2/docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/registry.k8s.io/hosts.toml\n content: |\n server = \"https://registry.k8s.io\"\n [host.\"http://192.168.0.30:5000/v2/registry.k8s.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/ghcr.io/hosts.toml\n content: |\n server = \"https://ghcr.io\"\n [host.\"http://192.168.0.30:5000/v2/ghcr.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/quay.io/hosts.toml\n content: |\n server = \"https://quay.io\"\n [host.\"http://192.168.0.30:5000/v2/quay.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true \n\n - path: /root/kubeadm-config.yaml\n content: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: InitConfiguration\n nodeRegistration:\n criSocket: unix:///run/containerd/containerd.sock\n ---\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: ClusterConfiguration\n networking:\n podSubnet: \"10.244.0.0/16\"\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n cgroupDriver: systemd\n\n - path: /etc/profile.d/kubectl.sh\n content: |\n alias k='kubectl'\n source \u003c(kubectl completion bash)\n complete -o default -F __start_kubectl k \n\nruncmd:\n - systemctl enable --now qemu-guest-agent\n - systemctl enable --now ssh\n\n # relevant to kubernetes\n - modprobe overlay\n - modprobe br_netfilter\n - sysctl --system\n\n # containerd\n - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg\n - echo \"deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" \u003e /etc/apt/sources.list.d/docker.list\n - apt-get update \u0026\u0026 apt-get install -y containerd.io\n - systemctl restart containerd\n\n # kubeadm/kubelet/kubectl v1.32\n - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg\n - echo \"deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /\" \u003e /etc/apt/sources.list.d/kubernetes.list\n - apt-get update \u0026\u0026 apt-get install -y kubelet kubeadm kubectl\n - apt-mark hold kubelet kubeadm kubectl\n\n # join cluster\n - kubeadm join 192.168.0.31:6443 --token ps457c.7g4koxq4awqigdwy --discovery-token-ca-cert-hash sha256:bd969ebcf4c68c424a76c8fc483f194ab8b32a2fd52f1cf992e164f496907447\n\n"},"sensitive_attributes":[],"dependencies":["data.local_file.join_command","libvirt_cloudinit_disk.commoninit","libvirt_domain.master","libvirt_volume.cloudinit","libvirt_volume.ubuntu_base","libvirt_volume.ubuntu_disk","null_resource.kubeadm_token"]}]},{"mode":"managed","type":"libvirt_domain","name":"master","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"autostart":true,"block_io_tune":null,"bootloader":null,"bootloader_args":null,"clock":null,"cpu":null,"cpu_tune":null,"create":null,"current_memory":null,"current_memory_unit":null,"default_io_thread":null,"description":null,"destroy":null,"devices":{"audios":null,"channels":null,"consoles":null,"controllers":null,"crypto":null,"disks":[{"acpi":null,"address":null,"alias":null,"auth":null,"backend_domain":null,"backing_store":null,"block_io":null,"boot":null,"device":null,"driver":{"ats":null,"cache":null,"copy_on_read":null,"detect_zeros":null,"discard":null,"discard_no_unref":null,"error_policy":null,"event_idx":null,"io":null,"io_event_fd":null,"io_thread":null,"io_threads":null,"iommu":null,"metadata_cache":null,"name":"qemu","packed":null,"page_per_vq":null,"queue_size":null,"queues":null,"rerror_policy":null,"type":"qcow2"},"encryption":null,"geometry":null,"io_tune":null,"mirror":null,"model":null,"product":null,"raw_io":null,"read_only":null,"serial":null,"sgio":null,"shareable":null,"snapshot":null,"source":{"block":null,"cookies":null,"data_store":null,"dir":null,"encryption":null,"file":{"fd_group":null,"file":"/srv/vms/kube-master-31.qcow2","sec_label":null},"index":null,"network":null,"nvme":null,"readahead":null,"reservations":null,"slices":null,"ssl":null,"startup_policy":null,"timeout":null,"vhost_user":null,"vhost_vdpa":null,"volume":null},"target":{"bus":"virtio","dev":"vda","removable":null,"rotation_rate":null,"tray":null},"throttle_filters":null,"transient":null,"vendor":null,"wwn":null},{"acpi":null,"address":null,"alias":null,"auth":null,"backend_domain":null,"backing_store":null,"block_io":null,"boot":null,"device":"cdrom","driver":{"ats":null,"cache":null,"copy_on_read":null,"detect_zeros":null,"discard":null,"discard_no_unref":null,"error_policy":null,"event_idx":null,"io":null,"io_event_fd":null,"io_thread":null,"io_threads":null,"iommu":null,"metadata_cache":null,"name":"qemu","packed":null,"page_per_vq":null,"queue_size":null,"queues":null,"rerror_policy":null,"type":"raw"},"encryption":null,"geometry":null,"io_tune":null,"mirror":null,"model":null,"product":null,"raw_io":null,"read_only":null,"serial":null,"sgio":null,"shareable":null,"snapshot":null,"source":{"block":null,"cookies":null,"data_store":null,"dir":null,"encryption":null,"file":{"fd_group":null,"file":"/srv/vms/kube-master-31-cloudinit.iso","sec_label":null},"index":null,"network":null,"nvme":null,"readahead":null,"reservations":null,"slices":null,"ssl":null,"startup_policy":null,"timeout":null,"vhost_user":null,"vhost_vdpa":null,"volume":null},"target":{"bus":"sata","dev":"sda","removable":null,"rotation_rate":null,"tray":null},"throttle_filters":null,"transient":null,"vendor":null,"wwn":null}],"emulator":null,"filesystems":null,"graphics":null,"hostdevs":null,"hubs":null,"inputs":null,"interfaces":[{"acpi":null,"address":null,"alias":null,"backend":null,"backend_domain":null,"bandwidth":null,"boot":null,"coalesce":null,"down_script":null,"driver":null,"filter_ref":null,"guest":null,"ip":null,"link":null,"mac":null,"managed":null,"model":{"type":"virtio"},"mtu":null,"port_forward":null,"port_options":null,"rom":null,"route":null,"script":null,"source":{"bridge":{"bridge":"br0"},"client":null,"direct":null,"ethernet":null,"hostdev":null,"internal":null,"mcast":null,"network":null,"null":null,"server":null,"udp":null,"user":null,"vdpa":null,"vds":null,"vhost_user":null},"target":null,"teaming":null,"trust_guest_rx_filters":null,"tune":null,"virtual_port":null,"vlan":null,"wait_for_ip":null}],"iommu":null,"leases":null,"mem_balloon":null,"memorydevs":null,"nvram":null,"panics":null,"parallels":null,"pstore":null,"redir_devs":null,"redir_filters":null,"rngs":null,"serials":[{"acpi":null,"address":null,"alias":null,"log":null,"protocol":null,"source":null,"target":null}],"shmems":null,"smartcards":null,"sounds":null,"tpms":null,"videos":null,"vsock":null,"watchdogs":null},"features":null,"gen_id":null,"hwuuid":null,"id":49,"id_map":null,"io_thread_i_ds":null,"io_threads":null,"key_wrap":null,"launch_security":null,"maximum_memory":null,"maximum_memory_slots":null,"maximum_memory_unit":null,"memory":2048,"memory_backing":null,"memory_dump_core":null,"memory_tune":null,"memory_unit":"MiB","metadata":null,"name":"kube-master-31","numa_tune":null,"on_crash":null,"on_poweroff":null,"on_reboot":null,"os":{"acpi":null,"bios":null,"boot_devices":null,"boot_menu":null,"cmdline":null,"dtb":null,"firmware":null,"firmware_info":null,"init":null,"init_args":null,"init_dir":null,"init_env":null,"init_group":null,"init_user":null,"initrd":null,"kernel":null,"loader":null,"loader_format":null,"loader_readonly":null,"loader_secure":null,"loader_stateless":null,"loader_type":null,"nv_ram":null,"shim":null,"sm_bios":null,"type":"hvm","type_arch":"x86_64","type_machine":"q35"},"perf":null,"pm":null,"resource":null,"running":true,"sec_label":null,"sys_info":null,"throttle_groups":null,"title":null,"type":"kvm","uuid":"1d3d2721-a2f0-49d0-a458-a905c4b6e5cd","vcpu":2,"vcpu_cpuset":null,"vcpu_current":null,"vcpu_placement":null,"vcpus":null},"sensitive_attributes":[],"dependencies":["libvirt_cloudinit_disk.commoninit","libvirt_volume.cloudinit","libvirt_volume.ubuntu_base","libvirt_volume.ubuntu_disk"]}]},{"mode":"managed","type":"libvirt_domain","name":"node_01","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"autostart":true,"block_io_tune":null,"bootloader":null,"bootloader_args":null,"clock":null,"cpu":null,"cpu_tune":null,"create":null,"current_memory":null,"current_memory_unit":null,"default_io_thread":null,"description":null,"destroy":null,"devices":{"audios":null,"channels":null,"consoles":null,"controllers":null,"crypto":null,"disks":[{"acpi":null,"address":null,"alias":null,"auth":null,"backend_domain":null,"backing_store":null,"block_io":null,"boot":null,"device":null,"driver":{"ats":null,"cache":null,"copy_on_read":null,"detect_zeros":null,"discard":null,"discard_no_unref":null,"error_policy":null,"event_idx":null,"io":null,"io_event_fd":null,"io_thread":null,"io_threads":null,"iommu":null,"metadata_cache":null,"name":"qemu","packed":null,"page_per_vq":null,"queue_size":null,"queues":null,"rerror_policy":null,"type":"qcow2"},"encryption":null,"geometry":null,"io_tune":null,"mirror":null,"model":null,"product":null,"raw_io":null,"read_only":null,"serial":null,"sgio":null,"shareable":null,"snapshot":null,"source":{"block":null,"cookies":null,"data_store":null,"dir":null,"encryption":null,"file":{"fd_group":null,"file":"/srv/vms/kube-node-32.qcow2","sec_label":null},"index":null,"network":null,"nvme":null,"readahead":null,"reservations":null,"slices":null,"ssl":null,"startup_policy":null,"timeout":null,"vhost_user":null,"vhost_vdpa":null,"volume":null},"target":{"bus":"virtio","dev":"vda","removable":null,"rotation_rate":null,"tray":null},"throttle_filters":null,"transient":null,"vendor":null,"wwn":null},{"acpi":null,"address":null,"alias":null,"auth":null,"backend_domain":null,"backing_store":null,"block_io":null,"boot":null,"device":"cdrom","driver":{"ats":null,"cache":null,"copy_on_read":null,"detect_zeros":null,"discard":null,"discard_no_unref":null,"error_policy":null,"event_idx":null,"io":null,"io_event_fd":null,"io_thread":null,"io_threads":null,"iommu":null,"metadata_cache":null,"name":"qemu","packed":null,"page_per_vq":null,"queue_size":null,"queues":null,"rerror_policy":null,"type":"raw"},"encryption":null,"geometry":null,"io_tune":null,"mirror":null,"model":null,"product":null,"raw_io":null,"read_only":null,"serial":null,"sgio":null,"shareable":null,"snapshot":null,"source":{"block":null,"cookies":null,"data_store":null,"dir":null,"encryption":null,"file":{"fd_group":null,"file":"/srv/vms/kube-node-32-cloudinit.iso","sec_label":null},"index":null,"network":null,"nvme":null,"readahead":null,"reservations":null,"slices":null,"ssl":null,"startup_policy":null,"timeout":null,"vhost_user":null,"vhost_vdpa":null,"volume":null},"target":{"bus":"sata","dev":"sda","removable":null,"rotation_rate":null,"tray":null},"throttle_filters":null,"transient":null,"vendor":null,"wwn":null}],"emulator":null,"filesystems":null,"graphics":null,"hostdevs":null,"hubs":null,"inputs":null,"interfaces":[{"acpi":null,"address":null,"alias":null,"backend":null,"backend_domain":null,"bandwidth":null,"boot":null,"coalesce":null,"down_script":null,"driver":null,"filter_ref":null,"guest":null,"ip":null,"link":null,"mac":null,"managed":null,"model":{"type":"virtio"},"mtu":null,"port_forward":null,"port_options":null,"rom":null,"route":null,"script":null,"source":{"bridge":{"bridge":"br0"},"client":null,"direct":null,"ethernet":null,"hostdev":null,"internal":null,"mcast":null,"network":null,"null":null,"server":null,"udp":null,"user":null,"vdpa":null,"vds":null,"vhost_user":null},"target":null,"teaming":null,"trust_guest_rx_filters":null,"tune":null,"virtual_port":null,"vlan":null,"wait_for_ip":null}],"iommu":null,"leases":null,"mem_balloon":null,"memorydevs":null,"nvram":null,"panics":null,"parallels":null,"pstore":null,"redir_devs":null,"redir_filters":null,"rngs":null,"serials":[{"acpi":null,"address":null,"alias":null,"log":null,"protocol":null,"source":null,"target":null}],"shmems":null,"smartcards":null,"sounds":null,"tpms":null,"videos":null,"vsock":null,"watchdogs":null},"features":null,"gen_id":null,"hwuuid":null,"id":50,"id_map":null,"io_thread_i_ds":null,"io_threads":null,"key_wrap":null,"launch_security":null,"maximum_memory":null,"maximum_memory_slots":null,"maximum_memory_unit":null,"memory":4096,"memory_backing":null,"memory_dump_core":null,"memory_tune":null,"memory_unit":"MiB","metadata":null,"name":"kube-node-32","numa_tune":null,"on_crash":null,"on_poweroff":null,"on_reboot":null,"os":{"acpi":null,"bios":null,"boot_devices":null,"boot_menu":null,"cmdline":null,"dtb":null,"firmware":null,"firmware_info":null,"init":null,"init_args":null,"init_dir":null,"init_env":null,"init_group":null,"init_user":null,"initrd":null,"kernel":null,"loader":null,"loader_format":null,"loader_readonly":null,"loader_secure":null,"loader_stateless":null,"loader_type":null,"nv_ram":null,"shim":null,"sm_bios":null,"type":"hvm","type_arch":"x86_64","type_machine":"q35"},"perf":null,"pm":null,"resource":null,"running":true,"sec_label":null,"sys_info":null,"throttle_groups":null,"title":null,"type":"kvm","uuid":"432c9f41-6033-4ee8-a77a-083b8cb6ef9a","vcpu":2,"vcpu_cpuset":null,"vcpu_current":null,"vcpu_placement":null,"vcpus":null},"sensitive_attributes":[],"dependencies":["data.local_file.join_command","libvirt_cloudinit_disk.commoninit","libvirt_cloudinit_disk.commoninit_node_01","libvirt_domain.master","libvirt_volume.cloudinit","libvirt_volume.cloudinit_node_01","libvirt_volume.node_01_disk","libvirt_volume.ubuntu_base","libvirt_volume.ubuntu_disk","null_resource.kubeadm_token"]}]},{"mode":"managed","type":"libvirt_volume","name":"cloudinit","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"allocation":49152,"allocation_unit":null,"backing_store":null,"capacity":49152,"capacity_unit":null,"create":{"content":{"url":"/var/folders/dq/h32yg2mx55b2m8hxmf1yh66r0000gn/T/terraform-provider-libvirt-cloudinit/cloudinit-9b5efd668378a2cc.iso"}},"id":"/srv/vms/kube-master-31-cloudinit.iso","key":"/srv/vms/kube-master-31-cloudinit.iso","name":"kube-master-31-cloudinit.iso","path":"/srv/vms/kube-master-31-cloudinit.iso","physical":49152,"physical_unit":null,"pool":"default","target":null,"type":null},"sensitive_attributes":[],"dependencies":["libvirt_cloudinit_disk.commoninit"]}]},{"mode":"managed","type":"libvirt_volume","name":"cloudinit_node_01","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"allocation":49152,"allocation_unit":null,"backing_store":null,"capacity":49152,"capacity_unit":null,"create":{"content":{"url":"/var/folders/dq/h32yg2mx55b2m8hxmf1yh66r0000gn/T/terraform-provider-libvirt-cloudinit/cloudinit-2d6abdd2982994a2.iso"}},"id":"/srv/vms/kube-node-32-cloudinit.iso","key":"/srv/vms/kube-node-32-cloudinit.iso","name":"kube-node-32-cloudinit.iso","path":"/srv/vms/kube-node-32-cloudinit.iso","physical":49152,"physical_unit":null,"pool":"default","target":null,"type":null},"sensitive_attributes":[],"dependencies":["data.local_file.join_command","libvirt_cloudinit_disk.commoninit","libvirt_cloudinit_disk.commoninit_node_01","libvirt_domain.master","libvirt_volume.cloudinit","libvirt_volume.ubuntu_base","libvirt_volume.ubuntu_disk","null_resource.kubeadm_token"]}]},{"mode":"managed","type":"libvirt_volume","name":"node_01_disk","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"allocation":503001088,"allocation_unit":null,"backing_store":{"format":{"type":"qcow2"},"path":"/srv/vms/ubuntu-24.04-base.qcow2","permissions":null},"capacity":21474836480,"capacity_unit":null,"create":null,"id":"/srv/vms/kube-node-32.qcow2","key":"/srv/vms/kube-node-32.qcow2","name":"kube-node-32.qcow2","path":"/srv/vms/kube-node-32.qcow2","physical":503054336,"physical_unit":null,"pool":"default","target":{"cluster_size":null,"cluster_size_unit":null,"compat":null,"encryption":null,"features":null,"format":{"type":"qcow2"},"path":"/srv/vms/kube-node-32.qcow2","permissions":null,"timestamps":null},"type":null},"sensitive_attributes":[],"dependencies":["libvirt_volume.ubuntu_base"]}]},{"mode":"managed","type":"libvirt_volume","name":"ubuntu_base","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"allocation":626663424,"allocation_unit":null,"backing_store":null,"capacity":3758096384,"capacity_unit":null,"create":{"content":{"url":"https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"}},"id":"/srv/vms/ubuntu-24.04-base.qcow2","key":"/srv/vms/ubuntu-24.04-base.qcow2","name":"ubuntu-24.04-base.qcow2","path":"/srv/vms/ubuntu-24.04-base.qcow2","physical":626655744,"physical_unit":null,"pool":"default","target":{"cluster_size":null,"cluster_size_unit":null,"compat":null,"encryption":null,"features":null,"format":{"type":"qcow2"},"path":"/srv/vms/ubuntu-24.04-base.qcow2","permissions":null,"timestamps":null},"type":null},"sensitive_attributes":[]}]},{"mode":"managed","type":"libvirt_volume","name":"ubuntu_disk","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"allocation":577777664,"allocation_unit":null,"backing_store":{"format":{"type":"qcow2"},"path":"/srv/vms/ubuntu-24.04-base.qcow2","permissions":null},"capacity":21474836480,"capacity_unit":null,"create":null,"id":"/srv/vms/kube-master-31.qcow2","key":"/srv/vms/kube-master-31.qcow2","name":"kube-master-31.qcow2","path":"/srv/vms/kube-master-31.qcow2","physical":577830912,"physical_unit":null,"pool":"default","target":{"cluster_size":null,"cluster_size_unit":null,"compat":null,"encryption":null,"features":null,"format":{"type":"qcow2"},"path":"/srv/vms/kube-master-31.qcow2","permissions":null,"timestamps":null},"type":null},"sensitive_attributes":[],"dependencies":["libvirt_volume.ubuntu_base"]}]},{"mode":"managed","type":"null_resource","name":"kubeadm_token","provider":"provider[\"registry.opentofu.org/hashicorp/null\"]","instances":[{"schema_version":0,"attributes":{"id":"1090738715192517178","triggers":null},"sensitive_attributes":[],"dependencies":["libvirt_cloudinit_disk.commoninit","libvirt_domain.master","libvirt_volume.cloudinit","libvirt_volume.ubuntu_base","libvirt_volume.ubuntu_disk"]}]},{"mode":"managed","type":"null_resource","name":"kubeconfig","provider":"provider[\"registry.opentofu.org/hashicorp/null\"]","instances":[{"schema_version":0,"attributes":{"id":"6762014692990027276","triggers":null},"sensitive_attributes":[]}]}],"check_results":null}
diff --git a/kubernetes-kvm-terraform/terraform.tfstate.backup b/kubernetes-kvm-terraform/terraform.tfstate.backup
new file mode 100644
index 0000000..5180af4
--- /dev/null
+++ b/kubernetes-kvm-terraform/terraform.tfstate.backup
@@ -0,0 +1 @@
+{"version":4,"terraform_version":"1.11.2","serial":64,"lineage":"69f81358-9017-b0fa-2e3d-404364b1f698","outputs":{"master_ip":{"value":"192.168.0.31","type":"string"},"node_01_ip":{"value":"192.168.0.32","type":"string"}},"resources":[{"mode":"data","type":"local_file","name":"join_command","provider":"provider[\"registry.opentofu.org/hashicorp/local\"]","instances":[{"schema_version":0,"attributes":{"content":"kubeadm join 192.168.0.31:6443 --token ps457c.7g4koxq4awqigdwy --discovery-token-ca-cert-hash sha256:bd969ebcf4c68c424a76c8fc483f194ab8b32a2fd52f1cf992e164f496907447 \n","content_base64":"a3ViZWFkbSBqb2luIDE5Mi4xNjguMC4zMTo2NDQzIC0tdG9rZW4gcHM0NTdjLjdnNGtveHE0YXdxaWdkd3kgLS1kaXNjb3ZlcnktdG9rZW4tY2EtY2VydC1oYXNoIHNoYTI1NjpiZDk2OWViY2Y0YzY4YzQyNGE3NmM4ZmM0ODNmMTk0YWI4YjMyYTJmZDUyZjFjZjk5MmUxNjRmNDk2OTA3NDQ3IAo=","content_base64sha256":"Nfe4cEuj9IcdF7E6B/NCohZAmoqEE1jkbv+5/fFg6Q4=","content_base64sha512":"/9NQ9J2SuaSzn2wG48VRTANdbi9tdr9AaVq9YLjYOGHRyJyrtntbjgJy636+fuhUvAcHXIad+t9ey4aEpOiWdg==","content_md5":"a50d45f29d7a87e59055f88ba97a6778","content_sha1":"431c014c1cef740ad9ae9e2e028d2ab426eb4f1d","content_sha256":"35f7b8704ba3f4871d17b13a07f342a216409a8a841358e46effb9fdf160e90e","content_sha512":"ffd350f49d92b9a4b39f6c06e3c5514c035d6e2f6d76bf40695abd60b8d83861d1c89cabb67b5b8e0272eb7ebe7ee854bc07075c869dfadf5ecb8684a4e89676","filename":"./join-command.txt","id":"431c014c1cef740ad9ae9e2e028d2ab426eb4f1d"},"sensitive_attributes":[]}]},{"mode":"managed","type":"libvirt_cloudinit_disk","name":"commoninit","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"id":"9b5efd668378a2cc","meta_data":"\"instance-id\": \"kube-master-31\"\n\"local-hostname\": \"kube-master-31\"\n","name":"kube-master-31-cloudinit.iso","network_config":"version: 2\nethernets:\n eth0:\n match:\n driver: virtio_net\n addresses:\n - 192.168.0.31/24\n routes:\n - to: default\n via: 192.168.0.4\n nameservers:\n addresses:\n - 8.8.8.8\n","path":"/var/folders/dq/h32yg2mx55b2m8hxmf1yh66r0000gn/T/terraform-provider-libvirt-cloudinit/cloudinit-9b5efd668378a2cc.iso","size":49152,"user_data":"#cloud-config\nhostname: kube-master-31\nusers:\n - name: ubuntu\n sudo: ALL=(ALL) NOPASSWD:ALL\n shell: /bin/bash\n ssh_authorized_keys:\n - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDTLxDOQMsumLpjIzMXvpN/hvoMfbVFcl7TDKXTYLBJIvTOLWO/elI3PeJkXNAlaIT1lgJQB7NwQED+DciPLjCLzkG3aSxuJdZRo0Z+vHm9CEUkWnq8icLTQs0zYadK6p24VpMqT61cGNv9L/riciMRWT6l2SdRN2ZBR6MhCXMZy/XgRRlcV9iN1n9T0Q9aZjaQ2CjgG59wTLdQ0bHRlFBSt2q6erFmCIiBHUJba4avDVRvYryvMgkopayL/eK2gxaloMjcMt4z+zIokxhQONPPU2AvqMcqQIsWGQYIzIkJ0UOO8ly/PAFCpgvukWJepmZR30vVBALFCRZCUN1zMbI3 jan.novak@Jans-MacBook-Air-9.local\n\nchpasswd:\n list: |\n ubuntu:yourpassword\n expire: false\nssh_pwauth: true\npackage_update: true\npackages:\n - qemu-guest-agent\n - openssh-server\n - apt-transport-https\n - ca-certificates\n - curl\n - gnupg \n\nwrite_files:\n - path: /etc/modules-load.d/k8s.conf\n content: |\n overlay\n br_netfilter\n\n - path: /etc/sysctl.d/k8s.conf\n content: |\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.ipv4.ip_forward = 1\n\n - path: /etc/containerd/config.toml\n content: |\n version = 2\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n runtime_type = \"io.containerd.runc.v2\"\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n SystemdCgroup = true\n\n # Update existing containerd config to enable registry config_path\n - path: /etc/containerd/config.toml\n content: |\n version = 2\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n runtime_type = \"io.containerd.runc.v2\"\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n SystemdCgroup = true\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n config_path = \"/etc/containerd/certs.d\"\n\n # Mirror configs for each upstream registry\n - path: /etc/containerd/certs.d/docker.io/hosts.toml\n content: |\n server = \"https://registry-1.docker.io\"\n [host.\"http://192.168.0.30:5000/v2/docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/registry.k8s.io/hosts.toml\n content: |\n server = \"https://registry.k8s.io\"\n [host.\"http://192.168.0.30:5000/v2/registry.k8s.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/ghcr.io/hosts.toml\n content: |\n server = \"https://ghcr.io\"\n [host.\"http://192.168.0.30:5000/v2/ghcr.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/quay.io/hosts.toml\n content: |\n server = \"https://quay.io\"\n [host.\"http://192.168.0.30:5000/v2/quay.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true \n\n - path: /root/kubeadm-config.yaml\n content: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: InitConfiguration\n nodeRegistration:\n criSocket: unix:///run/containerd/containerd.sock\n ---\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: ClusterConfiguration\n networking:\n podSubnet: \"10.244.0.0/16\"\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n cgroupDriver: systemd\n\n - path: /etc/profile.d/kubectl.sh\n content: |\n alias k='kubectl'\n source \u003c(kubectl completion bash)\n complete -o default -F __start_kubectl k \n\nruncmd:\n - systemctl enable --now qemu-guest-agent\n - systemctl enable --now ssh\n\n # relevant to kubernetes\n - modprobe overlay\n - modprobe br_netfilter\n - sysctl --system\n\n # containerd\n - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg\n - echo \"deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" \u003e /etc/apt/sources.list.d/docker.list\n - apt-get update \u0026\u0026 apt-get install -y containerd.io\n - systemctl restart containerd\n\n # kubeadm/kubelet/kubectl v1.32\n - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg\n - echo \"deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /\" \u003e /etc/apt/sources.list.d/kubernetes.list\n - apt-get update \u0026\u0026 apt-get install -y kubelet kubeadm kubectl\n - apt-mark hold kubelet kubeadm kubectl\n\n\n # init cluster\n - kubeadm init --config=/root/kubeadm-config.yaml --skip-phases=addon/kube-proxy\n \n # kubeconfig for root\n - mkdir -p /root/.kube \u0026\u0026 cp /etc/kubernetes/admin.conf /root/.kube/config\n\n # wait for API server\n - |\n echo \"Waiting for API server...\"\n until kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes ; do\n echo \"Waiting for API server...\"\n sleep 5\n done\n\n # CNI (cilium example, swap for flannel/calico as needed)\n - |\n CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)\n curl -L --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz\n tar xzvf cilium-linux-amd64.tar.gz -C /usr/local/bin\n cilium install --kubeconfig=/etc/kubernetes/admin.conf --set kubeProxyReplacement=true --wait \n"},"sensitive_attributes":[]}]},{"mode":"managed","type":"libvirt_cloudinit_disk","name":"commoninit_node_01","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"id":"2d6abdd2982994a2","meta_data":"\"instance-id\": \"kube-node-32\"\n\"local-hostname\": \"kube-node-32\"\n","name":"kube-node-32-cloudinit.iso","network_config":"version: 2\nethernets:\n eth0:\n match:\n driver: virtio_net\n addresses:\n - 192.168.0.32/24\n routes:\n - to: default\n via: 192.168.0.4\n nameservers:\n addresses:\n - 8.8.8.8\n","path":"/var/folders/dq/h32yg2mx55b2m8hxmf1yh66r0000gn/T/terraform-provider-libvirt-cloudinit/cloudinit-2d6abdd2982994a2.iso","size":49152,"user_data":"#cloud-config\nhostname: kube-node-32\nusers:\n - name: ubuntu\n sudo: ALL=(ALL) NOPASSWD:ALL\n shell: /bin/bash\n ssh_authorized_keys:\n - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDTLxDOQMsumLpjIzMXvpN/hvoMfbVFcl7TDKXTYLBJIvTOLWO/elI3PeJkXNAlaIT1lgJQB7NwQED+DciPLjCLzkG3aSxuJdZRo0Z+vHm9CEUkWnq8icLTQs0zYadK6p24VpMqT61cGNv9L/riciMRWT6l2SdRN2ZBR6MhCXMZy/XgRRlcV9iN1n9T0Q9aZjaQ2CjgG59wTLdQ0bHRlFBSt2q6erFmCIiBHUJba4avDVRvYryvMgkopayL/eK2gxaloMjcMt4z+zIokxhQONPPU2AvqMcqQIsWGQYIzIkJ0UOO8ly/PAFCpgvukWJepmZR30vVBALFCRZCUN1zMbI3 jan.novak@Jans-MacBook-Air-9.local\n\nchpasswd:\n list: |\n ubuntu:yourpassword\n expire: false\nssh_pwauth: true\npackage_update: true\npackages:\n - qemu-guest-agent\n - openssh-server\n - apt-transport-https\n - ca-certificates\n - curl\n - gnupg \n\nwrite_files:\n - path: /etc/modules-load.d/k8s.conf\n content: |\n overlay\n br_netfilter\n\n - path: /etc/sysctl.d/k8s.conf\n content: |\n net.bridge.bridge-nf-call-iptables = 1\n net.bridge.bridge-nf-call-ip6tables = 1\n net.ipv4.ip_forward = 1\n\n - path: /etc/containerd/config.toml\n content: |\n version = 2\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n runtime_type = \"io.containerd.runc.v2\"\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n SystemdCgroup = true\n\n # Update existing containerd config to enable registry config_path\n - path: /etc/containerd/config.toml\n content: |\n version = 2\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n runtime_type = \"io.containerd.runc.v2\"\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n SystemdCgroup = true\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n config_path = \"/etc/containerd/certs.d\"\n\n # Mirror configs for each upstream registry\n - path: /etc/containerd/certs.d/docker.io/hosts.toml\n content: |\n server = \"https://registry-1.docker.io\"\n [host.\"http://192.168.0.30:5000/v2/docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/registry.k8s.io/hosts.toml\n content: |\n server = \"https://registry.k8s.io\"\n [host.\"http://192.168.0.30:5000/v2/registry.k8s.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/ghcr.io/hosts.toml\n content: |\n server = \"https://ghcr.io\"\n [host.\"http://192.168.0.30:5000/v2/ghcr.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true\n\n - path: /etc/containerd/certs.d/quay.io/hosts.toml\n content: |\n server = \"https://quay.io\"\n [host.\"http://192.168.0.30:5000/v2/quay.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n override_path = true \n\n - path: /root/kubeadm-config.yaml\n content: |\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: InitConfiguration\n nodeRegistration:\n criSocket: unix:///run/containerd/containerd.sock\n ---\n apiVersion: kubeadm.k8s.io/v1beta3\n kind: ClusterConfiguration\n networking:\n podSubnet: \"10.244.0.0/16\"\n ---\n apiVersion: kubelet.config.k8s.io/v1beta1\n kind: KubeletConfiguration\n cgroupDriver: systemd\n\n - path: /etc/profile.d/kubectl.sh\n content: |\n alias k='kubectl'\n source \u003c(kubectl completion bash)\n complete -o default -F __start_kubectl k \n\nruncmd:\n - systemctl enable --now qemu-guest-agent\n - systemctl enable --now ssh\n\n # relevant to kubernetes\n - modprobe overlay\n - modprobe br_netfilter\n - sysctl --system\n\n # containerd\n - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg\n - echo \"deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" \u003e /etc/apt/sources.list.d/docker.list\n - apt-get update \u0026\u0026 apt-get install -y containerd.io\n - systemctl restart containerd\n\n # kubeadm/kubelet/kubectl v1.32\n - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg\n - echo \"deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /\" \u003e /etc/apt/sources.list.d/kubernetes.list\n - apt-get update \u0026\u0026 apt-get install -y kubelet kubeadm kubectl\n - apt-mark hold kubelet kubeadm kubectl\n\n # join cluster\n - kubeadm join 192.168.0.31:6443 --token ps457c.7g4koxq4awqigdwy --discovery-token-ca-cert-hash sha256:bd969ebcf4c68c424a76c8fc483f194ab8b32a2fd52f1cf992e164f496907447\n\n"},"sensitive_attributes":[],"dependencies":["data.local_file.join_command","libvirt_cloudinit_disk.commoninit","libvirt_domain.master","libvirt_volume.cloudinit","libvirt_volume.ubuntu_base","libvirt_volume.ubuntu_disk","null_resource.kubeadm_token"]}]},{"mode":"managed","type":"libvirt_domain","name":"master","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"autostart":true,"block_io_tune":null,"bootloader":null,"bootloader_args":null,"clock":null,"cpu":null,"cpu_tune":null,"create":null,"current_memory":null,"current_memory_unit":null,"default_io_thread":null,"description":null,"destroy":null,"devices":{"audios":null,"channels":null,"consoles":null,"controllers":null,"crypto":null,"disks":[{"acpi":null,"address":null,"alias":null,"auth":null,"backend_domain":null,"backing_store":null,"block_io":null,"boot":null,"device":null,"driver":{"ats":null,"cache":null,"copy_on_read":null,"detect_zeros":null,"discard":null,"discard_no_unref":null,"error_policy":null,"event_idx":null,"io":null,"io_event_fd":null,"io_thread":null,"io_threads":null,"iommu":null,"metadata_cache":null,"name":"qemu","packed":null,"page_per_vq":null,"queue_size":null,"queues":null,"rerror_policy":null,"type":"qcow2"},"encryption":null,"geometry":null,"io_tune":null,"mirror":null,"model":null,"product":null,"raw_io":null,"read_only":null,"serial":null,"sgio":null,"shareable":null,"snapshot":null,"source":{"block":null,"cookies":null,"data_store":null,"dir":null,"encryption":null,"file":{"fd_group":null,"file":"/srv/vms/kube-master-31.qcow2","sec_label":null},"index":null,"network":null,"nvme":null,"readahead":null,"reservations":null,"slices":null,"ssl":null,"startup_policy":null,"timeout":null,"vhost_user":null,"vhost_vdpa":null,"volume":null},"target":{"bus":"virtio","dev":"vda","removable":null,"rotation_rate":null,"tray":null},"throttle_filters":null,"transient":null,"vendor":null,"wwn":null},{"acpi":null,"address":null,"alias":null,"auth":null,"backend_domain":null,"backing_store":null,"block_io":null,"boot":null,"device":"cdrom","driver":{"ats":null,"cache":null,"copy_on_read":null,"detect_zeros":null,"discard":null,"discard_no_unref":null,"error_policy":null,"event_idx":null,"io":null,"io_event_fd":null,"io_thread":null,"io_threads":null,"iommu":null,"metadata_cache":null,"name":"qemu","packed":null,"page_per_vq":null,"queue_size":null,"queues":null,"rerror_policy":null,"type":"raw"},"encryption":null,"geometry":null,"io_tune":null,"mirror":null,"model":null,"product":null,"raw_io":null,"read_only":null,"serial":null,"sgio":null,"shareable":null,"snapshot":null,"source":{"block":null,"cookies":null,"data_store":null,"dir":null,"encryption":null,"file":{"fd_group":null,"file":"/srv/vms/kube-master-31-cloudinit.iso","sec_label":null},"index":null,"network":null,"nvme":null,"readahead":null,"reservations":null,"slices":null,"ssl":null,"startup_policy":null,"timeout":null,"vhost_user":null,"vhost_vdpa":null,"volume":null},"target":{"bus":"sata","dev":"sda","removable":null,"rotation_rate":null,"tray":null},"throttle_filters":null,"transient":null,"vendor":null,"wwn":null}],"emulator":null,"filesystems":null,"graphics":null,"hostdevs":null,"hubs":null,"inputs":null,"interfaces":[{"acpi":null,"address":null,"alias":null,"backend":null,"backend_domain":null,"bandwidth":null,"boot":null,"coalesce":null,"down_script":null,"driver":null,"filter_ref":null,"guest":null,"ip":null,"link":null,"mac":null,"managed":null,"model":{"type":"virtio"},"mtu":null,"port_forward":null,"port_options":null,"rom":null,"route":null,"script":null,"source":{"bridge":{"bridge":"br0"},"client":null,"direct":null,"ethernet":null,"hostdev":null,"internal":null,"mcast":null,"network":null,"null":null,"server":null,"udp":null,"user":null,"vdpa":null,"vds":null,"vhost_user":null},"target":null,"teaming":null,"trust_guest_rx_filters":null,"tune":null,"virtual_port":null,"vlan":null,"wait_for_ip":null}],"iommu":null,"leases":null,"mem_balloon":null,"memorydevs":null,"nvram":null,"panics":null,"parallels":null,"pstore":null,"redir_devs":null,"redir_filters":null,"rngs":null,"serials":[{"acpi":null,"address":null,"alias":null,"log":null,"protocol":null,"source":null,"target":null}],"shmems":null,"smartcards":null,"sounds":null,"tpms":null,"videos":null,"vsock":null,"watchdogs":null},"features":null,"gen_id":null,"hwuuid":null,"id":49,"id_map":null,"io_thread_i_ds":null,"io_threads":null,"key_wrap":null,"launch_security":null,"maximum_memory":null,"maximum_memory_slots":null,"maximum_memory_unit":null,"memory":2048,"memory_backing":null,"memory_dump_core":null,"memory_tune":null,"memory_unit":"MiB","metadata":null,"name":"kube-master-31","numa_tune":null,"on_crash":null,"on_poweroff":null,"on_reboot":null,"os":{"acpi":null,"bios":null,"boot_devices":null,"boot_menu":null,"cmdline":null,"dtb":null,"firmware":null,"firmware_info":null,"init":null,"init_args":null,"init_dir":null,"init_env":null,"init_group":null,"init_user":null,"initrd":null,"kernel":null,"loader":null,"loader_format":null,"loader_readonly":null,"loader_secure":null,"loader_stateless":null,"loader_type":null,"nv_ram":null,"shim":null,"sm_bios":null,"type":"hvm","type_arch":"x86_64","type_machine":"q35"},"perf":null,"pm":null,"resource":null,"running":true,"sec_label":null,"sys_info":null,"throttle_groups":null,"title":null,"type":"kvm","uuid":"1d3d2721-a2f0-49d0-a458-a905c4b6e5cd","vcpu":2,"vcpu_cpuset":null,"vcpu_current":null,"vcpu_placement":null,"vcpus":null},"sensitive_attributes":[],"dependencies":["libvirt_cloudinit_disk.commoninit","libvirt_volume.cloudinit","libvirt_volume.ubuntu_base","libvirt_volume.ubuntu_disk"]}]},{"mode":"managed","type":"libvirt_domain","name":"node_01","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"autostart":true,"block_io_tune":null,"bootloader":null,"bootloader_args":null,"clock":null,"cpu":null,"cpu_tune":null,"create":null,"current_memory":null,"current_memory_unit":null,"default_io_thread":null,"description":null,"destroy":null,"devices":{"audios":null,"channels":null,"consoles":null,"controllers":null,"crypto":null,"disks":[{"acpi":null,"address":null,"alias":null,"auth":null,"backend_domain":null,"backing_store":null,"block_io":null,"boot":null,"device":null,"driver":{"ats":null,"cache":null,"copy_on_read":null,"detect_zeros":null,"discard":null,"discard_no_unref":null,"error_policy":null,"event_idx":null,"io":null,"io_event_fd":null,"io_thread":null,"io_threads":null,"iommu":null,"metadata_cache":null,"name":"qemu","packed":null,"page_per_vq":null,"queue_size":null,"queues":null,"rerror_policy":null,"type":"qcow2"},"encryption":null,"geometry":null,"io_tune":null,"mirror":null,"model":null,"product":null,"raw_io":null,"read_only":null,"serial":null,"sgio":null,"shareable":null,"snapshot":null,"source":{"block":null,"cookies":null,"data_store":null,"dir":null,"encryption":null,"file":{"fd_group":null,"file":"/srv/vms/kube-node-32.qcow2","sec_label":null},"index":null,"network":null,"nvme":null,"readahead":null,"reservations":null,"slices":null,"ssl":null,"startup_policy":null,"timeout":null,"vhost_user":null,"vhost_vdpa":null,"volume":null},"target":{"bus":"virtio","dev":"vda","removable":null,"rotation_rate":null,"tray":null},"throttle_filters":null,"transient":null,"vendor":null,"wwn":null},{"acpi":null,"address":null,"alias":null,"auth":null,"backend_domain":null,"backing_store":null,"block_io":null,"boot":null,"device":"cdrom","driver":{"ats":null,"cache":null,"copy_on_read":null,"detect_zeros":null,"discard":null,"discard_no_unref":null,"error_policy":null,"event_idx":null,"io":null,"io_event_fd":null,"io_thread":null,"io_threads":null,"iommu":null,"metadata_cache":null,"name":"qemu","packed":null,"page_per_vq":null,"queue_size":null,"queues":null,"rerror_policy":null,"type":"raw"},"encryption":null,"geometry":null,"io_tune":null,"mirror":null,"model":null,"product":null,"raw_io":null,"read_only":null,"serial":null,"sgio":null,"shareable":null,"snapshot":null,"source":{"block":null,"cookies":null,"data_store":null,"dir":null,"encryption":null,"file":{"fd_group":null,"file":"/srv/vms/kube-node-32-cloudinit.iso","sec_label":null},"index":null,"network":null,"nvme":null,"readahead":null,"reservations":null,"slices":null,"ssl":null,"startup_policy":null,"timeout":null,"vhost_user":null,"vhost_vdpa":null,"volume":null},"target":{"bus":"sata","dev":"sda","removable":null,"rotation_rate":null,"tray":null},"throttle_filters":null,"transient":null,"vendor":null,"wwn":null}],"emulator":null,"filesystems":null,"graphics":null,"hostdevs":null,"hubs":null,"inputs":null,"interfaces":[{"acpi":null,"address":null,"alias":null,"backend":null,"backend_domain":null,"bandwidth":null,"boot":null,"coalesce":null,"down_script":null,"driver":null,"filter_ref":null,"guest":null,"ip":null,"link":null,"mac":null,"managed":null,"model":{"type":"virtio"},"mtu":null,"port_forward":null,"port_options":null,"rom":null,"route":null,"script":null,"source":{"bridge":{"bridge":"br0"},"client":null,"direct":null,"ethernet":null,"hostdev":null,"internal":null,"mcast":null,"network":null,"null":null,"server":null,"udp":null,"user":null,"vdpa":null,"vds":null,"vhost_user":null},"target":null,"teaming":null,"trust_guest_rx_filters":null,"tune":null,"virtual_port":null,"vlan":null,"wait_for_ip":null}],"iommu":null,"leases":null,"mem_balloon":null,"memorydevs":null,"nvram":null,"panics":null,"parallels":null,"pstore":null,"redir_devs":null,"redir_filters":null,"rngs":null,"serials":[{"acpi":null,"address":null,"alias":null,"log":null,"protocol":null,"source":null,"target":null}],"shmems":null,"smartcards":null,"sounds":null,"tpms":null,"videos":null,"vsock":null,"watchdogs":null},"features":null,"gen_id":null,"hwuuid":null,"id":50,"id_map":null,"io_thread_i_ds":null,"io_threads":null,"key_wrap":null,"launch_security":null,"maximum_memory":null,"maximum_memory_slots":null,"maximum_memory_unit":null,"memory":4096,"memory_backing":null,"memory_dump_core":null,"memory_tune":null,"memory_unit":"MiB","metadata":null,"name":"kube-node-32","numa_tune":null,"on_crash":null,"on_poweroff":null,"on_reboot":null,"os":{"acpi":null,"bios":null,"boot_devices":null,"boot_menu":null,"cmdline":null,"dtb":null,"firmware":null,"firmware_info":null,"init":null,"init_args":null,"init_dir":null,"init_env":null,"init_group":null,"init_user":null,"initrd":null,"kernel":null,"loader":null,"loader_format":null,"loader_readonly":null,"loader_secure":null,"loader_stateless":null,"loader_type":null,"nv_ram":null,"shim":null,"sm_bios":null,"type":"hvm","type_arch":"x86_64","type_machine":"q35"},"perf":null,"pm":null,"resource":null,"running":true,"sec_label":null,"sys_info":null,"throttle_groups":null,"title":null,"type":"kvm","uuid":"432c9f41-6033-4ee8-a77a-083b8cb6ef9a","vcpu":2,"vcpu_cpuset":null,"vcpu_current":null,"vcpu_placement":null,"vcpus":null},"sensitive_attributes":[],"dependencies":["data.local_file.join_command","libvirt_cloudinit_disk.commoninit","libvirt_cloudinit_disk.commoninit_node_01","libvirt_domain.master","libvirt_volume.cloudinit","libvirt_volume.cloudinit_node_01","libvirt_volume.node_01_disk","libvirt_volume.ubuntu_base","libvirt_volume.ubuntu_disk","null_resource.kubeadm_token"]}]},{"mode":"managed","type":"libvirt_volume","name":"cloudinit","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"allocation":49152,"allocation_unit":null,"backing_store":null,"capacity":49152,"capacity_unit":null,"create":{"content":{"url":"/var/folders/dq/h32yg2mx55b2m8hxmf1yh66r0000gn/T/terraform-provider-libvirt-cloudinit/cloudinit-9b5efd668378a2cc.iso"}},"id":"/srv/vms/kube-master-31-cloudinit.iso","key":"/srv/vms/kube-master-31-cloudinit.iso","name":"kube-master-31-cloudinit.iso","path":"/srv/vms/kube-master-31-cloudinit.iso","physical":49152,"physical_unit":null,"pool":"default","target":null,"type":null},"sensitive_attributes":[],"dependencies":["libvirt_cloudinit_disk.commoninit"]}]},{"mode":"managed","type":"libvirt_volume","name":"cloudinit_node_01","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"allocation":49152,"allocation_unit":null,"backing_store":null,"capacity":49152,"capacity_unit":null,"create":{"content":{"url":"/var/folders/dq/h32yg2mx55b2m8hxmf1yh66r0000gn/T/terraform-provider-libvirt-cloudinit/cloudinit-2d6abdd2982994a2.iso"}},"id":"/srv/vms/kube-node-32-cloudinit.iso","key":"/srv/vms/kube-node-32-cloudinit.iso","name":"kube-node-32-cloudinit.iso","path":"/srv/vms/kube-node-32-cloudinit.iso","physical":49152,"physical_unit":null,"pool":"default","target":null,"type":null},"sensitive_attributes":[],"dependencies":["data.local_file.join_command","libvirt_cloudinit_disk.commoninit","libvirt_cloudinit_disk.commoninit_node_01","libvirt_domain.master","libvirt_volume.cloudinit","libvirt_volume.ubuntu_base","libvirt_volume.ubuntu_disk","null_resource.kubeadm_token"]}]},{"mode":"managed","type":"libvirt_volume","name":"node_01_disk","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"allocation":200704,"allocation_unit":null,"backing_store":{"format":{"type":"qcow2"},"path":"/srv/vms/ubuntu-24.04-base.qcow2","permissions":null},"capacity":21474836480,"capacity_unit":null,"create":null,"id":"/srv/vms/kube-node-32.qcow2","key":"/srv/vms/kube-node-32.qcow2","name":"kube-node-32.qcow2","path":"/srv/vms/kube-node-32.qcow2","physical":196928,"physical_unit":null,"pool":"default","target":{"cluster_size":null,"cluster_size_unit":null,"compat":null,"encryption":null,"features":null,"format":{"type":"qcow2"},"path":"/srv/vms/kube-node-32.qcow2","permissions":null,"timestamps":null},"type":null},"sensitive_attributes":[],"dependencies":["libvirt_volume.ubuntu_base"]}]},{"mode":"managed","type":"libvirt_volume","name":"ubuntu_base","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"allocation":626663424,"allocation_unit":null,"backing_store":null,"capacity":3758096384,"capacity_unit":null,"create":{"content":{"url":"https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"}},"id":"/srv/vms/ubuntu-24.04-base.qcow2","key":"/srv/vms/ubuntu-24.04-base.qcow2","name":"ubuntu-24.04-base.qcow2","path":"/srv/vms/ubuntu-24.04-base.qcow2","physical":626655744,"physical_unit":null,"pool":"default","target":{"cluster_size":null,"cluster_size_unit":null,"compat":null,"encryption":null,"features":null,"format":{"type":"qcow2"},"path":"/srv/vms/ubuntu-24.04-base.qcow2","permissions":null,"timestamps":null},"type":null},"sensitive_attributes":[]}]},{"mode":"managed","type":"libvirt_volume","name":"ubuntu_disk","provider":"provider[\"registry.opentofu.org/dmacvicar/libvirt\"]","instances":[{"schema_version":0,"attributes":{"allocation":200704,"allocation_unit":null,"backing_store":{"format":{"type":"qcow2"},"path":"/srv/vms/ubuntu-24.04-base.qcow2","permissions":null},"capacity":21474836480,"capacity_unit":null,"create":null,"id":"/srv/vms/kube-master-31.qcow2","key":"/srv/vms/kube-master-31.qcow2","name":"kube-master-31.qcow2","path":"/srv/vms/kube-master-31.qcow2","physical":196928,"physical_unit":null,"pool":"default","target":{"cluster_size":null,"cluster_size_unit":null,"compat":null,"encryption":null,"features":null,"format":{"type":"qcow2"},"path":"/srv/vms/kube-master-31.qcow2","permissions":null,"timestamps":null},"type":null},"sensitive_attributes":[],"dependencies":["libvirt_volume.ubuntu_base"]}]},{"mode":"managed","type":"null_resource","name":"kubeadm_token","provider":"provider[\"registry.opentofu.org/hashicorp/null\"]","instances":[{"schema_version":0,"attributes":{"id":"1090738715192517178","triggers":null},"sensitive_attributes":[],"dependencies":["libvirt_cloudinit_disk.commoninit","libvirt_domain.master","libvirt_volume.cloudinit","libvirt_volume.ubuntu_base","libvirt_volume.ubuntu_disk"]}]},{"mode":"managed","type":"null_resource","name":"kubeconfig","provider":"provider[\"registry.opentofu.org/hashicorp/null\"]","instances":[{"status":"tainted","schema_version":0,"attributes":{"id":"4322620472031734347","triggers":null},"sensitive_attributes":[]}]}],"check_results":null}