diff --git a/.claude/settings.json b/.claude/settings.json index 5607d2e..ae23d68 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -18,7 +18,11 @@ "Bash(clusterctl generate *)", "Bash(ansible mgmt *)", "Bash(kubectl get *)", - "Bash(kubectl auth *)" + "Bash(kubectl auth *)", + "Bash(ansible pve *)", + "Read(//Users/jan.novak/**)", + "Bash(export KUBECONFIG=/Users/jan.novak/srv/go/cluster-autoscaler-with-capi/.claude/cluster-inspector/kubeconfigs/lab-mgmt.kubeconfig)", + "Bash(kubectl explain *)" ] } } diff --git a/configs/capi/lab1.yaml b/configs/capi/lab1.yaml new file mode 100644 index 0000000..9f0af82 --- /dev/null +++ b/configs/capi/lab1.yaml @@ -0,0 +1,294 @@ +# Workload cluster lab1 (phase-1 Step 3). Generated 2026-09-27 with clusterctl v1.12.11: +# +# PROXMOX_SOURCENODE=pv1-1 TEMPLATE_VMID=101 ALLOWED_NODES="[pv1-1]" \ +# VM_SSH_KEYS="$(cat ~/.ssh/id_ed25519.pub)" \ +# CONTROL_PLANE_ENDPOINT_IP=192.168.4.30 NODE_IP_RANGES="[192.168.4.31-192.168.4.39]" \ +# GATEWAY=192.168.4.8 IP_PREFIX=24 DNS_SERVERS="[8.8.8.8, 1.1.1.1]" BRIDGE=vmbr0 \ +# BOOT_VOLUME_DEVICE=scsi0 BOOT_VOLUME_SIZE=40 NUM_SOCKETS=1 NUM_CORES=2 MEMORY_MIB=4096 \ +# clusterctl generate cluster lab1 --infrastructure proxmox:v0.9.1 --kubernetes-version v1.35.9 \ +# --control-plane-machine-count 1 --worker-machine-count 1 --target-namespace default +# +# Hand edits (docs/phase-1-proxmox-capi.md Step 3): +# 1. pod CIDR 192.168.0.0/16 -> 10.244.0.0/16 (overlaps the lab LAN) +# 2. kube-vip v0.7.1 -> v1.2.4 (kube-vip-prepare.sh kept) +# 3. `format: qcow2` removed from both ProxmoxMachineTemplates (local-lvm is LVM-thin, raw only) +# 4. `vlan: 4` on net0 of both ProxmoxMachineTemplates +# 5. lab1-worker: 1 socket x 4 cores, 4096 MiB (owner's choice; runbook said 8192) +# 6. MachineDeployment lab1-workers: `replicas` removed (the autoscaler owns it) +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: lab1 + name: lab1 + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.244.0.0/16 + controlPlaneRef: + apiGroup: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: lab1-control-plane + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: ProxmoxCluster + name: lab1 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: ProxmoxCluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: lab1 + name: lab1 + namespace: default +spec: + allowedNodes: + - pv1-1 + controlPlaneEndpoint: + host: 192.168.4.30 + port: 6443 + dnsServers: + - 8.8.8.8 + - 1.1.1.1 + ipv4Config: + addresses: + - 192.168.4.31-192.168.4.39 + gateway: 192.168.4.8 + prefix: 24 +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlane +metadata: + name: lab1-control-plane + namespace: default +spec: + kubeadmConfigSpec: + files: + - content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: cp_enable + value: "true" + - name: vip_interface + value: "" + - name: address + value: 192.168.4.30 + - name: port + value: "6443" + - name: vip_arp + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + image: ghcr.io/kube-vip/kube-vip:v1.2.4 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - localhost + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + type: FileOrCreate + name: kubeconfig + status: {} + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + exit 0 + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: provider-id + value: proxmox://'{{ ds.meta_data.instance_id }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: provider-id + value: proxmox://'{{ ds.meta_data.instance_id }}' + preKubeadmCommands: + - /etc/kube-vip-prepare.sh + users: + - name: root + sshAuthorizedKeys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQxxkqmvtVI+8c5BkTaJ5c2HfBFRXJWMmEcevvfP9tV + jan.novak@Jans-MacBook-Air.local + machineTemplate: + spec: + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: ProxmoxMachineTemplate + name: lab1-control-plane + replicas: 1 + version: v1.35.9 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: ProxmoxMachineTemplate +metadata: + name: lab1-control-plane + namespace: default +spec: + template: + spec: + disks: + bootVolume: + disk: scsi0 + sizeGb: 40 + full: true + memoryMiB: 4096 + network: + networkDevices: + - bridge: vmbr0 + model: virtio + name: net0 + vlan: 4 + numCores: 2 + numSockets: 1 + sourceNode: pv1-1 + templateID: 101 +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: MachineDeployment +metadata: + labels: + cluster.x-k8s.io/cluster-name: lab1 + name: lab1-workers + namespace: default +spec: + clusterName: lab1 + selector: + matchLabels: null + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: lab1 + node-role.kubernetes.io/node: "" + spec: + bootstrap: + configRef: + apiGroup: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: lab1-worker + clusterName: lab1 + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: ProxmoxMachineTemplate + name: lab1-worker + version: v1.35.9 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: ProxmoxMachineTemplate +metadata: + name: lab1-worker + namespace: default +spec: + template: + spec: + disks: + bootVolume: + disk: scsi0 + sizeGb: 40 + full: true + memoryMiB: 4096 + network: + networkDevices: + - bridge: vmbr0 + model: virtio + name: net0 + vlan: 4 + numCores: 4 + numSockets: 1 + sourceNode: pv1-1 + templateID: 101 +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: lab1-worker + namespace: default +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: provider-id + value: proxmox://'{{ ds.meta_data.instance_id }}' + users: + - name: root + sshAuthorizedKeys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQxxkqmvtVI+8c5BkTaJ5c2HfBFRXJWMmEcevvfP9tV + jan.novak@Jans-MacBook-Air.local diff --git a/configs/test/inflate.yaml b/configs/test/inflate.yaml index 985dfba..d8b9ada 100644 --- a/configs/test/inflate.yaml +++ b/configs/test/inflate.yaml @@ -1,4 +1,6 @@ -# apply to the workload cluster; scale replicas to trigger scale-up (≈2 pods per 4 vCPU / 8 GiB worker) +# apply to the workload cluster; scale replicas to trigger scale-up +# 2 pods per 4 vCPU / 4 GiB worker: CPU-bound (2 × 1500m); 1Gi keeps memory from binding first +# (a 4 GiB node has ~3.3–3.6 GiB allocatable, which would fit only one 2Gi pod) apiVersion: apps/v1 kind: Deployment metadata: @@ -15,4 +17,4 @@ spec: - name: pause image: registry.k8s.io/pause:3.10 resources: - requests: { cpu: "1500m", memory: "2Gi" } + requests: { cpu: "1500m", memory: "1Gi" } diff --git a/docs/decisions.md b/docs/decisions.md index 9903e67..a957f82 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -23,3 +23,4 @@ image-builder fixes log; their dates are when the decision was recorded, not nec | 12 | 2026-09-27 | Management cluster built by one idempotent Ansible playbook, `ansible/mgmt.yml`, run from the laptop ([plan](plans/2026-09-27-0058-mgmt-cluster-ansible.md)) | cloud-init + k3s auto-deploy manifests; bash scripts with guards; OpenTofu + Helm | owner's choice; idempotent modules, one tool; only `ansible.builtin`, charts through k3s's helm-controller so nothing extra on the VM | active | | 13 | 2026-09-27 | CAPI providers installed by cluster-api-operator 0.28.0 (Provider CRs), cert-manager v1.21.2 pre-installed | guarded `clusterctl init` | declarative; re-apply is a no-op; upgrade = `spec.version` bump; 0.28.0 built against CAPI 1.12.10. Consequence: no `clusterctl init/upgrade/move/delete` | active | | 14 | 2026-09-27 | Management VM created by the playbook from the pinned Ubuntu 24.04 cloud image (VM 201, 192.168.4.21) | hand-made VM; the image-builder node template | reproducible from the repo; the image-builder template is a kubeadm node image, not meant for k3s | active | +| 15 | 2026-09-27 | `lab1` workers 4 vCPU / **4 GiB** (not 8 GiB); inflate test pods request 1500m / 1Gi so they stay CPU-bound at 2 per worker | 8 GiB workers with 2Gi inflate pods | owner's choice to save hypervisor RAM; scale tests keep the 1 → 3 worker shape ([configs/capi/lab1.yaml](../configs/capi/lab1.yaml)) | active | diff --git a/docs/phase-1-proxmox-capi.md b/docs/phase-1-proxmox-capi.md index e76bc8d..2769964 100644 --- a/docs/phase-1-proxmox-capi.md +++ b/docs/phase-1-proxmox-capi.md @@ -12,7 +12,7 @@ Versions: see [versions.md](versions.md). Template build: see [image-builder-pro | --- | --- | --- | | Management cluster | VM 201 `lab-mgmt` (192.168.4.21) on `pv1-1`, k3s v1.35.8+k3s1 | 2 vCPU, 4 GB RAM, 30 GB | | `lab1` control plane | 1 VM, VIP via kube-vip | 2 vCPU, 4 GB RAM | -| `lab1` workers | MachineDeployment `lab1-workers`, min 0 / max 5 | 4 vCPU, 8 GB RAM | +| `lab1` workers | MachineDeployment `lab1-workers`, min 0 / max 5 | 4 vCPU, 4 GB RAM | ## Checklist @@ -94,15 +94,11 @@ cluster-api-operator replaces `clusterctl init`: ## Step 3 — workload cluster `lab1` -```bash -clusterctl generate cluster lab1 \ - --infrastructure proxmox:v0.9.1 \ - --kubernetes-version v1.35.9 \ - --control-plane-machine-count 1 \ - --worker-machine-count 1 > lab1.yaml -``` - -Edit `lab1.yaml` before applying: +The generated and hand-edited manifest is committed as +[configs/capi/lab1.yaml](../configs/capi/lab1.yaml): template VMID 101, control-plane VIP +`192.168.4.30`, node IPs `192.168.4.31–39`. Its header holds the exact `clusterctl generate` +command (all variables as env vars, so no `~/.cluster-api/clusterctl.yaml` is needed). Regenerate +only when a version changes, then redo these edits: 1. **Pod CIDR** — `Cluster.spec.clusterNetwork.pods.cidrBlocks` defaults to `192.168.0.0/16`, which overlaps the lab network → use e.g. `10.244.0.0/16`. @@ -110,11 +106,12 @@ Edit `lab1.yaml` before applying: (handles `super-admin.conf` for kubeadm ≥ 1.29). Fall back to the template's version if the VIP flakes. 3. **Disk format** — delete `format: "qcow2"` from both `ProxmoxMachineTemplate`s (LVM-thin). 4. **Network** — `bridge: vmbr0` + `vlan: 4` on the network device. -5. **Worker size** — `ProxmoxMachineTemplate/lab1-worker`: `numSockets: 1`, `numCores: 4`, `memoryMiB: 8192`. +5. **Worker size** — `ProxmoxMachineTemplate/lab1-worker`: `numSockets: 1`, `numCores: 4`, `memoryMiB: 4096`. 6. **Replicas** — remove `spec.replicas` from `MachineDeployment/lab1-workers`; the autoscaler owns it. ```bash -kubectl apply -f lab1.yaml +export KUBECONFIG=~/.kube/lab-mgmt.kubeconfig +kubectl apply -f configs/capi/lab1.yaml clusterctl describe cluster lab1 --show-conditions all kubectl get proxmoxmachines,machines -w clusterctl get kubeconfig lab1 > lab1.kubeconfig @@ -143,7 +140,7 @@ metadata: cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0" cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" capacity.cluster-autoscaler.kubernetes.io/cpu: "4" # numSockets × numCores - capacity.cluster-autoscaler.kubernetes.io/memory: "8Gi" + capacity.cluster-autoscaler.kubernetes.io/memory: "4Gi" # memoryMiB 4096 capacity.cluster-autoscaler.kubernetes.io/ephemeral-disk: "40Gi" capacity.cluster-autoscaler.kubernetes.io/maximum-pods: "110" ``` @@ -167,7 +164,8 @@ The log must show node group `MachineDeployment/default/lab1-workers` with min 0 ## Step 7 — scale tests -Workload: [configs/test/inflate.yaml](../configs/test/inflate.yaml) (pause pods, 1500m / 2Gi each). +Workload: [configs/test/inflate.yaml](../configs/test/inflate.yaml) (pause pods, 1500m / 1Gi each → +CPU-bound at 2 pods per 4-core / 4 GiB worker). - [ ] Scale-up: `kubectl scale deploy/inflate --replicas=6` → workers 1 → 3; record pending → Running time. - [ ] Scale-down: `--replicas=0` → after `scale-down-unneeded-time` (5m) Machines and VMs disappear.