kacerrandClaude Opus 5.5
5e5b6c506a
Owner's pending work, committed ahead of the pv1-2 change it underlies: - lab1.yaml: worker template lab1-worker-2g (2048 MiB, decision #16), MD rollout maxSurge 0 / maxUnavailable 1, ProxmoxCluster schedulerHints.memoryAdjustment 160 (#17) - cluster-autoscaler values: RBAC for proxmoxmachinetemplates, aggressive scale-down profile, Cilium startup-taint; runbook Steps 4-7 and troubleshooting rows - inflate test pods 512Mi; decisions #16/#17; session note on the memory overcommit - docs/future-tasks.md convention (CLAUDE.md, README) with the namespace-per-cluster task - AGENTS.md; allowlist entries in .claude/settings.json Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
322 lines
10 KiB
YAML
322 lines
10 KiB
YAML
# Workload cluster lab1 (phase-1 Step 3). Generated 2026-09-27 with clusterctl v1.12.11:
|
||
#
|
||
# PROXMOX_SOURCENODE=pv1-1 TEMPLATE_VMID=101 ALLOWED_NODES="[pv1-1]" \
|
||
# VM_SSH_KEYS="$(cat ~/.ssh/id_ed25519.pub)" \
|
||
# CONTROL_PLANE_ENDPOINT_IP=192.168.4.30 NODE_IP_RANGES="[192.168.4.31-192.168.4.39]" \
|
||
# GATEWAY=192.168.4.8 IP_PREFIX=24 DNS_SERVERS="[8.8.8.8, 1.1.1.1]" BRIDGE=vmbr0 \
|
||
# BOOT_VOLUME_DEVICE=scsi0 BOOT_VOLUME_SIZE=40 NUM_SOCKETS=1 NUM_CORES=2 MEMORY_MIB=4096 \
|
||
# clusterctl generate cluster lab1 --infrastructure proxmox:v0.9.1 --kubernetes-version v1.35.9 \
|
||
# --control-plane-machine-count 1 --worker-machine-count 1 --target-namespace default
|
||
#
|
||
# Hand edits (docs/phase-1-proxmox-capi.md Step 3):
|
||
# 1. pod CIDR 192.168.0.0/16 -> 10.244.0.0/16 (overlaps the lab LAN)
|
||
# 2. kube-vip v0.7.1 -> v1.2.4 (kube-vip-prepare.sh kept)
|
||
# 3. `format: qcow2` removed from both ProxmoxMachineTemplates (local-lvm is LVM-thin, raw only)
|
||
# 4. `vlan: 4` on net0 of both ProxmoxMachineTemplates
|
||
# 5. worker template: 1 socket x 4 cores, 2048 MiB, named lab1-worker-2g (owner's choice, decision #16;
|
||
# first 4096 MiB as lab1-worker). Templates are never edited in place: a new size = a new
|
||
# template name + MD infrastructureRef, which rolls the workers.
|
||
# 6. MachineDeployment lab1-workers: `replicas` removed (the autoscaler owns it)
|
||
# 7. MachineDeployment lab1-workers: autoscaler min/max + capacity annotations (runbook Step 5)
|
||
# 8. MachineDeployment lab1-workers: rollout maxSurge 0 / maxUnavailable 1 (memory-tight host)
|
||
# 9. ProxmoxCluster lab1: schedulerHints.memoryAdjustment 160 (CAPMOX memory overcommit, decision #17)
|
||
apiVersion: cluster.x-k8s.io/v1beta2
|
||
kind: Cluster
|
||
metadata:
|
||
labels:
|
||
cluster.x-k8s.io/cluster-name: lab1
|
||
name: lab1
|
||
namespace: default
|
||
spec:
|
||
clusterNetwork:
|
||
pods:
|
||
cidrBlocks:
|
||
- 10.244.0.0/16
|
||
controlPlaneRef:
|
||
apiGroup: controlplane.cluster.x-k8s.io
|
||
kind: KubeadmControlPlane
|
||
name: lab1-control-plane
|
||
infrastructureRef:
|
||
apiGroup: infrastructure.cluster.x-k8s.io
|
||
kind: ProxmoxCluster
|
||
name: lab1
|
||
---
|
||
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
|
||
kind: ProxmoxCluster
|
||
metadata:
|
||
labels:
|
||
cluster.x-k8s.io/cluster-name: lab1
|
||
name: lab1
|
||
namespace: default
|
||
spec:
|
||
allowedNodes:
|
||
- pv1-1
|
||
controlPlaneEndpoint:
|
||
host: 192.168.4.30
|
||
port: 6443
|
||
dnsServers:
|
||
- 8.8.8.8
|
||
- 1.1.1.1
|
||
ipv4Config:
|
||
addresses:
|
||
- 192.168.4.31-192.168.4.39
|
||
gateway: 192.168.4.8
|
||
prefix: 24
|
||
# CAPMOX reserves the configured memory of every non-template VM, stopped ones included, against
|
||
# host total × memoryAdjustment%. 160% of 29.16 GiB = 46.7 GiB (~17.7 GiB headroom today).
|
||
# Real overcommit protection on pv1-1: KSM (ksmtuned) + 8 GiB swap.
|
||
schedulerHints:
|
||
memoryAdjustment: 160
|
||
---
|
||
apiVersion: controlplane.cluster.x-k8s.io/v1beta2
|
||
kind: KubeadmControlPlane
|
||
metadata:
|
||
name: lab1-control-plane
|
||
namespace: default
|
||
spec:
|
||
kubeadmConfigSpec:
|
||
files:
|
||
- content: |
|
||
apiVersion: v1
|
||
kind: Pod
|
||
metadata:
|
||
creationTimestamp: null
|
||
name: kube-vip
|
||
namespace: kube-system
|
||
spec:
|
||
containers:
|
||
- args:
|
||
- manager
|
||
env:
|
||
- name: cp_enable
|
||
value: "true"
|
||
- name: vip_interface
|
||
value: ""
|
||
- name: address
|
||
value: 192.168.4.30
|
||
- name: port
|
||
value: "6443"
|
||
- name: vip_arp
|
||
value: "true"
|
||
- name: vip_leaderelection
|
||
value: "true"
|
||
- name: vip_leaseduration
|
||
value: "15"
|
||
- name: vip_renewdeadline
|
||
value: "10"
|
||
- name: vip_retryperiod
|
||
value: "2"
|
||
image: ghcr.io/kube-vip/kube-vip:v1.2.4
|
||
imagePullPolicy: IfNotPresent
|
||
name: kube-vip
|
||
resources: {}
|
||
securityContext:
|
||
capabilities:
|
||
add:
|
||
- NET_ADMIN
|
||
- NET_RAW
|
||
volumeMounts:
|
||
- mountPath: /etc/kubernetes/admin.conf
|
||
name: kubeconfig
|
||
hostAliases:
|
||
- hostnames:
|
||
- localhost
|
||
- kubernetes
|
||
ip: 127.0.0.1
|
||
hostNetwork: true
|
||
volumes:
|
||
- hostPath:
|
||
path: /etc/kubernetes/admin.conf
|
||
type: FileOrCreate
|
||
name: kubeconfig
|
||
status: {}
|
||
owner: root:root
|
||
path: /etc/kubernetes/manifests/kube-vip.yaml
|
||
- content: |
|
||
#!/bin/bash
|
||
|
||
# Copyright 2020 The Kubernetes Authors.
|
||
#
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
|
||
set -e
|
||
|
||
# Configure the workaround required for kubeadm init with kube-vip:
|
||
# xref: https://github.com/kube-vip/kube-vip/issues/684
|
||
|
||
# Nothing to do for kubernetes < v1.29
|
||
KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)"
|
||
if [[ "$KUBEADM_MINOR" -lt "29" ]]; then
|
||
exit 0
|
||
fi
|
||
|
||
IS_KUBEADM_INIT="false"
|
||
|
||
# cloud-init kubeadm init
|
||
if [[ -f /run/kubeadm/kubeadm.yaml ]]; then
|
||
IS_KUBEADM_INIT="true"
|
||
fi
|
||
|
||
# ignition kubeadm init
|
||
if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then
|
||
IS_KUBEADM_INIT="true"
|
||
fi
|
||
|
||
if [[ "$IS_KUBEADM_INIT" == "true" ]]; then
|
||
sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \
|
||
/etc/kubernetes/manifests/kube-vip.yaml
|
||
fi
|
||
owner: root:root
|
||
path: /etc/kube-vip-prepare.sh
|
||
permissions: "0700"
|
||
initConfiguration:
|
||
nodeRegistration:
|
||
kubeletExtraArgs:
|
||
- name: provider-id
|
||
value: proxmox://'{{ ds.meta_data.instance_id }}'
|
||
joinConfiguration:
|
||
nodeRegistration:
|
||
kubeletExtraArgs:
|
||
- name: provider-id
|
||
value: proxmox://'{{ ds.meta_data.instance_id }}'
|
||
preKubeadmCommands:
|
||
- /etc/kube-vip-prepare.sh
|
||
users:
|
||
- name: root
|
||
sshAuthorizedKeys:
|
||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQxxkqmvtVI+8c5BkTaJ5c2HfBFRXJWMmEcevvfP9tV
|
||
jan.novak@Jans-MacBook-Air.local
|
||
machineTemplate:
|
||
spec:
|
||
infrastructureRef:
|
||
apiGroup: infrastructure.cluster.x-k8s.io
|
||
kind: ProxmoxMachineTemplate
|
||
name: lab1-control-plane
|
||
replicas: 1
|
||
version: v1.35.9
|
||
---
|
||
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
|
||
kind: ProxmoxMachineTemplate
|
||
metadata:
|
||
name: lab1-control-plane
|
||
namespace: default
|
||
spec:
|
||
template:
|
||
spec:
|
||
disks:
|
||
bootVolume:
|
||
disk: scsi0
|
||
sizeGb: 40
|
||
full: true
|
||
memoryMiB: 4096
|
||
network:
|
||
networkDevices:
|
||
- bridge: vmbr0
|
||
model: virtio
|
||
name: net0
|
||
vlan: 4
|
||
numCores: 2
|
||
numSockets: 1
|
||
sourceNode: pv1-1
|
||
templateID: 101
|
||
---
|
||
apiVersion: cluster.x-k8s.io/v1beta2
|
||
kind: MachineDeployment
|
||
metadata:
|
||
annotations:
|
||
cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0"
|
||
cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5"
|
||
# scale-from-zero: CAPMOX publishes no status.capacity; must match ProxmoxMachineTemplate/lab1-worker-2g
|
||
capacity.cluster-autoscaler.kubernetes.io/cpu: "4" # numSockets 1 × numCores 4
|
||
capacity.cluster-autoscaler.kubernetes.io/memory: "2Gi" # memoryMiB 2048
|
||
capacity.cluster-autoscaler.kubernetes.io/ephemeral-disk: "40Gi" # bootVolume sizeGb 40
|
||
capacity.cluster-autoscaler.kubernetes.io/maximum-pods: "110"
|
||
labels:
|
||
cluster.x-k8s.io/cluster-name: lab1
|
||
name: lab1-workers
|
||
namespace: default
|
||
spec:
|
||
clusterName: lab1
|
||
# Delete an old worker before creating its replacement. The default surge (1/0) deadlocks on a full
|
||
# host: CAPMOX can't reserve memory for the surge VM, and CA won't scale down the old workers while
|
||
# the MD is rolling out (clusterapi ScaleDownNodeUpgradeProcessor).
|
||
rollout:
|
||
strategy:
|
||
type: RollingUpdate
|
||
rollingUpdate:
|
||
maxSurge: 0
|
||
maxUnavailable: 1
|
||
selector:
|
||
matchLabels: null
|
||
template:
|
||
metadata:
|
||
labels:
|
||
cluster.x-k8s.io/cluster-name: lab1
|
||
node-role.kubernetes.io/node: ""
|
||
spec:
|
||
bootstrap:
|
||
configRef:
|
||
apiGroup: bootstrap.cluster.x-k8s.io
|
||
kind: KubeadmConfigTemplate
|
||
name: lab1-worker
|
||
clusterName: lab1
|
||
infrastructureRef:
|
||
apiGroup: infrastructure.cluster.x-k8s.io
|
||
kind: ProxmoxMachineTemplate
|
||
name: lab1-worker-2g
|
||
version: v1.35.9
|
||
---
|
||
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
|
||
kind: ProxmoxMachineTemplate
|
||
metadata:
|
||
name: lab1-worker-2g
|
||
namespace: default
|
||
spec:
|
||
template:
|
||
spec:
|
||
disks:
|
||
bootVolume:
|
||
disk: scsi0
|
||
sizeGb: 40
|
||
full: true
|
||
memoryMiB: 2048
|
||
network:
|
||
networkDevices:
|
||
- bridge: vmbr0
|
||
model: virtio
|
||
name: net0
|
||
vlan: 4
|
||
numCores: 4
|
||
numSockets: 1
|
||
sourceNode: pv1-1
|
||
templateID: 101
|
||
---
|
||
apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
|
||
kind: KubeadmConfigTemplate
|
||
metadata:
|
||
name: lab1-worker
|
||
namespace: default
|
||
spec:
|
||
template:
|
||
spec:
|
||
joinConfiguration:
|
||
nodeRegistration:
|
||
kubeletExtraArgs:
|
||
- name: provider-id
|
||
value: proxmox://'{{ ds.meta_data.instance_id }}'
|
||
users:
|
||
- name: root
|
||
sshAuthorizedKeys:
|
||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQxxkqmvtVI+8c5BkTaJ5c2HfBFRXJWMmEcevvfP9tV
|
||
jan.novak@Jans-MacBook-Air.local
|