lab1: 2 GiB workers, CAPMOX memory overcommit, CA install (Step 6), future-tasks
Owner's pending work, committed ahead of the pv1-2 change it underlies: - lab1.yaml: worker template lab1-worker-2g (2048 MiB, decision #16), MD rollout maxSurge 0 / maxUnavailable 1, ProxmoxCluster schedulerHints.memoryAdjustment 160 (#17) - cluster-autoscaler values: RBAC for proxmoxmachinetemplates, aggressive scale-down profile, Cilium startup-taint; runbook Steps 4-7 and troubleshooting rows - inflate test pods 512Mi; decisions #16/#17; session note on the memory overcommit - docs/future-tasks.md convention (CLAUDE.md, README) with the namespace-per-cluster task - AGENTS.md; allowlist entries in .claude/settings.json Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+24
-6
@@ -13,9 +13,13 @@
|
||||
# 2. kube-vip v0.7.1 -> v1.2.4 (kube-vip-prepare.sh kept)
|
||||
# 3. `format: qcow2` removed from both ProxmoxMachineTemplates (local-lvm is LVM-thin, raw only)
|
||||
# 4. `vlan: 4` on net0 of both ProxmoxMachineTemplates
|
||||
# 5. lab1-worker: 1 socket x 4 cores, 4096 MiB (owner's choice; runbook said 8192)
|
||||
# 5. worker template: 1 socket x 4 cores, 2048 MiB, named lab1-worker-2g (owner's choice, decision #16;
|
||||
# first 4096 MiB as lab1-worker). Templates are never edited in place: a new size = a new
|
||||
# template name + MD infrastructureRef, which rolls the workers.
|
||||
# 6. MachineDeployment lab1-workers: `replicas` removed (the autoscaler owns it)
|
||||
# 7. MachineDeployment lab1-workers: autoscaler min/max + capacity annotations (runbook Step 5)
|
||||
# 8. MachineDeployment lab1-workers: rollout maxSurge 0 / maxUnavailable 1 (memory-tight host)
|
||||
# 9. ProxmoxCluster lab1: schedulerHints.memoryAdjustment 160 (CAPMOX memory overcommit, decision #17)
|
||||
apiVersion: cluster.x-k8s.io/v1beta2
|
||||
kind: Cluster
|
||||
metadata:
|
||||
@@ -58,6 +62,11 @@ spec:
|
||||
- 192.168.4.31-192.168.4.39
|
||||
gateway: 192.168.4.8
|
||||
prefix: 24
|
||||
# CAPMOX reserves the configured memory of every non-template VM, stopped ones included, against
|
||||
# host total × memoryAdjustment%. 160% of 29.16 GiB = 46.7 GiB (~17.7 GiB headroom today).
|
||||
# Real overcommit protection on pv1-1: KSM (ksmtuned) + 8 GiB swap.
|
||||
schedulerHints:
|
||||
memoryAdjustment: 160
|
||||
---
|
||||
apiVersion: controlplane.cluster.x-k8s.io/v1beta2
|
||||
kind: KubeadmControlPlane
|
||||
@@ -227,9 +236,9 @@ metadata:
|
||||
annotations:
|
||||
cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0"
|
||||
cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5"
|
||||
# scale-from-zero: CAPMOX publishes no status.capacity; must match ProxmoxMachineTemplate/lab1-worker
|
||||
# scale-from-zero: CAPMOX publishes no status.capacity; must match ProxmoxMachineTemplate/lab1-worker-2g
|
||||
capacity.cluster-autoscaler.kubernetes.io/cpu: "4" # numSockets 1 × numCores 4
|
||||
capacity.cluster-autoscaler.kubernetes.io/memory: "4Gi" # memoryMiB 4096
|
||||
capacity.cluster-autoscaler.kubernetes.io/memory: "2Gi" # memoryMiB 2048
|
||||
capacity.cluster-autoscaler.kubernetes.io/ephemeral-disk: "40Gi" # bootVolume sizeGb 40
|
||||
capacity.cluster-autoscaler.kubernetes.io/maximum-pods: "110"
|
||||
labels:
|
||||
@@ -238,6 +247,15 @@ metadata:
|
||||
namespace: default
|
||||
spec:
|
||||
clusterName: lab1
|
||||
# Delete an old worker before creating its replacement. The default surge (1/0) deadlocks on a full
|
||||
# host: CAPMOX can't reserve memory for the surge VM, and CA won't scale down the old workers while
|
||||
# the MD is rolling out (clusterapi ScaleDownNodeUpgradeProcessor).
|
||||
rollout:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels: null
|
||||
template:
|
||||
@@ -255,13 +273,13 @@ spec:
|
||||
infrastructureRef:
|
||||
apiGroup: infrastructure.cluster.x-k8s.io
|
||||
kind: ProxmoxMachineTemplate
|
||||
name: lab1-worker
|
||||
name: lab1-worker-2g
|
||||
version: v1.35.9
|
||||
---
|
||||
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
|
||||
kind: ProxmoxMachineTemplate
|
||||
metadata:
|
||||
name: lab1-worker
|
||||
name: lab1-worker-2g
|
||||
namespace: default
|
||||
spec:
|
||||
template:
|
||||
@@ -271,7 +289,7 @@ spec:
|
||||
disk: scsi0
|
||||
sizeGb: 40
|
||||
full: true
|
||||
memoryMiB: 4096
|
||||
memoryMiB: 2048
|
||||
network:
|
||||
networkDevices:
|
||||
- bridge: vmbr0
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# helm install ca autoscaler/cluster-autoscaler --version 9.59.0 -n default -f values.yaml
|
||||
# helm upgrade --install ca cluster-autoscaler --repo https://kubernetes.github.io/autoscaler --version 9.59.0 -n default -f values.yaml
|
||||
# must be installed in the same namespace as the <cluster>-kubeconfig secret
|
||||
cloudProvider: clusterapi
|
||||
clusterAPIMode: kubeconfig-incluster # workload via kubeconfig, management in-cluster
|
||||
@@ -8,10 +8,26 @@ autoDiscovery:
|
||||
namespace: default
|
||||
image:
|
||||
tag: v1.35.2 # chart default appVersion is 1.35.0
|
||||
rbac:
|
||||
# CA reads the infrastructure machine template (capacity for scale-from-zero). The chart's
|
||||
# ClusterRole only covers cluster.x-k8s.io; without this the first main loop blocks forever on
|
||||
# "waiting for cache sync on infrastructure resource" and CA never scales.
|
||||
additionalRules:
|
||||
- apiGroups: [infrastructure.cluster.x-k8s.io]
|
||||
resources: [proxmoxmachinetemplates]
|
||||
verbs: [get, list, watch]
|
||||
extraArgs:
|
||||
expander: least-waste
|
||||
scale-down-unneeded-time: 5m
|
||||
scale-down-delay-after-add: 5m
|
||||
scale-down-utilization-threshold: "0.5"
|
||||
# Aggressive lab scale-down profile (CA 1.35 defaults in brackets): idle worker gone in ~3 min.
|
||||
# Keep delay-after-add >= node startup (~2-3 min here) or bursts flap up/down.
|
||||
scale-down-unneeded-time: 1m # [10m]
|
||||
scale-down-delay-after-add: 2m # [10m] cluster-wide cooldown after any scale-up
|
||||
scale-down-delay-after-failure: 1m # [3m]
|
||||
unremovable-node-recheck-timeout: 1m # [5m]
|
||||
scale-down-utilization-threshold: "0.6" # [0.5]
|
||||
max-graceful-termination-sec: "60" # [600] drain wait per pod
|
||||
balance-similar-node-groups: true
|
||||
# Cilium taints new nodes NoSchedule until its agent is up. Without this, CA simulates the freshly
|
||||
# registered (still tainted) node, finds the pending pods don't fit and adds one node too many.
|
||||
startup-taint: node.cilium.io/agent-not-ready
|
||||
v: 4
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# apply to the workload cluster; scale replicas to trigger scale-up
|
||||
# 2 pods per 4 vCPU / 4 GiB worker: CPU-bound (2 × 1500m); 1Gi keeps memory from binding first
|
||||
# (a 4 GiB node has ~3.3–3.6 GiB allocatable, which would fit only one 2Gi pod)
|
||||
# 2 pods per 4 vCPU / 2 GiB worker: CPU-bound (3 × 1500m > 4000m); 512Mi keeps memory from binding
|
||||
# first (a 2 GiB VM has ~1.78 GiB allocatable; measured 3.73 GiB on a 4 GiB VM)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -17,4 +17,4 @@ spec:
|
||||
- name: pause
|
||||
image: registry.k8s.io/pause:3.10
|
||||
resources:
|
||||
requests: { cpu: "1500m", memory: "1Gi" }
|
||||
requests: { cpu: "1500m", memory: "512Mi" }
|
||||
|
||||
Reference in New Issue
Block a user