diff --git a/.gitignore b/.gitignore index 0772492..264543f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ clusterctl.yaml # packer / image-builder output packer.log *.iso +!ansible/secrets.yml.example diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..87401a4 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,7 @@ +[defaults] +inventory = inventory.yml +interpreter_python = auto_silent + +[ssh_connection] +# accept-new: the mgmt VM is created by this playbook, so its host key is new on first run +ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o StrictHostKeyChecking=accept-new diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml new file mode 100644 index 0000000..27bfe00 --- /dev/null +++ b/ansible/group_vars/all.yml @@ -0,0 +1,38 @@ +# Pinned versions: docs/versions.md is the source of truth, keep in sync. +k3s_version: v1.35.8+k3s1 +cert_manager_chart_version: v1.21.2 +capi_operator_chart_version: 0.28.0 +capi_version: v1.12.11 +capmox_version: v0.9.1 +ipam_in_cluster_version: v1.1.1 + +ubuntu_image_url: https://cloud-images.ubuntu.com/noble/20260926/noble-server-cloudimg-amd64.img +ubuntu_image_sha256: 6a81c37564db9b1ee84e141922625e1d7c5b389b99bb3c572e0243607d5bb4d2 +ubuntu_image_path: /var/lib/vz/template/iso/noble-server-cloudimg-amd64-20260926.img + +# Proxmox (token secret lives in secrets.yml) +proxmox_url: https://192.168.0.8:8006 +proxmox_token_id: capmox@pve!capi +proxmox_storage: local-lvm +proxmox_bridge: vmbr0 +proxmox_vlan: 4 + +# Management VM (VLAN 4). Fill in the empty values before the first run. +mgmt_vmid: "" +mgmt_name: lab-mgmt +mgmt_ip: "" +mgmt_prefix: 24 +mgmt_gateway: "" +mgmt_dns: "" +mgmt_cores: 2 +mgmt_memory_mib: 4096 +mgmt_disk_size: 30G +mgmt_ssh_public_key: "{{ lookup('file', lookup('env', 'HOME') + '/.ssh/id_ed25519.pub') }}" + +# Installed by the cluster-api-operator; CR namespace = install namespace (clusterctl defaults). +capi_providers: + - { kind: CoreProvider, name: cluster-api, namespace: capi-system, version: "{{ capi_version }}" } + - { kind: BootstrapProvider, name: kubeadm, namespace: capi-kubeadm-bootstrap-system, version: "{{ capi_version }}" } + - { kind: ControlPlaneProvider, name: kubeadm, namespace: capi-kubeadm-control-plane-system, version: "{{ capi_version }}" } + - { kind: InfrastructureProvider, name: proxmox, namespace: capmox-system, version: "{{ capmox_version }}", config_secret: capmox-variables } + - { kind: IPAMProvider, name: in-cluster, namespace: capi-ipam-in-cluster-system, version: "{{ ipam_in_cluster_version }}" } diff --git a/ansible/inventory.yml b/ansible/inventory.yml new file mode 100644 index 0000000..9ab0a43 --- /dev/null +++ b/ansible/inventory.yml @@ -0,0 +1,12 @@ +all: + children: + pve: + hosts: + pv1-1: + ansible_host: 192.168.0.8 + ansible_user: root + mgmt: + hosts: + lab-mgmt: + ansible_host: "{{ mgmt_ip }}" + ansible_user: ubuntu diff --git a/ansible/mgmt.yml b/ansible/mgmt.yml new file mode 100644 index 0000000..56f1c1e --- /dev/null +++ b/ansible/mgmt.yml @@ -0,0 +1,258 @@ +# Management cluster: VM on Proxmox -> k3s -> cert-manager + cluster-api-operator -> CAPI providers. +# Idempotent: a second run reports changed=0. Run from this directory: +# ansible-playbook mgmt.yml [--tags vm|k3s|addons|capi|kubeconfig|inspector] + +- name: Management VM on Proxmox + hosts: pve + gather_facts: false + tags: [vm] + tasks: + - name: Required VM parameters are set in group_vars/all.yml + ansible.builtin.assert: + that: + - mgmt_vmid | string | length > 0 + - mgmt_ip | length > 0 + - mgmt_gateway | length > 0 + - mgmt_dns | length > 0 + + - name: Check whether the VM exists + ansible.builtin.command: qm status {{ mgmt_vmid }} + register: vm_status + changed_when: false + failed_when: false + + - name: Create the VM from the Ubuntu cloud image + when: vm_status.rc != 0 + block: + - name: Download the pinned cloud image + ansible.builtin.get_url: + url: "{{ ubuntu_image_url }}" + dest: "{{ ubuntu_image_path }}" + checksum: "sha256:{{ ubuntu_image_sha256 }}" + mode: "0644" + + - name: Stage the SSH public key for cloud-init + ansible.builtin.copy: + content: "{{ mgmt_ssh_public_key }}\n" + dest: /root/{{ mgmt_name }}.pub + mode: "0600" + + # raw disk on LVM-thin; bridge + separate VLAN tag, never vmbr0.4 + - name: Create the VM + ansible.builtin.command: >- + qm create {{ mgmt_vmid }} + --name {{ mgmt_name }} + --ostype l26 + --cores {{ mgmt_cores }} + --memory {{ mgmt_memory_mib }} + --net0 virtio,bridge={{ proxmox_bridge }},tag={{ proxmox_vlan }} + --scsihw virtio-scsi-pci + --scsi0 {{ proxmox_storage }}:0,import-from={{ ubuntu_image_path }} + --ide2 {{ proxmox_storage }}:cloudinit + --boot order=scsi0 + --serial0 socket --vga serial0 + --agent 1 + --ciuser ubuntu + --sshkeys /root/{{ mgmt_name }}.pub + --ipconfig0 ip={{ mgmt_ip }}/{{ mgmt_prefix }},gw={{ mgmt_gateway }} + --nameserver {{ mgmt_dns }} + changed_when: true + + - name: Grow the disk + ansible.builtin.command: qm resize {{ mgmt_vmid }} scsi0 {{ mgmt_disk_size }} + changed_when: true + + - name: Start the new VM + ansible.builtin.command: qm start {{ mgmt_vmid }} + changed_when: true + + - name: Start the VM if it exists but is stopped + ansible.builtin.command: qm start {{ mgmt_vmid }} + when: vm_status.rc == 0 and 'stopped' in vm_status.stdout + changed_when: true + + - name: Wait for SSH on the VM + ansible.builtin.wait_for: + host: "{{ mgmt_ip }}" + port: 22 + timeout: 300 + delegate_to: localhost + +- name: Install k3s on the management VM + hosts: mgmt + become: true + gather_facts: false + tags: [k3s] + handlers: + - name: Restart k3s + ansible.builtin.systemd_service: + name: k3s + state: restarted + tasks: + - name: Wait for cloud-init to finish (apt lock, SSH keys) + ansible.builtin.command: cloud-init status --wait + changed_when: false + + - name: Install qemu-guest-agent + ansible.builtin.apt: + name: qemu-guest-agent + update_cache: true + cache_valid_time: 86400 + + - name: Start qemu-guest-agent + ansible.builtin.systemd_service: + name: qemu-guest-agent + state: started + + - name: Create the k3s config directory + ansible.builtin.file: + path: /etc/rancher/k3s + state: directory + mode: "0755" + + - name: Write the k3s config + ansible.builtin.template: + src: k3s-config.yaml.j2 + dest: /etc/rancher/k3s/config.yaml + mode: "0644" + notify: Restart k3s + + - name: Installed k3s version + ansible.builtin.command: k3s --version + register: k3s_installed + changed_when: false + failed_when: false + + - name: Install or upgrade k3s to the pinned version + when: k3s_version not in k3s_installed.stdout | default('') + block: + - name: Download the k3s install script + ansible.builtin.get_url: + url: https://get.k3s.io + dest: /usr/local/bin/k3s-install.sh + mode: "0755" + + - name: Run the k3s install script + ansible.builtin.command: /usr/local/bin/k3s-install.sh + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" + changed_when: true + + - name: Apply pending k3s restart + ansible.builtin.meta: flush_handlers + + - name: Wait for the node to be Ready + ansible.builtin.command: k3s kubectl wait --for=condition=Ready node --all --timeout=30s + register: node_ready + until: node_ready.rc == 0 + retries: 12 + delay: 10 + changed_when: false + +- name: Install cert-manager and cluster-api-operator (k3s helm-controller) + hosts: mgmt + become: true + gather_facts: false + tags: [addons] + tasks: + - name: Write the cert-manager HelmChart + ansible.builtin.template: + src: helmchart-cert-manager.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/cert-manager.yaml + mode: "0600" + + - name: Wait for cert-manager deployments + ansible.builtin.command: k3s kubectl -n cert-manager wait --for=condition=Available deploy --all --timeout=30s + register: cm_ready + until: cm_ready.rc == 0 + retries: 30 + delay: 10 + changed_when: false + + - name: Write the cluster-api-operator HelmChart + ansible.builtin.template: + src: helmchart-capi-operator.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/cluster-api-operator.yaml + mode: "0600" + + - name: Wait for cluster-api-operator deployment + ansible.builtin.command: k3s kubectl -n capi-operator-system wait --for=condition=Available deploy --all --timeout=30s + register: op_ready + until: op_ready.rc == 0 + retries: 30 + delay: 10 + changed_when: false + + - name: Wait for the operator CRDs + ansible.builtin.command: k3s kubectl wait --for=condition=established crd/coreproviders.operator.cluster.x-k8s.io --timeout=60s + changed_when: false + +- name: CAPI providers + hosts: mgmt + become: true + gather_facts: false + tags: [capi] + vars_files: + - secrets.yml + tasks: + - name: Manifest directory + ansible.builtin.file: + path: /opt/lab-mgmt + state: directory + mode: "0700" + + - name: Provider manifests (namespaces, capmox-variables Secret, provider CRs) + ansible.builtin.template: + src: capi-providers.yaml.j2 + dest: /opt/lab-mgmt/capi-providers.yaml + mode: "0600" + diff: false + + - name: Apply provider manifests + ansible.builtin.command: k3s kubectl apply -f /opt/lab-mgmt/capi-providers.yaml + register: providers_apply + changed_when: "' created' in providers_apply.stdout or ' configured' in providers_apply.stdout" + + - name: Wait for provider deployments + ansible.builtin.command: k3s kubectl -n {{ item.namespace }} wait --for=condition=Available deploy --all --timeout=30s + loop: "{{ capi_providers }}" + loop_control: + label: "{{ item.kind }}/{{ item.name }}" + register: provider_ready + until: provider_ready.rc == 0 + retries: 40 + delay: 15 + changed_when: false + +- name: Kubeconfigs on the laptop + hosts: mgmt + become: true + gather_facts: false + tags: [kubeconfig] + tasks: + - name: Read the k3s admin kubeconfig + ansible.builtin.slurp: + src: /etc/rancher/k3s/k3s.yaml + register: k3s_kubeconfig + + - name: Write ~/.kube/lab-mgmt.kubeconfig (server = VM IP, names = lab-mgmt) + ansible.builtin.copy: + content: >- + {{ k3s_kubeconfig.content | b64decode + | replace('https://127.0.0.1:6443', 'https://' + mgmt_ip + ':6443') + | regex_replace('(?m): default$', ': lab-mgmt') }} + dest: "{{ lookup('env', 'HOME') }}/.kube/lab-mgmt.kubeconfig" + mode: "0600" + delegate_to: localhost + become: false + diff: false + + - name: Read-only kubeconfig for the cluster-inspector agent + ansible.builtin.command: >- + {{ playbook_dir }}/../.claude/cluster-inspector/make-kubeconfig.sh lab-mgmt + --admin-kubeconfig {{ lookup('env', 'HOME') }}/.kube/lab-mgmt.kubeconfig + --apply-rbac + delegate_to: localhost + become: false + changed_when: false + tags: [inspector] diff --git a/ansible/secrets.yml.example b/ansible/secrets.yml.example new file mode 100644 index 0000000..130b4f5 --- /dev/null +++ b/ansible/secrets.yml.example @@ -0,0 +1,2 @@ +# cp secrets.yml.example secrets.yml (gitignored) and fill in +proxmox_token_secret: "" diff --git a/ansible/templates/capi-providers.yaml.j2 b/ansible/templates/capi-providers.yaml.j2 new file mode 100644 index 0000000..1a51b5d --- /dev/null +++ b/ansible/templates/capi-providers.yaml.j2 @@ -0,0 +1,33 @@ +{% for p in capi_providers %} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ p.namespace }} +--- +{% endfor %} +# Read by CAPMOX v0.9.1 components; the operator does not watch it, so rotating the token +# means a new Secret name + updated configSecret reference. +apiVersion: v1 +kind: Secret +metadata: + name: capmox-variables + namespace: capmox-system +type: Opaque +stringData: + PROXMOX_URL: "{{ proxmox_url }}" + PROXMOX_TOKEN: "{{ proxmox_token_id }}" + PROXMOX_SECRET: "{{ proxmox_token_secret }}" +{% for p in capi_providers %} +--- +apiVersion: operator.cluster.x-k8s.io/v1alpha2 +kind: {{ p.kind }} +metadata: + name: {{ p.name }} + namespace: {{ p.namespace }} +spec: + version: {{ p.version }} +{% if p.config_secret is defined %} + configSecret: + name: {{ p.config_secret }} +{% endif %} +{% endfor %} diff --git a/ansible/templates/helmchart-capi-operator.yaml.j2 b/ansible/templates/helmchart-capi-operator.yaml.j2 new file mode 100644 index 0000000..598fc6e --- /dev/null +++ b/ansible/templates/helmchart-capi-operator.yaml.j2 @@ -0,0 +1,13 @@ +# Providers are NOT set in chart values (they would become Helm hooks, cluster-api-operator#1052); +# they are applied as separate CRs from capi-providers.yaml. +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cluster-api-operator + namespace: kube-system +spec: + repo: https://kubernetes-sigs.github.io/cluster-api-operator + chart: cluster-api-operator + version: {{ capi_operator_chart_version }} + targetNamespace: capi-operator-system + createNamespace: true diff --git a/ansible/templates/helmchart-cert-manager.yaml.j2 b/ansible/templates/helmchart-cert-manager.yaml.j2 new file mode 100644 index 0000000..aeff822 --- /dev/null +++ b/ansible/templates/helmchart-cert-manager.yaml.j2 @@ -0,0 +1,14 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cert-manager + namespace: kube-system +spec: + repo: https://charts.jetstack.io + chart: cert-manager + version: {{ cert_manager_chart_version }} + targetNamespace: cert-manager + createNamespace: true + valuesContent: |- + crds: + enabled: true diff --git a/ansible/templates/k3s-config.yaml.j2 b/ansible/templates/k3s-config.yaml.j2 new file mode 100644 index 0000000..269cc00 --- /dev/null +++ b/ansible/templates/k3s-config.yaml.j2 @@ -0,0 +1,5 @@ +write-kubeconfig-mode: "0600" +tls-san: + - {{ mgmt_ip }} +disable: + - traefik diff --git a/docs/plans-executions/.gitkeep b/docs/plans-executions/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/plans-executions/2026-09-27-0058-mgmt-cluster-ansible.md b/docs/plans-executions/2026-09-27-0058-mgmt-cluster-ansible.md new file mode 100644 index 0000000..6d5f0ef --- /dev/null +++ b/docs/plans-executions/2026-09-27-0058-mgmt-cluster-ansible.md @@ -0,0 +1,57 @@ +# Execution log: idempotent management cluster (Ansible + cluster-api-operator) + +Pairs with [docs/plans/2026-09-27-0058-mgmt-cluster-ansible.md](../plans/2026-09-27-0058-mgmt-cluster-ansible.md). + +## Status + +- [x] Step 0 — Initial import commit + plan commit +- [x] Step 1 — Scaffold `ansible/`, `.gitignore` exception, syntax/lint checks +- [ ] Step 2 — Run the playbook tag by tag against the lab (owner) +- [ ] Step 3 — Idempotency check (second full run `changed=0`) +- [ ] Step 4 — Docs, versions, decisions, changelog + +## Step 0 — Initial import + plan + +The repo had no commits, so the existing tree went in first as `5fd998c` (after a grep for +token/secret/key material found only placeholders), then the plan alone as `ac7d5a4`. + +## Step 1 — Scaffold `ansible/` (2026-09-27 01:02 CEST) + +Wrote `ansible/{ansible.cfg,inventory.yml,group_vars/all.yml,secrets.yml.example,mgmt.yml}` and +four templates as planned. `ansible/secrets.yml` was already ignored by `*secret*`; added +`!ansible/secrets.yml.example`. + +`ansible-core` is not installed on the laptop, so checks ran through `uvx` (cached, nothing +installed globally), pinned to ansible-core 2.18 (resolved 2.18.19): + +```bash +cd ansible +uvx --from 'ansible-core>=2.18,<2.19' ansible-playbook mgmt.yml --syntax-check +uvx --with 'ansible-core>=2.18,<2.19' ansible-lint --profile production mgmt.yml +``` + +First lint run: 6× `name[casing]`, 5× `no-changed-when`. The commands flagged are the +create/resize/start/install tasks that only run behind a `when:` guard, so they got +`changed_when: true` rather than a fake condition. Second run: production profile passed. + +Rendered all templates with dummy values (`-e mgmt_ip=192.0.2.10`, secret `DUMMY`) through a +throwaway local playbook: provider manifest parses as 11 documents (5 Namespaces, Secret, 5 +provider CRs), k3s config and kubeconfig rewrite (`server` → VM IP, every `default` name → +`lab-mgmt`) come out as intended. + +Connectivity to the PVE host: + +```bash +uvx --from 'ansible-core>=2.18,<2.19' ansible pve -m ping -o +# pv1-1 | UNREACHABLE!: ... root@192.168.0.8: Permission denied (publickey,password). +``` + +**Worth noting:** +- The plan's assumption "root SSH from the laptop to `pv1-1` works" does not hold yet; the host + answers but rejects the laptop's keys. Needed before Step 2. +- Condition-based waits: the playbook waits for `Available` Deployments in each provider + namespace instead of a condition on the provider CRs, so it doesn't depend on the operator's + condition naming (plan left that open). +- VM params (`mgmt_vmid`, `mgmt_ip`, `mgmt_gateway`, `mgmt_dns`) are empty in `group_vars/all.yml`; + Play 1 asserts on them. +- `clusterctl` is not installed on the laptop either (needed for phase-1 Step 3, not for this playbook).