cluster-autoscaler-with-capi/ansible/mgmt.yml

# Management cluster: VM on Proxmox -> k3s -> cert-manager + cluster-api-operator -> CAPI providers.
# Idempotent: a second run reports changed=0. Run from this directory:
#   ansible-playbook mgmt.yml [--tags vm|k3s|addons|capi|kubeconfig|inspector]

- name: Management VM on Proxmox
  hosts: pve
  gather_facts: false
  tags: [vm]
  tasks:
    - name: Required VM parameters are set in group_vars/all.yml
      ansible.builtin.assert:
        that:
          - mgmt_vmid | string | length > 0
          - mgmt_ip | length > 0
          - mgmt_gateway | length > 0
          - mgmt_dns | length > 0

    - name: Check whether the VM exists
      ansible.builtin.command: qm status {{ mgmt_vmid }}
      register: vm_status
      changed_when: false
      failed_when: false

    - name: Create the VM from the Ubuntu cloud image
      when: vm_status.rc != 0
      block:
        - name: Download the pinned cloud image
          ansible.builtin.get_url:
            url: "{{ ubuntu_image_url }}"
            dest: "{{ ubuntu_image_path }}"
            checksum: "sha256:{{ ubuntu_image_sha256 }}"
            mode: "0644"

        - name: Stage the SSH public key for cloud-init
          ansible.builtin.copy:
            content: "{{ mgmt_ssh_public_key }}\n"
            dest: /root/{{ mgmt_name }}.pub
            mode: "0600"

        # raw disk on LVM-thin; bridge + separate VLAN tag, never vmbr0.4
        - name: Create the VM
          ansible.builtin.command: >-
            qm create {{ mgmt_vmid }}
            --name {{ mgmt_name }}
            --ostype l26
            --cores {{ mgmt_cores }}
            --memory {{ mgmt_memory_mib }}
            --net0 virtio,bridge={{ proxmox_bridge }},tag={{ proxmox_vlan }}
            --scsihw virtio-scsi-pci
            --scsi0 {{ proxmox_storage }}:0,import-from={{ ubuntu_image_path }}
            --ide2 {{ proxmox_storage }}:cloudinit
            --boot order=scsi0
            --serial0 socket --vga serial0
            --agent 1
            --ciuser ubuntu
            --sshkeys /root/{{ mgmt_name }}.pub
            --ipconfig0 ip={{ mgmt_ip }}/{{ mgmt_prefix }},gw={{ mgmt_gateway }}
            --nameserver '{{ mgmt_dns | join(' ') }}'
          changed_when: true

        - name: Grow the disk
          ansible.builtin.command: qm resize {{ mgmt_vmid }} scsi0 {{ mgmt_disk_size }}
          changed_when: true

        - name: Start the new VM
          ansible.builtin.command: qm start {{ mgmt_vmid }}
          changed_when: true

    - name: Start the VM if it exists but is stopped
      ansible.builtin.command: qm start {{ mgmt_vmid }}
      when: vm_status.rc == 0 and 'stopped' in vm_status.stdout
      changed_when: true

    - name: Wait for SSH on the VM
      ansible.builtin.wait_for:
        host: "{{ mgmt_ip }}"
        port: 22
        timeout: 300
      delegate_to: localhost

- name: Install k3s on the management VM
  hosts: mgmt
  become: true
  gather_facts: false
  tags: [k3s]
  handlers:
    - name: Restart k3s
      ansible.builtin.systemd_service:
        name: k3s
        state: restarted
  tasks:
    - name: Wait for cloud-init to finish (apt lock, SSH keys)
      ansible.builtin.command: cloud-init status --wait
      register: cloud_init
      changed_when: false
      # rc 2 = done with recoverable errors; PVE's user-data triggers a deprecation warning ('user' as string)
      failed_when: cloud_init.rc not in [0, 2]

    - name: Install qemu-guest-agent
      ansible.builtin.apt:
        name: qemu-guest-agent
        update_cache: true
        cache_valid_time: 86400

    - name: Start qemu-guest-agent
      ansible.builtin.systemd_service:
        name: qemu-guest-agent
        state: started

    - name: Create the k3s config directory
      ansible.builtin.file:
        path: /etc/rancher/k3s
        state: directory
        mode: "0755"

    - name: Write the k3s config
      ansible.builtin.template:
        src: k3s-config.yaml.j2
        dest: /etc/rancher/k3s/config.yaml
        mode: "0644"
      notify: Restart k3s

    - name: Installed k3s version
      ansible.builtin.command: k3s --version
      register: k3s_installed
      changed_when: false
      failed_when: false

    - name: Install or upgrade k3s to the pinned version
      when: k3s_version not in k3s_installed.stdout | default('')
      block:
        - name: Download the k3s install script
          ansible.builtin.get_url:
            url: https://get.k3s.io
            dest: /usr/local/bin/k3s-install.sh
            mode: "0755"

        - name: Run the k3s install script
          ansible.builtin.command: /usr/local/bin/k3s-install.sh
          environment:
            INSTALL_K3S_VERSION: "{{ k3s_version }}"
          changed_when: true

    - name: Apply pending k3s restart
      ansible.builtin.meta: flush_handlers

    - name: Wait for the node to be Ready
      ansible.builtin.command: k3s kubectl wait --for=condition=Ready node --all --timeout=30s
      register: node_ready
      until: node_ready.rc == 0
      retries: 12
      delay: 10
      changed_when: false

- name: Install cert-manager and cluster-api-operator (k3s helm-controller)
  hosts: mgmt
  become: true
  gather_facts: false
  tags: [addons]
  tasks:
    - name: Write the cert-manager HelmChart
      ansible.builtin.template:
        src: helmchart-cert-manager.yaml.j2
        dest: /var/lib/rancher/k3s/server/manifests/cert-manager.yaml
        mode: "0600"

    - name: Wait for cert-manager deployments
      ansible.builtin.command: k3s kubectl -n cert-manager wait --for=condition=Available deploy --all --timeout=30s
      register: cm_ready
      until: cm_ready.rc == 0
      retries: 30
      delay: 10
      changed_when: false

    - name: Write the cluster-api-operator HelmChart
      ansible.builtin.template:
        src: helmchart-capi-operator.yaml.j2
        dest: /var/lib/rancher/k3s/server/manifests/cluster-api-operator.yaml
        mode: "0600"

    - name: Wait for cluster-api-operator deployment
      ansible.builtin.command: k3s kubectl -n capi-operator-system wait --for=condition=Available deploy --all --timeout=30s
      register: op_ready
      until: op_ready.rc == 0
      retries: 30
      delay: 10
      changed_when: false

    - name: Wait for the operator CRDs
      ansible.builtin.command: k3s kubectl wait --for=condition=established crd/coreproviders.operator.cluster.x-k8s.io --timeout=60s
      changed_when: false

- name: CAPI providers
  hosts: mgmt
  become: true
  gather_facts: false
  tags: [capi]
  vars_files:
    - secrets.yml
  tasks:
    - name: Manifest directory
      ansible.builtin.file:
        path: /opt/lab-mgmt
        state: directory
        mode: "0700"

    - name: Provider manifests (namespaces, capmox-variables Secret, provider CRs)
      ansible.builtin.template:
        src: capi-providers.yaml.j2
        dest: /opt/lab-mgmt/capi-providers.yaml
        mode: "0600"
      diff: false

    - name: Apply provider manifests
      ansible.builtin.command: k3s kubectl apply -f /opt/lab-mgmt/capi-providers.yaml
      register: providers_apply
      changed_when: "' created' in providers_apply.stdout or ' configured' in providers_apply.stdout"

    - name: Wait for provider deployments
      ansible.builtin.command: k3s kubectl -n {{ item.namespace }} wait --for=condition=Available deploy --all --timeout=30s
      loop: "{{ capi_providers }}"
      loop_control:
        label: "{{ item.kind }}/{{ item.name }}"
      register: provider_ready
      until: provider_ready.rc == 0
      retries: 40
      delay: 15
      changed_when: false

- name: Kubeconfigs on the laptop
  hosts: mgmt
  become: true
  gather_facts: false
  tags: [kubeconfig]
  tasks:
    - name: Read the k3s admin kubeconfig
      ansible.builtin.slurp:
        src: /etc/rancher/k3s/k3s.yaml
      register: k3s_kubeconfig

    - name: Write ~/.kube/lab-mgmt.kubeconfig (server = VM IP, names = lab-mgmt)
      ansible.builtin.copy:
        content: >-
          {{ k3s_kubeconfig.content | b64decode
             | replace('https://127.0.0.1:6443', 'https://' + mgmt_ip + ':6443')
             | regex_replace('(?m): default$', ': lab-mgmt') }}
        dest: "{{ lookup('env', 'HOME') }}/.kube/lab-mgmt.kubeconfig"
        mode: "0600"
      delegate_to: localhost
      become: false
      diff: false

    - name: Read-only kubeconfig for the cluster-inspector agent
      ansible.builtin.command: >-
        {{ playbook_dir }}/../.claude/cluster-inspector/make-kubeconfig.sh lab-mgmt
        --admin-kubeconfig {{ lookup('env', 'HOME') }}/.kube/lab-mgmt.kubeconfig
        --apply-rbac
      delegate_to: localhost
      become: false
      changed_when: false
      tags: [inspector]