proxmox: Create by cloning a template, seed upload, config, start
Create validates placement, then under the per-name lock adopts the VM a previous attempt left (by name, or at a taken VMID after the ownership check of spec entry 2) or full-clones the template onto the UID's deterministic VMID with the mark in the clone call. It then uploads the seed over any existing file, writes cores, memory, agent, tag, ide2 and net0 when configured, grows the boot disk when diskGiB is larger, starts, and waits for running/running. waitUnlocked no longer waits on a suspended lock: a hibernated VM is judged at once, so ours is Permanent and a foreign one is probed past. The template is checked only on the allocation path, and its shape (net0, boot disk size) is refused before any clone exists. Spec entries 2, 4 and 10 updated to match. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,425 @@
|
||||
package proxmox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// maxProbes bounds the VMID probe of spec entry 2.
|
||||
const maxProbes = 32
|
||||
|
||||
// diskKey is a Proxmox disk slot: the keys a boot disk, a CD-ROM or a
|
||||
// cloud-init drive can sit on.
|
||||
var diskKey = regexp.MustCompile(`^(ide|sata|scsi|virtio)[0-9]+$`)
|
||||
|
||||
// ownedVM is a VM Create has cloned or adopted: where it lives and its
|
||||
// config as last read, which the config and resize steps start from.
|
||||
type ownedVM struct {
|
||||
node string
|
||||
vmid int
|
||||
cfg vmConfig
|
||||
}
|
||||
|
||||
// Create is spec entry 10: validate placement, then, under the per-name
|
||||
// lock, adopt the VM a previous attempt left or clone a new one, and carry
|
||||
// it through seed, config, resize and start until it runs.
|
||||
func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
|
||||
node, mt, tmpl, err := p.placement(req)
|
||||
if err != nil {
|
||||
return "", p.wrapErr("create", "", err)
|
||||
}
|
||||
log := p.logger(ctx).WithValues("name", req.Name)
|
||||
log.V(2).Info("proxmox create resolved",
|
||||
"zone", node, "machineType", req.Placement.MachineType, "cores", mt.Cores, "memoryMiB", mt.MemoryMiB,
|
||||
"diskGiB", mt.DiskGiB, "image", req.Placement.Image, "templateVMID", tmpl.VMID, "storage", p.cfg.Storage,
|
||||
"isoStorage", p.cfg.ISOStorage, "bridge", p.cfg.Bridge, "vlanTag", p.cfg.VLANTag, "cloudInitBytes", len(req.CloudInit))
|
||||
|
||||
unlock, err := p.lockName(ctx, req.Name)
|
||||
if err != nil {
|
||||
return "", p.wrapErr("create", "", err)
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
vm, err := p.create(ctx, log, node, mt, tmpl, req)
|
||||
if err != nil {
|
||||
return "", p.wrapErr("create", "", err)
|
||||
}
|
||||
log.V(1).Info("proxmox vm ensured running", "vmid", vm.vmid, "node", vm.node)
|
||||
return formatProviderID(vm.vmid, req.Name), nil
|
||||
}
|
||||
|
||||
// placement is step 1: every catalogue key is checked before any backend
|
||||
// call, and an unknown one is Permanent naming the field and the value
|
||||
// (spec entry 3). Region is ignored.
|
||||
func (p *Provider) placement(req provider.CreateRequest) (string, provider.ProxmoxMachineType, provider.ProxmoxTemplate, error) {
|
||||
var (
|
||||
mt provider.ProxmoxMachineType
|
||||
tmpl provider.ProxmoxTemplate
|
||||
)
|
||||
if req.Name == "" {
|
||||
return "", mt, tmpl, permanentf("the request has no instance name")
|
||||
}
|
||||
node, err := p.node(req.Placement.Zone)
|
||||
if err != nil {
|
||||
return "", mt, tmpl, err
|
||||
}
|
||||
mt, ok := p.cfg.MachineTypes[req.Placement.MachineType]
|
||||
if !ok {
|
||||
return "", mt, tmpl, permanentf("placement.machineType %q is not in proxmox.machineTypes", req.Placement.MachineType)
|
||||
}
|
||||
tmpl, ok = p.cfg.Templates[req.Placement.Image]
|
||||
if !ok {
|
||||
return "", mt, tmpl, permanentf("placement.image %q is not in proxmox.templates", req.Placement.Image)
|
||||
}
|
||||
return node, mt, tmpl, nil
|
||||
}
|
||||
|
||||
// create is steps 2 to 9. The template is checked only when a clone is
|
||||
// needed: a VM a previous attempt created no longer depends on it (spec
|
||||
// entry 4), so a template removed since must not fail its adoption.
|
||||
func (p *Provider) create(ctx context.Context, log logr.Logger, node string, mt provider.ProxmoxMachineType, tmpl provider.ProxmoxTemplate, req provider.CreateRequest) (ownedVM, error) {
|
||||
rs, err := p.api.ClusterResources(ctx)
|
||||
if err != nil {
|
||||
return ownedVM{}, fmt.Errorf("listing cluster resources: %w", err)
|
||||
}
|
||||
vm, adopted, err := p.adoptByName(ctx, rs, req)
|
||||
if err != nil {
|
||||
return ownedVM{}, err
|
||||
}
|
||||
if !adopted {
|
||||
src, err := p.template(ctx, rs, req.Placement.Image, tmpl.VMID, mt)
|
||||
if err != nil {
|
||||
return ownedVM{}, err
|
||||
}
|
||||
if vm, adopted, err = p.allocate(ctx, log, rs, node, src, req); err != nil {
|
||||
return ownedVM{}, err
|
||||
}
|
||||
}
|
||||
log = log.WithValues("vmid", vm.vmid, "node", vm.node)
|
||||
if adopted {
|
||||
st, err := p.api.VMStatus(ctx, vm.node, vm.vmid)
|
||||
if err != nil {
|
||||
return ownedVM{}, fmt.Errorf("reading status of VM %d: %w", vm.vmid, err)
|
||||
}
|
||||
log.V(1).Info("proxmox vm adopted", "status", st.Status, "qmpstatus", st.QMPStatus)
|
||||
if st.Status == "running" {
|
||||
return vm, nil
|
||||
}
|
||||
}
|
||||
return vm, p.provision(ctx, log, vm, mt, req)
|
||||
}
|
||||
|
||||
// adoptByName is the by-name half of spec entry 2: two VMs with the name
|
||||
// are for an admin to sort out, one goes through the ownership check.
|
||||
func (p *Provider) adoptByName(ctx context.Context, rs []vmResource, req provider.CreateRequest) (ownedVM, bool, error) {
|
||||
var found []vmResource
|
||||
for _, r := range rs {
|
||||
if r.Type == "qemu" && r.Template == 0 && r.Name == req.Name {
|
||||
found = append(found, r)
|
||||
}
|
||||
}
|
||||
switch len(found) {
|
||||
case 0:
|
||||
return ownedVM{}, false, nil
|
||||
case 1:
|
||||
return p.inspect(ctx, found[0], req)
|
||||
}
|
||||
vmids := make([]string, len(found))
|
||||
for i, r := range found {
|
||||
vmids[i] = strconv.Itoa(r.VMID)
|
||||
}
|
||||
return ownedVM{}, false, permanentf("VMs %s are all named %s; remove all but one", strings.Join(vmids, ", "), req.Name)
|
||||
}
|
||||
|
||||
// inspect is the ownership check of spec entry 2 on a VMID that is taken.
|
||||
// It waits out a task's lock (a clone in progress is also nameless), then
|
||||
// reads the name and the mark: another name is someone else's VM, and our
|
||||
// name without our mark is never taken over.
|
||||
func (p *Provider) inspect(ctx context.Context, r vmResource, req provider.CreateRequest) (ownedVM, bool, error) {
|
||||
cfg, err := p.waitUnlocked(ctx, r.Node, r.VMID, p.budgets.clone)
|
||||
if err != nil {
|
||||
return ownedVM{}, false, err
|
||||
}
|
||||
if cfg["name"] != req.Name {
|
||||
return ownedVM{}, false, nil
|
||||
}
|
||||
mark := parseDescription(cfg["description"])
|
||||
switch {
|
||||
case !mark.Managed:
|
||||
return ownedVM{}, false, permanentf("VM %d is named %s but carries no proxy-operator mark; it was not created by this operator", r.VMID, req.Name)
|
||||
case mark.UID != "" && mark.UID != req.UID:
|
||||
return ownedVM{}, false, permanentf("VM %d named %s belongs to proxy uid %s, not %s", r.VMID, req.Name, mark.UID, req.UID)
|
||||
case cfg["lock"] == lockSuspended:
|
||||
return ownedVM{}, false, permanentf("VM %d named %s is hibernated (lock suspended); resume or destroy it in Proxmox", r.VMID, req.Name)
|
||||
}
|
||||
return ownedVM{node: r.Node, vmid: r.VMID, cfg: cfg}, true, nil
|
||||
}
|
||||
|
||||
// template finds the clone source and refuses one that is not a template
|
||||
// or brings a Proxmox cloud-init drive, which would compete with the seed
|
||||
// (spec entry 4). The clone copies the template's config, so what the
|
||||
// config or resize step would refuse on the clone is refused here, while no
|
||||
// VM exists that a fixed template could no longer help.
|
||||
func (p *Provider) template(ctx context.Context, rs []vmResource, image string, vmid int, mt provider.ProxmoxMachineType) (vmResource, error) {
|
||||
i := slices.IndexFunc(rs, func(r vmResource) bool { return r.VMID == vmid && r.Type == "qemu" })
|
||||
if i < 0 {
|
||||
return vmResource{}, permanentf("placement.image %q: template VM %d does not exist", image, vmid)
|
||||
}
|
||||
src := rs[i]
|
||||
cfg, err := p.api.VMConfig(ctx, src.Node, vmid)
|
||||
if err != nil {
|
||||
return vmResource{}, fmt.Errorf("reading config of template %d: %w", vmid, err)
|
||||
}
|
||||
if cfg["template"] != "1" {
|
||||
return vmResource{}, permanentf("placement.image %q: VM %d is not a template", image, vmid)
|
||||
}
|
||||
for _, k := range slices.Sorted(maps.Keys(cfg)) {
|
||||
if diskKey.MatchString(k) && strings.Contains(cfg[k], "cloudinit") {
|
||||
return vmResource{}, permanentf("placement.image %q: template %d has a Proxmox cloud-init drive on %s; remove it, the provider attaches its own seed", image, vmid, k)
|
||||
}
|
||||
}
|
||||
shape := ownedVM{node: src.Node, vmid: vmid, cfg: cfg}
|
||||
if _, err := p.configParams(shape, mt, ""); err != nil {
|
||||
return vmResource{}, err
|
||||
}
|
||||
if _, _, _, err := resizeTarget(shape, mt.DiskGiB); err != nil {
|
||||
return vmResource{}, err
|
||||
}
|
||||
return src, nil
|
||||
}
|
||||
|
||||
// allocate is the probe of spec entry 2: the attempt-th VMID for the UID
|
||||
// is cloned onto when the listing shows it free and put through the
|
||||
// ownership check when it is taken, or when the clone finds it taken after
|
||||
// all (F4). The boolean reports an adoption rather than a fresh clone.
|
||||
func (p *Provider) allocate(ctx context.Context, log logr.Logger, rs []vmResource, node string, src vmResource, req provider.CreateRequest) (ownedVM, bool, error) {
|
||||
taken := make(map[int]vmResource, len(rs))
|
||||
for _, r := range rs {
|
||||
taken[r.VMID] = r
|
||||
}
|
||||
rng := p.cfg.VMIDRange
|
||||
probes := min(maxProbes, rng.Max-rng.Min+1)
|
||||
for attempt := range probes {
|
||||
vmid := vmidFor(req.UID, rng.Min, rng.Max, attempt)
|
||||
r, isTaken := taken[vmid]
|
||||
if !isTaken {
|
||||
vm, err := p.clone(ctx, log, node, src, vmid, req)
|
||||
if !isAlreadyExists(err) {
|
||||
return vm, false, err
|
||||
}
|
||||
var found bool
|
||||
if r, found, err = p.resolve(ctx, vmid); err != nil {
|
||||
return ownedVM{}, false, err
|
||||
}
|
||||
if !found {
|
||||
log.V(1).Info("proxmox vmid taken by something other than a VM, probing on", "vmid", vmid, "attempt", attempt)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if r.Type == "qemu" && r.Template == 0 {
|
||||
vm, ours, err := p.inspect(ctx, r, req)
|
||||
if err != nil || ours {
|
||||
return vm, ours, err
|
||||
}
|
||||
}
|
||||
log.V(1).Info("proxmox vmid taken by another vm, probing on", "vmid", vmid, "attempt", attempt)
|
||||
}
|
||||
return ownedVM{}, false, transientf("no free VMID for %s after %d probes in %d..%d", req.Name, probes, rng.Min, rng.Max)
|
||||
}
|
||||
|
||||
// clone is step 4: a full clone carrying the mark, so the VM is never
|
||||
// unmarked (spec entry 8), then its unlocked config. A taken VMID comes
|
||||
// back as the API's error for allocate to judge.
|
||||
func (p *Provider) clone(ctx context.Context, log logr.Logger, node string, src vmResource, vmid int, req provider.CreateRequest) (ownedVM, error) {
|
||||
params := cloneParams{
|
||||
NewID: vmid,
|
||||
Name: req.Name,
|
||||
Description: formatDescription(req.UID, p.now()),
|
||||
Storage: p.cfg.Storage,
|
||||
}
|
||||
if node != src.Node {
|
||||
params.Target = node
|
||||
}
|
||||
upid, err := p.api.Clone(ctx, src.Node, src.VMID, params)
|
||||
if err != nil {
|
||||
return ownedVM{}, fmt.Errorf("cloning template %d to VM %d: %w", src.VMID, vmid, err)
|
||||
}
|
||||
if err := p.waitTask(ctx, log, upid, p.budgets.clone); err != nil {
|
||||
return ownedVM{}, fmt.Errorf("cloning template %d to VM %d: %w", src.VMID, vmid, err)
|
||||
}
|
||||
cfg, err := p.waitUnlocked(ctx, node, vmid, p.budgets.clone)
|
||||
if err != nil {
|
||||
return ownedVM{}, err
|
||||
}
|
||||
return ownedVM{node: node, vmid: vmid, cfg: cfg}, nil
|
||||
}
|
||||
|
||||
// provision is steps 5 to 9, each safe to repeat on a VM a previous attempt
|
||||
// left stopped: the upload overwrites, the config write replaces, and the
|
||||
// resize is skipped once the disk is large enough. Everything that can be
|
||||
// refused from the config alone is checked before the first write.
|
||||
func (p *Provider) provision(ctx context.Context, log logr.Logger, vm ownedVM, mt provider.ProxmoxMachineType, req provider.CreateRequest) error {
|
||||
params, err := p.configParams(vm, mt, req.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
disk, size, grow, err := resizeTarget(vm, mt.DiskGiB)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.uploadSeed(ctx, log, vm.node, req); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.api.SetVMConfig(ctx, vm.node, vm.vmid, params); err != nil {
|
||||
return fmt.Errorf("configuring VM %d: %w", vm.vmid, err)
|
||||
}
|
||||
if grow {
|
||||
upid, err := p.api.Resize(ctx, vm.node, vm.vmid, disk, size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resizing %s of VM %d to %s: %w", disk, vm.vmid, size, err)
|
||||
}
|
||||
if err := p.waitTask(ctx, log, upid, p.budgets.resize); err != nil {
|
||||
return fmt.Errorf("resizing %s of VM %d to %s: %w", disk, vm.vmid, size, err)
|
||||
}
|
||||
}
|
||||
upid, err := p.api.Start(ctx, vm.node, vm.vmid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("starting VM %d: %w", vm.vmid, err)
|
||||
}
|
||||
if err := p.waitTask(ctx, log, upid, p.budgets.start); err != nil {
|
||||
return fmt.Errorf("starting VM %d: %w", vm.vmid, err)
|
||||
}
|
||||
return p.waitRunning(ctx, vm.node, vm.vmid)
|
||||
}
|
||||
|
||||
// configParams is step 6's write: size, agent, tag and seed, and net0 only
|
||||
// when the config sets a bridge or a VLAN tag (spec entry 7).
|
||||
func (p *Provider) configParams(vm ownedVM, mt provider.ProxmoxMachineType, name string) (map[string]string, error) {
|
||||
params := map[string]string{
|
||||
"cores": strconv.Itoa(int(mt.Cores)),
|
||||
"memory": strconv.FormatInt(mt.MemoryMiB, 10),
|
||||
"agent": "1",
|
||||
"tags": proxyTag,
|
||||
"ide2": seedVolID(p.cfg.ISOStorage, name) + ",media=cdrom",
|
||||
}
|
||||
if p.cfg.Bridge == "" && p.cfg.VLANTag == 0 {
|
||||
return params, nil
|
||||
}
|
||||
net0 := vm.cfg["net0"]
|
||||
if net0 == "" {
|
||||
return nil, permanentf("VM %d has no net0 to apply proxmox.bridge or proxmox.vlanTag to; the template needs a network device", vm.vmid)
|
||||
}
|
||||
params["net0"] = rewriteNet0(net0, p.cfg.Bridge, p.cfg.VLANTag)
|
||||
return params, nil
|
||||
}
|
||||
|
||||
// rewriteNet0 replaces or appends bridge= and tag= in a net0 value, keeping
|
||||
// every other option, the model and the MAC in place. An empty bridge or a
|
||||
// zero tag leaves that option alone.
|
||||
func rewriteNet0(net0, bridge string, tag int) string {
|
||||
want := map[string]string{}
|
||||
if bridge != "" {
|
||||
want["bridge"] = bridge
|
||||
}
|
||||
if tag != 0 {
|
||||
want["tag"] = strconv.Itoa(tag)
|
||||
}
|
||||
opts := strings.Split(net0, ",")
|
||||
for i, opt := range opts {
|
||||
key, _, _ := strings.Cut(opt, "=")
|
||||
if v, ok := want[key]; ok {
|
||||
opts[i] = key + "=" + v
|
||||
delete(want, key)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"bridge", "tag"} {
|
||||
if v, ok := want[key]; ok {
|
||||
opts = append(opts, key+"="+v)
|
||||
}
|
||||
}
|
||||
return strings.Join(opts, ",")
|
||||
}
|
||||
|
||||
// resizeTarget decides step 7: grow the boot disk to diskGiB when that is
|
||||
// larger than its size= (spec entry 6). A disk is never shrunk.
|
||||
func resizeTarget(vm ownedVM, diskGiB int64) (disk, size string, grow bool, err error) {
|
||||
if diskGiB == 0 {
|
||||
return "", "", false, nil
|
||||
}
|
||||
disk, ok := bootDisk(vm.cfg)
|
||||
if !ok {
|
||||
return "", "", false, permanentf("VM %d has no boot disk to grow to %dG", vm.vmid, diskGiB)
|
||||
}
|
||||
cur, ok := diskBytes(vm.cfg[disk])
|
||||
if !ok {
|
||||
return "", "", false, permanentf("boot disk %s of VM %d has no readable size in %q", disk, vm.vmid, vm.cfg[disk])
|
||||
}
|
||||
if cur >= diskGiB<<30 {
|
||||
return "", "", false, nil
|
||||
}
|
||||
return disk, strconv.FormatInt(diskGiB, 10) + "G", true, nil
|
||||
}
|
||||
|
||||
// bootDisk is the first disk of boot: order=… that is not a CD-ROM, else
|
||||
// the legacy bootdisk key, else the first present of scsi0, virtio0,
|
||||
// sata0 and ide0. Proxmox appends the seed's ide2 to the order (F19), so
|
||||
// CD-ROMs are skipped.
|
||||
func bootDisk(cfg vmConfig) (string, bool) {
|
||||
isDisk := func(k string) bool {
|
||||
v, ok := cfg[k]
|
||||
return ok && diskKey.MatchString(k) && diskOption(v, "media") != "cdrom"
|
||||
}
|
||||
for dev := range strings.SplitSeq(diskOption(cfg["boot"], "order"), ";") {
|
||||
if isDisk(dev) {
|
||||
return dev, true
|
||||
}
|
||||
}
|
||||
if k := cfg["bootdisk"]; isDisk(k) {
|
||||
return k, true
|
||||
}
|
||||
for _, k := range []string{"scsi0", "virtio0", "sata0", "ide0"} {
|
||||
if isDisk(k) {
|
||||
return k, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// diskOption returns the value of key= among a config value's
|
||||
// comma-separated options.
|
||||
func diskOption(value, key string) string {
|
||||
for opt := range strings.SplitSeq(value, ",") {
|
||||
if v, ok := strings.CutPrefix(opt, key+"="); ok {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// diskBytes reads a disk value's size=, in Proxmox's <n>[KMGT] notation
|
||||
// (bytes without a suffix).
|
||||
func diskBytes(disk string) (int64, bool) {
|
||||
s := diskOption(disk, "size")
|
||||
if s == "" {
|
||||
return 0, false
|
||||
}
|
||||
shift := map[byte]uint{'K': 10, 'M': 20, 'G': 30, 'T': 40}
|
||||
unit, ok := shift[s[len(s)-1]]
|
||||
if ok {
|
||||
s = s[:len(s)-1]
|
||||
}
|
||||
n, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil || n < 0 {
|
||||
return 0, false
|
||||
}
|
||||
return int64(n * float64(int64(1)<<unit)), true
|
||||
}
|
||||
Reference in New Issue
Block a user