egress-proxies-operator/internal/provider/proxmox/create.go

package proxmox

import (
	"context"
	"fmt"
	"maps"
	"regexp"
	"slices"
	"strconv"
	"strings"

	"github.com/go-logr/logr"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

// maxProbes bounds the VMID probe of spec entry 2.
const maxProbes = 32

// diskKey is a Proxmox disk slot: the keys a boot disk, a CD-ROM or a
// cloud-init drive can sit on.
var diskKey = regexp.MustCompile(`^(ide|sata|scsi|virtio)[0-9]+$`)

// ownedVM is a VM Create has cloned or adopted: where it lives and its
// config as last read, which the config and resize steps start from.
type ownedVM struct {
	node string
	vmid int
	cfg  vmConfig
}

// Create is spec entry 10: validate placement, then, under the per-name
// lock, adopt the VM a previous attempt left or clone a new one, and carry
// it through seed, config, resize and start until it runs.
func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
	node, mt, tmpl, err := p.placement(req)
	if err != nil {
		return "", p.wrapErr("create", "", err)
	}
	log := p.logger(ctx).WithValues("name", req.Name)
	log.V(2).Info("proxmox create resolved",
		"zone", node, "machineType", req.Placement.MachineType, "cores", mt.Cores, "memoryMiB", mt.MemoryMiB,
		"diskGiB", mt.DiskGiB, "image", req.Placement.Image, "templateVMID", tmpl.VMID, "storage", p.cfg.Storage,
		"isoStorage", p.cfg.ISOStorage, "bridge", p.cfg.Bridge, "vlanTag", p.cfg.VLANTag, "cloudInitBytes", len(req.CloudInit))

	unlock, err := p.lockName(ctx, req.Name)
	if err != nil {
		return "", p.wrapErr("create", "", err)
	}
	defer unlock()

	vm, err := p.create(ctx, log, node, mt, tmpl, req)
	if err != nil {
		return "", p.wrapErr("create", "", err)
	}
	log.V(1).Info("proxmox vm ensured running", "vmid", vm.vmid, "node", vm.node)
	return formatProviderID(vm.vmid, req.Name), nil
}

// placement is step 1: every catalogue key is checked before any backend
// call, and an unknown one is Permanent naming the field and the value
// (spec entry 3). Region is ignored.
func (p *Provider) placement(req provider.CreateRequest) (string, provider.ProxmoxMachineType, provider.ProxmoxTemplate, error) {
	var (
		mt   provider.ProxmoxMachineType
		tmpl provider.ProxmoxTemplate
	)
	if req.Name == "" {
		return "", mt, tmpl, permanentf("the request has no instance name")
	}
	node, err := p.node(req.Placement.Zone)
	if err != nil {
		return "", mt, tmpl, err
	}
	mt, ok := p.cfg.MachineTypes[req.Placement.MachineType]
	if !ok {
		return "", mt, tmpl, permanentf("placement.machineType %q is not in proxmox.machineTypes", req.Placement.MachineType)
	}
	tmpl, ok = p.cfg.Templates[req.Placement.Image]
	if !ok {
		return "", mt, tmpl, permanentf("placement.image %q is not in proxmox.templates", req.Placement.Image)
	}
	return node, mt, tmpl, nil
}

// create is steps 2 to 9. The template is checked only when a clone is
// needed: a VM a previous attempt created no longer depends on it (spec
// entry 4), so a template removed since must not fail its adoption.
func (p *Provider) create(ctx context.Context, log logr.Logger, node string, mt provider.ProxmoxMachineType, tmpl provider.ProxmoxTemplate, req provider.CreateRequest) (ownedVM, error) {
	rs, err := p.api.ClusterResources(ctx)
	if err != nil {
		return ownedVM{}, fmt.Errorf("listing cluster resources: %w", err)
	}
	vm, adopted, err := p.adoptByName(ctx, rs, req)
	if err != nil {
		return ownedVM{}, err
	}
	if !adopted {
		src, err := p.template(ctx, rs, req.Placement.Image, tmpl.VMID, mt)
		if err != nil {
			return ownedVM{}, err
		}
		if vm, adopted, err = p.allocate(ctx, log, rs, node, src, req); err != nil {
			return ownedVM{}, err
		}
	}
	log = log.WithValues("vmid", vm.vmid, "node", vm.node)
	if adopted {
		st, err := p.api.VMStatus(ctx, vm.node, vm.vmid)
		if err != nil {
			return ownedVM{}, fmt.Errorf("reading status of VM %d: %w", vm.vmid, err)
		}
		log.V(1).Info("proxmox vm adopted", "status", st.Status, "qmpstatus", st.QMPStatus)
		if st.Status == "running" {
			return vm, nil
		}
	}
	return vm, p.provision(ctx, log, vm, mt, req)
}

// adoptByName is the by-name half of spec entry 2: two VMs with the name
// are for an admin to sort out, one goes through the ownership check.
func (p *Provider) adoptByName(ctx context.Context, rs []vmResource, req provider.CreateRequest) (ownedVM, bool, error) {
	var found []vmResource
	for _, r := range rs {
		if r.Type == "qemu" && r.Template == 0 && r.Name == req.Name {
			found = append(found, r)
		}
	}
	switch len(found) {
	case 0:
		return ownedVM{}, false, nil
	case 1:
		return p.inspect(ctx, found[0], req)
	}
	vmids := make([]string, len(found))
	for i, r := range found {
		vmids[i] = strconv.Itoa(r.VMID)
	}
	return ownedVM{}, false, permanentf("VMs %s are all named %s; remove all but one", strings.Join(vmids, ", "), req.Name)
}

// inspect is the ownership check of spec entry 2 on a VMID that is taken.
// It waits out a task's lock (a clone in progress is also nameless), then
// reads the name and the mark: another name is someone else's VM, and our
// name without our mark is never taken over.
func (p *Provider) inspect(ctx context.Context, r vmResource, req provider.CreateRequest) (ownedVM, bool, error) {
	cfg, err := p.waitUnlocked(ctx, r.Node, r.VMID, p.budgets.clone)
	if err != nil {
		return ownedVM{}, false, err
	}
	if cfg["name"] != req.Name {
		return ownedVM{}, false, nil
	}
	mark := parseDescription(cfg["description"])
	switch {
	case !mark.Managed:
		return ownedVM{}, false, permanentf("VM %d is named %s but carries no proxy-operator mark; it was not created by this operator", r.VMID, req.Name)
	case mark.UID != "" && mark.UID != req.UID:
		return ownedVM{}, false, permanentf("VM %d named %s belongs to proxy uid %s, not %s", r.VMID, req.Name, mark.UID, req.UID)
	case cfg["lock"] == lockSuspended:
		return ownedVM{}, false, permanentf("VM %d named %s is hibernated (lock suspended); resume or destroy it in Proxmox", r.VMID, req.Name)
	}
	return ownedVM{node: r.Node, vmid: r.VMID, cfg: cfg}, true, nil
}

// template finds the clone source and refuses one that is not a template
// or brings a Proxmox cloud-init drive, which would compete with the seed
// (spec entry 4). The clone copies the template's config, so what the
// config or resize step would refuse on the clone is refused here, while no
// VM exists that a fixed template could no longer help.
func (p *Provider) template(ctx context.Context, rs []vmResource, image string, vmid int, mt provider.ProxmoxMachineType) (vmResource, error) {
	i := slices.IndexFunc(rs, func(r vmResource) bool { return r.VMID == vmid && r.Type == "qemu" })
	if i < 0 {
		return vmResource{}, permanentf("placement.image %q: template VM %d does not exist", image, vmid)
	}
	src := rs[i]
	cfg, err := p.api.VMConfig(ctx, src.Node, vmid)
	if err != nil {
		return vmResource{}, fmt.Errorf("reading config of template %d: %w", vmid, err)
	}
	if cfg["template"] != "1" {
		return vmResource{}, permanentf("placement.image %q: VM %d is not a template", image, vmid)
	}
	for _, k := range slices.Sorted(maps.Keys(cfg)) {
		if diskKey.MatchString(k) && strings.Contains(cfg[k], "cloudinit") {
			return vmResource{}, permanentf("placement.image %q: template %d has a Proxmox cloud-init drive on %s; remove it, the provider attaches its own seed", image, vmid, k)
		}
	}
	shape := ownedVM{node: src.Node, vmid: vmid, cfg: cfg}
	if _, err := p.configParams(shape, mt, ""); err != nil {
		return vmResource{}, err
	}
	if _, _, _, err := resizeTarget(shape, mt.DiskGiB); err != nil {
		return vmResource{}, err
	}
	return src, nil
}

// allocate is the probe of spec entry 2: the attempt-th VMID for the UID
// is cloned onto when the listing shows it free, and put through the
// ownership check when it is taken, or when the clone finds it taken after
// all (F4), by a VM that is nameless or carries our name. The boolean
// reports an adoption rather than a fresh clone.
func (p *Provider) allocate(ctx context.Context, log logr.Logger, rs []vmResource, node string, src vmResource, req provider.CreateRequest) (ownedVM, bool, error) {
	taken := make(map[int]vmResource, len(rs))
	for _, r := range rs {
		taken[r.VMID] = r
	}
	rng := p.cfg.VMIDRange
	probes := min(maxProbes, rng.Max-rng.Min+1)
	for attempt := range probes {
		vmid := vmidFor(req.UID, rng.Min, rng.Max, attempt)
		r, isTaken := taken[vmid]
		if !isTaken {
			vm, err := p.clone(ctx, log, node, src, vmid, req)
			if !isAlreadyExists(err) {
				return vm, false, err
			}
			var found bool
			if r, found, err = p.resolve(ctx, vmid); err != nil {
				return ownedVM{}, false, err
			}
			if !found {
				log.V(1).Info("proxmox vmid taken by something other than a VM, probing on", "vmid", vmid, "attempt", attempt)
				continue
			}
		}
		// A listed name other than ours is someone else's VM whatever its
		// lock: only a clone in progress is nameless.
		if r.Type == "qemu" && r.Template == 0 && (r.Name == "" || r.Name == req.Name) {
			vm, ours, err := p.inspect(ctx, r, req)
			if err != nil || ours {
				return vm, ours, err
			}
		}
		log.V(1).Info("proxmox vmid taken by another vm, probing on", "vmid", vmid, "attempt", attempt)
	}
	return ownedVM{}, false, transientf("no free VMID for %s after %d probes in %d..%d", req.Name, probes, rng.Min, rng.Max)
}

// clone is step 4: a full clone carrying the mark, so the VM is never
// unmarked (spec entry 8), then its unlocked config. A taken VMID comes
// back as the API's error for allocate to judge.
func (p *Provider) clone(ctx context.Context, log logr.Logger, node string, src vmResource, vmid int, req provider.CreateRequest) (ownedVM, error) {
	params := cloneParams{
		NewID:       vmid,
		Name:        req.Name,
		Description: formatDescription(req.UID, p.now()),
		Storage:     p.cfg.Storage,
	}
	if node != src.Node {
		params.Target = node
	}
	upid, err := p.api.Clone(ctx, src.Node, src.VMID, params)
	if err != nil {
		return ownedVM{}, fmt.Errorf("cloning template %d to VM %d: %w", src.VMID, vmid, err)
	}
	if err := p.waitTask(ctx, log, upid, p.budgets.clone); err != nil {
		return ownedVM{}, fmt.Errorf("cloning template %d to VM %d: %w", src.VMID, vmid, err)
	}
	cfg, err := p.waitUnlocked(ctx, node, vmid, p.budgets.clone)
	if err != nil {
		return ownedVM{}, err
	}
	return ownedVM{node: node, vmid: vmid, cfg: cfg}, nil
}

// provision is steps 5 to 9, each safe to repeat on a VM a previous attempt
// left stopped: the upload overwrites, the config write replaces, and the
// resize is skipped once the disk is large enough. Everything that can be
// refused from the config alone is checked before the first write.
func (p *Provider) provision(ctx context.Context, log logr.Logger, vm ownedVM, mt provider.ProxmoxMachineType, req provider.CreateRequest) error {
	params, err := p.configParams(vm, mt, req.Name)
	if err != nil {
		return err
	}
	disk, size, grow, err := resizeTarget(vm, mt.DiskGiB)
	if err != nil {
		return err
	}
	if err := p.uploadSeed(ctx, log, vm.node, req); err != nil {
		return err
	}
	if err := p.api.SetVMConfig(ctx, vm.node, vm.vmid, params); err != nil {
		return fmt.Errorf("configuring VM %d: %w", vm.vmid, err)
	}
	if grow {
		upid, err := p.api.Resize(ctx, vm.node, vm.vmid, disk, size)
		if err != nil {
			return fmt.Errorf("resizing %s of VM %d to %s: %w", disk, vm.vmid, size, err)
		}
		if err := p.waitTask(ctx, log, upid, p.budgets.resize); err != nil {
			return fmt.Errorf("resizing %s of VM %d to %s: %w", disk, vm.vmid, size, err)
		}
	}
	upid, err := p.api.Start(ctx, vm.node, vm.vmid)
	if err != nil {
		return fmt.Errorf("starting VM %d: %w", vm.vmid, err)
	}
	if err := p.waitTask(ctx, log, upid, p.budgets.start); err != nil {
		return fmt.Errorf("starting VM %d: %w", vm.vmid, err)
	}
	return p.waitRunning(ctx, vm.node, vm.vmid)
}

// configParams is step 6's write: size, agent, tag and seed, and net0 only
// when the config sets a bridge or a VLAN tag (spec entry 7).
func (p *Provider) configParams(vm ownedVM, mt provider.ProxmoxMachineType, name string) (map[string]string, error) {
	params := map[string]string{
		"cores":  strconv.Itoa(int(mt.Cores)),
		"memory": strconv.FormatInt(mt.MemoryMiB, 10),
		"agent":  "1",
		"tags":   proxyTag,
		"ide2":   seedVolID(p.cfg.ISOStorage, name) + ",media=cdrom",
	}
	if p.cfg.Bridge == "" && p.cfg.VLANTag == 0 {
		return params, nil
	}
	net0 := vm.cfg["net0"]
	if net0 == "" {
		return nil, permanentf("VM %d has no net0 to apply proxmox.bridge or proxmox.vlanTag to; the template needs a network device", vm.vmid)
	}
	params["net0"] = rewriteNet0(net0, p.cfg.Bridge, p.cfg.VLANTag)
	return params, nil
}

// rewriteNet0 replaces or appends bridge= and tag= in a net0 value, keeping
// every other option, the model and the MAC in place. An empty bridge or a
// zero tag leaves that option alone.
func rewriteNet0(net0, bridge string, tag int) string {
	want := map[string]string{}
	if bridge != "" {
		want["bridge"] = bridge
	}
	if tag != 0 {
		want["tag"] = strconv.Itoa(tag)
	}
	opts := strings.Split(net0, ",")
	for i, opt := range opts {
		key, _, _ := strings.Cut(opt, "=")
		if v, ok := want[key]; ok {
			opts[i] = key + "=" + v
			delete(want, key)
		}
	}
	for _, key := range []string{"bridge", "tag"} {
		if v, ok := want[key]; ok {
			opts = append(opts, key+"="+v)
		}
	}
	return strings.Join(opts, ",")
}

// resizeTarget decides step 7: grow the boot disk to diskGiB when that is
// larger than its size= (spec entry 6). A disk is never shrunk.
func resizeTarget(vm ownedVM, diskGiB int64) (disk, size string, grow bool, err error) {
	if diskGiB == 0 {
		return "", "", false, nil
	}
	disk, ok := bootDisk(vm.cfg)
	if !ok {
		return "", "", false, permanentf("VM %d has no boot disk to grow to %dG", vm.vmid, diskGiB)
	}
	cur, ok := diskBytes(vm.cfg[disk])
	if !ok {
		return "", "", false, permanentf("boot disk %s of VM %d has no readable size in %q", disk, vm.vmid, vm.cfg[disk])
	}
	if cur >= diskGiB<<30 {
		return "", "", false, nil
	}
	return disk, strconv.FormatInt(diskGiB, 10) + "G", true, nil
}

// bootDisk is the first disk of boot: order=… that is not a CD-ROM, else
// the legacy bootdisk key, else the first present of scsi0, virtio0,
// sata0 and ide0. Proxmox appends the seed's ide2 to the order (F19), so
// CD-ROMs are skipped.
func bootDisk(cfg vmConfig) (string, bool) {
	isDisk := func(k string) bool {
		v, ok := cfg[k]
		return ok && diskKey.MatchString(k) && diskOption(v, "media") != "cdrom"
	}
	for dev := range strings.SplitSeq(diskOption(cfg["boot"], "order"), ";") {
		if isDisk(dev) {
			return dev, true
		}
	}
	if k := cfg["bootdisk"]; isDisk(k) {
		return k, true
	}
	for _, k := range []string{"scsi0", "virtio0", "sata0", "ide0"} {
		if isDisk(k) {
			return k, true
		}
	}
	return "", false
}

// diskOption returns the value of key= among a config value's
// comma-separated options.
func diskOption(value, key string) string {
	for opt := range strings.SplitSeq(value, ",") {
		if v, ok := strings.CutPrefix(opt, key+"="); ok {
			return v
		}
	}
	return ""
}

// diskBytes reads a disk value's size=, in Proxmox's <n>[KMGT] notation
// (bytes without a suffix).
func diskBytes(disk string) (int64, bool) {
	s := diskOption(disk, "size")
	if s == "" {
		return 0, false
	}
	shift := map[byte]uint{'K': 10, 'M': 20, 'G': 30, 'T': 40}
	unit, ok := shift[s[len(s)-1]]
	if ok {
		s = s[:len(s)-1]
	}
	n, err := strconv.ParseFloat(s, 64)
	if err != nil || n < 0 {
		return 0, false
	}
	return int64(n * float64(int64(1)<<unit)), true
}