package proxmox import ( "context" "fmt" "maps" "regexp" "slices" "strconv" "strings" "github.com/go-logr/logr" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider" ) // maxProbes bounds the VMID probe of spec entry 2. const maxProbes = 32 // diskKey is a Proxmox disk slot: the keys a boot disk, a CD-ROM or a // cloud-init drive can sit on. var diskKey = regexp.MustCompile(`^(ide|sata|scsi|virtio)[0-9]+$`) // ownedVM is a VM Create has cloned or adopted: where it lives and its // config as last read, which the config and resize steps start from. type ownedVM struct { node string vmid int cfg vmConfig } // Create is spec entry 10: validate placement, then, under the per-name // lock, adopt the VM a previous attempt left or clone a new one, and carry // it through seed, config, resize and start until it runs. func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) { node, mt, tmpl, err := p.placement(req) if err != nil { return "", p.wrapErr("create", "", err) } log := p.logger(ctx).WithValues("name", req.Name) log.V(2).Info("proxmox create resolved", "zone", node, "machineType", req.Placement.MachineType, "cores", mt.Cores, "memoryMiB", mt.MemoryMiB, "diskGiB", mt.DiskGiB, "image", req.Placement.Image, "templateVMID", tmpl.VMID, "storage", p.cfg.Storage, "isoStorage", p.cfg.ISOStorage, "bridge", p.cfg.Bridge, "vlanTag", p.cfg.VLANTag, "cloudInitBytes", len(req.CloudInit)) unlock, err := p.lockName(ctx, req.Name) if err != nil { return "", p.wrapErr("create", "", err) } defer unlock() vm, err := p.create(ctx, log, node, mt, tmpl, req) if err != nil { return "", p.wrapErr("create", "", err) } log.V(1).Info("proxmox vm ensured running", "vmid", vm.vmid, "node", vm.node) return formatProviderID(vm.vmid, req.Name), nil } // placement is step 1: every catalogue key is checked before any backend // call, and an unknown one is Permanent naming the field and the value // (spec entry 3). Region is ignored. func (p *Provider) placement(req provider.CreateRequest) (string, provider.ProxmoxMachineType, provider.ProxmoxTemplate, error) { var ( mt provider.ProxmoxMachineType tmpl provider.ProxmoxTemplate ) if req.Name == "" { return "", mt, tmpl, permanentf("the request has no instance name") } node, err := p.node(req.Placement.Zone) if err != nil { return "", mt, tmpl, err } mt, ok := p.cfg.MachineTypes[req.Placement.MachineType] if !ok { return "", mt, tmpl, permanentf("placement.machineType %q is not in proxmox.machineTypes", req.Placement.MachineType) } tmpl, ok = p.cfg.Templates[req.Placement.Image] if !ok { return "", mt, tmpl, permanentf("placement.image %q is not in proxmox.templates", req.Placement.Image) } return node, mt, tmpl, nil } // create is steps 2 to 9. The template is checked only when a clone is // needed: a VM a previous attempt created no longer depends on it (spec // entry 4), so a template removed since must not fail its adoption. func (p *Provider) create(ctx context.Context, log logr.Logger, node string, mt provider.ProxmoxMachineType, tmpl provider.ProxmoxTemplate, req provider.CreateRequest) (ownedVM, error) { rs, err := p.api.ClusterResources(ctx) if err != nil { return ownedVM{}, fmt.Errorf("listing cluster resources: %w", err) } vm, adopted, err := p.adoptByName(ctx, rs, req) if err != nil { return ownedVM{}, err } if !adopted { src, err := p.template(ctx, rs, req.Placement.Image, tmpl.VMID, mt) if err != nil { return ownedVM{}, err } if vm, adopted, err = p.allocate(ctx, log, rs, node, src, req); err != nil { return ownedVM{}, err } } log = log.WithValues("vmid", vm.vmid, "node", vm.node) if adopted { st, err := p.api.VMStatus(ctx, vm.node, vm.vmid) if err != nil { return ownedVM{}, fmt.Errorf("reading status of VM %d: %w", vm.vmid, err) } log.V(1).Info("proxmox vm adopted", "status", st.Status, "qmpstatus", st.QMPStatus) if st.Status == "running" { return vm, nil } } return vm, p.provision(ctx, log, vm, mt, req) } // adoptByName is the by-name half of spec entry 2: two VMs with the name // are for an admin to sort out, one goes through the ownership check. func (p *Provider) adoptByName(ctx context.Context, rs []vmResource, req provider.CreateRequest) (ownedVM, bool, error) { var found []vmResource for _, r := range rs { if r.Type == "qemu" && r.Template == 0 && r.Name == req.Name { found = append(found, r) } } switch len(found) { case 0: return ownedVM{}, false, nil case 1: return p.inspect(ctx, found[0], req) } vmids := make([]string, len(found)) for i, r := range found { vmids[i] = strconv.Itoa(r.VMID) } return ownedVM{}, false, permanentf("VMs %s are all named %s; remove all but one", strings.Join(vmids, ", "), req.Name) } // inspect is the ownership check of spec entry 2 on a VMID that is taken. // It waits out a task's lock (a clone in progress is also nameless), then // reads the name and the mark: another name is someone else's VM, and our // name without our mark is never taken over. func (p *Provider) inspect(ctx context.Context, r vmResource, req provider.CreateRequest) (ownedVM, bool, error) { cfg, err := p.waitUnlocked(ctx, r.Node, r.VMID, p.budgets.clone) if err != nil { return ownedVM{}, false, err } if cfg["name"] != req.Name { return ownedVM{}, false, nil } mark := parseDescription(cfg["description"]) switch { case !mark.Managed: return ownedVM{}, false, permanentf("VM %d is named %s but carries no proxy-operator mark; it was not created by this operator", r.VMID, req.Name) case mark.UID != "" && mark.UID != req.UID: return ownedVM{}, false, permanentf("VM %d named %s belongs to proxy uid %s, not %s", r.VMID, req.Name, mark.UID, req.UID) case cfg["lock"] == lockSuspended: return ownedVM{}, false, permanentf("VM %d named %s is hibernated (lock suspended); resume or destroy it in Proxmox", r.VMID, req.Name) } return ownedVM{node: r.Node, vmid: r.VMID, cfg: cfg}, true, nil } // template finds the clone source and refuses one that is not a template // or brings a Proxmox cloud-init drive, which would compete with the seed // (spec entry 4). The clone copies the template's config, so what the // config or resize step would refuse on the clone is refused here, while no // VM exists that a fixed template could no longer help. func (p *Provider) template(ctx context.Context, rs []vmResource, image string, vmid int, mt provider.ProxmoxMachineType) (vmResource, error) { i := slices.IndexFunc(rs, func(r vmResource) bool { return r.VMID == vmid && r.Type == "qemu" }) if i < 0 { return vmResource{}, permanentf("placement.image %q: template VM %d does not exist", image, vmid) } src := rs[i] cfg, err := p.api.VMConfig(ctx, src.Node, vmid) if err != nil { return vmResource{}, fmt.Errorf("reading config of template %d: %w", vmid, err) } if cfg["template"] != "1" { return vmResource{}, permanentf("placement.image %q: VM %d is not a template", image, vmid) } for _, k := range slices.Sorted(maps.Keys(cfg)) { if diskKey.MatchString(k) && strings.Contains(cfg[k], "cloudinit") { return vmResource{}, permanentf("placement.image %q: template %d has a Proxmox cloud-init drive on %s; remove it, the provider attaches its own seed", image, vmid, k) } } shape := ownedVM{node: src.Node, vmid: vmid, cfg: cfg} if _, err := p.configParams(shape, mt, ""); err != nil { return vmResource{}, err } if _, _, _, err := resizeTarget(shape, mt.DiskGiB); err != nil { return vmResource{}, err } return src, nil } // allocate is the probe of spec entry 2: the attempt-th VMID for the UID // is cloned onto when the listing shows it free, and put through the // ownership check when it is taken, or when the clone finds it taken after // all (F4), by a VM that is nameless or carries our name. The boolean // reports an adoption rather than a fresh clone. func (p *Provider) allocate(ctx context.Context, log logr.Logger, rs []vmResource, node string, src vmResource, req provider.CreateRequest) (ownedVM, bool, error) { taken := make(map[int]vmResource, len(rs)) for _, r := range rs { taken[r.VMID] = r } rng := p.cfg.VMIDRange probes := min(maxProbes, rng.Max-rng.Min+1) for attempt := range probes { vmid := vmidFor(req.UID, rng.Min, rng.Max, attempt) r, isTaken := taken[vmid] if !isTaken { vm, err := p.clone(ctx, log, node, src, vmid, req) if !isAlreadyExists(err) { return vm, false, err } var found bool if r, found, err = p.resolve(ctx, vmid); err != nil { return ownedVM{}, false, err } if !found { log.V(1).Info("proxmox vmid taken by something other than a VM, probing on", "vmid", vmid, "attempt", attempt) continue } } // A listed name other than ours is someone else's VM whatever its // lock: only a clone in progress is nameless. if r.Type == "qemu" && r.Template == 0 && (r.Name == "" || r.Name == req.Name) { vm, ours, err := p.inspect(ctx, r, req) if err != nil || ours { return vm, ours, err } } log.V(1).Info("proxmox vmid taken by another vm, probing on", "vmid", vmid, "attempt", attempt) } return ownedVM{}, false, transientf("no free VMID for %s after %d probes in %d..%d", req.Name, probes, rng.Min, rng.Max) } // clone is step 4: a full clone carrying the mark, so the VM is never // unmarked (spec entry 8), then its unlocked config. A taken VMID comes // back as the API's error for allocate to judge. func (p *Provider) clone(ctx context.Context, log logr.Logger, node string, src vmResource, vmid int, req provider.CreateRequest) (ownedVM, error) { params := cloneParams{ NewID: vmid, Name: req.Name, Description: formatDescription(req.UID, p.now()), Storage: p.cfg.Storage, } if node != src.Node { params.Target = node } upid, err := p.api.Clone(ctx, src.Node, src.VMID, params) if err != nil { return ownedVM{}, fmt.Errorf("cloning template %d to VM %d: %w", src.VMID, vmid, err) } if err := p.waitTask(ctx, log, upid, p.budgets.clone); err != nil { return ownedVM{}, fmt.Errorf("cloning template %d to VM %d: %w", src.VMID, vmid, err) } cfg, err := p.waitUnlocked(ctx, node, vmid, p.budgets.clone) if err != nil { return ownedVM{}, err } return ownedVM{node: node, vmid: vmid, cfg: cfg}, nil } // provision is steps 5 to 9, each safe to repeat on a VM a previous attempt // left stopped: the upload overwrites, the config write replaces, and the // resize is skipped once the disk is large enough. Everything that can be // refused from the config alone is checked before the first write. func (p *Provider) provision(ctx context.Context, log logr.Logger, vm ownedVM, mt provider.ProxmoxMachineType, req provider.CreateRequest) error { params, err := p.configParams(vm, mt, req.Name) if err != nil { return err } disk, size, grow, err := resizeTarget(vm, mt.DiskGiB) if err != nil { return err } if err := p.uploadSeed(ctx, log, vm.node, req); err != nil { return err } if err := p.api.SetVMConfig(ctx, vm.node, vm.vmid, params); err != nil { return fmt.Errorf("configuring VM %d: %w", vm.vmid, err) } if grow { upid, err := p.api.Resize(ctx, vm.node, vm.vmid, disk, size) if err != nil { return fmt.Errorf("resizing %s of VM %d to %s: %w", disk, vm.vmid, size, err) } if err := p.waitTask(ctx, log, upid, p.budgets.resize); err != nil { return fmt.Errorf("resizing %s of VM %d to %s: %w", disk, vm.vmid, size, err) } } upid, err := p.api.Start(ctx, vm.node, vm.vmid) if err != nil { return fmt.Errorf("starting VM %d: %w", vm.vmid, err) } if err := p.waitTask(ctx, log, upid, p.budgets.start); err != nil { return fmt.Errorf("starting VM %d: %w", vm.vmid, err) } return p.waitRunning(ctx, vm.node, vm.vmid) } // configParams is step 6's write: size, agent, tag and seed, and net0 only // when the config sets a bridge or a VLAN tag (spec entry 7). func (p *Provider) configParams(vm ownedVM, mt provider.ProxmoxMachineType, name string) (map[string]string, error) { params := map[string]string{ "cores": strconv.Itoa(int(mt.Cores)), "memory": strconv.FormatInt(mt.MemoryMiB, 10), "agent": "1", "tags": proxyTag, "ide2": seedVolID(p.cfg.ISOStorage, name) + ",media=cdrom", } if p.cfg.Bridge == "" && p.cfg.VLANTag == 0 { return params, nil } net0 := vm.cfg["net0"] if net0 == "" { return nil, permanentf("VM %d has no net0 to apply proxmox.bridge or proxmox.vlanTag to; the template needs a network device", vm.vmid) } params["net0"] = rewriteNet0(net0, p.cfg.Bridge, p.cfg.VLANTag) return params, nil } // rewriteNet0 replaces or appends bridge= and tag= in a net0 value, keeping // every other option, the model and the MAC in place. An empty bridge or a // zero tag leaves that option alone. func rewriteNet0(net0, bridge string, tag int) string { want := map[string]string{} if bridge != "" { want["bridge"] = bridge } if tag != 0 { want["tag"] = strconv.Itoa(tag) } opts := strings.Split(net0, ",") for i, opt := range opts { key, _, _ := strings.Cut(opt, "=") if v, ok := want[key]; ok { opts[i] = key + "=" + v delete(want, key) } } for _, key := range []string{"bridge", "tag"} { if v, ok := want[key]; ok { opts = append(opts, key+"="+v) } } return strings.Join(opts, ",") } // resizeTarget decides step 7: grow the boot disk to diskGiB when that is // larger than its size= (spec entry 6). A disk is never shrunk. func resizeTarget(vm ownedVM, diskGiB int64) (disk, size string, grow bool, err error) { if diskGiB == 0 { return "", "", false, nil } disk, ok := bootDisk(vm.cfg) if !ok { return "", "", false, permanentf("VM %d has no boot disk to grow to %dG", vm.vmid, diskGiB) } cur, ok := diskBytes(vm.cfg[disk]) if !ok { return "", "", false, permanentf("boot disk %s of VM %d has no readable size in %q", disk, vm.vmid, vm.cfg[disk]) } if cur >= diskGiB<<30 { return "", "", false, nil } return disk, strconv.FormatInt(diskGiB, 10) + "G", true, nil } // bootDisk is the first disk of boot: order=… that is not a CD-ROM, else // the legacy bootdisk key, else the first present of scsi0, virtio0, // sata0 and ide0. Proxmox appends the seed's ide2 to the order (F19), so // CD-ROMs are skipped. func bootDisk(cfg vmConfig) (string, bool) { isDisk := func(k string) bool { v, ok := cfg[k] return ok && diskKey.MatchString(k) && diskOption(v, "media") != "cdrom" } for dev := range strings.SplitSeq(diskOption(cfg["boot"], "order"), ";") { if isDisk(dev) { return dev, true } } if k := cfg["bootdisk"]; isDisk(k) { return k, true } for _, k := range []string{"scsi0", "virtio0", "sata0", "ide0"} { if isDisk(k) { return k, true } } return "", false } // diskOption returns the value of key= among a config value's // comma-separated options. func diskOption(value, key string) string { for opt := range strings.SplitSeq(value, ",") { if v, ok := strings.CutPrefix(opt, key+"="); ok { return v } } return "" } // diskBytes reads a disk value's size=, in Proxmox's [KMGT] notation // (bytes without a suffix). func diskBytes(disk string) (int64, bool) { s := diskOption(disk, "size") if s == "" { return 0, false } shift := map[byte]uint{'K': 10, 'M': 20, 'G': 30, 'T': 40} unit, ok := shift[s[len(s)-1]] if ok { s = s[:len(s)-1] } n, err := strconv.ParseFloat(s, 64) if err != nil || n < 0 { return 0, false } return int64(n * float64(int64(1)<