egress-proxies-operator/internal/provider/proxmox/delete.go

package proxmox

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/go-logr/logr"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

// Delete is spec entry 11: under the per-name lock, stop the VM when it is
// ours, delete its seed, then destroy it. Everything already gone is
// success, and nothing of ours is left when nil is returned, so Get
// afterwards is NotFound.
func (p *Provider) Delete(ctx context.Context, id string) error {
	vmid, name, err := parseProviderID(id)
	if err != nil {
		return p.wrapErr("delete", id, err)
	}
	log := p.logger(ctx).WithValues("vmid", vmid, "name", name)
	unlock, err := p.lockName(ctx, name)
	if err != nil {
		return p.wrapErr("delete", id, err)
	}
	defer unlock()

	node, err := p.destroy(ctx, log, vmid, name)
	if err != nil {
		return p.wrapErr("delete", id, err)
	}
	if node == "" {
		if err := p.deleteSeedEverywhere(ctx, log, name); err != nil {
			return p.wrapErr("delete", id, err)
		}
	} else {
		log = log.WithValues("node", node)
	}
	log.V(1).Info("proxmox vm deleted", "destroyed", node != "")
	return nil
}

// destroy is steps 2 to 7. It returns the node of the VM it destroyed, or
// "" when there was no VM of ours to destroy: then the seed's node is not
// known, and the seed is looked for on every node.
func (p *Provider) destroy(ctx context.Context, log logr.Logger, vmid int, name string) (string, error) {
	r, ok, err := p.resolve(ctx, vmid)
	if err != nil {
		return "", err
	}
	if !ok {
		log.V(1).Info("proxmox vm already gone")
		return "", nil
	}
	log = log.WithValues("node", r.Node)
	if r.Lock != "" && r.Lock != lockSuspended {
		return "", transientf("VM %d is locked (%s)", vmid, r.Lock)
	}
	if r.Status == "unknown" {
		return "", transientf("VM %d has status unknown: its node %s is not answering, or the VM is too new to be reported yet", vmid, r.Node)
	}
	cfg, err := p.api.VMConfig(ctx, r.Node, vmid)
	if isNotExist(err) {
		log.V(1).Info("proxmox vm already gone")
		return "", nil
	}
	if err != nil {
		return "", fmt.Errorf("reading config of VM %d: %w", vmid, err)
	}
	if cfg["name"] != name {
		log.V(1).Info("proxmox vm at this vmid is not ours, left alone", "vmName", cfg["name"])
		return "", nil
	}
	if !parseDescription(cfg["description"]).Managed {
		log.V(1).Info("proxmox vm carries no proxy-operator mark, left alone")
		return "", nil
	}
	// A hibernated VM is stopped with lock suspended, and the destroy
	// takes it, state volume included (captured on pv1-1).
	if lock := cfg["lock"]; lock != "" && lock != lockSuspended {
		return "", transientf("VM %d is locked (%s)", vmid, lock)
	}

	// The listing shows a paused VM as running, and Stop ends it too.
	if r.Status != "stopped" {
		if err := p.runTask(ctx, log, "stopping", vmid, p.budgets.stop, func() (string, error) { return p.api.Stop(ctx, r.Node, vmid) }); err != nil {
			return "", err
		}
	}
	// The seed goes while the VM is still there: once it is gone, Get is
	// NotFound and nothing calls Delete again to finish the job.
	if err := p.deleteSeed(ctx, log, r.Node, name); err != nil {
		return "", err
	}
	if err := p.runTask(ctx, log, "destroying", vmid, p.budgets.destroy, func() (string, error) { return p.api.Destroy(ctx, r.Node, vmid) }); err != nil {
		return "", err
	}
	return r.Node, nil
}

// runTask submits a task on a VM and waits for it. A VM that is gone by the
// time the task is submitted is success: it is what the task was for.
func (p *Provider) runTask(ctx context.Context, log logr.Logger, what string, vmid int, budget time.Duration, submit func() (string, error)) error {
	upid, err := submit()
	if isNotExist(err) {
		log.V(1).Info("proxmox vm already gone")
		return nil
	}
	if err != nil {
		return fmt.Errorf("%s VM %d: %w", what, vmid, err)
	}
	if err := p.waitTask(ctx, log, upid, budget); err != nil {
		return fmt.Errorf("%s VM %d: %w", what, vmid, err)
	}
	return nil
}

// deleteSeed is step 6 on one node. A missing volume still ends in a
// successful task (F13), so nothing is matched.
func (p *Provider) deleteSeed(ctx context.Context, log logr.Logger, node, name string) error {
	volid := seedVolID(p.cfg.ISOStorage, name)
	upid, err := p.api.DeleteVolume(ctx, node, p.cfg.ISOStorage, volid)
	if err != nil {
		return fmt.Errorf("deleting seed %s on %s: %w", volid, node, err)
	}
	if err := p.waitTask(ctx, log, upid, p.budgets.seedDelete); err != nil {
		return fmt.Errorf("deleting seed %s on %s: %w", volid, node, err)
	}
	return nil
}

// deleteSeedEverywhere is step 6 when the VM's node is not known: every
// configured node is tried, and one that fails is logged and skipped, so a
// node that is down or lacks the storage cannot hold the Delete up. An
// ended ctx fails the call instead, and so does a Permanent refusal such as
// a 403 for Datastore.Allocate, which every node would give and which
// would otherwise leave the seed behind unseen.
func (p *Provider) deleteSeedEverywhere(ctx context.Context, log logr.Logger, name string) error {
	for _, node := range p.cfg.Nodes {
		err := p.deleteSeed(ctx, log, node, name)
		if err == nil {
			continue
		}
		if ctx.Err() != nil || errors.Is(classify("delete", err), provider.ErrPermanent) {
			return err
		}
		log.V(1).Info("proxmox seed delete failed on a node, skipped", "node", node, "reason", err.Error())
	}
	return nil
}