egress-proxies-operator/internal/provider/proxmox/task.go

package proxmox

import (
	"context"
	"fmt"
	"regexp"
	"time"

	"github.com/go-logr/logr"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

// upidPattern is Proxmox's task id: UPID:<node>:<pid>:<pstart>:<starttime>:
// <type>:<id>:<user>: with hex pid, pstart and starttime.
var upidPattern = regexp.MustCompile(`^UPID:([a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?):[0-9A-Fa-f]{8}:[0-9A-Fa-f]{8,9}:[0-9A-Fa-f]{8}:[^:\s]+:[^:\s]*:[^:\s]+:$`)

// parseUPID returns the node a task runs on, which is where its status is
// read (spec entry 10). A malformed UPID is Permanent: Proxmox never sends
// one, so retrying cannot help.
func parseUPID(upid string) (string, error) {
	m := upidPattern.FindStringSubmatch(upid)
	if m == nil {
		return "", permanentf("malformed task id %q", upid)
	}
	return m[1], nil
}

// taskSucceeded reads a stopped task's exitstatus the way Proxmox does:
// "OK", or "WARNINGS: <n>" for a task that finished but logged warnings.
func taskSucceeded(exit string) bool {
	return exit == "OK" || taskWarnings.MatchString(exit)
}

var taskWarnings = regexp.MustCompile(`^WARNINGS: [0-9]+$`)

// pollUntil calls check every interval until it reports done or fails. Once
// budget has passed or ctx has ended it gives up with ErrTransient naming
// what it waited for; the backend keeps working, and the next attempt picks
// the result up.
func pollUntil(ctx context.Context, interval, budget time.Duration, what string, check func() (bool, error)) error {
	deadline := time.NewTimer(budget)
	defer deadline.Stop()
	tick := time.NewTicker(interval)
	defer tick.Stop()
	for {
		done, err := check()
		if err != nil || done {
			return err
		}
		select {
		case <-ctx.Done():
			return fmt.Errorf("%w: waiting for %s: %w", provider.ErrTransient, what, ctx.Err())
		case <-deadline.C:
			return transientf("waiting for %s: not done after %v", what, budget)
		case <-tick.C:
		}
	}
}

// waitTask polls a task on the node its UPID names until it stops. A
// failed task is ErrTransient carrying its exitstatus, and logged at V(1)
// (spec entries 10 and 20).
func (p *Provider) waitTask(ctx context.Context, log logr.Logger, upid string, budget time.Duration) error {
	node, err := parseUPID(upid)
	if err != nil {
		return err
	}
	var exit string
	err = pollUntil(ctx, p.budgets.poll, budget, "task "+upid, func() (bool, error) {
		s, err := p.api.TaskStatus(ctx, node, upid)
		if err != nil {
			return false, fmt.Errorf("reading task %s: %w", upid, err)
		}
		exit = s.ExitStatus
		return s.Status == "stopped", nil
	})
	if err != nil {
		return err
	}
	if !taskSucceeded(exit) {
		log.V(1).Info("proxmox task failed", "upid", upid, "exitstatus", exit)
		return transientf("proxmox task %s failed: %s", upid, exit)
	}
	if exit != "OK" {
		log.V(1).Info("proxmox task finished with warnings", "upid", upid, "exitstatus", exit)
	}
	return nil
}

// waitUnlocked polls a VM's config until no task holds its lock and returns
// that config, for Create's ownership check (spec entry 2). A suspended lock
// is returned at once: it marks a hibernated VM, not a task, and never
// clears on its own, so waiting would only burn the clone budget. A VM that
// disappears meanwhile, as a failed clone does, returns the does-not-exist
// error for the caller to judge.
func (p *Provider) waitUnlocked(ctx context.Context, node string, vmid int, budget time.Duration) (vmConfig, error) {
	var cfg vmConfig
	err := pollUntil(ctx, p.budgets.poll, budget, fmt.Sprintf("the lock on VM %d", vmid), func() (bool, error) {
		c, err := p.api.VMConfig(ctx, node, vmid)
		if err != nil {
			return false, fmt.Errorf("reading config of VM %d: %w", vmid, err)
		}
		cfg = c
		return c["lock"] == "" || c["lock"] == lockSuspended, nil
	})
	if err != nil {
		return nil, err
	}
	return cfg, nil
}

// waitRunning polls status/current after the start task until QEMU runs
// the guest (spec entry 10, step 9). A VM that is stopped again is
// Transient at once: it will not come up on its own, and the next attempt
// adopts and starts it.
func (p *Provider) waitRunning(ctx context.Context, node string, vmid int) error {
	return pollUntil(ctx, p.budgets.poll, p.budgets.running, fmt.Sprintf("VM %d to run", vmid), func() (bool, error) {
		st, err := p.api.VMStatus(ctx, node, vmid)
		if err != nil {
			return false, fmt.Errorf("reading status of VM %d: %w", vmid, err)
		}
		if st.Status == "stopped" {
			return false, transientf("VM %d stopped right after its start", vmid)
		}
		return st.Status == "running" && st.QMPStatus == "running", nil
	})
}