egress-proxies-operator/internal/provider/kvm2/create.go

package kvm2

import (
	"bytes"
	"fmt"
	"time"

	"context"

	"github.com/digitalocean/go-libvirt"
	"github.com/go-logr/logr"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

// The wait after DomainCreate (spec entry 18): Create returns only once the
// domain is running, so Get never sees a domain kvm2 defined but did not
// start (spec entry 9). DomainCreate is synchronous, so the first read
// normally succeeds; the cap only covers state settling and must not pin a
// reconcile worker for long.
const (
	defaultRunningWait = 15 * time.Second
	defaultRunningPoll = 500 * time.Millisecond
)

// Create is spec entry 10: validate placement, then, under the per-name
// lock, find or build the domain step by step, every step idempotent, and
// return once it is running.
func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
	host, err := p.host(req.Placement.Zone)
	if err != nil {
		if req.Placement.Zone != "" {
			err = permanentf("placement.zone %q is not in kvm2.hosts", req.Placement.Zone)
		}
		return "", p.wrapErr("create", "", err)
	}
	mt, ok := p.cfg.MachineTypes[req.Placement.MachineType]
	if !ok {
		return "", p.wrapErr("create", "", permanentf("placement.machineType %q is not in kvm2.machineTypes", req.Placement.MachineType))
	}
	src, err := resolveImage(p.name, p.cfg, req.Placement.Image)
	if err != nil {
		return "", p.wrapErr("create", "", err)
	}

	id := formatProviderID(host.Name, req.Name)
	log := p.logger(ctx).WithValues("host", host.Name, "name", req.Name)
	log.V(2).Info("kvm2 create resolved",
		"pool", host.Pool, "bridge", host.Bridge, "machineType", req.Placement.MachineType,
		"vcpus", mt.VCPUs, "memoryMiB", mt.MemoryMiB, "diskSizeGb", p.cfg.DiskSizeGB,
		"image", src.volumeName(), "firmware", src.Firmware, "cloudInitBytes", len(req.CloudInit))

	unlock, err := p.lockName(ctx, req.Name)
	if err != nil {
		return "", p.wrapErr("create", id, err)
	}
	defer unlock()

	be, err := p.hosts.get(ctx, host)
	if err != nil {
		logAPIError(log, "create", err)
		return "", p.wrapErr("create", id, err)
	}
	if err := p.create(ctx, log, be, host, req, mt, src); err != nil {
		logAPIError(log, "create", err)
		return "", p.wrapErr("create", id, err)
	}
	log.V(1).Info("kvm2 domain ensured running", "id", id)
	return id, nil
}

func (p *Provider) create(ctx context.Context, log logr.Logger, be backend, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType, src imageSource) error {
	dom, err := be.DomainLookupByName(ctx, req.Name)
	switch {
	case err == nil:
		meta, err := readMetadata(ctx, be, dom)
		if err != nil {
			return err
		}
		if !meta.managed() {
			return permanentf("domain %s exists but carries no kvm2 mark; it was not created by this provider", req.Name)
		}
		if meta.UID != "" && meta.UID != req.UID {
			return permanentf("domain %s belongs to proxy uid %s, not %s", req.Name, meta.UID, req.UID)
		}
		log.V(1).Info("kvm2 domain already defined", "uid", meta.UID)
		return p.ensureRunning(ctx, log, be, dom)
	case !hasCode(err, libvirt.ErrNoDomain):
		return fmt.Errorf("looking up domain %s: %w", req.Name, err)
	}

	pool, err := be.StoragePoolLookupByName(ctx, host.Pool)
	if err != nil {
		return fmt.Errorf("looking up storage pool %s: %w", host.Pool, err)
	}
	imageBytes, err := p.imageSizeIfUncached(ctx, be, pool, src)
	if err != nil {
		return err
	}
	if err := checkPoolSpace(ctx, be, pool, host.Pool, neededBytes(imageBytes, p.cfg.DiskSizeGB)); err != nil {
		return err
	}
	basePath, err := p.ensureImage(ctx, host, pool, be, src)
	if err != nil {
		return err
	}
	overlay, err := ensureOverlay(ctx, log, be, pool, req.Name, basePath, src.Format, p.cfg.DiskSizeGB)
	if err != nil {
		return err
	}
	seed, err := p.writeSeed(ctx, log, be, pool, req)
	if err != nil {
		return err
	}
	diskPath, err := be.StorageVolGetPath(ctx, overlay)
	if err != nil {
		return fmt.Errorf("reading the path of overlay volume %s: %w", overlay.Name, err)
	}
	seedPath, err := be.StorageVolGetPath(ctx, seed)
	if err != nil {
		return fmt.Errorf("reading the path of seed volume %s: %w", seed.Name, err)
	}

	xmlDoc, err := buildDomainXML(domainParams{
		Name:       req.Name,
		UID:        req.UID,
		CreatedAt:  p.now(),
		VCPUs:      mt.VCPUs,
		MemoryMiB:  mt.MemoryMiB,
		Firmware:   src.Firmware,
		Bridge:     host.Bridge,
		DiskPath:   diskPath,
		SeedPath:   seedPath,
		DiskFormat: provider.KVMFormatQcow2,
	})
	if err != nil {
		return err
	}
	dom, err = be.DomainDefineXML(ctx, xmlDoc)
	if err != nil {
		return fmt.Errorf("defining domain %s: %w", req.Name, err)
	}
	kv := []any{"uid", req.UID, "firmware", src.Firmware}
	if log.V(2).Enabled() {
		kv = append(kv, "xml", xmlDoc)
	}
	log.V(1).Info("domain defined", kv...)
	return p.ensureRunning(ctx, log, be, dom)
}

// ensureOverlay creates the per-VM qcow2 overlay backed by the base image,
// or reuses the one a previous attempt left: volume creation is atomic, so
// an existing overlay is complete.
func ensureOverlay(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, name, basePath, baseFormat string, diskSizeGB int64) (libvirt.StorageVol, error) {
	volName := overlayVolumeName(name)
	xmlDoc, err := buildVolumeXML(volumeSpec{
		Name:          volName,
		CapacityBytes: diskSizeGB << 30,
		Format:        provider.KVMFormatQcow2,
		BackingPath:   basePath,
		BackingFormat: baseFormat,
	})
	if err != nil {
		return libvirt.StorageVol{}, err
	}
	vol, err := be.StorageVolCreateXML(ctx, pool, xmlDoc, 0)
	switch {
	case err == nil:
		log.V(1).Info("overlay volume created", "volume", volName)
		return vol, nil
	case hasCode(err, libvirt.ErrStorageVolExist):
		vol, err = be.StorageVolLookupByName(ctx, pool, volName)
		if err != nil {
			return libvirt.StorageVol{}, fmt.Errorf("looking up overlay volume %s: %w", volName, err)
		}
		log.V(1).Info("overlay volume already exists", "volume", volName)
		return vol, nil
	default:
		return libvirt.StorageVol{}, fmt.Errorf("creating overlay volume %s: %w", volName, err)
	}
}

// writeSeed builds the NoCloud seed and uploads it as a raw volume,
// replacing whatever a previous attempt left (spec entry 5): a cut-off
// upload would boot a VM without cloud-init. The ISO is tens of KiB, so it
// is built in memory. Nothing about its content is logged or put in an
// error.
func (p *Provider) writeSeed(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, req provider.CreateRequest) (libvirt.StorageVol, error) {
	name := seedVolumeName(req.Name)
	if err := deleteVolumeIfPresent(ctx, be, pool, name); err != nil {
		return libvirt.StorageVol{}, err
	}
	installAgent := p.cfg.InstallGuestAgent == nil || *p.cfg.InstallGuestAgent
	vendorData, err := buildVendorData(p.cfg.VendorData, installAgent)
	if err != nil {
		return libvirt.StorageVol{}, permanentf("building vendor-data for %s: %v", req.Name, err)
	}
	var iso bytes.Buffer
	if err := buildSeedISO(req.Name, req.CloudInit, vendorData, &iso); err != nil {
		return libvirt.StorageVol{}, permanentf("building seed ISO for %s: %v", req.Name, err)
	}
	xmlDoc, err := buildVolumeXML(volumeSpec{Name: name, CapacityBytes: int64(iso.Len()), Format: provider.KVMFormatRaw})
	if err != nil {
		return libvirt.StorageVol{}, err
	}
	vol, err := be.StorageVolCreateXML(ctx, pool, xmlDoc, 0)
	if err != nil {
		return libvirt.StorageVol{}, fmt.Errorf("creating seed volume %s: %w", name, err)
	}
	if err := be.StorageVolUpload(ctx, vol, bytes.NewReader(iso.Bytes()), 0, uint64(iso.Len()), 0); err != nil {
		return libvirt.StorageVol{}, fmt.Errorf("uploading seed volume %s: %w", name, err)
	}
	log.V(1).Info("seed volume uploaded", "volume", name, "bytes", iso.Len())
	return vol, nil
}

// ensureRunning starts the domain unless it is active already, then waits
// until libvirt reports it running. ErrOperationInvalid from DomainCreate
// means another caller started it between the state read and the call. A
// paused or pmsuspended domain is an admin's doing and DomainCreate would
// refuse it; the verdict is Permanent so the reconciler does not retry it.
func (p *Provider) ensureRunning(ctx context.Context, log logr.Logger, be backend, dom libvirt.Domain) error {
	state, _, err := be.DomainGetState(ctx, dom, 0)
	if err != nil {
		return fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
	}
	switch s := libvirt.DomainState(state); {
	case isActive(s):
		return nil
	case s == libvirt.DomainPaused || s == libvirt.DomainPmsuspended:
		return permanentf("domain %s is %s and cannot be started by the operator; resume or destroy it on the host", dom.Name, s)
	}
	switch err := be.DomainCreate(ctx, dom); {
	case err == nil:
		log.V(1).Info("domain started")
	case hasCode(err, libvirt.ErrOperationInvalid):
		log.V(1).Info("domain start reported it already running")
	default:
		return fmt.Errorf("starting domain %s: %w", dom.Name, err)
	}
	return p.waitRunning(ctx, be, dom)
}

// waitRunning polls DomainGetState until the domain is active, the cap
// passes (Transient: the next Create attempt finds the domain defined and
// tries again) or ctx ends. Only nostate is worth another poll; a domain
// that is shut off, crashed or paused right after its start is not going
// to come up on its own, so that is Transient at once.
func (p *Provider) waitRunning(ctx context.Context, be backend, dom libvirt.Domain) error {
	deadline := time.NewTimer(p.runningWait)
	defer deadline.Stop()
	ticker := time.NewTicker(p.runningPoll)
	defer ticker.Stop()
	for {
		state, _, err := be.DomainGetState(ctx, dom, 0)
		if err != nil {
			return fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
		}
		switch s := libvirt.DomainState(state); {
		case isActive(s):
			return nil
		case s != libvirt.DomainNostate:
			return transientf("domain %s went %s right after start", dom.Name, s)
		}
		select {
		case <-ticker.C:
		case <-deadline.C:
			return transientf("domain %s did not reach running within %s (state %s)", dom.Name, p.runningWait, libvirt.DomainState(state))
		case <-ctx.Done():
			return fmt.Errorf("%w: waiting for domain %s to run: %w", provider.ErrTransient, dom.Name, ctx.Err())
		}
	}
}

// logAPIError is the V(1) line per failed backend call of spec entry 20.
func logAPIError(log logr.Logger, op string, err error) {
	kv := []any{"op", op, "error", err.Error()}
	kv = append(kv, libvirtErrorFields(err)...)
	log.V(1).Info("kvm2 backend call failed", kv...)
}