kvm2: Create, Get, Delete and ListByTag with the contract tests

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-16 19:23:49 +00:00
parent 6b3c985b4a
commit 794a2aef3d
14 changed files with 2120 additions and 63 deletions
@@ -0,0 +1,265 @@
+package kvm2
+
+import (
+	"bytes"
+	"fmt"
+	"time"
+
+	"context"
+
+	"github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// The wait after DomainCreate (spec entry 18): Create returns only once the
+// domain is running, so Get never sees a domain kvm2 defined but did not
+// start (spec entry 9).
+const (
+	defaultRunningWait = 2 * time.Minute
+	defaultRunningPoll = 500 * time.Millisecond
+)
+
+// Create is spec entry 10: validate placement, then, under the per-name
+// lock, find or build the domain step by step, every step idempotent, and
+// return once it is running.
+func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
+	host, err := p.host(req.Placement.Zone)
+	if err != nil {
+		if req.Placement.Zone != "" {
+			err = permanentf("placement.zone %q is not in kvm2.hosts", req.Placement.Zone)
+		}
+		return "", p.wrapErr("create", "", err)
+	}
+	mt, ok := p.cfg.MachineTypes[req.Placement.MachineType]
+	if !ok {
+		return "", p.wrapErr("create", "", permanentf("placement.machineType %q is not in kvm2.machineTypes", req.Placement.MachineType))
+	}
+	src, err := resolveImage(p.cfg, req.Placement.Image)
+	if err != nil {
+		return "", p.wrapErr("create", "", err)
+	}
+
+	id := formatProviderID(host.Name, req.Name)
+	log := p.logger(ctx).WithValues("host", host.Name, "name", req.Name)
+	log.V(2).Info("kvm2 create resolved",
+		"pool", host.Pool, "bridge", host.Bridge, "machineType", req.Placement.MachineType,
+		"vcpus", mt.VCPUs, "memoryMiB", mt.MemoryMiB, "diskSizeGb", p.cfg.DiskSizeGB,
+		"image", src.volumeName(), "firmware", src.Firmware, "cloudInitBytes", len(req.CloudInit))
+
+	unlock, err := p.lockName(ctx, req.Name)
+	if err != nil {
+		return "", p.wrapErr("create", id, err)
+	}
+	defer unlock()
+
+	be, err := p.hosts.get(ctx, host)
+	if err != nil {
+		logAPIError(log, "create", err)
+		return "", p.wrapErr("create", id, err)
+	}
+	if err := p.create(ctx, log, be, host, req, mt, src); err != nil {
+		logAPIError(log, "create", err)
+		return "", p.wrapErr("create", id, err)
+	}
+	log.V(1).Info("kvm2 domain ensured running", "id", id)
+	return id, nil
+}
+
+func (p *Provider) create(ctx context.Context, log logr.Logger, be backend, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType, src imageSource) error {
+	dom, err := be.DomainLookupByName(ctx, req.Name)
+	switch {
+	case err == nil:
+		meta, err := readMetadata(ctx, be, dom)
+		if err != nil {
+			return err
+		}
+		if meta.UID != "" && meta.UID != req.UID {
+			return permanentf("domain %s belongs to proxy uid %s, not %s", req.Name, meta.UID, req.UID)
+		}
+		log.V(1).Info("kvm2 domain already defined", "uid", meta.UID)
+		return p.ensureRunning(ctx, log, be, dom)
+	case !hasCode(err, libvirt.ErrNoDomain):
+		return fmt.Errorf("looking up domain %s: %w", req.Name, err)
+	}
+
+	pool, err := be.StoragePoolLookupByName(ctx, host.Pool)
+	if err != nil {
+		return fmt.Errorf("looking up storage pool %s: %w", host.Pool, err)
+	}
+	imageBytes, err := p.imageSizeIfUncached(ctx, be, pool, src)
+	if err != nil {
+		return err
+	}
+	if err := checkPoolSpace(ctx, be, pool, host.Pool, neededBytes(imageBytes, p.cfg.DiskSizeGB)); err != nil {
+		return err
+	}
+	basePath, err := p.ensureImage(ctx, host, pool, be, src)
+	if err != nil {
+		return err
+	}
+	overlay, err := ensureOverlay(ctx, log, be, pool, req.Name, basePath, src.Format, p.cfg.DiskSizeGB)
+	if err != nil {
+		return err
+	}
+	seed, err := p.writeSeed(ctx, log, be, pool, req)
+	if err != nil {
+		return err
+	}
+	diskPath, err := be.StorageVolGetPath(ctx, overlay)
+	if err != nil {
+		return fmt.Errorf("reading the path of overlay volume %s: %w", overlay.Name, err)
+	}
+	seedPath, err := be.StorageVolGetPath(ctx, seed)
+	if err != nil {
+		return fmt.Errorf("reading the path of seed volume %s: %w", seed.Name, err)
+	}
+
+	xmlDoc, err := buildDomainXML(domainParams{
+		Name:       req.Name,
+		UID:        req.UID,
+		CreatedAt:  p.now(),
+		VCPUs:      mt.VCPUs,
+		MemoryMiB:  mt.MemoryMiB,
+		Firmware:   src.Firmware,
+		Bridge:     host.Bridge,
+		DiskPath:   diskPath,
+		SeedPath:   seedPath,
+		DiskFormat: provider.KVMFormatQcow2,
+	})
+	if err != nil {
+		return err
+	}
+	dom, err = be.DomainDefineXML(ctx, xmlDoc)
+	if err != nil {
+		return fmt.Errorf("defining domain %s: %w", req.Name, err)
+	}
+	kv := []any{"uid", req.UID, "firmware", src.Firmware}
+	if log.V(2).Enabled() {
+		kv = append(kv, "xml", xmlDoc)
+	}
+	log.V(1).Info("domain defined", kv...)
+	return p.ensureRunning(ctx, log, be, dom)
+}
+
+// ensureOverlay creates the per-VM qcow2 overlay backed by the base image,
+// or reuses the one a previous attempt left: volume creation is atomic, so
+// an existing overlay is complete.
+func ensureOverlay(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, name, basePath, baseFormat string, diskSizeGB int64) (libvirt.StorageVol, error) {
+	volName := overlayVolumeName(name)
+	xmlDoc, err := buildVolumeXML(volumeSpec{
+		Name:          volName,
+		CapacityBytes: diskSizeGB << 30,
+		Format:        provider.KVMFormatQcow2,
+		BackingPath:   basePath,
+		BackingFormat: baseFormat,
+	})
+	if err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	vol, err := be.StorageVolCreateXML(ctx, pool, xmlDoc, 0)
+	switch {
+	case err == nil:
+		log.V(1).Info("overlay volume created", "volume", volName)
+		return vol, nil
+	case hasCode(err, libvirt.ErrStorageVolExist):
+		vol, err = be.StorageVolLookupByName(ctx, pool, volName)
+		if err != nil {
+			return libvirt.StorageVol{}, fmt.Errorf("looking up overlay volume %s: %w", volName, err)
+		}
+		log.V(1).Info("overlay volume already exists", "volume", volName)
+		return vol, nil
+	default:
+		return libvirt.StorageVol{}, fmt.Errorf("creating overlay volume %s: %w", volName, err)
+	}
+}
+
+// writeSeed builds the NoCloud seed and uploads it as a raw volume,
+// replacing whatever a previous attempt left (spec entry 5): a cut-off
+// upload would boot a VM without cloud-init. The ISO is tens of KiB, so it
+// is built in memory. Nothing about its content is logged or put in an
+// error.
+func (p *Provider) writeSeed(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, req provider.CreateRequest) (libvirt.StorageVol, error) {
+	name := seedVolumeName(req.Name)
+	if err := deleteVolumeIfPresent(ctx, be, pool, name); err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	installAgent := p.cfg.InstallGuestAgent == nil || *p.cfg.InstallGuestAgent
+	vendorData, err := buildVendorData(p.cfg.VendorData, installAgent)
+	if err != nil {
+		return libvirt.StorageVol{}, permanentf("building vendor-data for %s: %v", req.Name, err)
+	}
+	var iso bytes.Buffer
+	if err := buildSeedISO(req.Name, req.CloudInit, vendorData, &iso); err != nil {
+		return libvirt.StorageVol{}, permanentf("building seed ISO for %s: %v", req.Name, err)
+	}
+	xmlDoc, err := buildVolumeXML(volumeSpec{Name: name, CapacityBytes: int64(iso.Len()), Format: provider.KVMFormatRaw})
+	if err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	vol, err := be.StorageVolCreateXML(ctx, pool, xmlDoc, 0)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("creating seed volume %s: %w", name, err)
+	}
+	if err := be.StorageVolUpload(ctx, vol, bytes.NewReader(iso.Bytes()), 0, uint64(iso.Len()), 0); err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("uploading seed volume %s: %w", name, err)
+	}
+	log.V(1).Info("seed volume uploaded", "volume", name, "bytes", iso.Len())
+	return vol, nil
+}
+
+// ensureRunning starts the domain unless it is active already, then waits
+// until libvirt reports it running. ErrOperationInvalid from DomainCreate
+// means another caller started it between the state read and the call.
+func (p *Provider) ensureRunning(ctx context.Context, log logr.Logger, be backend, dom libvirt.Domain) error {
+	state, _, err := be.DomainGetState(ctx, dom, 0)
+	if err != nil {
+		return fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
+	}
+	if isActive(libvirt.DomainState(state)) {
+		return nil
+	}
+	switch err := be.DomainCreate(ctx, dom); {
+	case err == nil:
+		log.V(1).Info("domain started")
+	case hasCode(err, libvirt.ErrOperationInvalid):
+		log.V(1).Info("domain start reported it already running")
+	default:
+		return fmt.Errorf("starting domain %s: %w", dom.Name, err)
+	}
+	return p.waitRunning(ctx, be, dom)
+}
+
+// waitRunning polls DomainGetState until the domain is active, the cap
+// passes (Transient: the next Create attempt finds the domain defined and
+// tries again) or ctx ends.
+func (p *Provider) waitRunning(ctx context.Context, be backend, dom libvirt.Domain) error {
+	deadline := time.NewTimer(p.runningWait)
+	defer deadline.Stop()
+	ticker := time.NewTicker(p.runningPoll)
+	defer ticker.Stop()
+	for {
+		state, _, err := be.DomainGetState(ctx, dom, 0)
+		if err != nil {
+			return fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
+		}
+		if isActive(libvirt.DomainState(state)) {
+			return nil
+		}
+		select {
+		case <-ticker.C:
+		case <-deadline.C:
+			return transientf("domain %s did not reach running within %s (state %s)", dom.Name, p.runningWait, libvirt.DomainState(state))
+		case <-ctx.Done():
+			return fmt.Errorf("%w: waiting for domain %s to run: %w", provider.ErrTransient, dom.Name, ctx.Err())
+		}
+	}
+}
+
+// logAPIError is the V(1) line per failed backend call of spec entry 20.
+func logAPIError(log logr.Logger, op string, err error) {
+	kv := []any{"op", op, "error", err.Error()}
+	kv = append(kv, libvirtErrorFields(err)...)
+	log.V(1).Info("kvm2 backend call failed", kv...)
+}