package kvm2 import ( "bytes" "fmt" "time" "context" "github.com/digitalocean/go-libvirt" "github.com/go-logr/logr" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/nocloud" ) // The wait after DomainCreate (spec entry 18): Create returns only once the // domain is running, so Get never sees a domain kvm2 defined but did not // start (spec entry 9). DomainCreate is synchronous, so the first read // normally succeeds; the cap only covers state settling and must not pin a // reconcile worker for long. const ( defaultRunningWait = 15 * time.Second defaultRunningPoll = 500 * time.Millisecond ) // Create is spec entry 10: validate placement, then, under the per-name // lock, find or build the domain step by step, every step idempotent, and // return once it is running. func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) { host, err := p.host(req.Placement.Zone) if err != nil { if req.Placement.Zone != "" { err = permanentf("placement.zone %q is not in kvm2.hosts", req.Placement.Zone) } return "", p.wrapErr("create", "", err) } mt, ok := p.cfg.MachineTypes[req.Placement.MachineType] if !ok { return "", p.wrapErr("create", "", permanentf("placement.machineType %q is not in kvm2.machineTypes", req.Placement.MachineType)) } src, err := resolveImage(p.name, p.cfg, req.Placement.Image) if err != nil { return "", p.wrapErr("create", "", err) } id := formatProviderID(host.Name, req.Name) log := p.logger(ctx).WithValues("host", host.Name, "name", req.Name) log.V(2).Info("kvm2 create resolved", "pool", host.Pool, "bridge", host.Bridge, "machineType", req.Placement.MachineType, "vcpus", mt.VCPUs, "memoryMiB", mt.MemoryMiB, "diskSizeGb", p.cfg.DiskSizeGB, "image", src.volumeName(), "firmware", src.Firmware, "cloudInitBytes", len(req.CloudInit)) unlock, err := p.lockName(ctx, req.Name) if err != nil { return "", p.wrapErr("create", id, err) } defer unlock() be, err := p.hosts.get(ctx, host) if err != nil { logAPIError(log, "create", err) return "", p.wrapErr("create", id, err) } if err := p.create(ctx, log, be, host, req, mt, src); err != nil { logAPIError(log, "create", err) return "", p.wrapErr("create", id, err) } log.V(1).Info("kvm2 domain ensured running", "id", id) return id, nil } func (p *Provider) create(ctx context.Context, log logr.Logger, be backend, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType, src imageSource) error { dom, err := be.DomainLookupByName(ctx, req.Name) switch { case err == nil: meta, err := readMetadata(ctx, be, dom) if err != nil { return err } if !meta.managed() { return permanentf("domain %s exists but carries no kvm2 mark; it was not created by this provider", req.Name) } if meta.UID != "" && meta.UID != req.UID { return permanentf("domain %s belongs to proxy uid %s, not %s", req.Name, meta.UID, req.UID) } log.V(1).Info("kvm2 domain already defined", "uid", meta.UID) return p.ensureRunning(ctx, log, be, dom) case !hasCode(err, libvirt.ErrNoDomain): return fmt.Errorf("looking up domain %s: %w", req.Name, err) } pool, err := be.StoragePoolLookupByName(ctx, host.Pool) if err != nil { return fmt.Errorf("looking up storage pool %s: %w", host.Pool, err) } imageBytes, err := p.imageSizeIfUncached(ctx, be, pool, src) if err != nil { return err } if err := checkPoolSpace(ctx, be, pool, host.Pool, neededBytes(imageBytes, p.cfg.DiskSizeGB)); err != nil { return err } basePath, err := p.ensureImage(ctx, host, pool, be, src) if err != nil { return err } overlay, err := ensureOverlay(ctx, log, be, pool, req.Name, basePath, src.Format, p.cfg.DiskSizeGB) if err != nil { return err } seed, err := p.writeSeed(ctx, log, be, pool, req) if err != nil { return err } diskPath, err := be.StorageVolGetPath(ctx, overlay) if err != nil { return fmt.Errorf("reading the path of overlay volume %s: %w", overlay.Name, err) } seedPath, err := be.StorageVolGetPath(ctx, seed) if err != nil { return fmt.Errorf("reading the path of seed volume %s: %w", seed.Name, err) } xmlDoc, err := buildDomainXML(domainParams{ Name: req.Name, UID: req.UID, CreatedAt: p.now(), VCPUs: mt.VCPUs, MemoryMiB: mt.MemoryMiB, Firmware: src.Firmware, Bridge: host.Bridge, DiskPath: diskPath, SeedPath: seedPath, DiskFormat: provider.KVMFormatQcow2, }) if err != nil { return err } dom, err = be.DomainDefineXML(ctx, xmlDoc) if err != nil { return fmt.Errorf("defining domain %s: %w", req.Name, err) } kv := []any{"uid", req.UID, "firmware", src.Firmware} if log.V(2).Enabled() { kv = append(kv, "xml", xmlDoc) } log.V(1).Info("domain defined", kv...) return p.ensureRunning(ctx, log, be, dom) } // ensureOverlay creates the per-VM qcow2 overlay backed by the base image, // or reuses the one a previous attempt left: volume creation is atomic, so // an existing overlay is complete. func ensureOverlay(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, name, basePath, baseFormat string, diskSizeGB int64) (libvirt.StorageVol, error) { volName := overlayVolumeName(name) xmlDoc, err := buildVolumeXML(volumeSpec{ Name: volName, CapacityBytes: diskSizeGB << 30, Format: provider.KVMFormatQcow2, BackingPath: basePath, BackingFormat: baseFormat, }) if err != nil { return libvirt.StorageVol{}, err } vol, err := be.StorageVolCreateXML(ctx, pool, xmlDoc, 0) switch { case err == nil: log.V(1).Info("overlay volume created", "volume", volName) return vol, nil case hasCode(err, libvirt.ErrStorageVolExist): vol, err = be.StorageVolLookupByName(ctx, pool, volName) if err != nil { return libvirt.StorageVol{}, fmt.Errorf("looking up overlay volume %s: %w", volName, err) } log.V(1).Info("overlay volume already exists", "volume", volName) return vol, nil default: return libvirt.StorageVol{}, fmt.Errorf("creating overlay volume %s: %w", volName, err) } } // writeSeed builds the NoCloud seed and uploads it as a raw volume, // replacing whatever a previous attempt left (spec entry 5): a cut-off // upload would boot a VM without cloud-init. The ISO is tens of KiB, so it // is built in memory. Nothing about its content is logged or put in an // error. func (p *Provider) writeSeed(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, req provider.CreateRequest) (libvirt.StorageVol, error) { name := seedVolumeName(req.Name) if err := deleteVolumeIfPresent(ctx, be, pool, name); err != nil { return libvirt.StorageVol{}, err } installAgent := p.cfg.InstallGuestAgent == nil || *p.cfg.InstallGuestAgent vendorData, err := nocloud.BuildVendorData(p.cfg.VendorData, installAgent) if err != nil { return libvirt.StorageVol{}, permanentf("building vendor-data for %s: %v", req.Name, err) } var iso bytes.Buffer if err := nocloud.BuildSeedISO(req.Name, req.CloudInit, vendorData, &iso); err != nil { return libvirt.StorageVol{}, permanentf("building seed ISO for %s: %v", req.Name, err) } xmlDoc, err := buildVolumeXML(volumeSpec{Name: name, CapacityBytes: int64(iso.Len()), Format: provider.KVMFormatRaw}) if err != nil { return libvirt.StorageVol{}, err } vol, err := be.StorageVolCreateXML(ctx, pool, xmlDoc, 0) if err != nil { return libvirt.StorageVol{}, fmt.Errorf("creating seed volume %s: %w", name, err) } if err := be.StorageVolUpload(ctx, vol, bytes.NewReader(iso.Bytes()), 0, uint64(iso.Len()), 0); err != nil { return libvirt.StorageVol{}, fmt.Errorf("uploading seed volume %s: %w", name, err) } log.V(1).Info("seed volume uploaded", "volume", name, "bytes", iso.Len()) return vol, nil } // ensureRunning starts the domain unless it is active already, then waits // until libvirt reports it running. ErrOperationInvalid from DomainCreate // means another caller started it between the state read and the call. A // paused or pmsuspended domain is an admin's doing and DomainCreate would // refuse it; the verdict is Permanent so the reconciler does not retry it. func (p *Provider) ensureRunning(ctx context.Context, log logr.Logger, be backend, dom libvirt.Domain) error { state, _, err := be.DomainGetState(ctx, dom, 0) if err != nil { return fmt.Errorf("reading state of domain %s: %w", dom.Name, err) } switch s := libvirt.DomainState(state); { case isActive(s): return nil case s == libvirt.DomainPaused || s == libvirt.DomainPmsuspended: return permanentf("domain %s is %s and cannot be started by the operator; resume or destroy it on the host", dom.Name, s) } switch err := be.DomainCreate(ctx, dom); { case err == nil: log.V(1).Info("domain started") case hasCode(err, libvirt.ErrOperationInvalid): log.V(1).Info("domain start reported it already running") default: return fmt.Errorf("starting domain %s: %w", dom.Name, err) } return p.waitRunning(ctx, be, dom) } // waitRunning polls DomainGetState until the domain is active, the cap // passes (Transient: the next Create attempt finds the domain defined and // tries again) or ctx ends. Only nostate is worth another poll; a domain // that is shut off, crashed or paused right after its start is not going // to come up on its own, so that is Transient at once. func (p *Provider) waitRunning(ctx context.Context, be backend, dom libvirt.Domain) error { deadline := time.NewTimer(p.runningWait) defer deadline.Stop() ticker := time.NewTicker(p.runningPoll) defer ticker.Stop() for { state, _, err := be.DomainGetState(ctx, dom, 0) if err != nil { return fmt.Errorf("reading state of domain %s: %w", dom.Name, err) } switch s := libvirt.DomainState(state); { case isActive(s): return nil case s != libvirt.DomainNostate: return transientf("domain %s went %s right after start", dom.Name, s) } select { case <-ticker.C: case <-deadline.C: return transientf("domain %s did not reach running within %s (state %s)", dom.Name, p.runningWait, libvirt.DomainState(state)) case <-ctx.Done(): return fmt.Errorf("%w: waiting for domain %s to run: %w", provider.ErrTransient, dom.Name, ctx.Err()) } } } // logAPIError is the V(1) line per failed backend call of spec entry 20. func logAPIError(log logr.Logger, op string, err error) { kv := []any{"op", op, "error", err.Error()} kv = append(kv, libvirtErrorFields(err)...) log.V(1).Info("kvm2 backend call failed", kv...) }