egress-proxies-operator/internal/provider/kvm/kvm.go

// Package kvm implements the provider contract on libvirt/KVM hypervisors.
// Each configured host is reached over SSH with the libvirt unix socket
// forwarded through the connection, so the hypervisors need no libvirtd TCP
// or TLS listener and the operator image needs no libvirt libraries. VMs
// boot a stock cloud image (fetched into the host's storage pool on first
// use) through a qcow2 overlay, with the Proxy's cloud-init delivered on a
// NoCloud seed ISO built in-process.
package kvm

import (
	"bytes"
	"context"
	"fmt"
	"net"
	"net/http"
	"time"

	libvirt "github.com/digitalocean/go-libvirt"
	"github.com/go-logr/logr"
	logf "sigs.k8s.io/controller-runtime/pkg/log"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

const (
	defaultDiskSizeGB = 10
	defaultSocket     = "/var/run/libvirt/libvirt-sock"
	defaultPool       = "default"
	formatQcow2       = "qcow2"
	formatRaw         = "raw"
	firmwareUEFI      = "uefi"
	firmwareBIOS      = "bios"

	// undefineFlags removes everything libvirt may keep alongside a domain
	// definition, so an undefine never fails on leftover state.
	undefineFlags = libvirt.DomainUndefineManagedSave |
		libvirt.DomainUndefineSnapshotsMetadata |
		libvirt.DomainUndefineNvram
)

func withDefaults(cfg provider.KVMConfig) provider.KVMConfig {
	if cfg.DiskSizeGB == 0 {
		cfg.DiskSizeGB = defaultDiskSizeGB
	}
	images := make(map[string]provider.KVMImage, len(cfg.Images))
	for k, img := range cfg.Images {
		if img.Format == "" {
			img.Format = formatQcow2
		}
		if img.Firmware == "" {
			img.Firmware = firmwareUEFI
		}
		images[k] = img
	}
	cfg.Images = images
	hosts := make([]provider.KVMHost, len(cfg.Hosts))
	for i, h := range cfg.Hosts {
		if h.Socket == "" {
			h.Socket = defaultSocket
		}
		if h.Pool == "" {
			h.Pool = defaultPool
		}
		hosts[i] = h
	}
	cfg.Hosts = hosts
	return cfg
}

// Provider implements provider.Provider on one or more libvirt hosts.
type Provider struct {
	name   string
	cfg    provider.KVMConfig
	hosts  *hostPool
	images *imageCache
	now    func() time.Time
}

// New builds a Provider that reaches its hosts over SSH. The key and
// known_hosts files are read on the first connection, not here: they come
// from an optional Secret mount, and a constructor that failed without it
// would abort the whole provider set and crash-loop the manager, taking
// the kubernetes and gcp providers down with it. Unreadable files are
// still reported at startup, as a warning. Satisfies registry.Constructor.
func New(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
	cfg := provider.KVMConfig{}
	if pc.KVM != nil {
		cfg = *pc.KVM
	}
	if _, err := newSSHAuth(cfg); err != nil {
		logf.FromContext(ctx).WithName("kvm").WithValues("provider", pc.Name).Info(
			"WARNING: kvm ssh material is not usable; proxies on this provider will fail until it is",
			"error", err.Error())
	}
	return newWithDialer(pc, lazyDialer(cfg), nil), nil
}

// newWithDialer is the test seam: the dialer decides what libvirtAPI each
// host resolves to, and the HTTP client is what image downloads go through.
func newWithDialer(pc provider.ProviderConfig, dial hostDialer, httpClient *http.Client) *Provider {
	cfg := provider.KVMConfig{}
	if pc.KVM != nil {
		cfg = *pc.KVM
	}
	return &Provider{
		name:   pc.Name,
		cfg:    withDefaults(cfg),
		hosts:  newHostPool(dial),
		images: newImageCache(httpClient),
		now:    time.Now,
	}
}

func (p *Provider) logger(ctx context.Context) logr.Logger {
	return logf.FromContext(ctx).WithName("kvm").WithValues("provider", p.name)
}

func (p *Provider) wrapErr(op, id string, err error) error {
	return provider.Wrap(classify(err), op, p.name, id, err)
}

// host resolves placement.zone (or a providerID's host segment) to a
// configured host. An empty zone is allowed only when exactly one host is
// configured; anything else is a spec error.
func (p *Provider) host(zone string) (provider.KVMHost, error) {
	if zone == "" {
		switch len(p.cfg.Hosts) {
		case 0:
			return provider.KVMHost{}, fmt.Errorf("%w: kvm.hosts is empty", provider.ErrPermanent)
		case 1:
			return p.cfg.Hosts[0], nil
		default:
			return provider.KVMHost{}, fmt.Errorf("%w: placement.zone is required when kvm.hosts lists more than one host",
				provider.ErrPermanent)
		}
	}
	for _, h := range p.cfg.Hosts {
		if h.Name == zone {
			return h, nil
		}
	}
	return provider.KVMHost{}, fmt.Errorf("%w: host %q is not in kvm.hosts", provider.ErrPermanent, zone)
}

// Create makes the VM exist and running on the placed host, and is
// idempotent: a defined domain is only started if needed, its volumes
// untouched. Before the domain exists nothing can be using the volumes,
// so an existing overlay is reused (its creation is atomic) but the seed
// is rebuilt from scratch — a previous attempt may have been cut off mid
// upload, and a truncated seed would boot a VM without cloud-init. The
// first Create for an image on a host also downloads that image into the
// host's pool, synchronously.
func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
	pl := req.Placement
	host, err := p.host(pl.Zone)
	if err != nil {
		return "", p.wrapErr("create", "", err)
	}
	mt, ok := p.cfg.MachineTypes[pl.MachineType]
	if !ok {
		return "", p.wrapErr("create", "", fmt.Errorf("%w: placement.machineType %q is not in kvm.machineTypes",
			provider.ErrPermanent, pl.MachineType))
	}
	src, err := resolveImage(p.cfg, pl.Image)
	if err != nil {
		return "", p.wrapErr("create", "", err)
	}
	id := formatProviderID(host.Name, req.Name)
	log := p.logger(ctx).WithValues("host", host.Name, "name", req.Name)
	log.V(2).Info("KVM create resolved",
		"pool", host.Pool, "bridge", host.Bridge, "machineType", pl.MachineType,
		"vcpus", mt.VCPUs, "memoryMiB", mt.MemoryMiB, "diskSizeGb", p.cfg.DiskSizeGB,
		"image", src.volumeName(), "cloudInitBytes", len(req.CloudInit))

	err = p.hosts.call(ctx, log, host, func(api libvirtAPI) error {
		dom, err := api.DomainLookupByName(req.Name)
		if err == nil {
			log.V(1).Info("domain already defined")
			return p.ensureRunning(log, api, dom)
		}
		if !hasCode(err, libvirt.ErrNoDomain) {
			return fmt.Errorf("looking up domain: %w", err)
		}
		pool, err := api.StoragePoolLookupByName(host.Pool)
		if err != nil {
			return fmt.Errorf("looking up pool %s: %w", host.Pool, err)
		}
		basePath, err := p.images.ensure(ctx, log, api, host.Name, pool, src)
		if err != nil {
			return err
		}
		overlay, err := p.ensureOverlay(log, api, pool, req.Name, basePath, src.Format)
		if err != nil {
			return err
		}
		seed, err := p.writeSeed(log, api, pool, req.Name, req.CloudInit)
		if err != nil {
			return err
		}
		diskPath, err := api.StorageVolGetPath(overlay)
		if err != nil {
			return fmt.Errorf("resolving overlay path: %w", err)
		}
		seedPath, err := api.StorageVolGetPath(seed)
		if err != nil {
			return fmt.Errorf("resolving seed path: %w", err)
		}
		dom, err = p.defineDomain(log, api, host, req, mt, src.Firmware, diskPath, seedPath)
		if err != nil {
			return err
		}
		return p.ensureRunning(log, api, dom)
	})
	if err != nil {
		logAPIError(log, "create", err)
		return "", p.wrapErr("create", id, err)
	}
	log.V(1).Info("KVM domain ensured running", "id", id)
	return id, nil
}

// ensureOverlay creates the per-VM overlay volume, or returns the existing
// one: creation is atomic, so an overlay that exists is complete.
func (p *Provider) ensureOverlay(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, basePath, baseFormat string) (libvirt.StorageVol, error) {
	xml, err := buildVolumeXML(overlayVolume(name, p.cfg.DiskSizeGB, basePath, baseFormat))
	if err != nil {
		return libvirt.StorageVol{}, err
	}
	vol, err := api.StorageVolCreateXML(pool, xml, 0)
	switch {
	case err == nil:
		log.V(1).Info("overlay volume created", "volume", overlayVolumeName(name), "base", basePath)
		return vol, nil
	case hasCode(err, libvirt.ErrStorageVolExist):
		log.V(1).Info("overlay volume already exists", "volume", overlayVolumeName(name))
		vol, err := api.StorageVolLookupByName(pool, overlayVolumeName(name))
		if err != nil {
			return libvirt.StorageVol{}, fmt.Errorf("looking up existing overlay volume: %w", err)
		}
		return vol, nil
	default:
		return libvirt.StorageVol{}, fmt.Errorf("creating overlay volume: %w", err)
	}
}

// writeSeed replaces the seed volume with a freshly built NoCloud ISO.
// Only called while no domain references it.
func (p *Provider) writeSeed(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, userData string) (libvirt.StorageVol, error) {
	seedName := seedVolumeName(name)
	if err := deleteIfExists(api, pool, seedName); err != nil {
		return libvirt.StorageVol{}, err
	}
	vendorData, err := buildVendorData(p.cfg.VendorData, p.cfg.InstallGuestAgent == nil || *p.cfg.InstallGuestAgent)
	if err != nil {
		return libvirt.StorageVol{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
	}
	iso, err := buildSeedISO(name, userData, vendorData)
	if err != nil {
		return libvirt.StorageVol{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
	}
	xml, err := buildVolumeXML(rawVolume(seedName, int64(len(iso))))
	if err != nil {
		return libvirt.StorageVol{}, err
	}
	vol, err := api.StorageVolCreateXML(pool, xml, 0)
	if err != nil {
		return libvirt.StorageVol{}, fmt.Errorf("creating seed volume: %w", err)
	}
	if err := api.StorageVolUpload(vol, bytes.NewReader(iso), 0, 0, 0); err != nil {
		return libvirt.StorageVol{}, fmt.Errorf("uploading seed volume: %w", err)
	}
	log.V(1).Info("seed volume uploaded", "volume", seedName, "bytes", len(iso))
	return vol, nil
}

func (p *Provider) defineDomain(log logr.Logger, api libvirtAPI, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType, firmware, diskPath, seedPath string) (libvirt.Domain, error) {
	xml, err := buildDomainXML(domainParams{
		Name:      req.Name,
		UID:       req.UID,
		CreatedAt: p.now(),
		VCPUs:     mt.VCPUs,
		MemoryMiB: mt.MemoryMiB,
		Firmware:  firmware,
		Bridge:    host.Bridge,
		DiskPath:  diskPath,
		SeedPath:  seedPath,
	})
	if err != nil {
		return libvirt.Domain{}, err
	}
	dom, err := api.DomainDefineXML(xml)
	if err != nil {
		return libvirt.Domain{}, fmt.Errorf("defining domain: %w", err)
	}
	log.V(1).Info("domain defined", "uid", req.UID, "firmware", firmware)
	return dom, nil
}

func (p *Provider) ensureRunning(log logr.Logger, api libvirtAPI, dom libvirt.Domain) error {
	state, _, err := api.DomainGetState(dom, 0)
	if err != nil {
		return fmt.Errorf("reading domain state: %w", err)
	}
	if isActive(libvirt.DomainState(state)) {
		return nil
	}
	err = api.DomainCreate(dom)
	switch {
	case err == nil:
		log.V(1).Info("domain started")
	case hasCode(err, libvirt.ErrOperationInvalid):
		log.V(1).Info("domain already running, start treated as success")
	default:
		return fmt.Errorf("starting domain: %w", err)
	}
	return nil
}

// Get reads the domain's state, metadata and addresses. The providerID
// carries the host, so this looks at the right hypervisor even after a
// placement.zone edit.
func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instance, error) {
	hostName, name, err := parseProviderID(providerID)
	if err != nil {
		return nil, provider.Wrap(provider.ErrPermanent, "get", p.name, providerID, err)
	}
	host, err := p.host(hostName)
	if err != nil {
		return nil, p.wrapErr("get", providerID, err)
	}
	log := p.logger(ctx).WithValues("host", host.Name, "name", name)
	var inst *provider.Instance
	err = p.hosts.call(ctx, log, host, func(api libvirtAPI) error {
		dom, err := api.DomainLookupByName(name)
		if err != nil {
			return fmt.Errorf("looking up domain: %w", err)
		}
		meta, err := p.metadata(api, dom)
		if err != nil {
			return err
		}
		inst, err = p.inspect(log, api, host.Name, dom, meta)
		return err
	})
	if err != nil {
		logAPIError(log, "get", err)
		return nil, p.wrapErr("get", providerID, err)
	}
	log.V(1).Info("KVM domain fetched", "state", inst.State, "ip", inst.IP, "uid", inst.UID)
	return inst, nil
}

// inspect assembles an Instance from a domain that is known to exist and
// whose metadata has already been read (the zero value for a domain
// without the operator's element, which reports an empty UID as the
// contract requires).
func (p *Provider) inspect(log logr.Logger, api libvirtAPI, hostName string, dom libvirt.Domain, meta instanceMetadata) (*provider.Instance, error) {
	state, _, err := api.DomainGetState(dom, 0)
	if err != nil {
		return nil, fmt.Errorf("reading domain state: %w", err)
	}
	var ip string
	if isActive(libvirt.DomainState(state)) {
		ip = p.address(log, api, dom)
	}
	return &provider.Instance{
		ID:        formatProviderID(hostName, dom.Name),
		IP:        ip,
		State:     mapState(libvirt.DomainState(state), ip),
		UID:       meta.UID,
		CreatedAt: meta.createdAt(),
	}, nil
}

// metadata returns the operator's element, or the zero value when the
// domain has none (it was not created by this operator).
func (p *Provider) metadata(api libvirtAPI, dom libvirt.Domain) (instanceMetadata, error) {
	raw, err := api.DomainGetMetadata(dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, libvirt.DomainAffectCurrent)
	if hasCode(err, libvirt.ErrNoDomainMetadata) {
		return instanceMetadata{}, nil
	}
	if err != nil {
		return instanceMetadata{}, fmt.Errorf("reading domain metadata: %w", err)
	}
	meta, err := parseInstanceMetadata(raw)
	if err != nil {
		return instanceMetadata{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
	}
	return meta, nil
}

// address returns the domain's first routable IPv4, asking the host's ARP
// table first and the guest agent second. Both sources fail routinely while
// the guest is still booting (no traffic seen yet, agent not installed
// yet), so their errors mean "no address yet", not "Get failed".
func (p *Provider) address(log logr.Logger, api libvirtAPI, dom libvirt.Domain) string {
	for _, source := range []libvirt.DomainInterfaceAddressesSource{
		libvirt.DomainInterfaceAddressesSrcArp,
		libvirt.DomainInterfaceAddressesSrcAgent,
	} {
		ifaces, err := api.DomainInterfaceAddresses(dom, uint32(source), 0)
		if err != nil {
			log.V(2).Info("no addresses from source", "source", source.String(), "error", err.Error())
			continue
		}
		if ip := firstIPv4(ifaces); ip != "" {
			return ip
		}
	}
	return ""
}

func firstIPv4(ifaces []libvirt.DomainInterface) string {
	for _, iface := range ifaces {
		for _, a := range iface.Addrs {
			if a.Type != int32(libvirt.IPAddrTypeIpv4) {
				continue
			}
			ip := net.ParseIP(a.Addr)
			if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsUnspecified() {
				continue
			}
			return a.Addr
		}
	}
	return ""
}

// isActive reports whether the guest is executing (or blocked on the
// hypervisor, which is still "up" from the outside).
func isActive(s libvirt.DomainState) bool {
	return s == libvirt.DomainRunning || s == libvirt.DomainBlocked
}

// mapState collapses libvirt's states onto the provider's. An active
// domain without an address is still Provisioning: an empty IP must never
// be published as Running. "No state" is also Provisioning — the
// reconciler answers Stopped with delete-and-recreate, too drastic for a
// state libvirt could not determine; the next poll decides. Everything
// else — paused, suspended, shut off, crashed, shutting down — is Stopped.
func mapState(s libvirt.DomainState, ip string) provider.InstanceState {
	if s == libvirt.DomainNostate {
		return provider.StateProvisioning
	}
	if !isActive(s) {
		return provider.StateStopped
	}
	if ip == "" {
		return provider.StateProvisioning
	}
	return provider.StateRunning
}

// Delete tears the VM down and is idempotent at every step: a stopped
// domain is not destroyed again, a missing domain is not undefined, and
// the per-VM volumes are removed by name even when the domain is already
// gone, so a crash between volume creation and define is cleaned up on the
// next reconcile. A providerID naming a host that is no longer configured
// is a no-op success rather than an error.
func (p *Provider) Delete(ctx context.Context, providerID string) error {
	hostName, name, err := parseProviderID(providerID)
	if err != nil {
		return provider.Wrap(provider.ErrPermanent, "delete", p.name, providerID, err)
	}
	host, err := p.host(hostName)
	if err != nil {
		// The host is no longer configured, so this operator has no way to
		// reach the domain and nothing left to clean. Same reasoning as
		// removeVolumes over a missing pool: refusing would pin the Proxy's
		// finalizer. Whatever is still running there is the admin's to remove,
		// and orphan GC cannot see it either — see docs/kvm-provider.md.
		p.logger(ctx).Info("host is not configured; delete has nothing to clean up",
			"host", hostName, "providerID", providerID)
		return nil
	}
	log := p.logger(ctx).WithValues("host", host.Name, "name", name)
	err = p.hosts.call(ctx, log, host, func(api libvirtAPI) error {
		if err := p.removeDomain(log, api, name); err != nil {
			return err
		}
		return p.removeVolumes(log, api, host.Pool, name)
	})
	if err != nil {
		logAPIError(log, "delete", err)
		return p.wrapErr("delete", providerID, err)
	}
	log.V(1).Info("KVM domain deleted")
	return nil
}

func (p *Provider) removeDomain(log logr.Logger, api libvirtAPI, name string) error {
	dom, err := api.DomainLookupByName(name)
	if hasCode(err, libvirt.ErrNoDomain) {
		log.V(1).Info("domain already gone")
		return nil
	}
	if err != nil {
		return fmt.Errorf("looking up domain: %w", err)
	}
	err = api.DomainDestroy(dom)
	switch {
	case err == nil:
		log.V(1).Info("domain destroyed")
	case hasCode(err, libvirt.ErrOperationInvalid):
		log.V(1).Info("domain not running, destroy skipped")
	case hasCode(err, libvirt.ErrNoDomain):
		return nil
	default:
		return fmt.Errorf("destroying domain: %w", err)
	}
	if err := api.DomainUndefineFlags(dom, undefineFlags); err != nil && !hasCode(err, libvirt.ErrNoDomain) {
		return fmt.Errorf("undefining domain: %w", err)
	}
	log.V(1).Info("domain undefined")
	return nil
}

// removeVolumes deletes the overlay and seed. A missing pool means there
// is nothing left to clean, not a failure: refusing to finish a Delete
// over a pool that no longer exists would pin the Proxy's finalizer.
func (p *Provider) removeVolumes(log logr.Logger, api libvirtAPI, poolName, name string) error {
	pool, err := api.StoragePoolLookupByName(poolName)
	if hasCode(err, libvirt.ErrNoStoragePool) {
		log.Info("pool not found, no volumes to delete", "pool", poolName)
		return nil
	}
	if err != nil {
		return fmt.Errorf("looking up pool %s: %w", poolName, err)
	}
	for _, vol := range []string{overlayVolumeName(name), seedVolumeName(name)} {
		if err := deleteIfExists(api, pool, vol); err != nil {
			return err
		}
	}
	log.V(1).Info("per-VM volumes deleted")
	return nil
}

// ListByTag inspects every domain on every host and keeps those carrying
// the operator's managed metadata. One unreachable host fails the whole
// sweep, so orphan GC never acts on a partial view of the fleet; a single
// domain that cannot be read is skipped instead, which is safe in the
// same direction — a domain missing from the list is never collected.
func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
	log := p.logger(ctx)
	var out []provider.Instance
	for _, host := range p.cfg.Hosts {
		hlog := log.WithValues("host", host.Name)
		err := p.hosts.call(ctx, hlog, host, func(api libvirtAPI) error {
			doms, _, err := api.ConnectListAllDomains(1, 0)
			if err != nil {
				return fmt.Errorf("listing domains: %w", err)
			}
			for _, dom := range doms {
				if inst := p.managedInstance(hlog, api, host.Name, dom); inst != nil {
					out = append(out, *inst)
				}
			}
			return nil
		})
		if err != nil {
			logAPIError(hlog, "list", err)
			return nil, p.wrapErr("list", "", err)
		}
	}
	log.V(1).Info("KVM domains listed", "hosts", len(p.cfg.Hosts), "managed", len(out))
	for _, inst := range out {
		log.V(2).Info("KVM listed instance", "id", inst.ID, "state", inst.State, "uid", inst.UID, "createdAt", inst.CreatedAt)
	}
	return out, nil
}

// managedInstance returns nil for domains the operator does not own, for
// domains that vanished between the listing and the inspection, and for
// domains whose metadata or state cannot be read (corrupt element, mid
// teardown): one bad domain must not block GC for every other one. A
// transport failure would also land here, but it fails the next RPC in
// the same sweep too, so the host still counts as unreachable.
func (p *Provider) managedInstance(log logr.Logger, api libvirtAPI, hostName string, dom libvirt.Domain) *provider.Instance {
	log = log.WithValues("name", dom.Name)
	meta, err := p.metadata(api, dom)
	if err != nil {
		log.V(1).Info("skipping domain with unreadable metadata", "error", err.Error())
		return nil
	}
	if !meta.managed() {
		return nil
	}
	inst, err := p.inspect(log, api, hostName, dom, meta)
	if err != nil {
		log.V(1).Info("skipping domain that could not be inspected", "error", err.Error())
		return nil
	}
	return inst
}