egress-proxies-operator/internal/provider/kvm2/check.go

package kvm2

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/digitalocean/go-libvirt"
	"golang.org/x/crypto/ssh/knownhosts"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

var _ provider.Checker = (*Provider)(nil)

// Reasons only this provider can tell apart (spec entry 16). Each names a
// failure an admin fixes somewhere other than where the shared reasons
// point.
const (
	// reasonCredentialsUnavailable: the key or known_hosts file is missing
	// or unparsable; fix the Secret in the cluster, not the host.
	reasonCredentialsUnavailable = "CredentialsUnavailable"
	// reasonPoolNotFound: the configured pool is not defined on the host;
	// compare with `virsh pool-list --all`.
	reasonPoolNotFound = "PoolNotFound"
	// reasonPoolNotActive: the pool exists but is not running; `virsh
	// pool-start <pool>` (and autostart) brings it back.
	reasonPoolNotActive = "PoolNotActive"
)

// Check verifies every configured host concurrently, each on a fresh
// dedicated connection: dial, libvirt version, hostname, pool lookup, pool
// info, close. The pooled connection is never touched, so a check that runs
// out of time closes only its own connection and cannot fail a reconcile's
// call in flight. Every target gets a verdict, in config order.
func (p *Provider) Check(ctx context.Context) []provider.TargetStatus {
	out := make([]provider.TargetStatus, len(p.cfg.Hosts))
	var wg sync.WaitGroup
	for i, host := range p.cfg.Hosts {
		wg.Go(func() { out[i] = p.checkHost(ctx, host) })
	}
	wg.Wait()
	return out
}

func (p *Provider) checkHost(ctx context.Context, host provider.KVMHost) provider.TargetStatus {
	start := time.Now()
	log := p.logger(ctx).WithValues("host", host.Name)
	ts := provider.TargetStatus{
		Name:    host.Name,
		Address: host.Address,
		Details: map[string]string{"user": host.User, "pool": host.Pool},
	}
	err := p.probeHost(ctx, host, ts.Details)
	ts.Latency = time.Since(start)
	switch {
	case err == nil:
		ts.Ready = true
		ts.Reason = provider.ReasonReachable
	default:
		var inactive *poolNotActiveError
		switch {
		case ctx.Err() != nil:
			// The round ran out of time, whichever stage was running: the
			// error's shape (a dial that saw the ctx end, an abandoned
			// call) says nothing the admin should act on.
			ts.Reason = provider.ReasonTimeout
		case errors.As(err, &inactive):
			ts.Reason = reasonPoolNotActive
		default:
			ts.Reason = checkReason(err)
		}
		ts.Message = err.Error()
	}
	log.V(1).Info("kvm2 host checked", "ready", ts.Ready, "reason", ts.Reason, "latency", ts.Latency)
	return ts
}

// probeHost runs the check sequence on its own connection and fills details
// as facts arrive, so a failure late in the sequence still reports what the
// host said before it.
func (p *Provider) probeHost(ctx context.Context, host provider.KVMHost, details map[string]string) error {
	b, err := p.hosts.dialFresh(ctx, host)
	if err != nil {
		return err
	}
	defer func() { _ = b.Close() }()

	version, err := b.ConnectGetLibVersion(ctx)
	if err != nil {
		return err
	}
	details["libvirtVersion"] = formatLibVersion(version)
	hostname, err := b.ConnectGetHostname(ctx)
	if err != nil {
		return err
	}
	details["hostname"] = hostname
	pool, err := b.StoragePoolLookupByName(ctx, host.Pool)
	if err != nil {
		return err
	}
	state, capacity, _, available, err := b.StoragePoolGetInfo(ctx, pool)
	if err != nil {
		return err
	}
	details["poolState"] = poolStateName(state)
	details["poolCapacityGiB"] = formatGiB(capacity)
	details["poolAvailableGiB"] = formatGiB(available)
	if libvirt.StoragePoolState(state) != libvirt.StoragePoolRunning {
		return &poolNotActiveError{pool: host.Pool, state: poolStateName(state)}
	}
	return nil
}

// poolNotActiveError is the one verdict that is not an error from the
// backend: the pool answered, but is not usable.
type poolNotActiveError struct {
	pool  string
	state string
}

func (e *poolNotActiveError) Error() string {
	return fmt.Sprintf("storage pool %q is %s", e.pool, e.state)
}

// checkReason maps a failed probe to a reason once checkHost has ruled out
// the round's ctx ending. A dial failure is judged by its stage, so a TCP
// connect or an SSH handshake that ran out of the dial budget on its own is
// Unreachable, not Timeout: the host did not answer. Timeout is for a
// budget expiring once the connection was up.
func checkReason(err error) string {
	var de *dialError
	if errors.As(err, &de) {
		switch de.stage {
		case stageCredentials:
			return reasonCredentialsUnavailable
		case stageSSH:
			var keyErr *knownhosts.KeyError
			if errors.As(de.err, &keyErr) || strings.Contains(de.err.Error(), sshAuthFailed) {
				return provider.ReasonUnauthorized
			}
		}
		return provider.ReasonUnreachable
	}
	switch {
	case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
		return provider.ReasonTimeout
	case hasCode(err, libvirt.ErrAccessDenied), hasCode(err, libvirt.ErrAuthFailed):
		return provider.ReasonForbidden
	case hasCode(err, libvirt.ErrNoStoragePool):
		return reasonPoolNotFound
	}
	if _, ok := libvirtCode(err); ok {
		return provider.ReasonAPIError
	}
	return provider.ReasonUnreachable
}

// formatLibVersion decodes ConnectGetLibVersion's major*1_000_000 +
// minor*1_000 + micro.
func formatLibVersion(v uint64) string {
	return fmt.Sprintf("%d.%d.%d", v/1_000_000, v/1_000%1_000, v%1_000)
}

// poolStateName uses virsh's words for a pool state.
func poolStateName(state uint8) string {
	switch libvirt.StoragePoolState(state) {
	case libvirt.StoragePoolInactive:
		return "inactive"
	case libvirt.StoragePoolBuilding:
		return "building"
	case libvirt.StoragePoolRunning:
		return "running"
	case libvirt.StoragePoolDegraded:
		return "degraded"
	case libvirt.StoragePoolInaccessible:
		return "inaccessible"
	default:
		return fmt.Sprintf("state %d", state)
	}
}

func formatGiB(bytes uint64) string {
	return fmt.Sprintf("%.1f", float64(bytes)/(1<<30))
}