kvm2: health checker on dedicated connections; kvm2 in the ProviderHealth type comment

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

internal/provider/kvm2/check.go

@@ -0,0 +1,182 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+var _ provider.Checker = (*Provider)(nil)
+
+// Reasons only this provider can tell apart (spec entry 16). Each names a
+// failure an admin fixes somewhere other than where the shared reasons
+// point.
+const (
+	// reasonCredentialsUnavailable: the key or known_hosts file is missing
+	// or unparsable; fix the Secret in the cluster, not the host.
+	reasonCredentialsUnavailable = "CredentialsUnavailable"
+	// reasonPoolNotFound: the configured pool is not defined on the host;
+	// compare with `virsh pool-list --all`.
+	reasonPoolNotFound = "PoolNotFound"
+	// reasonPoolNotActive: the pool exists but is not running; `virsh
+	// pool-start <pool>` (and autostart) brings it back.
+	reasonPoolNotActive = "PoolNotActive"
+)
+
+// Check verifies every configured host concurrently, each on a fresh
+// dedicated connection: dial, libvirt version, hostname, pool lookup, pool
+// info, close. The pooled connection is never touched, so a check that runs
+// out of time closes only its own connection and cannot fail a reconcile's
+// call in flight. Every target gets a verdict, in config order.
+func (p *Provider) Check(ctx context.Context) []provider.TargetStatus {
+	out := make([]provider.TargetStatus, len(p.cfg.Hosts))
+	var wg sync.WaitGroup
+	for i, host := range p.cfg.Hosts {
+		wg.Go(func() { out[i] = p.checkHost(ctx, host) })
+	}
+	wg.Wait()
+	return out
+}
+
+func (p *Provider) checkHost(ctx context.Context, host provider.KVMHost) provider.TargetStatus {
+	start := time.Now()
+	log := p.logger(ctx).WithValues("host", host.Name)
+	ts := provider.TargetStatus{
+		Name:    host.Name,
+		Address: host.Address,
+		Details: map[string]string{"user": host.User, "pool": host.Pool},
+	}
+	err := p.probeHost(ctx, host, ts.Details)
+	ts.Latency = time.Since(start)
+	switch {
+	case err == nil:
+		ts.Ready = true
+		ts.Reason = provider.ReasonReachable
+	default:
+		var inactive *poolNotActiveError
+		if errors.As(err, &inactive) {
+			ts.Reason = reasonPoolNotActive
+		} else {
+			ts.Reason = checkReason(err)
+		}
+		ts.Message = err.Error()
+	}
+	log.V(1).Info("kvm2 host checked", "ready", ts.Ready, "reason", ts.Reason, "latency", ts.Latency)
+	return ts
+}
+
+// probeHost runs the check sequence on its own connection and fills details
+// as facts arrive, so a failure late in the sequence still reports what the
+// host said before it.
+func (p *Provider) probeHost(ctx context.Context, host provider.KVMHost, details map[string]string) error {
+	b, err := p.hosts.dialFresh(ctx, host)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = b.Close() }()
+
+	version, err := b.ConnectGetLibVersion(ctx)
+	if err != nil {
+		return err
+	}
+	details["libvirtVersion"] = formatLibVersion(version)
+	hostname, err := b.ConnectGetHostname(ctx)
+	if err != nil {
+		return err
+	}
+	details["hostname"] = hostname
+	pool, err := b.StoragePoolLookupByName(ctx, host.Pool)
+	if err != nil {
+		return err
+	}
+	state, capacity, _, available, err := b.StoragePoolGetInfo(ctx, pool)
+	if err != nil {
+		return err
+	}
+	details["poolState"] = poolStateName(state)
+	details["poolCapacityGiB"] = formatGiB(capacity)
+	details["poolAvailableGiB"] = formatGiB(available)
+	if libvirt.StoragePoolState(state) != libvirt.StoragePoolRunning {
+		return &poolNotActiveError{pool: host.Pool, state: poolStateName(state)}
+	}
+	return nil
+}
+
+// poolNotActiveError is the one verdict that is not an error from the
+// backend: the pool answered, but is not usable.
+type poolNotActiveError struct {
+	pool  string
+	state string
+}
+
+func (e *poolNotActiveError) Error() string {
+	return fmt.Sprintf("storage pool %q is %s", e.pool, e.state)
+}
+
+// checkReason maps a failed probe to a reason. A dial failure is judged by
+// its stage, so a TCP connect or an SSH handshake that ran out of time is
+// Unreachable, not Timeout: the host did not answer. Timeout is for ctx
+// ending or a budget expiring once the connection was up.
+func checkReason(err error) string {
+	var de *dialError
+	if errors.As(err, &de) {
+		switch de.stage {
+		case stageCredentials:
+			return reasonCredentialsUnavailable
+		case stageSSH:
+			var keyErr *knownhosts.KeyError
+			if errors.As(de.err, &keyErr) || strings.Contains(de.err.Error(), sshAuthFailed) {
+				return provider.ReasonUnauthorized
+			}
+		}
+		return provider.ReasonUnreachable
+	}
+	switch {
+	case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
+		return provider.ReasonTimeout
+	case hasCode(err, libvirt.ErrAccessDenied), hasCode(err, libvirt.ErrAuthFailed):
+		return provider.ReasonForbidden
+	case hasCode(err, libvirt.ErrNoStoragePool):
+		return reasonPoolNotFound
+	}
+	if _, ok := libvirtCode(err); ok {
+		return provider.ReasonAPIError
+	}
+	return provider.ReasonUnreachable
+}
+
+// formatLibVersion decodes ConnectGetLibVersion's major*1_000_000 +
+// minor*1_000 + micro.
+func formatLibVersion(v uint64) string {
+	return fmt.Sprintf("%d.%d.%d", v/1_000_000, v/1_000%1_000, v%1_000)
+}
+
+// poolStateName uses virsh's words for a pool state.
+func poolStateName(state uint8) string {
+	switch libvirt.StoragePoolState(state) {
+	case libvirt.StoragePoolInactive:
+		return "inactive"
+	case libvirt.StoragePoolBuilding:
+		return "building"
+	case libvirt.StoragePoolRunning:
+		return "running"
+	case libvirt.StoragePoolDegraded:
+		return "degraded"
+	case libvirt.StoragePoolInaccessible:
+		return "inaccessible"
+	default:
+		return fmt.Sprintf("state %d", state)
+	}
+}
+
+func formatGiB(bytes uint64) string {
+	return fmt.Sprintf("%.1f", float64(bytes)/(1<<30))
+}