package kvm2 import ( "context" "errors" "fmt" "strings" "sync" "time" "github.com/digitalocean/go-libvirt" "golang.org/x/crypto/ssh/knownhosts" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider" ) var _ provider.Checker = (*Provider)(nil) // Reasons only this provider can tell apart (spec entry 16). Each names a // failure an admin fixes somewhere other than where the shared reasons // point. const ( // reasonCredentialsUnavailable: the key or known_hosts file is missing // or unparsable; fix the Secret in the cluster, not the host. reasonCredentialsUnavailable = "CredentialsUnavailable" // reasonPoolNotFound: the configured pool is not defined on the host; // compare with `virsh pool-list --all`. reasonPoolNotFound = "PoolNotFound" // reasonPoolNotActive: the pool exists but is not running; `virsh // pool-start ` (and autostart) brings it back. reasonPoolNotActive = "PoolNotActive" ) // Check verifies every configured host concurrently, each on a fresh // dedicated connection: dial, libvirt version, hostname, pool lookup, pool // info, close. The pooled connection is never touched, so a check that runs // out of time closes only its own connection and cannot fail a reconcile's // call in flight. Every target gets a verdict, in config order. func (p *Provider) Check(ctx context.Context) []provider.TargetStatus { out := make([]provider.TargetStatus, len(p.cfg.Hosts)) var wg sync.WaitGroup for i, host := range p.cfg.Hosts { wg.Go(func() { out[i] = p.checkHost(ctx, host) }) } wg.Wait() return out } func (p *Provider) checkHost(ctx context.Context, host provider.KVMHost) provider.TargetStatus { start := time.Now() log := p.logger(ctx).WithValues("host", host.Name) ts := provider.TargetStatus{ Name: host.Name, Address: host.Address, Details: map[string]string{"user": host.User, "pool": host.Pool}, } err := p.probeHost(ctx, host, ts.Details) ts.Latency = time.Since(start) switch { case err == nil: ts.Ready = true ts.Reason = provider.ReasonReachable default: var inactive *poolNotActiveError if errors.As(err, &inactive) { ts.Reason = reasonPoolNotActive } else { ts.Reason = checkReason(err) } ts.Message = err.Error() } log.V(1).Info("kvm2 host checked", "ready", ts.Ready, "reason", ts.Reason, "latency", ts.Latency) return ts } // probeHost runs the check sequence on its own connection and fills details // as facts arrive, so a failure late in the sequence still reports what the // host said before it. func (p *Provider) probeHost(ctx context.Context, host provider.KVMHost, details map[string]string) error { b, err := p.hosts.dialFresh(ctx, host) if err != nil { return err } defer func() { _ = b.Close() }() version, err := b.ConnectGetLibVersion(ctx) if err != nil { return err } details["libvirtVersion"] = formatLibVersion(version) hostname, err := b.ConnectGetHostname(ctx) if err != nil { return err } details["hostname"] = hostname pool, err := b.StoragePoolLookupByName(ctx, host.Pool) if err != nil { return err } state, capacity, _, available, err := b.StoragePoolGetInfo(ctx, pool) if err != nil { return err } details["poolState"] = poolStateName(state) details["poolCapacityGiB"] = formatGiB(capacity) details["poolAvailableGiB"] = formatGiB(available) if libvirt.StoragePoolState(state) != libvirt.StoragePoolRunning { return &poolNotActiveError{pool: host.Pool, state: poolStateName(state)} } return nil } // poolNotActiveError is the one verdict that is not an error from the // backend: the pool answered, but is not usable. type poolNotActiveError struct { pool string state string } func (e *poolNotActiveError) Error() string { return fmt.Sprintf("storage pool %q is %s", e.pool, e.state) } // checkReason maps a failed probe to a reason. A dial failure is judged by // its stage, so a TCP connect or an SSH handshake that ran out of time is // Unreachable, not Timeout: the host did not answer. Timeout is for ctx // ending or a budget expiring once the connection was up. func checkReason(err error) string { var de *dialError if errors.As(err, &de) { switch de.stage { case stageCredentials: return reasonCredentialsUnavailable case stageSSH: var keyErr *knownhosts.KeyError if errors.As(de.err, &keyErr) || strings.Contains(de.err.Error(), sshAuthFailed) { return provider.ReasonUnauthorized } } return provider.ReasonUnreachable } switch { case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): return provider.ReasonTimeout case hasCode(err, libvirt.ErrAccessDenied), hasCode(err, libvirt.ErrAuthFailed): return provider.ReasonForbidden case hasCode(err, libvirt.ErrNoStoragePool): return reasonPoolNotFound } if _, ok := libvirtCode(err); ok { return provider.ReasonAPIError } return provider.ReasonUnreachable } // formatLibVersion decodes ConnectGetLibVersion's major*1_000_000 + // minor*1_000 + micro. func formatLibVersion(v uint64) string { return fmt.Sprintf("%d.%d.%d", v/1_000_000, v/1_000%1_000, v%1_000) } // poolStateName uses virsh's words for a pool state. func poolStateName(state uint8) string { switch libvirt.StoragePoolState(state) { case libvirt.StoragePoolInactive: return "inactive" case libvirt.StoragePoolBuilding: return "building" case libvirt.StoragePoolRunning: return "running" case libvirt.StoragePoolDegraded: return "degraded" case libvirt.StoragePoolInaccessible: return "inaccessible" default: return fmt.Sprintf("state %d", state) } } func formatGiB(bytes uint64) string { return fmt.Sprintf("%.1f", float64(bytes)/(1<<30)) }