egress-proxies-operator/internal/provider/proxmox/check.go

package proxmox

import (
	"context"
	"errors"
	"fmt"
	"io"
	"maps"
	"net"
	"net/http"
	"slices"
	"strconv"
	"strings"
	"sync"
	"time"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

var _ provider.Checker = (*Provider)(nil)

// Reasons only this provider can tell apart (spec entry 16). Each names a
// failure an admin fixes somewhere other than where the shared reasons
// point.
const (
	// reasonCertificateInvalid: TLS verification failed or the CA file is
	// unusable; fix url or caFile, not the host.
	reasonCertificateInvalid = "CertificateInvalid"
	// reasonCredentialsUnavailable: the token file is missing or malformed;
	// fix the Secret, not Proxmox.
	reasonCredentialsUnavailable = "CredentialsUnavailable"
	// reasonStorageNotFound: isoStorage is missing, inactive, or holds no
	// iso content on the node; fix the storage on the node.
	reasonStorageNotFound = "StorageNotFound"
	// reasonTemplateNotFound: a configured template VMID does not exist or
	// is not a template; fix the template or the catalogue.
	reasonTemplateNotFound = "TemplateNotFound"
)

// pveproxy's own answers when it cannot reach the node behind it.
const (
	statusProxyConnect = 595
	statusProxyBroken  = 596
)

// Check verifies every configured node concurrently and returns one verdict
// per node in config order. The requests share the provider's HTTP client,
// but each is its own request, so a round that runs out of time cancels
// only its own requests and never a reconcile's.
func (p *Provider) Check(ctx context.Context) []provider.TargetStatus {
	out := make([]provider.TargetStatus, len(p.cfg.Nodes))
	var wg sync.WaitGroup
	for i, node := range p.cfg.Nodes {
		wg.Go(func() { out[i] = p.checkNode(ctx, node) })
	}
	wg.Wait()
	return out
}

func (p *Provider) checkNode(ctx context.Context, node string) provider.TargetStatus {
	start := time.Now()
	log := p.logger(ctx).WithValues("node", node)
	ts := provider.TargetStatus{
		Name:    node,
		Address: p.cfg.URL,
		Details: map[string]string{"isoStorage": p.cfg.ISOStorage},
	}
	err := p.probeNode(ctx, node, ts.Details)
	ts.Latency = time.Since(start)
	switch {
	case err == nil:
		ts.Ready = true
		ts.Reason = provider.ReasonReachable
	case ctx.Err() != nil:
		// The round ran out of time, whichever request was running.
		ts.Reason = provider.ReasonTimeout
		ts.Message = err.Error()
	default:
		ts.Reason = checkReason(err)
		ts.Message = err.Error()
	}
	log.V(1).Info("proxmox node checked", "ready", ts.Ready, "reason", ts.Reason, "latency", ts.Latency)
	return ts
}

// probeNode runs spec entry 16's four requests and fills details as facts
// arrive, so a failure late in the sequence still reports what came before.
func (p *Provider) probeNode(ctx context.Context, node string, details map[string]string) error {
	// The token file is read here only for its id: the secret after "="
	// is dropped at once and never reaches details, logs or errors.
	token, err := readToken(p.cfg.TokenFile)
	if err != nil {
		return err
	}
	tokenID, _, _ := strings.Cut(token, "=")
	details["tokenID"] = tokenID

	v, err := p.api.Version(ctx)
	if err != nil {
		return fmt.Errorf("reading the API version: %w", err)
	}
	details["pveVersion"] = v.Version
	ns, err := p.api.NodeStatus(ctx, node)
	if err != nil {
		return fmt.Errorf("reading the status of node %s: %w", node, err)
	}
	details["kernel"] = ns.CurrentKernel.Release
	if err := p.checkStorage(ctx, node, details); err != nil {
		return err
	}
	return p.checkTemplates(ctx, details)
}

// checkStorage reads isoStorage on the node. A storage the cluster does
// not know is a 400 on the storage parameter (captured). A 500 is taken
// the same way, whatever its wording: Proxmox cannot use that storage on
// this node (not enabled here, or offline; not captured). The message
// carries Proxmox's text.
func (p *Provider) checkStorage(ctx context.Context, node string, details map[string]string) error {
	storage := p.cfg.ISOStorage
	st, err := p.api.StorageStatus(ctx, node, storage)
	var ae *apiError
	if errors.As(err, &ae) {
		switch {
		case isNoSuchStorage(ae):
			return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s on node %s: %s", storage, node, ae.Errors["storage"])}
		case ae.Status == http.StatusInternalServerError:
			return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s on node %s: %s", storage, node, ae.Message)}
		}
	}
	if err != nil {
		return fmt.Errorf("reading storage %s on node %s: %w", storage, node, err)
	}
	details["isoStorageAvailGiB"] = formatGiB(st.Avail)
	if st.Active == 0 {
		return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s is not active on node %s", storage, node)}
	}
	if !slices.Contains(strings.Split(st.Content, ","), "iso") {
		return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s on node %s does not hold iso content (content: %s)", storage, node, st.Content)}
	}
	return nil
}

// checkTemplates looks every catalogue entry up in the cluster's resources:
// a template may sit on any node, since a clone can target another.
func (p *Provider) checkTemplates(ctx context.Context, details map[string]string) error {
	rs, err := p.api.ClusterResources(ctx)
	if err != nil {
		return fmt.Errorf("listing cluster resources: %w", err)
	}
	isTemplate := make(map[int]bool)
	for _, r := range rs {
		if r.Type == "qemu" {
			isTemplate[r.VMID] = r.Template == 1
		}
	}
	var problems []string
	for _, key := range slices.Sorted(maps.Keys(p.cfg.Templates)) {
		vmid := p.cfg.Templates[key].VMID
		switch tmpl, found := isTemplate[vmid]; {
		case !found:
			problems = append(problems, fmt.Sprintf("template %q: VM %d does not exist", key, vmid))
		case !tmpl:
			problems = append(problems, fmt.Sprintf("template %q: VM %d is not a template", key, vmid))
		}
	}
	details["templates"] = fmt.Sprintf("%d/%d", len(p.cfg.Templates)-len(problems), len(p.cfg.Templates))
	if len(problems) > 0 {
		return &checkFailure{reason: reasonTemplateNotFound, msg: strings.Join(problems, "; ")}
	}
	return nil
}

// checkFailure is a verdict the backend answered but the check refuses: the
// storage or a template is not usable.
type checkFailure struct {
	reason string
	msg    string
}

func (e *checkFailure) Error() string { return e.msg }

// checkReason maps a failed probe to a reason once checkNode has ruled out
// the round's ctx ending. A failed connect or a cut connection is
// Unreachable; a request timeout with the round still running is Timeout.
func checkReason(err error) string {
	var (
		cf *checkFailure
		ae *apiError
		op *net.OpError
		ne net.Error
	)
	switch {
	case errors.As(err, &cf):
		return cf.reason
	case errors.Is(err, errCredentials):
		return reasonCredentialsUnavailable
	case errors.Is(err, errCAFile), isTLSVerification(err):
		return reasonCertificateInvalid
	case errors.As(err, &ae):
		switch ae.Status {
		case http.StatusUnauthorized:
			return provider.ReasonUnauthorized
		case http.StatusForbidden:
			return provider.ReasonForbidden
		case statusProxyConnect, statusProxyBroken:
			return provider.ReasonUnreachable
		}
		return provider.ReasonAPIError
	case errors.As(err, &op):
		// Checked before the deadline: a dial that runs out of its own
		// budget is a node that did not answer.
		return provider.ReasonUnreachable
	case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
		return provider.ReasonTimeout
	case errors.As(err, &ne), errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
		return provider.ReasonUnreachable
	}
	return provider.ReasonAPIError
}

func formatGiB(bytes int64) string {
	return strconv.FormatFloat(float64(bytes)/(1<<30), 'f', 1, 64)
}