package proxmox import ( "context" "errors" "fmt" "io" "maps" "net" "net/http" "slices" "strconv" "strings" "sync" "time" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider" ) var _ provider.Checker = (*Provider)(nil) // Reasons only this provider can tell apart (spec entry 16). Each names a // failure an admin fixes somewhere other than where the shared reasons // point. const ( // reasonCertificateInvalid: TLS verification failed or the CA file is // unusable; fix url or caFile, not the host. reasonCertificateInvalid = "CertificateInvalid" // reasonCredentialsUnavailable: the token file is missing or malformed; // fix the Secret, not Proxmox. reasonCredentialsUnavailable = "CredentialsUnavailable" // reasonStorageNotFound: isoStorage is missing, inactive, or holds no // iso content on the node; fix the storage on the node. reasonStorageNotFound = "StorageNotFound" // reasonTemplateNotFound: a configured template VMID does not exist or // is not a template; fix the template or the catalogue. reasonTemplateNotFound = "TemplateNotFound" ) // pveproxy's own answers when it cannot reach the node behind it. const ( statusProxyConnect = 595 statusProxyBroken = 596 ) // Check verifies every configured node concurrently and returns one verdict // per node in config order. The requests share the provider's HTTP client, // but each is its own request, so a round that runs out of time cancels // only its own requests and never a reconcile's. func (p *Provider) Check(ctx context.Context) []provider.TargetStatus { out := make([]provider.TargetStatus, len(p.cfg.Nodes)) var wg sync.WaitGroup for i, node := range p.cfg.Nodes { wg.Go(func() { out[i] = p.checkNode(ctx, node) }) } wg.Wait() return out } func (p *Provider) checkNode(ctx context.Context, node string) provider.TargetStatus { start := time.Now() log := p.logger(ctx).WithValues("node", node) ts := provider.TargetStatus{ Name: node, Address: p.cfg.URL, Details: map[string]string{"isoStorage": p.cfg.ISOStorage}, } err := p.probeNode(ctx, node, ts.Details) ts.Latency = time.Since(start) switch { case err == nil: ts.Ready = true ts.Reason = provider.ReasonReachable case ctx.Err() != nil: // The round ran out of time, whichever request was running. ts.Reason = provider.ReasonTimeout ts.Message = err.Error() default: ts.Reason = checkReason(err) ts.Message = err.Error() } log.V(1).Info("proxmox node checked", "ready", ts.Ready, "reason", ts.Reason, "latency", ts.Latency) return ts } // probeNode runs spec entry 16's four requests and fills details as facts // arrive, so a failure late in the sequence still reports what came before. func (p *Provider) probeNode(ctx context.Context, node string, details map[string]string) error { // The token file is read here only for its id: the secret after "=" // is dropped at once and never reaches details, logs or errors. token, err := readToken(p.cfg.TokenFile) if err != nil { return err } tokenID, _, _ := strings.Cut(token, "=") details["tokenID"] = tokenID v, err := p.api.Version(ctx) if err != nil { return fmt.Errorf("reading the API version: %w", err) } details["pveVersion"] = v.Version ns, err := p.api.NodeStatus(ctx, node) if err != nil { return fmt.Errorf("reading the status of node %s: %w", node, err) } details["kernel"] = ns.CurrentKernel.Release if err := p.checkStorage(ctx, node, details); err != nil { return err } return p.checkTemplates(ctx, details) } // checkStorage reads isoStorage on the node. A storage the cluster does // not know is a 400 on the storage parameter (captured). A 500 is taken // the same way, whatever its wording: Proxmox cannot use that storage on // this node (not enabled here, or offline; not captured). The message // carries Proxmox's text. func (p *Provider) checkStorage(ctx context.Context, node string, details map[string]string) error { storage := p.cfg.ISOStorage st, err := p.api.StorageStatus(ctx, node, storage) var ae *apiError if errors.As(err, &ae) { switch { case isNoSuchStorage(ae): return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s on node %s: %s", storage, node, ae.Errors["storage"])} case ae.Status == http.StatusInternalServerError: return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s on node %s: %s", storage, node, ae.Message)} } } if err != nil { return fmt.Errorf("reading storage %s on node %s: %w", storage, node, err) } details["isoStorageAvailGiB"] = formatGiB(st.Avail) if st.Active == 0 { return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s is not active on node %s", storage, node)} } if !slices.Contains(strings.Split(st.Content, ","), "iso") { return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s on node %s does not hold iso content (content: %s)", storage, node, st.Content)} } return nil } // checkTemplates looks every catalogue entry up in the cluster's resources: // a template may sit on any node, since a clone can target another. func (p *Provider) checkTemplates(ctx context.Context, details map[string]string) error { rs, err := p.api.ClusterResources(ctx) if err != nil { return fmt.Errorf("listing cluster resources: %w", err) } isTemplate := make(map[int]bool) for _, r := range rs { if r.Type == "qemu" { isTemplate[r.VMID] = r.Template == 1 } } var problems []string for _, key := range slices.Sorted(maps.Keys(p.cfg.Templates)) { vmid := p.cfg.Templates[key].VMID switch tmpl, found := isTemplate[vmid]; { case !found: problems = append(problems, fmt.Sprintf("template %q: VM %d does not exist", key, vmid)) case !tmpl: problems = append(problems, fmt.Sprintf("template %q: VM %d is not a template", key, vmid)) } } details["templates"] = fmt.Sprintf("%d/%d", len(p.cfg.Templates)-len(problems), len(p.cfg.Templates)) if len(problems) > 0 { return &checkFailure{reason: reasonTemplateNotFound, msg: strings.Join(problems, "; ")} } return nil } // checkFailure is a verdict the backend answered but the check refuses: the // storage or a template is not usable. type checkFailure struct { reason string msg string } func (e *checkFailure) Error() string { return e.msg } // checkReason maps a failed probe to a reason once checkNode has ruled out // the round's ctx ending. A failed connect or a cut connection is // Unreachable; a request timeout with the round still running is Timeout. func checkReason(err error) string { var ( cf *checkFailure ae *apiError op *net.OpError ne net.Error ) switch { case errors.As(err, &cf): return cf.reason case errors.Is(err, errCredentials): return reasonCredentialsUnavailable case errors.Is(err, errCAFile), isTLSVerification(err): return reasonCertificateInvalid case errors.As(err, &ae): switch ae.Status { case http.StatusUnauthorized: return provider.ReasonUnauthorized case http.StatusForbidden: return provider.ReasonForbidden case statusProxyConnect, statusProxyBroken: return provider.ReasonUnreachable } return provider.ReasonAPIError case errors.As(err, &op): // Checked before the deadline: a dial that runs out of its own // budget is a node that did not answer. return provider.ReasonUnreachable case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): return provider.ReasonTimeout case errors.As(err, &ne), errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF): return provider.ReasonUnreachable } return provider.ReasonAPIError } func formatGiB(bytes int64) string { return strconv.FormatFloat(float64(bytes)/(1<<30), 'f', 1, 64) }