egress-proxies-operator/internal/provider/proxmox/check_test.go

package proxmox

import (
	"context"
	"crypto/x509"
	"errors"
	"fmt"
	"io"
	"maps"
	"net"
	"net/http"
	"net/url"
	"path/filepath"
	"slices"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

// checkSecret is the secret half of the Checker's token file; it must never
// reach a verdict or a log line.
const checkSecret = "SENTINEL-token-secret-4c1d"

// newCheckFixture is an observed create fixture at V(2) with a token file
// whose secret is checkSecret.
func newCheckFixture(t *testing.T) *fixture {
	t.Helper()
	fx := newCreateFixture(t, 2)
	fx.p.api = observe(fx.fake, testProvider)
	fx.p.cfg.TokenFile = filepath.Join(t.TempDir(), "token")
	writeFile(t, fx.p.cfg.TokenFile, "proxy-operator@pve!operator="+checkSecret+"\n")
	return fx
}

// assertNoSecret checks every verdict and every log line for checkSecret.
func assertNoSecret(t *testing.T, fx *fixture, got []provider.TargetStatus) {
	t.Helper()
	for _, ts := range got {
		if strings.Contains(ts.Message, checkSecret) || slices.ContainsFunc(slices.Collect(maps.Values(ts.Details)), func(v string) bool { return strings.Contains(v, checkSecret) }) {
			t.Errorf("verdict carries the token secret: %+v", ts)
		}
	}
	for _, l := range fx.logs.all() {
		if strings.Contains(l, checkSecret) {
			t.Errorf("log line carries the token secret: %s", l)
		}
	}
}

func TestCheck_oneEntryPerNodeInConfigOrder(t *testing.T) {
	t.Parallel()
	fx := newCheckFixture(t)
	fx.p.cfg.Nodes = []string{"pv1-3", fakeNode, "pv1-2"}
	fx.fake.nodes["pv1-3"] = fx.fake.nodes[fakeNode]
	got := fx.p.Check(fx.ctx)
	if len(got) != len(fx.p.cfg.Nodes) {
		t.Fatalf("Check() returned %d targets, want %d", len(got), len(fx.p.cfg.Nodes))
	}
	for i, node := range fx.p.cfg.Nodes {
		ts := got[i]
		if ts.Name != node || ts.Address != fx.p.cfg.URL {
			t.Errorf("target %d = %s (%s), want %s (%s)", i, ts.Name, ts.Address, node, fx.p.cfg.URL)
		}
		if wantReady := node != "pv1-2"; ts.Ready != wantReady {
			t.Errorf("target %s ready = %v, want %v (%s: %s)", node, ts.Ready, wantReady, ts.Reason, ts.Message)
		}
	}
	if got[2].Reason != provider.ReasonAPIError || !strings.Contains(got[2].Message, "pv1-2") {
		t.Errorf("pv1-2 verdict = %s: %s, want APIError naming the node", got[2].Reason, got[2].Message)
	}
}

// TestCheck_nodesCheckedConcurrently holds every node's first request
// until all of them have arrived; checked one at a time, the first would
// give up and fail its verdict.
func TestCheck_nodesCheckedConcurrently(t *testing.T) {
	t.Parallel()
	fx := newCheckFixture(t)
	fx.p.cfg.Nodes = []string{fakeNode, "pv1-2", "pv1-3"}
	for _, n := range fx.p.cfg.Nodes[1:] {
		fx.fake.nodes[n] = fx.fake.nodes[fakeNode]
	}
	var (
		mu      sync.Mutex
		arrived int
	)
	all := make(chan struct{})
	fx.fake.hookOn("Version", func() error {
		mu.Lock()
		if arrived++; arrived == len(fx.p.cfg.Nodes) {
			close(all)
		}
		mu.Unlock()
		select {
		case <-all:
			return nil
		case <-time.After(2 * time.Second):
			return errors.New("barrier: the nodes were not checked concurrently")
		}
	})
	for _, ts := range fx.p.Check(fx.ctx) {
		if !ts.Ready {
			t.Errorf("target %s = %s: %s, want Ready", ts.Name, ts.Reason, ts.Message)
		}
	}
}

func TestCheck_expiredContextReturnsPromptly(t *testing.T) {
	t.Parallel()
	fx := newCheckFixture(t)
	fx.p.cfg.Nodes = []string{fakeNode, "pv1-2"}
	fx.fake.blockOn("Version")
	ctx, cancel := context.WithTimeout(fx.ctx, 0)
	defer cancel()
	start := time.Now()
	var got []provider.TargetStatus
	within(t, 5*time.Second, func() { got = fx.p.Check(ctx) })
	if elapsed := time.Since(start); elapsed > time.Second {
		t.Errorf("Check() took %v with an expired ctx, want under 1s", elapsed)
	}
	if len(got) != 2 {
		t.Fatalf("Check() returned %d targets, want 2", len(got))
	}
	for _, ts := range got {
		if ts.Ready || ts.Reason != provider.ReasonTimeout || !strings.Contains(ts.Message, context.DeadlineExceeded.Error()) {
			t.Errorf("target %s = ready %v reason %s (%s), want Timeout naming the deadline", ts.Name, ts.Ready, ts.Reason, ts.Message)
		}
	}
}

func TestCheck_reasonsTable(t *testing.T) {
	t.Parallel()
	urlErr := func(err error) error {
		return &url.Error{Op: "Get", URL: "https://192.0.2.8:8006/api2/json/version", Err: err}
	}
	tests := []struct {
		name    string
		prep    func(fx *fixture)
		timeout time.Duration
		cancel  bool

		wantReady   bool
		wantReason  string
		wantMessage string
	}{
		{name: "ok", wantReady: true, wantReason: provider.ReasonReachable},
		{
			name:        "token file missing",
			prep:        func(fx *fixture) { fx.p.cfg.TokenFile = filepath.Join(fx.t.TempDir(), "absent") },
			wantReason:  reasonCredentialsUnavailable,
			wantMessage: "does not exist yet",
		},
		{
			name:        "token file malformed",
			prep:        func(fx *fixture) { writeFile(fx.t, fx.p.cfg.TokenFile, checkSecret+"\n") },
			wantReason:  reasonCredentialsUnavailable,
			wantMessage: "is not <user>@<realm>!<tokenid>=<secret>",
		},
		{
			name: "token file gone by the time the client reads it",
			prep: func(fx *fixture) {
				fx.fake.failWith("NodeStatus", fmt.Errorf("%w: %w: /etc/proxy-operator/proxmox/token does not exist yet", provider.ErrTransient, errCredentials))
			},
			wantReason: reasonCredentialsUnavailable,
		},
		{
			name:       "unknown certificate authority",
			prep:       func(fx *fixture) { fx.fake.failWith("Version", urlErr(x509.UnknownAuthorityError{})) },
			wantReason: reasonCertificateInvalid,
		},
		{
			name:       "host not in the certificate",
			prep:       func(fx *fixture) { fx.fake.failWith("Version", urlErr(x509.HostnameError{Host: "192.168.2.8"})) },
			wantReason: reasonCertificateInvalid,
		},
		{
			name: "CA file unusable",
			prep: func(fx *fixture) {
				fx.fake.failWith("Version", fmt.Errorf("%w: %w: reading CA file: %w", provider.ErrTransient, errCAFile, syscall.ENOENT))
			},
			wantReason: reasonCertificateInvalid,
		},
		{
			name:       "wrong secret",
			prep:       func(fx *fixture) { fx.fake.failWith("Version", loadAPIError(fx.t, "err_unauthorized")) },
			wantReason: provider.ReasonUnauthorized,
		},
		{
			name:       "no credentials sent",
			prep:       func(fx *fixture) { fx.fake.failWith("Version", loadAPIError(fx.t, "err_no_credentials")) },
			wantReason: provider.ReasonUnauthorized,
		},
		{
			name:        "privilege missing",
			prep:        func(fx *fixture) { fx.fake.failWith("NodeStatus", loadAPIError(fx.t, "err_forbidden")) },
			wantReason:  provider.ReasonForbidden,
			wantMessage: "Sys.Syslog",
		},
		{
			name:       "pveproxy cannot reach the node",
			prep:       func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(595, "Connection refused")) },
			wantReason: provider.ReasonUnreachable,
		},
		{
			name:       "pveproxy broken pipe",
			prep:       func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(596, "Broken pipe")) },
			wantReason: provider.ReasonUnreachable,
		},
		{
			name: "connection refused",
			prep: func(fx *fixture) {
				fx.fake.failWith("Version", urlErr(&net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}))
			},
			wantReason: provider.ReasonUnreachable,
		},
		{
			name:       "node status 500",
			prep:       func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(500, "fake: pve-cluster not running")) },
			wantReason: provider.ReasonAPIError,
		},
		{
			name: "iso storage missing",
			prep: func(fx *fixture) {
				fx.fake.mu.Lock()
				delete(fx.fake.storages, fakeISOStorage)
				fx.fake.mu.Unlock()
			},
			wantReason:  reasonStorageNotFound,
			wantMessage: "storage proxy-seeds on node pv1-1: No such storage.",
		},
		{
			// Not captured: a storage Proxmox knows but cannot use here.
			name: "iso storage unusable on the node",
			prep: func(fx *fixture) {
				fx.fake.failWith("StorageStatus", fakeErr(500, "fake: storage 'proxy-seeds' is not online"))
			},
			wantReason:  reasonStorageNotFound,
			wantMessage: "storage proxy-seeds on node pv1-1: fake: storage 'proxy-seeds' is not online",
		},
		{
			name:       "another bad parameter is not a missing storage",
			prep:       func(fx *fixture) { fx.fake.failWith("StorageStatus", loadAPIError(fx.t, "err_bad_param")) },
			wantReason: provider.ReasonAPIError,
		},
		{
			name:        "iso storage inactive",
			prep:        func(fx *fixture) { setStorage(fx, func(s *storageStatus) { s.Active = 0 }) },
			wantReason:  reasonStorageNotFound,
			wantMessage: "not active",
		},
		{
			name:        "iso storage without iso content",
			prep:        func(fx *fixture) { setStorage(fx, func(s *storageStatus) { s.Content = "images,rootdir,isoish" }) },
			wantReason:  reasonStorageNotFound,
			wantMessage: "does not hold iso content (content: images,rootdir,isoish)",
		},
		{
			name:       "iso storage forbidden",
			prep:       func(fx *fixture) { fx.fake.failWith("StorageStatus", loadAPIError(fx.t, "err_forbidden")) },
			wantReason: provider.ReasonForbidden,
		},
		{
			name: "template missing",
			prep: func(fx *fixture) {
				fx.fake.mu.Lock()
				delete(fx.fake.vms, testTemplateVMID)
				fx.fake.mu.Unlock()
			},
			wantReason:  reasonTemplateNotFound,
			wantMessage: `template "debian-13": VM 9001 does not exist`,
		},
		{
			name:        "template VMID holds a plain VM",
			prep:        func(fx *fixture) { fx.fake.vms[testTemplateVMID].template = false },
			wantReason:  reasonTemplateNotFound,
			wantMessage: `template "debian-13": VM 9001 is not a template`,
		},
		{
			name:        "template VMID holds a container",
			prep:        func(fx *fixture) { fx.fake.vms[testTemplateVMID].kind = "lxc" },
			wantReason:  reasonTemplateNotFound,
			wantMessage: "does not exist",
		},
		{
			name: "resources listing fails",
			prep: func(fx *fixture) {
				fx.fake.failWith("ClusterResources", fakeErr(500, "fake: cluster filesystem not ready"))
			},
			wantReason: provider.ReasonAPIError,
		},
		{
			name:       "request outlives the deadline",
			prep:       func(fx *fixture) { fx.fake.blockOn("StorageStatus") },
			timeout:    50 * time.Millisecond,
			wantReason: provider.ReasonTimeout,
		},
		{
			// The real client's dial sees the round's ctx end and fails as
			// a dial error, which checkReason alone would call Unreachable.
			name: "dial cut short by the round's deadline",
			prep: func(fx *fixture) {
				fx.fake.hookOn("Version", func() error {
					time.Sleep(200 * time.Millisecond)
					return urlErr(&net.OpError{Op: "dial", Net: "tcp", Err: context.DeadlineExceeded})
				})
			},
			timeout:    20 * time.Millisecond,
			wantReason: provider.ReasonTimeout,
		},
		{
			name:       "ctx cancelled mid check",
			prep:       func(fx *fixture) { fx.fake.blockOn("ClusterResources") },
			cancel:     true,
			wantReason: provider.ReasonTimeout,
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fx := newCheckFixture(t)
			if tc.prep != nil {
				tc.prep(fx)
			}
			ctx := fx.ctx
			switch {
			case tc.timeout > 0:
				var cancel context.CancelFunc
				ctx, cancel = context.WithTimeout(ctx, tc.timeout)
				defer cancel()
			case tc.cancel:
				var cancel context.CancelFunc
				ctx, cancel = context.WithCancel(ctx)
				time.AfterFunc(50*time.Millisecond, cancel)
			}
			var got []provider.TargetStatus
			within(t, 5*time.Second, func() { got = fx.p.Check(ctx) })
			if len(got) != 1 {
				t.Fatalf("Check() returned %d targets, want 1", len(got))
			}
			ts := got[0]
			if ts.Ready != tc.wantReady || ts.Reason != tc.wantReason {
				t.Errorf("verdict = ready %v reason %s (%s), want ready %v reason %s", ts.Ready, ts.Reason, ts.Message, tc.wantReady, tc.wantReason)
			}
			if !tc.wantReady && ts.Message == "" {
				t.Error("failed check has no message")
			}
			if tc.wantReady && ts.Message != "" {
				t.Errorf("successful check has a message: %q", ts.Message)
			}
			if tc.wantMessage != "" && !strings.Contains(ts.Message, tc.wantMessage) {
				t.Errorf("message = %q, want it to contain %q", ts.Message, tc.wantMessage)
			}
			if ts.Details["isoStorage"] != fakeISOStorage {
				t.Errorf("details = %v, want isoStorage whatever failed", ts.Details)
			}
			assertNoSecret(t, fx, got)
		})
	}
}

func setStorage(fx *fixture, fn func(s *storageStatus)) {
	fx.fake.mu.Lock()
	defer fx.fake.mu.Unlock()
	s := fx.fake.storages[fakeISOStorage]
	fn(&s)
	fx.fake.storages[fakeISOStorage] = s
}

// TestCheck_reasonsOverHTTPClient takes the transport-level reasons from
// the real client, so the errors checkReason sees are the ones net/http and
// crypto/tls produce, and the Ready row replays the Phase 0 captures.
func TestCheck_reasonsOverHTTPClient(t *testing.T) {
	t.Parallel()
	captures := func(t *testing.T) http.HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request) {
			path := strings.TrimPrefix(r.URL.Path, "/api2/json/")
			name := map[string]string{
				"version":                          "version",
				"nodes/pv1-1/status":               "node_status",
				"nodes/pv1-1/storage/local/status": "storage_status",
				"cluster/resources":                "cluster_resources",
			}[path]
			if name == "" {
				t.Errorf("unexpected request %s", path)
				http.NotFound(w, r)
				return
			}
			serveJSON(w, readCapture(t, name).Body)
		}
	}
	tests := []struct {
		name       string
		handler    func(t *testing.T) http.HandlerFunc
		mutate     func(t *testing.T, cfx *clientFixture)
		wantReason string
	}{
		{name: "captures", handler: captures, wantReason: provider.ReasonReachable},
		{name: "unknown authority", handler: captures, mutate: func(_ *testing.T, cfx *clientFixture) { cfx.cfg.CAFile = "" }, wantReason: reasonCertificateInvalid},
		{
			name:    "host not in the certificate",
			handler: captures,
			mutate: func(_ *testing.T, cfx *clientFixture) {
				cfx.cfg.URL = strings.Replace(cfx.srv.URL, "127.0.0.1", "localhost", 1)
			},
			wantReason: reasonCertificateInvalid,
		},
		{
			name:       "CA file missing",
			handler:    captures,
			mutate:     func(t *testing.T, cfx *clientFixture) { cfx.cfg.CAFile = filepath.Join(t.TempDir(), "ca.pem") },
			wantReason: reasonCertificateInvalid,
		},
		{
			name: "401 from pveproxy",
			handler: func(t *testing.T) http.HandlerFunc {
				return func(w http.ResponseWriter, _ *http.Request) { serveCapture(t, w, readCapture(t, "err_unauthorized")) }
			},
			wantReason: provider.ReasonUnauthorized,
		},
		{
			name: "596 from pveproxy",
			handler: func(t *testing.T) http.HandlerFunc {
				return func(w http.ResponseWriter, _ *http.Request) {
					serveCapture(t, w, capture{Status: 596, Reason: "Broken pipe"})
				}
			},
			wantReason: provider.ReasonUnreachable,
		},
		{
			name:       "server gone",
			handler:    captures,
			mutate:     func(_ *testing.T, cfx *clientFixture) { cfx.srv.Close() },
			wantReason: provider.ReasonUnreachable,
		},
		{
			name: "connection cut mid response",
			handler: func(t *testing.T) http.HandlerFunc {
				return func(w http.ResponseWriter, _ *http.Request) {
					conn, _, err := w.(http.Hijacker).Hijack()
					if err != nil {
						t.Errorf("hijack: %v", err)
						return
					}
					_, _ = io.WriteString(conn, "HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{\"data\":")
					_ = conn.Close()
				}
			},
			wantReason: provider.ReasonUnreachable,
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			cfx := newClientFixture(t, tc.handler(t))
			if tc.mutate != nil {
				tc.mutate(t, cfx)
			}
			cfg := testConfig()
			cfg.ISOStorage = "local"
			cfg.URL, cfg.TokenFile, cfg.CAFile = cfx.cfg.URL, cfx.cfg.TokenFile, cfx.cfg.CAFile
			p := newProvider(testProvider, cfg, cfx.api(t))
			got := p.Check(context.Background())
			if len(got) != 1 {
				t.Fatalf("Check() returned %d targets, want 1", len(got))
			}
			if ts := got[0]; ts.Reason != tc.wantReason || ts.Ready != (tc.wantReason == provider.ReasonReachable) {
				t.Errorf("verdict = ready %v reason %s (%s), want %s", ts.Ready, ts.Reason, ts.Message, tc.wantReason)
			}
		})
	}
}

func TestCheck_detailsKeys(t *testing.T) {
	t.Parallel()
	full := map[string]string{
		"tokenID": "proxy-operator@pve!operator", "isoStorage": fakeISOStorage, "pveVersion": "9.2.20",
		"kernel": "7.0.14-17-pve", "isoStorageAvailGiB": "90.0", "templates": "1/1",
	}
	tests := []struct {
		name string
		prep func(fx *fixture)
		want []string
	}{
		{name: "success", want: slices.Sorted(maps.Keys(full))},
		{name: "token unusable", prep: func(fx *fixture) { writeFile(fx.t, fx.p.cfg.TokenFile, "junk") }, want: []string{"isoStorage"}},
		{name: "node status fails", prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(596, "Broken pipe")) }, want: []string{"isoStorage", "pveVersion", "tokenID"}},
		{name: "storage inactive", prep: func(fx *fixture) { setStorage(fx, func(s *storageStatus) { s.Active = 0 }) }, want: []string{"isoStorage", "isoStorageAvailGiB", "kernel", "pveVersion", "tokenID"}},
		{name: "template missing", prep: func(fx *fixture) { fx.fake.vms[testTemplateVMID].template = false }, want: slices.Sorted(maps.Keys(full))},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fx := newCheckFixture(t)
			if tc.prep != nil {
				tc.prep(fx)
			}
			got := fx.p.Check(fx.ctx)[0]
			if keys := slices.Sorted(maps.Keys(got.Details)); !slices.Equal(keys, tc.want) {
				t.Errorf("details keys = %v, want %v", keys, tc.want)
			}
			for k, v := range got.Details {
				if want, ok := full[k]; ok && tc.name == "success" && v != want {
					t.Errorf("details[%s] = %q, want %q", k, v, want)
				}
				if v == "" {
					t.Errorf("details[%s] is empty", k)
				}
			}
		})
	}
	t.Run("templates counts found over configured", func(t *testing.T) {
		t.Parallel()
		fx := newCheckFixture(t)
		fx.p.cfg.Templates["ubuntu"] = provider.ProxmoxTemplate{VMID: 9002}
		fx.p.cfg.Templates["debian-again"] = provider.ProxmoxTemplate{VMID: testTemplateVMID}
		got := fx.p.Check(fx.ctx)[0]
		if got.Details["templates"] != "2/3" || got.Reason != reasonTemplateNotFound || !strings.Contains(got.Message, `template "ubuntu": VM 9002`) {
			t.Errorf("verdict = %s %q, details %v; want TemplateNotFound naming ubuntu, templates 2/3", got.Reason, got.Message, got.Details)
		}
	})
}

func TestCheck_logsVerdictPerNode(t *testing.T) {
	t.Parallel()
	fx := newCheckFixture(t)
	fx.p.cfg.Nodes = []string{fakeNode, "pv1-2"}
	fx.p.Check(fx.ctx)
	checked := fx.logs.matching(`"msg"="proxmox node checked"`)
	if len(checked) != 2 {
		t.Fatalf("got %d verdict lines, want 2: %v", len(checked), checked)
	}
	for _, l := range checked {
		switch {
		case strings.Contains(l, `"node"="pv1-1"`):
			if !strings.Contains(l, `"ready"=true`) || !strings.Contains(l, `"reason"="Reachable"`) || !strings.Contains(l, `"latency"=`) {
				t.Errorf("pv1-1 verdict line = %s", l)
			}
		case strings.Contains(l, `"node"="pv1-2"`):
			if !strings.Contains(l, `"ready"=false`) || !strings.Contains(l, `"reason"="APIError"`) {
				t.Errorf("pv1-2 verdict line = %s", l)
			}
		default:
			t.Errorf("verdict line without a node: %s", l)
		}
		if !strings.Contains(l, `"provider"="`+testProvider+`"`) || !strings.Contains(l, `"level"=1`) {
			t.Errorf("verdict line lacks the provider or is not V(1): %s", l)
		}
	}
}

func TestCheckReason_table(t *testing.T) {
	t.Parallel()
	dial := &url.Error{Op: "Get", URL: "https://192.0.2.8:8006/api2/json/version", Err: &net.OpError{Op: "dial", Net: "tcp", Err: context.DeadlineExceeded}}
	tests := []struct {
		name string
		err  error
		want string
	}{
		{name: "storage verdict", err: &checkFailure{reason: reasonStorageNotFound, msg: "x"}, want: reasonStorageNotFound},
		{name: "template verdict wrapped", err: fmt.Errorf("check: %w", &checkFailure{reason: reasonTemplateNotFound, msg: "x"}), want: reasonTemplateNotFound},
		{name: "token missing", err: fmt.Errorf("%w: %w: /t does not exist yet", provider.ErrTransient, errCredentials), want: reasonCredentialsUnavailable},
		{name: "token malformed", err: fmt.Errorf("%w: %w: /t is not a token", provider.ErrPermanent, errCredentials), want: reasonCredentialsUnavailable},
		{name: "CA file", err: fmt.Errorf("%w: %w: /ca holds no PEM certificate", provider.ErrPermanent, errCAFile), want: reasonCertificateInvalid},
		{name: "unknown authority", err: &url.Error{Err: x509.UnknownAuthorityError{}}, want: reasonCertificateInvalid},
		{name: "certificate invalid", err: &url.Error{Err: x509.CertificateInvalidError{Reason: x509.Expired}}, want: reasonCertificateInvalid},
		{name: "401", err: loadAPIError(t, "err_unauthorized"), want: provider.ReasonUnauthorized},
		{name: "403 wrapped", err: fmt.Errorf("reading: %w", loadAPIError(t, "err_forbidden_volume_delete")), want: provider.ReasonForbidden},
		{name: "595", err: fakeErr(595, "Connection refused"), want: provider.ReasonUnreachable},
		{name: "596", err: fakeErr(596, "Broken pipe"), want: provider.ReasonUnreachable},
		{name: "400", err: loadAPIError(t, "err_bad_param"), want: provider.ReasonAPIError},
		{name: "500", err: loadAPIError(t, "err_vm_missing"), want: provider.ReasonAPIError},
		{name: "dial ran out of its own budget", err: dial, want: provider.ReasonUnreachable},
		{name: "connection reset", err: &url.Error{Err: &net.OpError{Op: "read", Net: "tcp", Err: syscall.ECONNRESET}}, want: provider.ReasonUnreachable},
		{name: "request timeout", err: fmt.Errorf("reading: %w", &url.Error{Err: context.DeadlineExceeded}), want: provider.ReasonTimeout},
		{name: "cancelled", err: context.Canceled, want: provider.ReasonTimeout},
		{name: "server closed mid response", err: &url.Error{Err: io.ErrUnexpectedEOF}, want: provider.ReasonUnreachable},
		{name: "eof", err: io.EOF, want: provider.ReasonUnreachable},
		{name: "decode failure", err: errors.New("decoding GET version: invalid character '<'"), want: provider.ReasonAPIError},
		{name: "nil", err: nil, want: provider.ReasonAPIError},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			if got := checkReason(tc.err); got != tc.want {
				t.Errorf("checkReason(%v) = %s, want %s", tc.err, got, tc.want)
			}
		})
	}
}