package proxmox import ( "context" "crypto/x509" "errors" "fmt" "io" "maps" "net" "net/http" "net/url" "path/filepath" "slices" "strings" "sync" "syscall" "testing" "time" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider" ) // checkSecret is the secret half of the Checker's token file; it must never // reach a verdict or a log line. const checkSecret = "SENTINEL-token-secret-4c1d" // newCheckFixture is an observed create fixture at V(2) with a token file // whose secret is checkSecret. func newCheckFixture(t *testing.T) *fixture { t.Helper() fx := newCreateFixture(t, 2) fx.p.api = observe(fx.fake, testProvider) fx.p.cfg.TokenFile = filepath.Join(t.TempDir(), "token") writeFile(t, fx.p.cfg.TokenFile, "proxy-operator@pve!operator="+checkSecret+"\n") return fx } // assertNoSecret checks every verdict and every log line for checkSecret. func assertNoSecret(t *testing.T, fx *fixture, got []provider.TargetStatus) { t.Helper() for _, ts := range got { if strings.Contains(ts.Message, checkSecret) || slices.ContainsFunc(slices.Collect(maps.Values(ts.Details)), func(v string) bool { return strings.Contains(v, checkSecret) }) { t.Errorf("verdict carries the token secret: %+v", ts) } } for _, l := range fx.logs.all() { if strings.Contains(l, checkSecret) { t.Errorf("log line carries the token secret: %s", l) } } } func TestCheck_oneEntryPerNodeInConfigOrder(t *testing.T) { t.Parallel() fx := newCheckFixture(t) fx.p.cfg.Nodes = []string{"pv1-3", fakeNode, "pv1-2"} fx.fake.nodes["pv1-3"] = fx.fake.nodes[fakeNode] got := fx.p.Check(fx.ctx) if len(got) != len(fx.p.cfg.Nodes) { t.Fatalf("Check() returned %d targets, want %d", len(got), len(fx.p.cfg.Nodes)) } for i, node := range fx.p.cfg.Nodes { ts := got[i] if ts.Name != node || ts.Address != fx.p.cfg.URL { t.Errorf("target %d = %s (%s), want %s (%s)", i, ts.Name, ts.Address, node, fx.p.cfg.URL) } if wantReady := node != "pv1-2"; ts.Ready != wantReady { t.Errorf("target %s ready = %v, want %v (%s: %s)", node, ts.Ready, wantReady, ts.Reason, ts.Message) } } if got[2].Reason != provider.ReasonAPIError || !strings.Contains(got[2].Message, "pv1-2") { t.Errorf("pv1-2 verdict = %s: %s, want APIError naming the node", got[2].Reason, got[2].Message) } } // TestCheck_nodesCheckedConcurrently holds every node's first request // until all of them have arrived; checked one at a time, the first would // give up and fail its verdict. func TestCheck_nodesCheckedConcurrently(t *testing.T) { t.Parallel() fx := newCheckFixture(t) fx.p.cfg.Nodes = []string{fakeNode, "pv1-2", "pv1-3"} for _, n := range fx.p.cfg.Nodes[1:] { fx.fake.nodes[n] = fx.fake.nodes[fakeNode] } var ( mu sync.Mutex arrived int ) all := make(chan struct{}) fx.fake.hookOn("Version", func() error { mu.Lock() if arrived++; arrived == len(fx.p.cfg.Nodes) { close(all) } mu.Unlock() select { case <-all: return nil case <-time.After(2 * time.Second): return errors.New("barrier: the nodes were not checked concurrently") } }) for _, ts := range fx.p.Check(fx.ctx) { if !ts.Ready { t.Errorf("target %s = %s: %s, want Ready", ts.Name, ts.Reason, ts.Message) } } } func TestCheck_expiredContextReturnsPromptly(t *testing.T) { t.Parallel() fx := newCheckFixture(t) fx.p.cfg.Nodes = []string{fakeNode, "pv1-2"} fx.fake.blockOn("Version") ctx, cancel := context.WithTimeout(fx.ctx, 0) defer cancel() start := time.Now() got := fx.p.Check(ctx) if elapsed := time.Since(start); elapsed > time.Second { t.Errorf("Check() took %v with an expired ctx, want under 1s", elapsed) } if len(got) != 2 { t.Fatalf("Check() returned %d targets, want 2", len(got)) } for _, ts := range got { if ts.Ready || ts.Reason != provider.ReasonTimeout || !strings.Contains(ts.Message, context.DeadlineExceeded.Error()) { t.Errorf("target %s = ready %v reason %s (%s), want Timeout naming the deadline", ts.Name, ts.Ready, ts.Reason, ts.Message) } } } func TestCheck_reasonsTable(t *testing.T) { t.Parallel() urlErr := func(err error) error { return &url.Error{Op: "Get", URL: "https://192.0.2.8:8006/api2/json/version", Err: err} } tests := []struct { name string prep func(fx *fixture) timeout time.Duration cancel bool wantReady bool wantReason string wantMessage string }{ {name: "ok", wantReady: true, wantReason: provider.ReasonReachable}, { name: "token file missing", prep: func(fx *fixture) { fx.p.cfg.TokenFile = filepath.Join(fx.t.TempDir(), "absent") }, wantReason: reasonCredentialsUnavailable, wantMessage: "does not exist yet", }, { name: "token file malformed", prep: func(fx *fixture) { writeFile(fx.t, fx.p.cfg.TokenFile, checkSecret+"\n") }, wantReason: reasonCredentialsUnavailable, wantMessage: "is not @!=", }, { name: "token file gone by the time the client reads it", prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", fmt.Errorf("%w: %w: /etc/proxy-operator/proxmox/token does not exist yet", provider.ErrTransient, errCredentials)) }, wantReason: reasonCredentialsUnavailable, }, { name: "unknown certificate authority", prep: func(fx *fixture) { fx.fake.failWith("Version", urlErr(x509.UnknownAuthorityError{})) }, wantReason: reasonCertificateInvalid, }, { name: "host not in the certificate", prep: func(fx *fixture) { fx.fake.failWith("Version", urlErr(x509.HostnameError{Host: "192.168.2.8"})) }, wantReason: reasonCertificateInvalid, }, { name: "CA file unusable", prep: func(fx *fixture) { fx.fake.failWith("Version", fmt.Errorf("%w: %w: reading CA file: %w", provider.ErrTransient, errCAFile, syscall.ENOENT)) }, wantReason: reasonCertificateInvalid, }, { name: "wrong secret", prep: func(fx *fixture) { fx.fake.failWith("Version", loadAPIError(fx.t, "err_unauthorized")) }, wantReason: provider.ReasonUnauthorized, }, { name: "no credentials sent", prep: func(fx *fixture) { fx.fake.failWith("Version", loadAPIError(fx.t, "err_no_credentials")) }, wantReason: provider.ReasonUnauthorized, }, { name: "privilege missing", prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", loadAPIError(fx.t, "err_forbidden")) }, wantReason: provider.ReasonForbidden, wantMessage: "Sys.Syslog", }, { name: "pveproxy cannot reach the node", prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(595, "Connection refused")) }, wantReason: provider.ReasonUnreachable, }, { name: "pveproxy broken pipe", prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(596, "Broken pipe")) }, wantReason: provider.ReasonUnreachable, }, { name: "connection refused", prep: func(fx *fixture) { fx.fake.failWith("Version", urlErr(&net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED})) }, wantReason: provider.ReasonUnreachable, }, { name: "node status 500", prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(500, "fake: pve-cluster not running")) }, wantReason: provider.ReasonAPIError, }, { name: "iso storage missing", prep: func(fx *fixture) { fx.fake.mu.Lock() delete(fx.fake.storages, fakeISOStorage) fx.fake.mu.Unlock() }, wantReason: reasonStorageNotFound, wantMessage: "storage proxy-seeds on node pv1-1: fake: storage 'proxy-seeds' does not exist", }, { name: "iso storage inactive", prep: func(fx *fixture) { setStorage(fx, func(s *storageStatus) { s.Active = 0 }) }, wantReason: reasonStorageNotFound, wantMessage: "not active", }, { name: "iso storage without iso content", prep: func(fx *fixture) { setStorage(fx, func(s *storageStatus) { s.Content = "images,rootdir,isoish" }) }, wantReason: reasonStorageNotFound, wantMessage: "does not hold iso content (content: images,rootdir,isoish)", }, { name: "iso storage forbidden", prep: func(fx *fixture) { fx.fake.failWith("StorageStatus", loadAPIError(fx.t, "err_forbidden")) }, wantReason: provider.ReasonForbidden, }, { name: "template missing", prep: func(fx *fixture) { fx.fake.mu.Lock() delete(fx.fake.vms, testTemplateVMID) fx.fake.mu.Unlock() }, wantReason: reasonTemplateNotFound, wantMessage: `template "debian-13": VM 9001 does not exist`, }, { name: "template VMID holds a plain VM", prep: func(fx *fixture) { fx.fake.vms[testTemplateVMID].template = false }, wantReason: reasonTemplateNotFound, wantMessage: `template "debian-13": VM 9001 is not a template`, }, { name: "template VMID holds a container", prep: func(fx *fixture) { fx.fake.vms[testTemplateVMID].kind = "lxc" }, wantReason: reasonTemplateNotFound, wantMessage: "does not exist", }, { name: "resources listing fails", prep: func(fx *fixture) { fx.fake.failWith("ClusterResources", fakeErr(500, "fake: cluster filesystem not ready")) }, wantReason: provider.ReasonAPIError, }, { name: "request outlives the deadline", prep: func(fx *fixture) { fx.fake.blockOn("StorageStatus") }, timeout: 50 * time.Millisecond, wantReason: provider.ReasonTimeout, }, { // The real client's dial sees the round's ctx end and fails as // a dial error, which checkReason alone would call Unreachable. name: "dial cut short by the round's deadline", prep: func(fx *fixture) { fx.fake.hookOn("Version", func() error { time.Sleep(200 * time.Millisecond) return urlErr(&net.OpError{Op: "dial", Net: "tcp", Err: context.DeadlineExceeded}) }) }, timeout: 20 * time.Millisecond, wantReason: provider.ReasonTimeout, }, { name: "ctx cancelled mid check", prep: func(fx *fixture) { fx.fake.blockOn("ClusterResources") }, cancel: true, wantReason: provider.ReasonTimeout, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() fx := newCheckFixture(t) if tc.prep != nil { tc.prep(fx) } ctx := fx.ctx switch { case tc.timeout > 0: var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, tc.timeout) defer cancel() case tc.cancel: var cancel context.CancelFunc ctx, cancel = context.WithCancel(ctx) time.AfterFunc(50*time.Millisecond, cancel) } got := fx.p.Check(ctx) if len(got) != 1 { t.Fatalf("Check() returned %d targets, want 1", len(got)) } ts := got[0] if ts.Ready != tc.wantReady || ts.Reason != tc.wantReason { t.Errorf("verdict = ready %v reason %s (%s), want ready %v reason %s", ts.Ready, ts.Reason, ts.Message, tc.wantReady, tc.wantReason) } if !tc.wantReady && ts.Message == "" { t.Error("failed check has no message") } if tc.wantReady && ts.Message != "" { t.Errorf("successful check has a message: %q", ts.Message) } if tc.wantMessage != "" && !strings.Contains(ts.Message, tc.wantMessage) { t.Errorf("message = %q, want it to contain %q", ts.Message, tc.wantMessage) } if ts.Details["isoStorage"] != fakeISOStorage { t.Errorf("details = %v, want isoStorage whatever failed", ts.Details) } assertNoSecret(t, fx, got) }) } } func setStorage(fx *fixture, fn func(s *storageStatus)) { fx.fake.mu.Lock() defer fx.fake.mu.Unlock() s := fx.fake.storages[fakeISOStorage] fn(&s) fx.fake.storages[fakeISOStorage] = s } // TestCheck_reasonsOverHTTPClient takes the transport-level reasons from // the real client, so the errors checkReason sees are the ones net/http and // crypto/tls produce, and the Ready row replays the Phase 0 captures. func TestCheck_reasonsOverHTTPClient(t *testing.T) { t.Parallel() captures := func(t *testing.T) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { path := strings.TrimPrefix(r.URL.Path, "/api2/json/") name := map[string]string{ "version": "version", "nodes/pv1-1/status": "node_status", "nodes/pv1-1/storage/local/status": "storage_status", "cluster/resources": "cluster_resources", }[path] if name == "" { t.Errorf("unexpected request %s", path) http.NotFound(w, r) return } serveJSON(w, readCapture(t, name).Body) } } tests := []struct { name string handler func(t *testing.T) http.HandlerFunc mutate func(t *testing.T, cfx *clientFixture) wantReason string }{ {name: "captures", handler: captures, wantReason: provider.ReasonReachable}, {name: "unknown authority", handler: captures, mutate: func(_ *testing.T, cfx *clientFixture) { cfx.cfg.CAFile = "" }, wantReason: reasonCertificateInvalid}, { name: "host not in the certificate", handler: captures, mutate: func(_ *testing.T, cfx *clientFixture) { cfx.cfg.URL = strings.Replace(cfx.srv.URL, "127.0.0.1", "localhost", 1) }, wantReason: reasonCertificateInvalid, }, { name: "CA file missing", handler: captures, mutate: func(t *testing.T, cfx *clientFixture) { cfx.cfg.CAFile = filepath.Join(t.TempDir(), "ca.pem") }, wantReason: reasonCertificateInvalid, }, { name: "401 from pveproxy", handler: func(t *testing.T) http.HandlerFunc { return func(w http.ResponseWriter, _ *http.Request) { serveCapture(t, w, readCapture(t, "err_unauthorized")) } }, wantReason: provider.ReasonUnauthorized, }, { name: "596 from pveproxy", handler: func(t *testing.T) http.HandlerFunc { return func(w http.ResponseWriter, _ *http.Request) { serveCapture(t, w, capture{Status: 596, Reason: "Broken pipe"}) } }, wantReason: provider.ReasonUnreachable, }, { name: "server gone", handler: captures, mutate: func(_ *testing.T, cfx *clientFixture) { cfx.srv.Close() }, wantReason: provider.ReasonUnreachable, }, { name: "connection cut mid response", handler: func(t *testing.T) http.HandlerFunc { return func(w http.ResponseWriter, _ *http.Request) { conn, _, err := w.(http.Hijacker).Hijack() if err != nil { t.Errorf("hijack: %v", err) return } _, _ = io.WriteString(conn, "HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{\"data\":") _ = conn.Close() } }, wantReason: provider.ReasonUnreachable, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() cfx := newClientFixture(t, tc.handler(t)) if tc.mutate != nil { tc.mutate(t, cfx) } cfg := testConfig() cfg.ISOStorage = "local" cfg.URL, cfg.TokenFile, cfg.CAFile = cfx.cfg.URL, cfx.cfg.TokenFile, cfx.cfg.CAFile p := newProvider(testProvider, cfg, cfx.api(t)) got := p.Check(context.Background()) if len(got) != 1 { t.Fatalf("Check() returned %d targets, want 1", len(got)) } if ts := got[0]; ts.Reason != tc.wantReason || ts.Ready != (tc.wantReason == provider.ReasonReachable) { t.Errorf("verdict = ready %v reason %s (%s), want %s", ts.Ready, ts.Reason, ts.Message, tc.wantReason) } }) } } func TestCheck_detailsKeys(t *testing.T) { t.Parallel() full := map[string]string{ "tokenID": "proxy-operator@pve!operator", "isoStorage": fakeISOStorage, "pveVersion": "9.2.20", "kernel": "7.0.14-17-pve", "isoStorageAvailGiB": "90.0", "templates": "1/1", } tests := []struct { name string prep func(fx *fixture) want []string }{ {name: "success", want: slices.Sorted(maps.Keys(full))}, {name: "token unusable", prep: func(fx *fixture) { writeFile(fx.t, fx.p.cfg.TokenFile, "junk") }, want: []string{"isoStorage"}}, {name: "node status fails", prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(596, "Broken pipe")) }, want: []string{"isoStorage", "pveVersion", "tokenID"}}, {name: "storage inactive", prep: func(fx *fixture) { setStorage(fx, func(s *storageStatus) { s.Active = 0 }) }, want: []string{"isoStorage", "isoStorageAvailGiB", "kernel", "pveVersion", "tokenID"}}, {name: "template missing", prep: func(fx *fixture) { fx.fake.vms[testTemplateVMID].template = false }, want: slices.Sorted(maps.Keys(full))}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() fx := newCheckFixture(t) if tc.prep != nil { tc.prep(fx) } got := fx.p.Check(fx.ctx)[0] if keys := slices.Sorted(maps.Keys(got.Details)); !slices.Equal(keys, tc.want) { t.Errorf("details keys = %v, want %v", keys, tc.want) } for k, v := range got.Details { if want, ok := full[k]; ok && tc.name == "success" && v != want { t.Errorf("details[%s] = %q, want %q", k, v, want) } if v == "" { t.Errorf("details[%s] is empty", k) } } }) } t.Run("templates counts found over configured", func(t *testing.T) { t.Parallel() fx := newCheckFixture(t) fx.p.cfg.Templates["ubuntu"] = provider.ProxmoxTemplate{VMID: 9002} fx.p.cfg.Templates["debian-again"] = provider.ProxmoxTemplate{VMID: testTemplateVMID} got := fx.p.Check(fx.ctx)[0] if got.Details["templates"] != "2/3" || got.Reason != reasonTemplateNotFound || !strings.Contains(got.Message, `template "ubuntu": VM 9002`) { t.Errorf("verdict = %s %q, details %v; want TemplateNotFound naming ubuntu, templates 2/3", got.Reason, got.Message, got.Details) } }) } func TestCheck_logsVerdictPerNode(t *testing.T) { t.Parallel() fx := newCheckFixture(t) fx.p.cfg.Nodes = []string{fakeNode, "pv1-2"} fx.p.Check(fx.ctx) checked := fx.logs.matching(`"msg"="proxmox node checked"`) if len(checked) != 2 { t.Fatalf("got %d verdict lines, want 2: %v", len(checked), checked) } for _, l := range checked { switch { case strings.Contains(l, `"node"="pv1-1"`): if !strings.Contains(l, `"ready"=true`) || !strings.Contains(l, `"reason"="Reachable"`) || !strings.Contains(l, `"latency"=`) { t.Errorf("pv1-1 verdict line = %s", l) } case strings.Contains(l, `"node"="pv1-2"`): if !strings.Contains(l, `"ready"=false`) || !strings.Contains(l, `"reason"="APIError"`) { t.Errorf("pv1-2 verdict line = %s", l) } default: t.Errorf("verdict line without a node: %s", l) } if !strings.Contains(l, `"provider"="`+testProvider+`"`) || !strings.Contains(l, `"level"=1`) { t.Errorf("verdict line lacks the provider or is not V(1): %s", l) } } } func TestCheckReason_table(t *testing.T) { t.Parallel() dial := &url.Error{Op: "Get", URL: "https://192.0.2.8:8006/api2/json/version", Err: &net.OpError{Op: "dial", Net: "tcp", Err: context.DeadlineExceeded}} tests := []struct { name string err error want string }{ {name: "storage verdict", err: &checkFailure{reason: reasonStorageNotFound, msg: "x"}, want: reasonStorageNotFound}, {name: "template verdict wrapped", err: fmt.Errorf("check: %w", &checkFailure{reason: reasonTemplateNotFound, msg: "x"}), want: reasonTemplateNotFound}, {name: "token missing", err: fmt.Errorf("%w: %w: /t does not exist yet", provider.ErrTransient, errCredentials), want: reasonCredentialsUnavailable}, {name: "token malformed", err: fmt.Errorf("%w: %w: /t is not a token", provider.ErrPermanent, errCredentials), want: reasonCredentialsUnavailable}, {name: "CA file", err: fmt.Errorf("%w: %w: /ca holds no PEM certificate", provider.ErrPermanent, errCAFile), want: reasonCertificateInvalid}, {name: "unknown authority", err: &url.Error{Err: x509.UnknownAuthorityError{}}, want: reasonCertificateInvalid}, {name: "certificate invalid", err: &url.Error{Err: x509.CertificateInvalidError{Reason: x509.Expired}}, want: reasonCertificateInvalid}, {name: "401", err: loadAPIError(t, "err_unauthorized"), want: provider.ReasonUnauthorized}, {name: "403 wrapped", err: fmt.Errorf("reading: %w", loadAPIError(t, "err_forbidden_volume_delete")), want: provider.ReasonForbidden}, {name: "595", err: fakeErr(595, "Connection refused"), want: provider.ReasonUnreachable}, {name: "596", err: fakeErr(596, "Broken pipe"), want: provider.ReasonUnreachable}, {name: "400", err: loadAPIError(t, "err_bad_param"), want: provider.ReasonAPIError}, {name: "500", err: loadAPIError(t, "err_vm_missing"), want: provider.ReasonAPIError}, {name: "dial ran out of its own budget", err: dial, want: provider.ReasonUnreachable}, {name: "connection reset", err: &url.Error{Err: &net.OpError{Op: "read", Net: "tcp", Err: syscall.ECONNRESET}}, want: provider.ReasonUnreachable}, {name: "request timeout", err: fmt.Errorf("reading: %w", &url.Error{Err: context.DeadlineExceeded}), want: provider.ReasonTimeout}, {name: "cancelled", err: context.Canceled, want: provider.ReasonTimeout}, {name: "server closed mid response", err: &url.Error{Err: io.ErrUnexpectedEOF}, want: provider.ReasonUnreachable}, {name: "eof", err: io.EOF, want: provider.ReasonUnreachable}, {name: "decode failure", err: errors.New("decoding GET version: invalid character '<'"), want: provider.ReasonAPIError}, {name: "nil", err: nil, want: provider.ReasonAPIError}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() if got := checkReason(tc.err); got != tc.want { t.Errorf("checkReason(%v) = %s, want %s", tc.err, got, tc.want) } }) } }