egress-proxies-operator/internal/provider/kvm2/provider_test.go

package kvm2

import (
	"context"
	"errors"
	"fmt"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/digitalocean/go-libvirt"
	"github.com/go-logr/logr/funcr"
	logf "sigs.k8s.io/controller-runtime/pkg/log"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

const (
	testProvider = "kvm2-home"
	testHost     = "beelink"
	testUID      = "0f1e2d3c-uid-1"
	testName     = "proxy-abcdefghijklmnop"
)

// fixture is one configured host backed by a fake, an image server the
// catalogue points at, a log capture at a chosen verbosity and a provider
// with a fast running-wait, so a Create runs end to end in memory.
type fixture struct {
	t      *testing.T
	p      *Provider
	fake   *fakeBackend
	dialer *stubDialer
	logs   *logCapture
	ctx    context.Context
	image  []byte
}

func newFixture(t *testing.T, verbosity int, opts ...providerOption) *fixture {
	t.Helper()
	fx := &fixture{t: t, fake: newFakeBackend(), logs: &logCapture{}, image: []byte("QFI\xfb fake base image bytes")}
	srv := httptest.NewServer(serveBytes(fx.image))
	t.Cleanup(srv.Close)
	fx.dialer = &stubDialer{backend: fx.fake}
	cfg := provider.KVMConfig{
		Images:       map[string]provider.KVMImage{imageKey: {URL: srv.URL + "/debian-13-generic-amd64.qcow2"}},
		MachineTypes: map[string]provider.KVMMachineType{"small": {VCPUs: 1, MemoryMiB: 1024}},
		Hosts:        []provider.KVMHost{{Name: testHost, Address: "beelink:22", User: "virt", Bridge: "br0"}},
	}
	opts = append([]providerOption{withHTTPClient(srv.Client()), withRunningWait(time.Millisecond, 500*time.Millisecond)}, opts...)
	fx.p = newProvider(testProvider, cfg, fx.dialer, opts...)
	log := funcr.New(func(prefix, args string) {
		fx.logs.mu.Lock()
		defer fx.logs.mu.Unlock()
		fx.logs.lines = append(fx.logs.lines, prefix+" "+args)
	}, funcr.Options{Verbosity: verbosity})
	fx.ctx = logf.IntoContext(context.Background(), log)
	return fx
}

func (fx *fixture) req(name string) provider.CreateRequest {
	return provider.CreateRequest{
		Name:      name,
		UID:       testUID,
		Namespace: "default",
		ProxyName: "egress",
		Placement: provider.Placement{Zone: testHost, MachineType: "small", Image: imageKey},
		CloudInit: "#cloud-config\npackages: [squid]\n",
		Port:      3128,
	}
}

func (fx *fixture) create(name string) string {
	fx.t.Helper()
	id, err := fx.p.Create(fx.ctx, fx.req(name))
	if err != nil {
		fx.t.Fatalf("Create(%s) error = %v", name, err)
	}
	if want := formatProviderID(testHost, name); id != want {
		fx.t.Fatalf("Create(%s) = %q, want %q", name, id, want)
	}
	return id
}

func (fx *fixture) get(id string) *provider.Instance {
	fx.t.Helper()
	inst, err := fx.p.Get(fx.ctx, id)
	if err != nil {
		fx.t.Fatalf("Get(%s) error = %v", id, err)
	}
	return inst
}

// withDomain runs fn on the fake's domain under its lock.
func (fx *fixture) withDomain(name string, fn func(d *fakeDomain)) {
	fx.t.Helper()
	fx.fake.mu.Lock()
	defer fx.fake.mu.Unlock()
	d, ok := fx.fake.domains[name]
	if !ok {
		fx.t.Fatalf("fake has no domain %s", name)
	}
	fn(d)
}

func (fx *fixture) hasDomain(name string) bool {
	fx.fake.mu.Lock()
	defer fx.fake.mu.Unlock()
	_, ok := fx.fake.domains[name]
	return ok
}

func (fx *fixture) setAddrs(name string, src libvirt.DomainInterfaceAddressesSource, ips ...string) {
	fx.withDomain(name, func(d *fakeDomain) {
		var addrs []libvirt.DomainIPAddr
		for _, ip := range ips {
			addrs = append(addrs, libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv4), Addr: ip, Prefix: 24})
		}
		d.addrs[uint32(src)] = []libvirt.DomainInterface{{Name: "enp1s0", Addrs: addrs}}
	})
}

// putVol places a volume in the fake's pool directly.
func (fx *fixture) putVol(name string, content []byte) *fakeVol {
	fx.fake.mu.Lock()
	defer fx.fake.mu.Unlock()
	v := &fakeVol{content: content, capacity: uint64(len(content))}
	fx.fake.pools[fakePoolName].vols[name] = v
	return v
}

// baseName is the cached base image's volume name for this fixture's
// provider entry and image key.
func (fx *fixture) baseName() string {
	fx.t.Helper()
	src, err := resolveImage(fx.p.name, fx.p.cfg, imageKey)
	if err != nil {
		fx.t.Fatal(err)
	}
	return src.volumeName()
}

// cacheImage places a complete base image and its marker, as a previous
// Create on the host would have.
func (fx *fixture) cacheImage() {
	src, err := resolveImage(fx.p.name, fx.p.cfg, imageKey)
	if err != nil {
		fx.t.Fatal(err)
	}
	fx.putVol(src.volumeName(), fx.image)
	fx.putVol(src.markerName(), []byte(digestOf(fx.image)))
}

func managedMetadata(uid string, createdAt time.Time) string {
	s, err := marshalInstanceMetadata(newInstanceMetadata(uid, createdAt))
	if err != nil {
		panic(err)
	}
	return s
}

// assertProviderError checks err is a *provider.Error from the configured
// provider with the given op and class.
func assertProviderError(t *testing.T, err error, op string, class error) {
	t.Helper()
	if err == nil {
		t.Fatalf("error = nil, want op %s class %v", op, class)
	}
	var pe *provider.Error
	if !errors.As(err, &pe) {
		t.Fatalf("error = %T (%v), want *provider.Error", err, err)
	}
	if pe.Provider != testProvider {
		t.Errorf("error.Provider = %q, want %q", pe.Provider, testProvider)
	}
	if pe.Op != op {
		t.Errorf("error.Op = %q, want %q", pe.Op, op)
	}
	if !errors.Is(err, class) {
		t.Errorf("error = %v, want class %v", err, class)
	}
	for _, other := range []error{provider.ErrNotFound, provider.ErrQuotaExceeded, provider.ErrPermanent, provider.ErrTransient} {
		if other != class && errors.Is(err, other) {
			t.Errorf("error = %v, also matches %v", err, other)
		}
	}
}

// hostDialer hands out one backend per host name and fails hosts in errs.
type hostDialer struct {
	backends map[string]backend
	errs     map[string]error
}

func (d *hostDialer) dial(_ context.Context, host provider.KVMHost) (backend, error) {
	if err := d.errs[host.Name]; err != nil {
		return nil, err
	}
	b, ok := d.backends[host.Name]
	if !ok {
		return nil, fmt.Errorf("hostDialer: no backend for %s", host.Name)
	}
	return b, nil
}

// waitFor polls cond until it holds or the deadline passes.
func waitFor(t *testing.T, what string, cond func() bool) {
	t.Helper()
	deadline := time.Now().Add(5 * time.Second)
	for !cond() {
		if time.Now().After(deadline) {
			t.Fatalf("timed out waiting for %s", what)
		}
		time.Sleep(time.Millisecond)
	}
}

func TestProvider_cancelledContextReturnsPromptly(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name     string
		firstRPC string
		op       string
		call     func(ctx context.Context, fx *fixture) error
	}{
		{
			name: "create", firstRPC: "DomainLookupByName", op: "create",
			call: func(ctx context.Context, fx *fixture) error { _, err := fx.p.Create(ctx, fx.req(testName)); return err },
		},
		{
			name: "get", firstRPC: "DomainLookupByName", op: "get",
			call: func(ctx context.Context, fx *fixture) error {
				_, err := fx.p.Get(ctx, formatProviderID(testHost, testName))
				return err
			},
		},
		{
			name: "delete", firstRPC: "DomainLookupByName", op: "delete",
			call: func(ctx context.Context, fx *fixture) error {
				return fx.p.Delete(ctx, formatProviderID(testHost, testName))
			},
		},
		{
			name: "list", firstRPC: "ConnectListAllDomains", op: "list",
			call: func(ctx context.Context, fx *fixture) error { _, err := fx.p.ListByTag(ctx); return err },
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			release := fx.fake.blockOn(tc.firstRPC)
			defer close(release)
			ctx, cancel := context.WithCancel(fx.ctx)
			done := make(chan error, 1)
			go func() { done <- tc.call(ctx, fx) }()
			waitFor(t, "the first RPC to block", func() bool { return fx.fake.callCount(tc.firstRPC) == 1 })
			cancel()
			select {
			case err := <-done:
				assertProviderError(t, err, tc.op, provider.ErrTransient)
			case <-time.After(time.Second):
				t.Fatalf("%s did not return within 1s of cancellation", tc.name)
			}
		})
	}
}

func TestProvider_everyErrorIsWrappedWithConfiguredName(t *testing.T) {
	t.Parallel()
	dialErr := errors.New("dial tcp 192.0.2.1:22: connect: connection refused")
	tests := []struct {
		name  string
		setup func(fx *fixture)
		call  func(fx *fixture) error
		op    string
		class error
	}{
		{
			name: "create unknown zone",
			call: func(fx *fixture) error {
				req := fx.req(testName)
				req.Placement.Zone = "nas9"
				_, err := fx.p.Create(fx.ctx, req)
				return err
			},
			op: "create", class: provider.ErrPermanent,
		},
		{
			name:  "create dial failure",
			setup: func(fx *fixture) { fx.dialer.err = dialErr },
			call:  func(fx *fixture) error { _, err := fx.p.Create(fx.ctx, fx.req(testName)); return err },
			op:    "create", class: provider.ErrTransient,
		},
		{
			name: "create pool out of space",
			setup: func(fx *fixture) {
				fx.fake.mu.Lock()
				fx.fake.pools[fakePoolName].available = 1
				fx.fake.mu.Unlock()
			},
			call: func(fx *fixture) error { _, err := fx.p.Create(fx.ctx, fx.req(testName)); return err },
			op:   "create", class: provider.ErrQuotaExceeded,
		},
		{
			name:  "create vanished volume",
			setup: func(fx *fixture) { fx.fake.failWith("StorageVolGetPath", fakeErr(libvirt.ErrNoStorageVol, "gone")) },
			call:  func(fx *fixture) error { _, err := fx.p.Create(fx.ctx, fx.req(testName)); return err },
			op:    "create", class: provider.ErrTransient,
		},
		{
			name: "get malformed id",
			call: func(fx *fixture) error { _, err := fx.p.Get(fx.ctx, "beelink/"+testName); return err },
			op:   "get", class: provider.ErrPermanent,
		},
		{
			name: "get missing domain",
			call: func(fx *fixture) error { _, err := fx.p.Get(fx.ctx, formatProviderID(testHost, testName)); return err },
			op:   "get", class: provider.ErrNotFound,
		},
		{
			name: "get busy backend",
			setup: func(fx *fixture) {
				fx.create(testName)
				fx.fake.failWith("DomainGetState", fakeErr(libvirt.ErrOperationInvalid, "busy"))
			},
			call: func(fx *fixture) error { _, err := fx.p.Get(fx.ctx, formatProviderID(testHost, testName)); return err },
			op:   "get", class: provider.ErrTransient,
		},
		{
			name: "delete malformed id",
			call: func(fx *fixture) error { return fx.p.Delete(fx.ctx, "hosts//domains/"+testName) },
			op:   "delete", class: provider.ErrPermanent,
		},
		{
			name:  "delete dial failure",
			setup: func(fx *fixture) { fx.dialer.err = dialErr },
			call:  func(fx *fixture) error { return fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)) },
			op:    "delete", class: provider.ErrTransient,
		},
		{
			name:  "delete access denied",
			setup: func(fx *fixture) { fx.fake.failWith("DomainLookupByName", fakeErr(libvirt.ErrAccessDenied, "denied")) },
			call:  func(fx *fixture) error { return fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)) },
			op:    "delete", class: provider.ErrPermanent,
		},
		{
			name:  "list dial failure",
			setup: func(fx *fixture) { fx.dialer.err = dialErr },
			call:  func(fx *fixture) error { _, err := fx.p.ListByTag(fx.ctx); return err },
			op:    "list", class: provider.ErrTransient,
		},
		{
			name: "list rpc failure",
			setup: func(fx *fixture) {
				fx.fake.failWith("ConnectListAllDomains", fakeErr(libvirt.ErrInternalError, "boom"))
			},
			call: func(fx *fixture) error { _, err := fx.p.ListByTag(fx.ctx); return err },
			op:   "list", class: provider.ErrTransient,
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			if tc.setup != nil {
				tc.setup(fx)
			}
			assertProviderError(t, tc.call(fx), tc.op, tc.class)
		})
	}
}

func TestLogging_v0SilentV1OutcomeV2Detail(t *testing.T) {
	t.Parallel()
	outcomes := []string{"overlay volume created", "seed volume uploaded", "domain defined", "kvm2 domain ensured running"}
	t.Run("v0 is silent", func(t *testing.T) {
		t.Parallel()
		fx := newFixture(t, 0)
		fx.create(testName)
		if lines := fx.logs.all(); len(lines) != 0 {
			t.Errorf("Create logged %d lines at V(0), want none:\n%s", len(lines), strings.Join(lines, "\n"))
		}
	})
	t.Run("v1 has outcomes without xml", func(t *testing.T) {
		t.Parallel()
		fx := newFixture(t, 1)
		fx.create(testName)
		for _, want := range outcomes {
			if len(fx.logs.matching(want)) != 1 {
				t.Errorf("V(1) lines matching %q = %d, want 1:\n%s", want, len(fx.logs.matching(want)), strings.Join(fx.logs.all(), "\n"))
			}
		}
		if got := fx.logs.matching("<domain"); len(got) != 0 {
			t.Errorf("V(1) carries domain XML:\n%s", strings.Join(got, "\n"))
		}
		if got := fx.logs.matching("kvm2 create resolved"); len(got) != 0 {
			t.Errorf("V(1) carries the V(2) detail line:\n%s", strings.Join(got, "\n"))
		}
		for _, l := range fx.logs.all() {
			if !strings.Contains(l, `"provider"="kvm2-home"`) || !strings.Contains(l, `"host"="beelink"`) {
				t.Errorf("line lacks provider or host: %s", l)
			}
			if !strings.Contains(l, `"name"="`+testName+`"`) && !strings.Contains(l, `"volume"="base-`) {
				t.Errorf("instance line lacks the name: %s", l)
			}
		}
	})
	t.Run("v2 has detail", func(t *testing.T) {
		t.Parallel()
		fx := newFixture(t, 2)
		fx.create(testName)
		if got := fx.logs.matching("<domain"); len(got) == 0 {
			t.Errorf("V(2) lacks domain XML:\n%s", strings.Join(fx.logs.all(), "\n"))
		}
		resolved := fx.logs.matching("kvm2 create resolved")
		if len(resolved) != 1 {
			t.Fatalf("V(2) lines matching resolved = %d, want 1", len(resolved))
		}
		for _, key := range []string{`"cloudInitBytes"=`, `"image"="base-` + testProvider + `-debian-13.qcow2"`, `"firmware"="uefi"`, `"vcpus"=1`, `"memoryMiB"=1024`, `"machineType"="small"`, `"bridge"="br0"`, `"pool"="default"`} {
			if !strings.Contains(resolved[0], key) {
				t.Errorf("resolved line lacks %s: %s", key, resolved[0])
			}
		}
	})
}