egress-proxies-operator/internal/provider/kvm2/backend_test.go

package kvm2

import (
	"context"
	"errors"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/digitalocean/go-libvirt"
	"github.com/digitalocean/go-libvirt/libvirttest"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

// blockingFn returns a func that blocks until release is closed, plus a
// counter of how often it was started.
func blockingFn(t *testing.T) (fn func() error, started *atomic.Int32) {
	t.Helper()
	release := make(chan struct{})
	t.Cleanup(func() { close(release) })
	started = &atomic.Int32{}
	return func() error {
		started.Add(1)
		<-release
		return nil
	}, started
}

func TestCall_deadlineClosesConnectionAndReturnsTransient(t *testing.T) {
	t.Parallel()
	var killed atomic.Int32
	l := newLink("beelink", func() { killed.Add(1) })
	fn, started := blockingFn(t)

	start := time.Now()
	err := l.run(context.Background(), "DomainCreate", 50*time.Millisecond, fn)
	elapsed := time.Since(start)
	if !errors.Is(err, provider.ErrTransient) {
		t.Fatalf("run() error = %v, want ErrTransient", err)
	}
	if !errors.Is(err, context.DeadlineExceeded) {
		t.Errorf("run() error = %v, want it to wrap context.DeadlineExceeded", err)
	}
	if !strings.Contains(err.Error(), "DomainCreate") || !strings.Contains(err.Error(), "beelink") {
		t.Errorf("run() error = %q, want the call name and host in it", err)
	}
	if elapsed > time.Second {
		t.Errorf("run() took %v, want about the 50ms budget", elapsed)
	}
	if killed.Load() != 1 {
		t.Errorf("kill hook called %d times, want 1", killed.Load())
	}
	if !l.isDead() {
		t.Error("link not dead after an abandoned call")
	}
	if !strings.Contains(l.deadReason(), "budget") {
		t.Errorf("deadReason() = %q, want the budget mentioned", l.deadReason())
	}

	start = time.Now()
	err = l.run(context.Background(), "DomainGetState", time.Minute, fn)
	if !errors.Is(err, provider.ErrTransient) {
		t.Fatalf("second run() error = %v, want ErrTransient", err)
	}
	if time.Since(start) > 100*time.Millisecond {
		t.Errorf("second run() took %v, want immediate", time.Since(start))
	}
	if started.Load() != 1 {
		t.Errorf("fn started %d times, want 1: a dead link must not touch the socket", started.Load())
	}
	if killed.Load() != 1 {
		t.Errorf("kill hook called %d times, want still 1", killed.Load())
	}
}

func TestCall_cancelUnblocksRPC(t *testing.T) {
	t.Parallel()
	var killed atomic.Int32
	l := newLink("beelink", func() { killed.Add(1) })
	fn, _ := blockingFn(t)
	ctx, cancel := context.WithCancel(context.Background())
	go func() {
		time.Sleep(10 * time.Millisecond)
		cancel()
	}()

	start := time.Now()
	err := l.run(ctx, "ConnectListAllDomains", time.Minute, fn)
	if time.Since(start) > 100*time.Millisecond {
		t.Errorf("run() took %v after cancel, want under 100ms", time.Since(start))
	}
	if !errors.Is(err, provider.ErrTransient) || !errors.Is(err, context.Canceled) {
		t.Errorf("run() error = %v, want ErrTransient wrapping context.Canceled", err)
	}
	if killed.Load() != 1 {
		t.Errorf("kill hook called %d times, want 1", killed.Load())
	}
}

func TestCall_alreadyCancelledDoesNotStart(t *testing.T) {
	t.Parallel()
	l := newLink("beelink", nil)
	fn, started := blockingFn(t)
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	err := l.run(ctx, "DomainGetState", time.Minute, fn)
	if !errors.Is(err, provider.ErrTransient) || !errors.Is(err, context.Canceled) {
		t.Errorf("run() error = %v, want ErrTransient wrapping context.Canceled", err)
	}
	if started.Load() != 0 {
		t.Error("fn started although ctx was already cancelled")
	}
	if l.isDead() {
		t.Error("link marked dead although nothing was abandoned")
	}
}

func TestCall_returnsFnResult(t *testing.T) {
	t.Parallel()
	l := newLink("beelink", nil)
	want := libvirt.Error{Code: uint32(libvirt.ErrNoDomain), Message: "gone"}
	err := l.run(context.Background(), "DomainLookupByName", time.Second, func() error { return want })
	if !hasCode(err, libvirt.ErrNoDomain) {
		t.Errorf("run() error = %v, want the libvirt error through unchanged", err)
	}
	if l.isDead() {
		t.Error("a backend error must not kill the link")
	}
}

func TestCall_killedByAnotherCallerFailsTransient(t *testing.T) {
	t.Parallel()
	l := newLink("beelink", nil)
	fn, _ := blockingFn(t)
	done := make(chan error, 1)
	go func() { done <- l.run(context.Background(), "DomainGetState", time.Minute, fn) }()
	time.Sleep(10 * time.Millisecond)
	l.markDead("keepalive failed")
	select {
	case err := <-done:
		if !errors.Is(err, provider.ErrTransient) || !strings.Contains(err.Error(), "keepalive failed") {
			t.Errorf("run() error = %v, want ErrTransient with the death reason", err)
		}
	case <-time.After(time.Second):
		t.Fatal("run() did not return after the link was killed")
	}
}

// TestMarkDead_slowKillDoesNotBlockCallers pins the guarantee the pooled
// connection rests on: a kill that takes its time — closing a transport can
// block for as long as the kernel retransmits to a host that went away —
// holds up neither the callers waiting on dead nor the next markDead. The
// call's own fn signals when it is in flight, so the parked caller is
// genuinely parked rather than refused by the dead check on the way in.
func TestMarkDead_slowKillDoesNotBlockCallers(t *testing.T) {
	t.Parallel()
	release := make(chan struct{})
	t.Cleanup(func() { close(release) })
	killing := make(chan struct{})
	var killed atomic.Int32
	l := newLink("beelink", func() {
		killed.Add(1)
		close(killing)
		<-release
	})

	inflight := make(chan struct{})
	waiting := make(chan error, 1)
	go func() {
		waiting <- l.run(context.Background(), "StorageVolUpload", time.Minute, func() error {
			close(inflight)
			<-release
			return nil
		})
	}()
	<-inflight

	go l.markDead("keepalive failed")
	<-killing

	select {
	case <-l.dead():
	case <-time.After(time.Second):
		t.Fatal("link not signalled dead while its kill is still running")
	}
	select {
	case err := <-waiting:
		if !errors.Is(err, provider.ErrTransient) || !strings.Contains(err.Error(), "keepalive failed") {
			t.Errorf("parked call error = %v, want ErrTransient with the death reason", err)
		}
	case <-time.After(time.Second):
		t.Fatal("parked call not released while the kill is still running")
	}

	second := make(chan struct{})
	go func() {
		l.markDead("closed")
		close(second)
	}()
	select {
	case <-second:
	case <-time.After(time.Second):
		t.Fatal("second markDead blocked behind the running kill")
	}

	if killed.Load() != 1 {
		t.Errorf("kill hook called %d times, want 1", killed.Load())
	}
	if !strings.Contains(l.deadReason(), "keepalive failed") {
		t.Errorf("deadReason() = %q, want the first reason", l.deadReason())
	}
}

func TestBudgets_upload(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name   string
		length uint64
		want   time.Duration
	}{
		{name: "seed sized", length: 400 << 10, want: defaultBudgets.seedUpload},
		{name: "exactly the limit", length: smallUploadBytes, want: defaultBudgets.seedUpload},
		{name: "one over the limit", length: smallUploadBytes + 1, want: defaultBudgets.transfer},
		{name: "base image", length: 435 << 20, want: defaultBudgets.transfer},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			if got := defaultBudgets.upload(tc.length); got != tc.want {
				t.Errorf("upload(%d) = %v, want %v", tc.length, got, tc.want)
			}
		})
	}
}

// mockAdapter wires a libvirtBackend to go-libvirt's own mock server over a
// net.Pipe, so the adapter is exercised with the real RPC client.
func mockAdapter(t *testing.T, b budgets) *libvirtBackend {
	t.Helper()
	m := libvirttest.New()
	conn, err := m.Dial()
	if err != nil {
		t.Fatalf("mock dial: %v", err)
	}
	ad := newLibvirtBackend(libvirt.New(conn), "mock", conn, b)
	if err := ad.connect(context.Background()); err != nil {
		t.Fatalf("connect: %v", err)
	}
	t.Cleanup(func() { _ = ad.Close() })
	return ad
}

func TestLibvirtBackend_roundTripThroughMock(t *testing.T) {
	t.Parallel()
	ad := mockAdapter(t, defaultBudgets)
	v, err := ad.ConnectGetLibVersion(context.Background())
	if err != nil {
		t.Fatalf("ConnectGetLibVersion() error = %v", err)
	}
	if v == 0 {
		t.Error("ConnectGetLibVersion() = 0, want the mock's version")
	}
	pool, err := ad.StoragePoolLookupByName(context.Background(), "default")
	if err != nil {
		t.Fatalf("StoragePoolLookupByName() error = %v", err)
	}
	if pool.Name == "" {
		t.Error("StoragePoolLookupByName() returned an empty pool")
	}
	if isDead(ad) {
		t.Error("adapter dead after successful calls")
	}
}

func TestLibvirtBackend_unansweredCallIsAbandoned(t *testing.T) {
	t.Parallel()
	b := defaultBudgets
	b.rpc = 100 * time.Millisecond
	ad := mockAdapter(t, b)
	// The mock does not implement ConnectGetHostname, so it never replies.
	_, err := ad.ConnectGetHostname(context.Background())
	if !errors.Is(err, provider.ErrTransient) || !errors.Is(err, context.DeadlineExceeded) {
		t.Fatalf("ConnectGetHostname() error = %v, want ErrTransient wrapping DeadlineExceeded", err)
	}
	if !isDead(ad) {
		t.Error("adapter not dead after an abandoned call")
	}
	select {
	case <-ad.lv.Disconnected():
	case <-time.After(time.Second):
		t.Error("go-libvirt did not notice the closed socket")
	}
	if _, err := ad.ConnectGetLibVersion(context.Background()); !errors.Is(err, provider.ErrTransient) {
		t.Errorf("call on dead adapter error = %v, want ErrTransient", err)
	}
	if err := ad.Close(); err != nil {
		t.Errorf("Close() on dead adapter error = %v, want nil", err)
	}
}

func TestLibvirtBackend_closeIsIdempotent(t *testing.T) {
	t.Parallel()
	ad := mockAdapter(t, defaultBudgets)
	first := ad.Close()
	second := ad.Close()
	if first != nil || second != nil {
		t.Errorf("Close() errors = %v, %v, want nil, nil", first, second)
	}
	if !isDead(ad) {
		t.Error("adapter not dead after Close")
	}
	if _, err := ad.ConnectGetLibVersion(context.Background()); !errors.Is(err, provider.ErrTransient) {
		t.Errorf("call after Close error = %v, want ErrTransient", err)
	}
}