kvm2: SSH transport, ctx-first libvirt seam, host table and span decorator
Every go-libvirt call runs under its own budget and a cancelled or overlong call kills the connection; credentials are read on every dial; one span and one log line per RPC come from a decorator applied where connections are made. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,252 @@
|
||||
package kvm2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/digitalocean/go-libvirt"
|
||||
"github.com/digitalocean/go-libvirt/libvirttest"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// blockingFn returns a func that blocks until release is closed, plus a
|
||||
// counter of how often it was started.
|
||||
func blockingFn(t *testing.T) (fn func() error, started *atomic.Int32) {
|
||||
t.Helper()
|
||||
release := make(chan struct{})
|
||||
t.Cleanup(func() { close(release) })
|
||||
started = &atomic.Int32{}
|
||||
return func() error {
|
||||
started.Add(1)
|
||||
<-release
|
||||
return nil
|
||||
}, started
|
||||
}
|
||||
|
||||
func TestCall_deadlineClosesConnectionAndReturnsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
var killed atomic.Int32
|
||||
l := newLink("beelink", func() { killed.Add(1) })
|
||||
fn, started := blockingFn(t)
|
||||
|
||||
start := time.Now()
|
||||
err := l.run(context.Background(), "DomainCreate", 50*time.Millisecond, fn)
|
||||
elapsed := time.Since(start)
|
||||
if !errors.Is(err, provider.ErrTransient) {
|
||||
t.Fatalf("run() error = %v, want ErrTransient", err)
|
||||
}
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Errorf("run() error = %v, want it to wrap context.DeadlineExceeded", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "DomainCreate") || !strings.Contains(err.Error(), "beelink") {
|
||||
t.Errorf("run() error = %q, want the call name and host in it", err)
|
||||
}
|
||||
if elapsed > time.Second {
|
||||
t.Errorf("run() took %v, want about the 50ms budget", elapsed)
|
||||
}
|
||||
if killed.Load() != 1 {
|
||||
t.Errorf("kill hook called %d times, want 1", killed.Load())
|
||||
}
|
||||
if !l.isDead() {
|
||||
t.Error("link not dead after an abandoned call")
|
||||
}
|
||||
if !strings.Contains(l.deadReason(), "budget") {
|
||||
t.Errorf("deadReason() = %q, want the budget mentioned", l.deadReason())
|
||||
}
|
||||
|
||||
start = time.Now()
|
||||
err = l.run(context.Background(), "DomainGetState", time.Minute, fn)
|
||||
if !errors.Is(err, provider.ErrTransient) {
|
||||
t.Fatalf("second run() error = %v, want ErrTransient", err)
|
||||
}
|
||||
if time.Since(start) > 100*time.Millisecond {
|
||||
t.Errorf("second run() took %v, want immediate", time.Since(start))
|
||||
}
|
||||
if started.Load() != 1 {
|
||||
t.Errorf("fn started %d times, want 1: a dead link must not touch the socket", started.Load())
|
||||
}
|
||||
if killed.Load() != 1 {
|
||||
t.Errorf("kill hook called %d times, want still 1", killed.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCall_cancelUnblocksRPC(t *testing.T) {
|
||||
t.Parallel()
|
||||
var killed atomic.Int32
|
||||
l := newLink("beelink", func() { killed.Add(1) })
|
||||
fn, _ := blockingFn(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
start := time.Now()
|
||||
err := l.run(ctx, "ConnectListAllDomains", time.Minute, fn)
|
||||
if time.Since(start) > 100*time.Millisecond {
|
||||
t.Errorf("run() took %v after cancel, want under 100ms", time.Since(start))
|
||||
}
|
||||
if !errors.Is(err, provider.ErrTransient) || !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("run() error = %v, want ErrTransient wrapping context.Canceled", err)
|
||||
}
|
||||
if killed.Load() != 1 {
|
||||
t.Errorf("kill hook called %d times, want 1", killed.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCall_alreadyCancelledDoesNotStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
l := newLink("beelink", nil)
|
||||
fn, started := blockingFn(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
err := l.run(ctx, "DomainGetState", time.Minute, fn)
|
||||
if !errors.Is(err, provider.ErrTransient) || !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("run() error = %v, want ErrTransient wrapping context.Canceled", err)
|
||||
}
|
||||
if started.Load() != 0 {
|
||||
t.Error("fn started although ctx was already cancelled")
|
||||
}
|
||||
if l.isDead() {
|
||||
t.Error("link marked dead although nothing was abandoned")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCall_returnsFnResult(t *testing.T) {
|
||||
t.Parallel()
|
||||
l := newLink("beelink", nil)
|
||||
want := libvirt.Error{Code: uint32(libvirt.ErrNoDomain), Message: "gone"}
|
||||
err := l.run(context.Background(), "DomainLookupByName", time.Second, func() error { return want })
|
||||
if !hasCode(err, libvirt.ErrNoDomain) {
|
||||
t.Errorf("run() error = %v, want the libvirt error through unchanged", err)
|
||||
}
|
||||
if l.isDead() {
|
||||
t.Error("a backend error must not kill the link")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCall_killedByAnotherCallerFailsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
l := newLink("beelink", nil)
|
||||
fn, _ := blockingFn(t)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- l.run(context.Background(), "DomainGetState", time.Minute, fn) }()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
l.markDead("keepalive failed")
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, provider.ErrTransient) || !strings.Contains(err.Error(), "keepalive failed") {
|
||||
t.Errorf("run() error = %v, want ErrTransient with the death reason", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("run() did not return after the link was killed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgets_upload(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
length uint64
|
||||
want time.Duration
|
||||
}{
|
||||
{name: "seed sized", length: 400 << 10, want: defaultBudgets.seedUpload},
|
||||
{name: "exactly the limit", length: smallUploadBytes, want: defaultBudgets.seedUpload},
|
||||
{name: "one over the limit", length: smallUploadBytes + 1, want: defaultBudgets.transfer},
|
||||
{name: "base image", length: 435 << 20, want: defaultBudgets.transfer},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := defaultBudgets.upload(tc.length); got != tc.want {
|
||||
t.Errorf("upload(%d) = %v, want %v", tc.length, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// mockAdapter wires a libvirtBackend to go-libvirt's own mock server over a
|
||||
// net.Pipe, so the adapter is exercised with the real RPC client.
|
||||
func mockAdapter(t *testing.T, b budgets) *libvirtBackend {
|
||||
t.Helper()
|
||||
m := libvirttest.New()
|
||||
conn, err := m.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("mock dial: %v", err)
|
||||
}
|
||||
ad := newLibvirtBackend(libvirt.New(conn), "mock", conn, b)
|
||||
if err := ad.connect(context.Background()); err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ad.Close() })
|
||||
return ad
|
||||
}
|
||||
|
||||
func TestLibvirtBackend_roundTripThroughMock(t *testing.T) {
|
||||
t.Parallel()
|
||||
ad := mockAdapter(t, defaultBudgets)
|
||||
v, err := ad.ConnectGetLibVersion(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ConnectGetLibVersion() error = %v", err)
|
||||
}
|
||||
if v == 0 {
|
||||
t.Error("ConnectGetLibVersion() = 0, want the mock's version")
|
||||
}
|
||||
pool, err := ad.StoragePoolLookupByName(context.Background(), "default")
|
||||
if err != nil {
|
||||
t.Fatalf("StoragePoolLookupByName() error = %v", err)
|
||||
}
|
||||
if pool.Name == "" {
|
||||
t.Error("StoragePoolLookupByName() returned an empty pool")
|
||||
}
|
||||
if isDead(ad) {
|
||||
t.Error("adapter dead after successful calls")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibvirtBackend_unansweredCallIsAbandoned(t *testing.T) {
|
||||
t.Parallel()
|
||||
b := defaultBudgets
|
||||
b.rpc = 100 * time.Millisecond
|
||||
ad := mockAdapter(t, b)
|
||||
// The mock does not implement ConnectGetHostname, so it never replies.
|
||||
_, err := ad.ConnectGetHostname(context.Background())
|
||||
if !errors.Is(err, provider.ErrTransient) || !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("ConnectGetHostname() error = %v, want ErrTransient wrapping DeadlineExceeded", err)
|
||||
}
|
||||
if !isDead(ad) {
|
||||
t.Error("adapter not dead after an abandoned call")
|
||||
}
|
||||
select {
|
||||
case <-ad.lv.Disconnected():
|
||||
case <-time.After(time.Second):
|
||||
t.Error("go-libvirt did not notice the closed socket")
|
||||
}
|
||||
if _, err := ad.ConnectGetLibVersion(context.Background()); !errors.Is(err, provider.ErrTransient) {
|
||||
t.Errorf("call on dead adapter error = %v, want ErrTransient", err)
|
||||
}
|
||||
if err := ad.Close(); err != nil {
|
||||
t.Errorf("Close() on dead adapter error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibvirtBackend_closeIsIdempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
ad := mockAdapter(t, defaultBudgets)
|
||||
first := ad.Close()
|
||||
second := ad.Close()
|
||||
if first != nil || second != nil {
|
||||
t.Errorf("Close() errors = %v, %v, want nil, nil", first, second)
|
||||
}
|
||||
if !isDead(ad) {
|
||||
t.Error("adapter not dead after Close")
|
||||
}
|
||||
if _, err := ad.ConnectGetLibVersion(context.Background()); !errors.Is(err, provider.ErrTransient) {
|
||||
t.Errorf("call after Close error = %v, want ErrTransient", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user