Add KVM provider integration test against a real hypervisor
Behind -tags=integration and self-skipping unless KVM_TEST_* names a host: one sequential lifecycle (image fetch, Create, idempotent re-Create, Running with IP, ListByTag, Delete, pool state afterwards), driven through the provider directly so it needs no Step 5 wiring. Adds `make test-integration-kvm` and documents the run and the deviation from the testcontainers convention in docs/testing.md. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,291 @@
|
||||
//go:build integration
|
||||
|
||||
package kvm
|
||||
|
||||
// Integration test against a real libvirt/KVM hypervisor. It deviates from
|
||||
// the repo's testcontainers convention on purpose: libvirt needs /dev/kvm
|
||||
// and a bridge, neither of which a container provides, so this runs
|
||||
// against whatever host the KVM_TEST_* variables name. It creates one VM
|
||||
// from a stock cloud image, waits for it to report an address, deletes it,
|
||||
// and checks the pool afterwards. The base image stays cached on the host.
|
||||
//
|
||||
// Required:
|
||||
//
|
||||
// KVM_TEST_HOST SSH endpoint, host:port
|
||||
// KVM_TEST_USER SSH user (member of the libvirt group)
|
||||
// KVM_TEST_SSH_KEY path to the private key
|
||||
// KVM_TEST_BRIDGE host bridge for the VM's NIC
|
||||
// KVM_TEST_IMAGE_URL cloud image to fetch (http/https)
|
||||
//
|
||||
// Optional:
|
||||
//
|
||||
// KVM_TEST_KNOWN_HOSTS known_hosts file; without it host keys are NOT verified
|
||||
// KVM_TEST_IMAGE_SHA256 expected digest of the image
|
||||
// KVM_TEST_POOL storage pool (default "default")
|
||||
// KVM_TEST_SOCKET libvirt socket path on the host
|
||||
// KVM_TEST_TIMEOUT overall budget incl. the image download (default 20m)
|
||||
// KVM_TEST_KEEP set to anything to leave the VM behind for inspection
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
const (
|
||||
integrationHostName = "kvm-test"
|
||||
integrationImageKey = "test-image"
|
||||
integrationPoll = 5 * time.Second
|
||||
)
|
||||
|
||||
type integrationEnv struct {
|
||||
cfg provider.ProviderConfig
|
||||
timeout time.Duration
|
||||
keep bool
|
||||
}
|
||||
|
||||
func loadIntegrationEnv(t *testing.T) integrationEnv {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("integration test skipped under -short")
|
||||
}
|
||||
required := map[string]string{}
|
||||
for _, key := range []string{"KVM_TEST_HOST", "KVM_TEST_USER", "KVM_TEST_SSH_KEY", "KVM_TEST_BRIDGE", "KVM_TEST_IMAGE_URL"} {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
t.Skipf("%s not set; skipping KVM integration test", key)
|
||||
}
|
||||
required[key] = v
|
||||
}
|
||||
cfg := &provider.KVMConfig{
|
||||
SSHKeyFile: required["KVM_TEST_SSH_KEY"],
|
||||
KnownHostsFile: os.Getenv("KVM_TEST_KNOWN_HOSTS"),
|
||||
Images: map[string]provider.KVMImage{
|
||||
integrationImageKey: {URL: required["KVM_TEST_IMAGE_URL"], SHA256: os.Getenv("KVM_TEST_IMAGE_SHA256")},
|
||||
},
|
||||
MachineTypes: map[string]provider.KVMMachineType{"small": {VCPUs: 1, MemoryMiB: 1024}},
|
||||
Hosts: []provider.KVMHost{{
|
||||
Name: integrationHostName,
|
||||
Address: required["KVM_TEST_HOST"],
|
||||
User: required["KVM_TEST_USER"],
|
||||
Socket: os.Getenv("KVM_TEST_SOCKET"),
|
||||
Pool: os.Getenv("KVM_TEST_POOL"),
|
||||
Bridge: required["KVM_TEST_BRIDGE"],
|
||||
}},
|
||||
}
|
||||
if cfg.KnownHostsFile == "" {
|
||||
cfg.InsecureIgnoreHostKey = true
|
||||
t.Log("KVM_TEST_KNOWN_HOSTS not set: host key verification disabled for this run")
|
||||
}
|
||||
pc := provider.ProviderConfig{Name: "kvm-integration", Type: "kvm", KVM: cfg}
|
||||
timeout := 20 * time.Minute
|
||||
if v := os.Getenv("KVM_TEST_TIMEOUT"); v != "" {
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil {
|
||||
t.Fatalf("KVM_TEST_TIMEOUT: %v", err)
|
||||
}
|
||||
timeout = d
|
||||
}
|
||||
return integrationEnv{cfg: pc, timeout: timeout, keep: os.Getenv("KVM_TEST_KEEP") != ""}
|
||||
}
|
||||
|
||||
func randomUID(t *testing.T) string {
|
||||
t.Helper()
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := hex.EncodeToString(b[:])
|
||||
return fmt.Sprintf("%s-%s-%s-%s-%s", h[0:8], h[8:12], h[12:16], h[16:20], h[20:32])
|
||||
}
|
||||
|
||||
// TestKVM_lifecycle is one sequential story, not parallel subtests: each
|
||||
// stage depends on the state the previous one left on the hypervisor.
|
||||
func TestKVM_lifecycle(t *testing.T) {
|
||||
env := loadIntegrationEnv(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), env.timeout)
|
||||
defer cancel()
|
||||
|
||||
prov, err := New(ctx, env.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
p := prov.(*Provider)
|
||||
host := p.cfg.Hosts[0]
|
||||
|
||||
uid := randomUID(t)
|
||||
req := provider.CreateRequest{
|
||||
Name: provider.NameFromUID(types.UID(uid)),
|
||||
UID: uid,
|
||||
Namespace: "integration",
|
||||
ProxyName: "kvm-integration",
|
||||
Placement: provider.Placement{Zone: integrationHostName, MachineType: "small", Image: integrationImageKey},
|
||||
CloudInit: "#cloud-config\n",
|
||||
Port: 3128,
|
||||
}
|
||||
t.Logf("host %s (%s), pool %q, bridge %s, domain %s", host.Name, host.Address, host.Pool, host.Bridge, req.Name)
|
||||
|
||||
// Cleanup runs even when a stage fails, with its own budget: the test
|
||||
// ctx may already be spent by then.
|
||||
t.Cleanup(func() {
|
||||
if env.keep {
|
||||
t.Logf("KVM_TEST_KEEP set: leaving domain %s and its volumes on %s", req.Name, host.Address)
|
||||
return
|
||||
}
|
||||
cctx, ccancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer ccancel()
|
||||
if err := p.Delete(cctx, formatProviderID(host.Name, req.Name)); err != nil {
|
||||
t.Errorf("cleanup Delete: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// --- create (includes the image fetch on a cold host) ---
|
||||
start := time.Now()
|
||||
id, err := p.Create(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
t.Logf("Create returned %s after %s", id, time.Since(start).Round(time.Second))
|
||||
if want := formatProviderID(host.Name, req.Name); id != want {
|
||||
t.Fatalf("providerID = %s, want %s", id, want)
|
||||
}
|
||||
|
||||
// --- create again: must find what it made, not duplicate it ---
|
||||
id2, err := p.Create(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("second Create: %v", err)
|
||||
}
|
||||
if id2 != id {
|
||||
t.Fatalf("second Create returned %s, want %s", id2, id)
|
||||
}
|
||||
|
||||
// --- wait for the guest to come up with an address ---
|
||||
inst := waitForRunning(ctx, t, p, id)
|
||||
t.Logf("domain Running with IP %s (state reported after %s)", inst.IP, time.Since(start).Round(time.Second))
|
||||
if inst.UID != uid {
|
||||
t.Errorf("UID read back = %q, want %q (metadata round-trip through libvirt)", inst.UID, uid)
|
||||
}
|
||||
if age := time.Since(inst.CreatedAt); inst.CreatedAt.IsZero() || age < 0 || age > time.Hour {
|
||||
t.Errorf("CreatedAt = %s (age %s), want a recent timestamp", inst.CreatedAt, age)
|
||||
}
|
||||
if conn, err := net.DialTimeout("tcp", net.JoinHostPort(inst.IP, "22"), 5*time.Second); err != nil {
|
||||
t.Logf("note: tcp %s:22 not reachable from this machine (%v); fine if the bridge network is not routed here", inst.IP, err)
|
||||
} else {
|
||||
conn.Close()
|
||||
t.Logf("tcp %s:22 reachable from this machine", inst.IP)
|
||||
}
|
||||
|
||||
// --- the GC view must include it ---
|
||||
listed, err := p.ListByTag(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListByTag: %v", err)
|
||||
}
|
||||
if !containsID(listed, id) {
|
||||
t.Errorf("ListByTag = %+v, want it to include %s", listed, id)
|
||||
}
|
||||
|
||||
// --- delete, then verify nothing of the VM remains but the base image does ---
|
||||
if err := p.Delete(ctx, id); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
if _, err := p.Get(ctx, id); !errors.Is(err, provider.ErrNotFound) {
|
||||
t.Errorf("Get after Delete = %v, want ErrNotFound", err)
|
||||
}
|
||||
if err := p.Delete(ctx, id); err != nil {
|
||||
t.Errorf("second Delete: %v, want nil (idempotent)", err)
|
||||
}
|
||||
listed, err = p.ListByTag(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListByTag after Delete: %v", err)
|
||||
}
|
||||
if containsID(listed, id) {
|
||||
t.Errorf("ListByTag still includes %s after Delete", id)
|
||||
}
|
||||
|
||||
src, err := resolveImage(p.cfg, integrationImageKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
|
||||
pool, err := api.StoragePoolLookupByName(host.Pool)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, gone := range []string{overlayVolumeName(req.Name), seedVolumeName(req.Name)} {
|
||||
if _, err := api.StorageVolLookupByName(pool, gone); !hasCode(err, libvirt.ErrNoStorageVol) {
|
||||
t.Errorf("volume %s after Delete: lookup err = %v, want ErrNoStorageVol", gone, err)
|
||||
}
|
||||
}
|
||||
for _, kept := range []string{src.volumeName(), src.markerName()} {
|
||||
if _, err := api.StorageVolLookupByName(pool, kept); err != nil {
|
||||
t.Errorf("cached image volume %s: %v, want it kept after Delete", kept, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("inspecting pool after Delete: %v", err)
|
||||
}
|
||||
// KVM_TEST_KEEP is for inspecting a failed run; a passed run has
|
||||
// nothing left to keep, so let cleanup's Delete confirm that.
|
||||
env.keep = false
|
||||
}
|
||||
|
||||
// waitForRunning polls Get until the instance is Running with an IP or ctx
|
||||
// ends. Transient errors are retried; anything else fails the test.
|
||||
func waitForRunning(ctx context.Context, t *testing.T, p *Provider, id string) *provider.Instance {
|
||||
t.Helper()
|
||||
var last provider.InstanceState
|
||||
ticker := time.NewTicker(integrationPoll)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
inst, err := p.Get(ctx, id)
|
||||
switch {
|
||||
case err == nil:
|
||||
if inst.State != last {
|
||||
t.Logf("state %s ip %q", inst.State, inst.IP)
|
||||
last = inst.State
|
||||
}
|
||||
if inst.State == provider.StateRunning {
|
||||
return inst
|
||||
}
|
||||
if inst.State == provider.StateStopped {
|
||||
t.Fatalf("domain reported %s while waiting for Running; check `virsh console %s` on the host", inst.State, idName(id))
|
||||
}
|
||||
case errors.Is(err, provider.ErrTransient):
|
||||
t.Logf("Get: transient error, retrying: %v", err)
|
||||
default:
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("timed out waiting for Running (last state %s): %v", last, ctx.Err())
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func idName(id string) string {
|
||||
_, name, _ := parseProviderID(id)
|
||||
return name
|
||||
}
|
||||
|
||||
func containsID(list []provider.Instance, id string) bool {
|
||||
for _, inst := range list {
|
||||
if inst.ID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user