kacerr/egress-proxies-operator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
10ca71331ec30fbbe6ef3fa96342c707d0a9dbc0
egress-proxies-operator/internal/provider/kvm/kvm_test.go
T
kacerrandClaude Opus 5 10ca71331e Load the KVM ssh material on first use instead of at startup 
The kvm-ssh Secret is mounted optional, but kvm.New read the key and
known_hosts eagerly and registry.Build aborts the whole provider set on a
constructor error, so a missing or misnamed Secret exited the manager
before any controller started and took the kubernetes and gcp providers
down with it.

lazyAuth loads the material on the first connection and caches it once it
works, so a Secret created after the manager starts heals the affected
proxies on the next reconcile with no restart. A missing file classifies
transient for that reason; a key that does not parse is permanent.

New still checks both files and logs a WARNING: line when they are
unreadable, keeping the startup diagnostic the eager load was there for.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-10 22:34:28 +00:00

1018 lines
33 KiB
Go
Raw Blame History

package kvm
import (
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"strings"
"testing"
"time"
libvirt "github.com/digitalocean/go-libvirt"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
func TestWithDefaults(t *testing.T) {
t.Parallel()
cfg := withDefaults(provider.KVMConfig{
Images: map[string]provider.KVMImage{
"debian": {URL: "https://x/d.qcow2"},
"raw": {URL: "https://x/r.img", Format: "raw", Firmware: "bios"},
},
Hosts: []provider.KVMHost{
{Name: "a", Address: "a:22", User: "virt", Bridge: "br0"},
{Name: "b", Address: "b:22", User: "virt", Bridge: "br1", Socket: "/run/s", Pool: "fast"},
},
})
if cfg.DiskSizeGB != defaultDiskSizeGB {
t.Errorf("diskSizeGb = %d, want %d", cfg.DiskSizeGB, defaultDiskSizeGB)
}
if cfg.VendorData != "" {
t.Errorf("vendorData = %q, want none: the guest-agent bootstrap is merged into the user-data, not defaulted here", cfg.VendorData)
}
if cfg.Images["debian"].Format != "qcow2" || cfg.Images["raw"].Format != "raw" {
t.Errorf("image formats = %q/%q, want qcow2 default and raw kept", cfg.Images["debian"].Format, cfg.Images["raw"].Format)
}
if cfg.Images["debian"].Firmware != firmwareUEFI || cfg.Images["raw"].Firmware != firmwareBIOS {
t.Errorf("image firmware = %q/%q, want uefi default and bios kept", cfg.Images["debian"].Firmware, cfg.Images["raw"].Firmware)
}
if a := cfg.Hosts[0]; a.Socket != defaultSocket || a.Pool != defaultPool {
t.Errorf("host a = %+v, want default socket and pool", a)
}
if b := cfg.Hosts[1]; b.Socket != "/run/s" || b.Pool != "fast" {
t.Errorf("host b = %+v, want explicit socket and pool kept", b)
}
}
func TestWithDefaults_keepsExplicitValues(t *testing.T) {
t.Parallel()
cfg := withDefaults(provider.KVMConfig{DiskSizeGB: 42, VendorData: "#cloud-config\n"})
if cfg.DiskSizeGB != 42 || cfg.VendorData != "#cloud-config\n" {
t.Errorf("cfg = %+v, want explicit values kept", cfg)
}
}
// --- provider over fake hosts ---
const (
testName = "proxy-abc123def456ghij"
testUID = "11111111-2222-3333-4444-555555555555"
testImageURL = "https://images.example/debian-13.qcow2"
testCloudInit = "#cloud-config\npackages: [squid]\n"
)
var (
testNow = time.Date(2026, 9, 8, 10, 0, 0, 0, time.UTC)
errHostOff = errors.New("ssh dial: connection refused")
)
// hostFakes routes each configured host name to its own fakeAPI; a host
// with no entry is unreachable.
type hostFakes map[string]*fakeAPI
func (h hostFakes) dial(_ context.Context, host provider.KVMHost) (*hostClient, error) {
api, ok := h[host.Name]
if !ok {
return nil, errHostOff
}
return &hostClient{api: api, close: func() {}}, nil
}
func testConfig(hosts ...string) provider.ProviderConfig {
cfg := &provider.KVMConfig{
SSHKeyFile: "/k",
KnownHostsFile: "/kh",
Images: map[string]provider.KVMImage{"debian-13": {URL: testImageURL}},
MachineTypes: map[string]provider.KVMMachineType{"small": {VCPUs: 2, MemoryMiB: 2048}},
}
for _, h := range hosts {
cfg.Hosts = append(cfg.Hosts, provider.KVMHost{Name: h, Address: h + ":22", User: "virt", Bridge: "br0"})
}
return provider.ProviderConfig{Name: "kvm-home", Type: "kvm", KVM: cfg}
}
func newTestProvider(fakes hostFakes, hosts ...string) *Provider {
p := newWithDialer(testConfig(hosts...), fakes.dial, nil)
p.now = func() time.Time { return testNow }
return p
}
// cacheImage pre-populates the pool with the configured image and its
// marker so Create never touches the network.
func cacheImage(api *fakeAPI) {
src := imageSource{Key: "debian-13", URL: testImageURL, Format: formatQcow2}
api.putVol("default", src.volumeName(), formatQcow2, []byte("qcow2"))
api.putVol("default", src.markerName(), formatRaw, []byte(testImageSHA()))
}
func readyHost() *fakeAPI {
api := newFakeAPI("default")
cacheImage(api)
return api
}
func testRequest() provider.CreateRequest {
return provider.CreateRequest{
Name: testName,
UID: testUID,
Namespace: "default",
ProxyName: "home-proxy-1",
Placement: provider.Placement{Zone: "nas1", MachineType: "small", Image: "debian-13"},
CloudInit: testCloudInit,
Port: 3128,
}
}
func metadataXML(t *testing.T, uid string, created time.Time) string {
t.Helper()
out, err := xml.Marshal(newInstanceMetadata(uid, created))
if err != nil {
t.Fatal(err)
}
return string(out)
}
func ipv4(addr string) libvirt.DomainIPAddr {
return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv4), Addr: addr, Prefix: 24}
}
func ipv6(addr string) libvirt.DomainIPAddr {
return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv6), Addr: addr, Prefix: 64}
}
// managedDomain seeds a domain as Create would have left it.
func managedDomain(t *testing.T, api *fakeAPI, name, uid string, running bool, addrs map[uint32][]libvirt.DomainInterface) {
t.Helper()
api.putDomain(&fakeDomain{name: name, metadata: metadataXML(t, uid, testNow), running: running, addrs: addrs})
api.putVol("default", overlayVolumeName(name), formatQcow2, nil)
api.putVol("default", seedVolumeName(name), formatRaw, []byte("iso"))
}
func wantClass(t *testing.T, err error, class error) {
t.Helper()
if err == nil {
t.Fatalf("got nil error, want %v", class)
}
if !errors.Is(err, class) {
t.Fatalf("error class = %v, want %v (err: %v)", provider.Class(err), class, err)
}
}
func assertCreated(t *testing.T, api *fakeAPI) {
t.Helper()
dom := api.domain(testName)
if dom == nil {
t.Fatal("domain not defined")
}
if !dom.running {
t.Error("domain not running")
}
overlay := api.vol("default", overlayVolumeName(testName))
if overlay == nil {
t.Fatal("overlay volume missing")
}
if overlay.backing != "/var/lib/libvirt/images/base-debian-13.qcow2" || overlay.format != formatQcow2 {
t.Errorf("overlay = %+v, want qcow2 backed by the cached base", overlay)
}
if overlay.capacity != defaultDiskSizeGB*bytesPerGiB {
t.Errorf("overlay capacity = %d, want %d", overlay.capacity, defaultDiskSizeGB*bytesPerGiB)
}
seed := api.vol("default", seedVolumeName(testName))
if seed == nil {
t.Fatal("seed volume missing")
}
if seed.format != formatRaw || len(seed.data) == 0 || int64(len(seed.data)) != seed.capacity {
t.Errorf("seed = format %s, %d bytes, capacity %d; want raw with capacity == uploaded size", seed.format, len(seed.data), seed.capacity)
}
_, files := readSeed(t, seed.data)
if ud := files["user-data"]; !strings.Contains(ud, "squid") || !strings.Contains(ud, guestAgentPackage) {
t.Errorf("seed user-data = %q, want the Proxy's cloud-config with the guest agent merged in", ud)
}
}
func TestCreate_freshHost(t *testing.T) {
t.Parallel()
api := readyHost()
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
id, err := p.Create(context.Background(), testRequest())
if err != nil {
t.Fatalf("Create: %v", err)
}
if want := "hosts/nas1/domains/" + testName; id != want {
t.Errorf("providerID = %s, want %s", id, want)
}
assertCreated(t, api)
var d domainXML
if err := xml.Unmarshal([]byte(api.domain(testName).xml), &d); err != nil {
t.Fatal(err)
}
if d.VCPU != 2 || d.Memory.Value != 2048 || d.Memory.Unit != "MiB" {
t.Errorf("domain size = %d vcpu / %d %s, want 2 / 2048 MiB", d.VCPU, d.Memory.Value, d.Memory.Unit)
}
if d.Devices.Interfaces[0].Source.Bridge != "br0" {
t.Errorf("bridge = %q, want br0", d.Devices.Interfaces[0].Source.Bridge)
}
wantDisk, wantSeed := api.poolPath+"/"+overlayVolumeName(testName), api.poolPath+"/"+seedVolumeName(testName)
if got := d.Devices.Disks[0].Source.File; got != wantDisk {
t.Errorf("overlay disk source = %q, want the path libvirt reported, %q", got, wantDisk)
}
if got := d.Devices.Disks[1].Source.File; got != wantSeed {
t.Errorf("seed cdrom source = %q, want the path libvirt reported, %q", got, wantSeed)
}
if m := d.Metadata.Instance; m.UID != testUID || !m.managed() || !m.createdAt().Equal(testNow) {
t.Errorf("metadata = %+v, want managed, uid %s, createdAt %s", m, testUID, testNow)
}
if api.hasCall("StorageVolDownload") || api.callCount("StorageVolUpload") != 1 {
t.Errorf("calls = %v, want no image download and exactly one upload (the seed)", api.calls)
}
}
func TestCreate_fetchesImageOnFirstUse(t *testing.T) {
t.Parallel()
srv := newImageServer(t)
api := newFakeAPI("default")
cfg := testConfig("nas1")
cfg.KVM.Images["debian-13"] = provider.KVMImage{URL: srv.URL + "/debian.qcow2", SHA256: testImageSHA()}
p := newWithDialer(cfg, hostFakes{"nas1": api}.dial, nil)
if _, err := p.Create(context.Background(), testRequest()); err != nil {
t.Fatalf("Create: %v", err)
}
if srv.hits.Load() != 1 {
t.Errorf("image server hits = %d, want 1", srv.hits.Load())
}
base := api.vol("default", "base-debian-13.qcow2")
if base == nil || string(base.data) != string(testImage) {
t.Fatal("base image not cached in the pool")
}
if api.domain(testName) == nil || !api.domain(testName).running {
t.Error("domain not running after image fetch")
}
// A second Proxy with the same image on the same host reuses the cache.
req := testRequest()
req.Name, req.UID = "proxy-second", "uid-2"
if _, err := p.Create(context.Background(), req); err != nil {
t.Fatalf("second Create: %v", err)
}
if srv.hits.Load() != 1 {
t.Errorf("image server hits after second Create = %d, want still 1", srv.hits.Load())
}
}
func TestCreate_isIdempotentWhenComplete(t *testing.T) {
t.Parallel()
api := readyHost()
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
if _, err := p.Create(context.Background(), testRequest()); err != nil {
t.Fatal(err)
}
before := len(api.calls)
id, err := p.Create(context.Background(), testRequest())
if err != nil {
t.Fatalf("second Create: %v", err)
}
if id != "hosts/nas1/domains/"+testName {
t.Errorf("providerID = %s", id)
}
for _, m := range []string{"DomainDefineXML", "DomainCreate", "StorageVolUpload"} {
if n := api.callCount(m); n != 1 {
t.Errorf("%s called %d times across two Creates, want 1", m, n)
}
}
if len(api.calls) == before {
t.Error("second Create made no calls at all; expected lookups")
}
}
// Each case breaks Create after a different step, then re-runs it and
// expects the second attempt to pick up where the first stopped without
// duplicating what already exists.
func TestCreate_resumesAfterPartialFailure(t *testing.T) {
t.Parallel()
cases := []struct {
name string
sabotage func(api *fakeAPI)
class error
check func(t *testing.T, api *fakeAPI)
}{
{
name: "seed upload fails after overlay exists",
sabotage: func(api *fakeAPI) {
api.uploadHook = func(vol string, _ io.Reader) error {
api.uploadHook = nil
return io.ErrUnexpectedEOF
}
},
class: provider.ErrTransient,
check: func(t *testing.T, api *fakeAPI) {
if n := api.callCount("StorageVolUpload"); n != 2 {
t.Errorf("seed uploads = %d, want 2 (failed, then redone)", n)
}
if n := api.callCount("StorageVolCreateXML"); n != 4 {
t.Errorf("volume creates = %d, want 4 (overlay, seed; overlay rejected as existing, seed again)", n)
}
if names := api.volNames("default"); len(names) != 4 {
t.Errorf("pool holds %v, want base, marker, one overlay, one seed", names)
}
},
},
{
name: "define fails after volumes exist",
sabotage: func(api *fakeAPI) { api.failOnce["DomainDefineXML"] = lvError(libvirt.ErrInternalError, "boom") },
class: provider.ErrTransient,
check: func(t *testing.T, api *fakeAPI) {
if n := api.callCount("StorageVolUpload"); n != 2 {
t.Errorf("seed uploads = %d, want 2 (seed rebuilt while no domain exists)", n)
}
if n := api.callCount("DomainDefineXML"); n != 2 {
t.Errorf("defines = %d, want 2", n)
}
},
},
{
name: "start fails after define",
sabotage: func(api *fakeAPI) { api.failOnce["DomainCreate"] = lvError(libvirt.ErrInternalError, "boom") },
class: provider.ErrTransient,
check: func(t *testing.T, api *fakeAPI) {
if n := api.callCount("DomainDefineXML"); n != 1 {
t.Errorf("defines = %d, want 1 (existing domain reused)", n)
}
if n := api.callCount("DomainCreate"); n != 2 {
t.Errorf("starts = %d, want 2", n)
}
},
},
{
name: "transport drops mid-create",
sabotage: func(api *fakeAPI) { api.failOnce["DomainLookupByName"] = io.EOF },
class: provider.ErrTransient,
check: func(t *testing.T, api *fakeAPI) {
if n := api.callCount("DomainDefineXML"); n != 1 {
t.Errorf("defines = %d, want 1", n)
}
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
api := readyHost()
tc.sabotage(api)
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
_, err := p.Create(context.Background(), testRequest())
wantClass(t, err, tc.class)
if _, err := p.Create(context.Background(), testRequest()); err != nil {
t.Fatalf("retry Create: %v", err)
}
assertCreated(t, api)
tc.check(t, api)
})
}
}
// A reconcile cancelled mid-upload leaves a seed volume whose content is
// short; the compensating delete cannot run because the pool has already
// closed the connection. The next Create must not trust that volume.
func TestCreate_replacesLeftoverSeed(t *testing.T) {
t.Parallel()
api := readyHost()
api.putVol("default", seedVolumeName(testName), formatRaw, []byte("trunc"))
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
if _, err := p.Create(context.Background(), testRequest()); err != nil {
t.Fatal(err)
}
assertCreated(t, api)
if seed := api.vol("default", seedVolumeName(testName)); string(seed.data) == "trunc" {
t.Error("leftover seed was kept")
}
if n := api.callCount("StorageVolUpload"); n != 1 {
t.Errorf("uploads = %d, want 1 (the rebuilt seed)", n)
}
}
func TestCreate_startsStoppedDomain(t *testing.T) {
t.Parallel()
api := readyHost()
managedDomain(t, api, testName, testUID, false, nil)
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
if _, err := p.Create(context.Background(), testRequest()); err != nil {
t.Fatal(err)
}
if !api.domain(testName).running {
t.Error("domain still stopped")
}
if api.hasCall("DomainDefineXML") || api.hasCall("StorageVolUpload") || api.hasCall("StorageVolDelete") {
t.Errorf("calls = %v, want no define, upload or volume delete for an existing domain", api.calls)
}
}
func TestCreate_singleHostNeedsNoZone(t *testing.T) {
t.Parallel()
api := readyHost()
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
req := testRequest()
req.Placement.Zone = ""
id, err := p.Create(context.Background(), req)
if err != nil {
t.Fatal(err)
}
if id != "hosts/nas1/domains/"+testName {
t.Errorf("providerID = %s, want the single host filled in", id)
}
}
func TestCreate_placementErrors(t *testing.T) {
t.Parallel()
cases := []struct {
name string
hosts []string
edit func(*provider.CreateRequest)
want string
}{
{"unknown host", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Zone = "nas9" }, "not in kvm.hosts"},
{"empty zone with two hosts", []string{"nas1", "nas2"}, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "placement.zone is required"},
{"empty zone with no hosts", nil, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "kvm.hosts is empty"},
{"unknown machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "huge" }, "not in kvm.machineTypes"},
{"empty machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "" }, "not in kvm.machineTypes"},
{"unknown image", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Image = "windows" }, "neither a key"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fakes := hostFakes{}
for _, h := range tc.hosts {
fakes[h] = readyHost()
}
p := newTestProvider(fakes, tc.hosts...)
req := testRequest()
tc.edit(&req)
_, err := p.Create(context.Background(), req)
wantClass(t, err, provider.ErrPermanent)
if !strings.Contains(err.Error(), tc.want) {
t.Errorf("error = %q, want it to mention %q", err, tc.want)
}
for _, api := range fakes {
if len(api.calls) != 0 {
t.Errorf("host was called: %v", api.calls)
}
}
})
}
}
func TestCreate_hostFailures(t *testing.T) {
t.Parallel()
cases := []struct {
name string
setup func() (hostFakes, string)
class error
}{
{
name: "host unreachable",
setup: func() (hostFakes, string) { return hostFakes{}, "nas1" },
class: provider.ErrTransient,
},
{
name: "pool missing",
setup: func() (hostFakes, string) {
api := newFakeAPI("other")
return hostFakes{"nas1": api}, "nas1"
},
class: provider.ErrPermanent,
},
{
name: "domain xml rejected",
setup: func() (hostFakes, string) {
api := readyHost()
api.fail["DomainDefineXML"] = lvError(libvirt.ErrConfigUnsupported, "no q35 here")
return hostFakes{"nas1": api}, "nas1"
},
class: provider.ErrPermanent,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fakes, host := tc.setup()
p := newTestProvider(fakes, host)
_, err := p.Create(context.Background(), testRequest())
wantClass(t, err, tc.class)
})
}
}
func TestCreate_cancelledContext(t *testing.T) {
t.Parallel()
api := readyHost()
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := p.Create(ctx, testRequest())
wantClass(t, err, provider.ErrTransient)
if !errors.Is(err, context.Canceled) {
t.Errorf("error = %v, want it to unwrap to context.Canceled", err)
}
}
func TestGet(t *testing.T) {
t.Parallel()
arp := map[uint32][]libvirt.DomainInterface{
uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Name: "vnet0", Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.50")}}},
}
agentOnly := map[uint32][]libvirt.DomainInterface{
uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
{Name: "lo", Addrs: []libvirt.DomainIPAddr{ipv4("127.0.0.1"), ipv6("::1")}},
{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv6("fe80::1"), ipv4("192.168.1.51")}},
},
}
linkLocalOnly := map[uint32][]libvirt.DomainInterface{
uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv4("169.254.10.10")}},
},
}
cases := []struct {
name string
running bool
addrs map[uint32][]libvirt.DomainInterface
state provider.InstanceState
ip string
}{
{"running with arp address", true, arp, provider.StateRunning, "192.168.1.50"},
{"running, agent only, skips loopback and v6", true, agentOnly, provider.StateRunning, "192.168.1.51"},
{"running, no address yet", true, nil, provider.StateProvisioning, ""},
{"running, only link-local", true, linkLocalOnly, provider.StateProvisioning, ""},
{"shut off", false, arp, provider.StateStopped, ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
api := readyHost()
managedDomain(t, api, testName, testUID, tc.running, tc.addrs)
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
inst, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
if err != nil {
t.Fatalf("Get: %v", err)
}
want := provider.Instance{ID: "hosts/nas1/domains/" + testName, IP: tc.ip, State: tc.state, UID: testUID, CreatedAt: testNow}
if *inst != want {
t.Errorf("instance = %+v, want %+v", *inst, want)
}
})
}
}
func TestGet_unmanagedDomainHasNoUID(t *testing.T) {
t.Parallel()
api := readyHost()
api.putDomain(&fakeDomain{name: "hand-made", running: true})
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
inst, err := p.Get(context.Background(), "hosts/nas1/domains/hand-made")
if err != nil {
t.Fatalf("Get: %v", err)
}
if inst.UID != "" || !inst.CreatedAt.IsZero() {
t.Errorf("instance = %+v, want empty UID and zero CreatedAt", inst)
}
}
func TestGet_errors(t *testing.T) {
t.Parallel()
cases := []struct {
name string
id string
setup func(t *testing.T, api *fakeAPI)
class error
}{
{"missing domain", "hosts/nas1/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrNotFound},
{"malformed id", "zones/x/instances/y", func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
{"host not configured", "hosts/nas9/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
{
"state read fails", "hosts/nas1/domains/" + testName,
func(t *testing.T, api *fakeAPI) {
managedDomain(t, api, testName, testUID, true, nil)
api.fail["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
},
provider.ErrTransient,
},
{
"corrupt metadata", "hosts/nas1/domains/" + testName,
func(_ *testing.T, api *fakeAPI) {
api.putDomain(&fakeDomain{name: testName, metadata: "<instance", running: true})
},
provider.ErrPermanent,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
api := readyHost()
tc.setup(t, api)
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
_, err := p.Get(context.Background(), tc.id)
wantClass(t, err, tc.class)
})
}
}
func TestGet_hostUnreachable(t *testing.T) {
t.Parallel()
p := newTestProvider(hostFakes{}, "nas1")
_, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
wantClass(t, err, provider.ErrTransient)
}
func assertGone(t *testing.T, api *fakeAPI) {
t.Helper()
if api.domain(testName) != nil {
t.Error("domain still defined")
}
if names := api.volNames("default"); len(names) != 2 {
t.Errorf("pool holds %v, want only the cached base image and its marker", names)
}
}
func TestDelete(t *testing.T) {
t.Parallel()
cases := []struct {
name string
setup func(t *testing.T, api *fakeAPI)
check func(t *testing.T, api *fakeAPI)
}{
{
name: "running domain",
setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, true, nil) },
check: func(t *testing.T, api *fakeAPI) {
if !api.hasCall("DomainDestroy") || !api.hasCall("DomainUndefineFlags") {
t.Errorf("calls = %v, want destroy and undefine", api.calls)
}
},
},
{
name: "stopped domain",
setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, false, nil) },
check: func(t *testing.T, api *fakeAPI) {
if !api.hasCall("DomainUndefineFlags") {
t.Errorf("calls = %v, want undefine despite destroy being invalid", api.calls)
}
},
},
{
name: "domain gone, volumes left",
setup: func(_ *testing.T, api *fakeAPI) {
api.putVol("default", overlayVolumeName(testName), formatQcow2, nil)
api.putVol("default", seedVolumeName(testName), formatRaw, nil)
},
check: func(t *testing.T, api *fakeAPI) {
if api.hasCall("DomainDestroy") || api.hasCall("DomainUndefineFlags") {
t.Errorf("calls = %v, want no domain operations", api.calls)
}
},
},
{
name: "nothing exists",
setup: func(*testing.T, *fakeAPI) {},
check: func(t *testing.T, api *fakeAPI) {
if api.hasCall("StorageVolDelete") {
t.Errorf("calls = %v, want no deletes", api.calls)
}
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
api := readyHost()
tc.setup(t, api)
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
t.Fatalf("Delete: %v", err)
}
assertGone(t, api)
tc.check(t, api)
if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
t.Fatalf("second Delete: %v", err)
}
})
}
}
func TestDelete_missingPoolIsNotAnError(t *testing.T) {
t.Parallel()
api := newFakeAPI("other")
api.putDomain(&fakeDomain{name: testName, metadata: metadataXML(t, testUID, testNow), running: true})
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
t.Fatalf("Delete: %v", err)
}
if api.domain(testName) != nil {
t.Error("domain still defined")
}
}
// A host dropped from the config leaves the operator no way to reach the
// domain, so Delete has nothing left to clean. Failing instead would pin
// the Proxy's finalizer forever, since the reconciler's delete path cannot
// make progress without a Delete that can succeed.
func TestDelete_unconfiguredHostIsNotAnError(t *testing.T) {
t.Parallel()
api := readyHost()
managedDomain(t, api, testName, testUID, true, nil)
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
if err := p.Delete(context.Background(), "hosts/nas9/domains/"+testName); err != nil {
t.Fatalf("Delete over an unconfigured host: %v", err)
}
if api.domain(testName) == nil {
t.Error("domain on the configured host was touched; only the named host's state may change")
}
// Get still refuses, so the live path cannot mistake a config typo for a
// missing VM and recreate it somewhere else.
if _, err := p.Get(context.Background(), "hosts/nas9/domains/"+testName); provider.Class(err) != provider.ErrPermanent {
t.Errorf("Get class = %v, want ErrPermanent", provider.Class(err))
}
}
func TestDelete_errors(t *testing.T) {
t.Parallel()
cases := []struct {
name string
id string
fakes func(t *testing.T) hostFakes
class error
}{
{"malformed id", "nope", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
{"host unreachable", "hosts/nas1/domains/x", func(*testing.T) hostFakes { return hostFakes{} }, provider.ErrTransient},
{
"undefine fails", "hosts/nas1/domains/" + testName,
func(t *testing.T) hostFakes {
api := readyHost()
managedDomain(t, api, testName, testUID, true, nil)
api.fail["DomainUndefineFlags"] = lvError(libvirt.ErrInternalError, "boom")
return hostFakes{"nas1": api}
},
provider.ErrTransient,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
p := newTestProvider(tc.fakes(t), "nas1")
wantClass(t, p.Delete(context.Background(), tc.id), tc.class)
})
}
}
func TestListByTag_acrossHosts(t *testing.T) {
t.Parallel()
nas1, nas2 := readyHost(), readyHost()
managedDomain(t, nas1, "proxy-one", "uid-1", true, map[uint32][]libvirt.DomainInterface{
uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.10")}}},
})
nas1.putDomain(&fakeDomain{name: "hand-made", running: true})
managedDomain(t, nas2, "proxy-two", "uid-2", false, nil)
nas2.putDomain(&fakeDomain{name: "foreign", metadata: `<instance xmlns="` + metadataURI + `"><managed>false</managed></instance>`})
p := newTestProvider(hostFakes{"nas1": nas1, "nas2": nas2}, "nas1", "nas2")
got, err := p.ListByTag(context.Background())
if err != nil {
t.Fatalf("ListByTag: %v", err)
}
want := []provider.Instance{
{ID: "hosts/nas1/domains/proxy-one", IP: "192.168.1.10", State: provider.StateRunning, UID: "uid-1", CreatedAt: testNow},
{ID: "hosts/nas2/domains/proxy-two", State: provider.StateStopped, UID: "uid-2", CreatedAt: testNow},
}
if len(got) != len(want) {
t.Fatalf("got %d instances %+v, want %d", len(got), got, len(want))
}
for i := range want {
if got[i] != want[i] {
t.Errorf("instance[%d] = %+v, want %+v", i, got[i], want[i])
}
}
}
func TestListByTag_oneHostDownFailsSweep(t *testing.T) {
t.Parallel()
nas1 := readyHost()
managedDomain(t, nas1, "proxy-one", "uid-1", true, nil)
p := newTestProvider(hostFakes{"nas1": nas1}, "nas1", "nas2")
got, err := p.ListByTag(context.Background())
wantClass(t, err, provider.ErrTransient)
if got != nil {
t.Errorf("got %+v alongside the error, want nil (no partial view)", got)
}
}
// A domain that cannot be read is left out of the sweep rather than
// failing it: GC never deletes what it does not see, so skipping is the
// safe direction, while one bad domain blocking GC for all others is not.
func TestListByTag_skipsUnreadableDomain(t *testing.T) {
t.Parallel()
cases := []struct {
name string
setup func(t *testing.T, api *fakeAPI)
}{
{
"deleted mid-sweep",
func(t *testing.T, api *fakeAPI) {
managedDomain(t, api, "proxy-one", "uid-1", true, nil)
api.failOnce["DomainGetMetadata"] = lvError(libvirt.ErrNoDomain, "gone")
},
},
{
"corrupt metadata",
func(_ *testing.T, api *fakeAPI) {
api.putDomain(&fakeDomain{name: "proxy-one", metadata: "<instance", running: true})
},
},
{
"state unreadable",
func(t *testing.T, api *fakeAPI) {
managedDomain(t, api, "proxy-one", "uid-1", true, nil)
api.failOnce["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
nas1 := readyHost()
tc.setup(t, nas1)
managedDomain(t, nas1, "proxy-two", "uid-2", false, nil)
p := newTestProvider(hostFakes{"nas1": nas1}, "nas1")
got, err := p.ListByTag(context.Background())
if err != nil {
t.Fatalf("ListByTag: %v", err)
}
if len(got) != 1 || got[0].UID != "uid-2" {
t.Errorf("got %+v, want only proxy-two", got)
}
})
}
}
func TestListByTag_noHosts(t *testing.T) {
t.Parallel()
p := newTestProvider(hostFakes{})
got, err := p.ListByTag(context.Background())
if err != nil || len(got) != 0 {
t.Errorf("got %v, %v; want empty, nil", got, err)
}
}
func TestMapState(t *testing.T) {
t.Parallel()
cases := []struct {
state libvirt.DomainState
ip string
want provider.InstanceState
}{
{libvirt.DomainRunning, "10.0.0.1", provider.StateRunning},
{libvirt.DomainRunning, "", provider.StateProvisioning},
{libvirt.DomainBlocked, "10.0.0.1", provider.StateRunning},
{libvirt.DomainPaused, "10.0.0.1", provider.StateStopped},
{libvirt.DomainShutdown, "", provider.StateStopped},
{libvirt.DomainShutoff, "", provider.StateStopped},
{libvirt.DomainCrashed, "", provider.StateStopped},
{libvirt.DomainPmsuspended, "", provider.StateStopped},
{libvirt.DomainNostate, "", provider.StateProvisioning},
}
for _, tc := range cases {
t.Run(fmt.Sprintf("%s/%q", tc.state, tc.ip), func(t *testing.T) {
t.Parallel()
if got := mapState(tc.state, tc.ip); got != tc.want {
t.Errorf("mapState(%s, %q) = %s, want %s", tc.state, tc.ip, got, tc.want)
}
})
}
}
// A bad key path no longer fails the constructor: registry.Build aborts the
// whole provider set on a constructor error, so that took the manager down
// with every other provider. It degrades per-proxy instead — see
// TestNew_missingSSHMaterialStillBuilds and TestLazyAuth_healsWhenTheKeyAppears.
func TestNew_badKeyPathDefersToFirstUse(t *testing.T) {
t.Parallel()
cfg := testConfig("nas1")
cfg.KVM.SSHKeyFile = t.TempDir() + "/missing"
p, err := New(context.Background(), cfg)
if err != nil {
t.Fatalf("New() error = %v, want a provider that fails only when used", err)
}
if err := p.Delete(context.Background(), "hosts/nas1/domains/proxy-x"); provider.Class(err) != provider.ErrTransient {
t.Errorf("Delete class = %v, want ErrTransient", provider.Class(err))
}
}
// runAllOps drives one full lifecycle over a ready fake host so logging
// tests see every provider method and every libvirt call.
func runAllOps(t *testing.T, ctx context.Context, req provider.CreateRequest) {
t.Helper()
p := newTestProvider(hostFakes{"nas1": readyHost()}, "nas1")
id, err := p.Create(ctx, req)
if err != nil {
t.Fatalf("Create: %v", err)
}
if _, err := p.Get(ctx, id); err != nil {
t.Fatalf("Get: %v", err)
}
if _, err := p.ListByTag(ctx); err != nil {
t.Fatalf("ListByTag: %v", err)
}
if err := p.Delete(ctx, id); err != nil {
t.Fatalf("Delete: %v", err)
}
}
func TestLogging_verbosityTiers(t *testing.T) {
t.Parallel()
tests := []struct {
name string
verbosity int
wantLines []string
absentLines []string
}{
{
name: "v0 stays silent",
verbosity: 0,
absentLines: []string{"libvirt call", "KVM domain ensured running"},
},
{
name: "v1 logs one line per libvirt call plus the operation outcomes",
verbosity: 1,
wantLines: []string{
`kvm/libvirt "level"=1 "msg"="libvirt host connected"`,
`"msg"="libvirt call"`,
`"rpc"="virDomainLookupByName"`,
`"libvirtError"="ErrNoDomain"`,
`"rpc"="virStorageVolUpload"`,
`"rpc"="virDomainDefineXML"`,
`"rpc"="virDomainCreate"`,
`"rpc"="virDomainInterfaceAddresses"`,
`"rpc"="virConnectListAllDomains"`,
`"rpc"="virDomainUndefineFlags"`,
`"rpc"="virStorageVolDelete"`,
`"provider"="kvm-home"`,
`"host"="nas1"`,
`"name"="` + testName + `"`,
`"msg"="KVM domain ensured running"`,
},
absentLines: []string{`"xml"=`, `"metadata"=`},
},
{
name: "v2 adds xml, metadata and addresses on the same lines",
verbosity: 2,
wantLines: []string{`"xml"="<domain`, `"xml"="<volume`, `"metadata"="<instance`, `"names"=[`},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx, snapshot := captureContext(tc.verbosity)
req := testRequest()
const sentinel = "SENTINEL-cloud-init-must-never-be-logged"
req.CloudInit = "#cloud-config\nhostname: " + sentinel + "\n"
runAllOps(t, ctx, req)
lines := snapshot()
joined := strings.Join(lines, "\n")
if tc.verbosity == 0 && len(lines) != 0 {
t.Errorf("verbosity 0 logged %d lines:\n%s", len(lines), joined)
}
for _, want := range tc.wantLines {
if !strings.Contains(joined, want) {
t.Errorf("output missing %s:\n%s", want, joined)
}
}
for _, absent := range tc.absentLines {
if strings.Contains(joined, absent) {
t.Errorf("output unexpectedly contains %s:\n%s", absent, joined)
}
}
if strings.Contains(joined, sentinel) {
t.Errorf("cloud-init content leaked into logs:\n%s", joined)
}
})
}
}
Reference in New Issue View Git Blame Copy Permalink