egress-proxies-operator/internal/provider/kvm/kvm_test.go

package kvm

import (
	"context"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"strings"
	"testing"
	"time"

	libvirt "github.com/digitalocean/go-libvirt"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

func TestWithDefaults(t *testing.T) {
	t.Parallel()
	cfg := withDefaults(provider.KVMConfig{
		Images: map[string]provider.KVMImage{
			"debian": {URL: "https://x/d.qcow2"},
			"raw":    {URL: "https://x/r.img", Format: "raw", Firmware: "bios"},
		},
		Hosts: []provider.KVMHost{
			{Name: "a", Address: "a:22", User: "virt", Bridge: "br0"},
			{Name: "b", Address: "b:22", User: "virt", Bridge: "br1", Socket: "/run/s", Pool: "fast"},
		},
	})
	if cfg.DiskSizeGB != defaultDiskSizeGB {
		t.Errorf("diskSizeGb = %d, want %d", cfg.DiskSizeGB, defaultDiskSizeGB)
	}
	if cfg.VendorData != defaultVendorData {
		t.Errorf("vendorData = %q, want the default", cfg.VendorData)
	}
	if cfg.Images["debian"].Format != "qcow2" || cfg.Images["raw"].Format != "raw" {
		t.Errorf("image formats = %q/%q, want qcow2 default and raw kept", cfg.Images["debian"].Format, cfg.Images["raw"].Format)
	}
	if cfg.Images["debian"].Firmware != firmwareUEFI || cfg.Images["raw"].Firmware != firmwareBIOS {
		t.Errorf("image firmware = %q/%q, want uefi default and bios kept", cfg.Images["debian"].Firmware, cfg.Images["raw"].Firmware)
	}
	if a := cfg.Hosts[0]; a.Socket != defaultSocket || a.Pool != defaultPool {
		t.Errorf("host a = %+v, want default socket and pool", a)
	}
	if b := cfg.Hosts[1]; b.Socket != "/run/s" || b.Pool != "fast" {
		t.Errorf("host b = %+v, want explicit socket and pool kept", b)
	}
}

func TestWithDefaults_keepsExplicitValues(t *testing.T) {
	t.Parallel()
	cfg := withDefaults(provider.KVMConfig{DiskSizeGB: 42, VendorData: "#cloud-config\n"})
	if cfg.DiskSizeGB != 42 || cfg.VendorData != "#cloud-config\n" {
		t.Errorf("cfg = %+v, want explicit values kept", cfg)
	}
}

// --- provider over fake hosts ---

const (
	testName      = "proxy-abc123def456ghij"
	testUID       = "11111111-2222-3333-4444-555555555555"
	testImageURL  = "https://images.example/debian-13.qcow2"
	testCloudInit = "#cloud-config\npackages: [squid]\n"
)

var (
	testNow    = time.Date(2026, 9, 8, 10, 0, 0, 0, time.UTC)
	errHostOff = errors.New("ssh dial: connection refused")
)

// hostFakes routes each configured host name to its own fakeAPI; a host
// with no entry is unreachable.
type hostFakes map[string]*fakeAPI

func (h hostFakes) dial(_ context.Context, host provider.KVMHost) (*hostClient, error) {
	api, ok := h[host.Name]
	if !ok {
		return nil, errHostOff
	}
	return &hostClient{api: api, close: func() {}}, nil
}

func testConfig(hosts ...string) provider.ProviderConfig {
	cfg := &provider.KVMConfig{
		SSHKeyFile:     "/k",
		KnownHostsFile: "/kh",
		Images:         map[string]provider.KVMImage{"debian-13": {URL: testImageURL}},
		MachineTypes:   map[string]provider.KVMMachineType{"small": {VCPUs: 2, MemoryMiB: 2048}},
	}
	for _, h := range hosts {
		cfg.Hosts = append(cfg.Hosts, provider.KVMHost{Name: h, Address: h + ":22", User: "virt", Bridge: "br0"})
	}
	return provider.ProviderConfig{Name: "kvm-home", Type: "kvm", KVM: cfg}
}

func newTestProvider(fakes hostFakes, hosts ...string) *Provider {
	p := newWithDialer(testConfig(hosts...), fakes.dial, nil)
	p.now = func() time.Time { return testNow }
	return p
}

// cacheImage pre-populates the pool with the configured image and its
// marker so Create never touches the network.
func cacheImage(api *fakeAPI) {
	src := imageSource{Key: "debian-13", URL: testImageURL, Format: formatQcow2}
	api.putVol("default", src.volumeName(), formatQcow2, []byte("qcow2"))
	api.putVol("default", src.markerName(), formatRaw, []byte(testImageSHA()))
}

func readyHost() *fakeAPI {
	api := newFakeAPI("default")
	cacheImage(api)
	return api
}

func testRequest() provider.CreateRequest {
	return provider.CreateRequest{
		Name:      testName,
		UID:       testUID,
		Namespace: "default",
		ProxyName: "home-proxy-1",
		Placement: provider.Placement{Zone: "nas1", MachineType: "small", Image: "debian-13"},
		CloudInit: testCloudInit,
		Port:      3128,
	}
}

func metadataXML(t *testing.T, uid string, created time.Time) string {
	t.Helper()
	out, err := xml.Marshal(newInstanceMetadata(uid, created))
	if err != nil {
		t.Fatal(err)
	}
	return string(out)
}

func ipv4(addr string) libvirt.DomainIPAddr {
	return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv4), Addr: addr, Prefix: 24}
}

func ipv6(addr string) libvirt.DomainIPAddr {
	return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv6), Addr: addr, Prefix: 64}
}

// managedDomain seeds a domain as Create would have left it.
func managedDomain(t *testing.T, api *fakeAPI, name, uid string, running bool, addrs map[uint32][]libvirt.DomainInterface) {
	t.Helper()
	api.putDomain(&fakeDomain{name: name, metadata: metadataXML(t, uid, testNow), running: running, addrs: addrs})
	api.putVol("default", overlayVolumeName(name), formatQcow2, nil)
	api.putVol("default", seedVolumeName(name), formatRaw, []byte("iso"))
}

func wantClass(t *testing.T, err error, class error) {
	t.Helper()
	if err == nil {
		t.Fatalf("got nil error, want %v", class)
	}
	if !errors.Is(err, class) {
		t.Fatalf("error class = %v, want %v (err: %v)", provider.Class(err), class, err)
	}
}

func assertCreated(t *testing.T, api *fakeAPI) {
	t.Helper()
	dom := api.domain(testName)
	if dom == nil {
		t.Fatal("domain not defined")
	}
	if !dom.running {
		t.Error("domain not running")
	}
	overlay := api.vol("default", overlayVolumeName(testName))
	if overlay == nil {
		t.Fatal("overlay volume missing")
	}
	if overlay.backing != "/var/lib/libvirt/images/base-debian-13.qcow2" || overlay.format != formatQcow2 {
		t.Errorf("overlay = %+v, want qcow2 backed by the cached base", overlay)
	}
	if overlay.capacity != defaultDiskSizeGB*bytesPerGiB {
		t.Errorf("overlay capacity = %d, want %d", overlay.capacity, defaultDiskSizeGB*bytesPerGiB)
	}
	seed := api.vol("default", seedVolumeName(testName))
	if seed == nil {
		t.Fatal("seed volume missing")
	}
	if seed.format != formatRaw || len(seed.data) == 0 || int64(len(seed.data)) != seed.capacity {
		t.Errorf("seed = format %s, %d bytes, capacity %d; want raw with capacity == uploaded size", seed.format, len(seed.data), seed.capacity)
	}
}

func TestCreate_freshHost(t *testing.T) {
	t.Parallel()
	api := readyHost()
	p := newTestProvider(hostFakes{"nas1": api}, "nas1")

	id, err := p.Create(context.Background(), testRequest())
	if err != nil {
		t.Fatalf("Create: %v", err)
	}
	if want := "hosts/nas1/domains/" + testName; id != want {
		t.Errorf("providerID = %s, want %s", id, want)
	}
	assertCreated(t, api)

	var d domainXML
	if err := xml.Unmarshal([]byte(api.domain(testName).xml), &d); err != nil {
		t.Fatal(err)
	}
	if d.VCPU != 2 || d.Memory.Value != 2048 || d.Memory.Unit != "MiB" {
		t.Errorf("domain size = %d vcpu / %d %s, want 2 / 2048 MiB", d.VCPU, d.Memory.Value, d.Memory.Unit)
	}
	if d.Devices.Interfaces[0].Source.Bridge != "br0" {
		t.Errorf("bridge = %q, want br0", d.Devices.Interfaces[0].Source.Bridge)
	}
	wantDisk, wantSeed := api.poolPath+"/"+overlayVolumeName(testName), api.poolPath+"/"+seedVolumeName(testName)
	if got := d.Devices.Disks[0].Source.File; got != wantDisk {
		t.Errorf("overlay disk source = %q, want the path libvirt reported, %q", got, wantDisk)
	}
	if got := d.Devices.Disks[1].Source.File; got != wantSeed {
		t.Errorf("seed cdrom source = %q, want the path libvirt reported, %q", got, wantSeed)
	}
	if m := d.Metadata.Instance; m.UID != testUID || !m.managed() || !m.createdAt().Equal(testNow) {
		t.Errorf("metadata = %+v, want managed, uid %s, createdAt %s", m, testUID, testNow)
	}
	if api.hasCall("StorageVolDownload") || api.callCount("StorageVolUpload") != 1 {
		t.Errorf("calls = %v, want no image download and exactly one upload (the seed)", api.calls)
	}
}

func TestCreate_fetchesImageOnFirstUse(t *testing.T) {
	t.Parallel()
	srv := newImageServer(t)
	api := newFakeAPI("default")
	cfg := testConfig("nas1")
	cfg.KVM.Images["debian-13"] = provider.KVMImage{URL: srv.URL + "/debian.qcow2", SHA256: testImageSHA()}
	p := newWithDialer(cfg, hostFakes{"nas1": api}.dial, nil)

	if _, err := p.Create(context.Background(), testRequest()); err != nil {
		t.Fatalf("Create: %v", err)
	}
	if srv.hits.Load() != 1 {
		t.Errorf("image server hits = %d, want 1", srv.hits.Load())
	}
	base := api.vol("default", "base-debian-13.qcow2")
	if base == nil || string(base.data) != string(testImage) {
		t.Fatal("base image not cached in the pool")
	}
	if api.domain(testName) == nil || !api.domain(testName).running {
		t.Error("domain not running after image fetch")
	}

	// A second Proxy with the same image on the same host reuses the cache.
	req := testRequest()
	req.Name, req.UID = "proxy-second", "uid-2"
	if _, err := p.Create(context.Background(), req); err != nil {
		t.Fatalf("second Create: %v", err)
	}
	if srv.hits.Load() != 1 {
		t.Errorf("image server hits after second Create = %d, want still 1", srv.hits.Load())
	}
}

func TestCreate_isIdempotentWhenComplete(t *testing.T) {
	t.Parallel()
	api := readyHost()
	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
	if _, err := p.Create(context.Background(), testRequest()); err != nil {
		t.Fatal(err)
	}
	before := len(api.calls)

	id, err := p.Create(context.Background(), testRequest())
	if err != nil {
		t.Fatalf("second Create: %v", err)
	}
	if id != "hosts/nas1/domains/"+testName {
		t.Errorf("providerID = %s", id)
	}
	for _, m := range []string{"DomainDefineXML", "DomainCreate", "StorageVolUpload"} {
		if n := api.callCount(m); n != 1 {
			t.Errorf("%s called %d times across two Creates, want 1", m, n)
		}
	}
	if len(api.calls) == before {
		t.Error("second Create made no calls at all; expected lookups")
	}
}

// Each case breaks Create after a different step, then re-runs it and
// expects the second attempt to pick up where the first stopped without
// duplicating what already exists.
func TestCreate_resumesAfterPartialFailure(t *testing.T) {
	t.Parallel()
	cases := []struct {
		name     string
		sabotage func(api *fakeAPI)
		class    error
		check    func(t *testing.T, api *fakeAPI)
	}{
		{
			name: "seed upload fails after overlay exists",
			sabotage: func(api *fakeAPI) {
				api.uploadHook = func(vol string, _ io.Reader) error {
					api.uploadHook = nil
					return io.ErrUnexpectedEOF
				}
			},
			class: provider.ErrTransient,
			check: func(t *testing.T, api *fakeAPI) {
				if n := api.callCount("StorageVolUpload"); n != 2 {
					t.Errorf("seed uploads = %d, want 2 (failed, then redone)", n)
				}
				if n := api.callCount("StorageVolCreateXML"); n != 4 {
					t.Errorf("volume creates = %d, want 4 (overlay, seed; overlay rejected as existing, seed again)", n)
				}
				if names := api.volNames("default"); len(names) != 4 {
					t.Errorf("pool holds %v, want base, marker, one overlay, one seed", names)
				}
			},
		},
		{
			name:     "define fails after volumes exist",
			sabotage: func(api *fakeAPI) { api.failOnce["DomainDefineXML"] = lvError(libvirt.ErrInternalError, "boom") },
			class:    provider.ErrTransient,
			check: func(t *testing.T, api *fakeAPI) {
				if n := api.callCount("StorageVolUpload"); n != 2 {
					t.Errorf("seed uploads = %d, want 2 (seed rebuilt while no domain exists)", n)
				}
				if n := api.callCount("DomainDefineXML"); n != 2 {
					t.Errorf("defines = %d, want 2", n)
				}
			},
		},
		{
			name:     "start fails after define",
			sabotage: func(api *fakeAPI) { api.failOnce["DomainCreate"] = lvError(libvirt.ErrInternalError, "boom") },
			class:    provider.ErrTransient,
			check: func(t *testing.T, api *fakeAPI) {
				if n := api.callCount("DomainDefineXML"); n != 1 {
					t.Errorf("defines = %d, want 1 (existing domain reused)", n)
				}
				if n := api.callCount("DomainCreate"); n != 2 {
					t.Errorf("starts = %d, want 2", n)
				}
			},
		},
		{
			name:     "transport drops mid-create",
			sabotage: func(api *fakeAPI) { api.failOnce["DomainLookupByName"] = io.EOF },
			class:    provider.ErrTransient,
			check: func(t *testing.T, api *fakeAPI) {
				if n := api.callCount("DomainDefineXML"); n != 1 {
					t.Errorf("defines = %d, want 1", n)
				}
			},
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			api := readyHost()
			tc.sabotage(api)
			p := newTestProvider(hostFakes{"nas1": api}, "nas1")

			_, err := p.Create(context.Background(), testRequest())
			wantClass(t, err, tc.class)
			if _, err := p.Create(context.Background(), testRequest()); err != nil {
				t.Fatalf("retry Create: %v", err)
			}
			assertCreated(t, api)
			tc.check(t, api)
		})
	}
}

// A reconcile cancelled mid-upload leaves a seed volume whose content is
// short; the compensating delete cannot run because the pool has already
// closed the connection. The next Create must not trust that volume.
func TestCreate_replacesLeftoverSeed(t *testing.T) {
	t.Parallel()
	api := readyHost()
	api.putVol("default", seedVolumeName(testName), formatRaw, []byte("trunc"))
	p := newTestProvider(hostFakes{"nas1": api}, "nas1")

	if _, err := p.Create(context.Background(), testRequest()); err != nil {
		t.Fatal(err)
	}
	assertCreated(t, api)
	if seed := api.vol("default", seedVolumeName(testName)); string(seed.data) == "trunc" {
		t.Error("leftover seed was kept")
	}
	if n := api.callCount("StorageVolUpload"); n != 1 {
		t.Errorf("uploads = %d, want 1 (the rebuilt seed)", n)
	}
}

func TestCreate_startsStoppedDomain(t *testing.T) {
	t.Parallel()
	api := readyHost()
	managedDomain(t, api, testName, testUID, false, nil)
	p := newTestProvider(hostFakes{"nas1": api}, "nas1")

	if _, err := p.Create(context.Background(), testRequest()); err != nil {
		t.Fatal(err)
	}
	if !api.domain(testName).running {
		t.Error("domain still stopped")
	}
	if api.hasCall("DomainDefineXML") || api.hasCall("StorageVolUpload") || api.hasCall("StorageVolDelete") {
		t.Errorf("calls = %v, want no define, upload or volume delete for an existing domain", api.calls)
	}
}

func TestCreate_singleHostNeedsNoZone(t *testing.T) {
	t.Parallel()
	api := readyHost()
	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
	req := testRequest()
	req.Placement.Zone = ""

	id, err := p.Create(context.Background(), req)
	if err != nil {
		t.Fatal(err)
	}
	if id != "hosts/nas1/domains/"+testName {
		t.Errorf("providerID = %s, want the single host filled in", id)
	}
}

func TestCreate_placementErrors(t *testing.T) {
	t.Parallel()
	cases := []struct {
		name  string
		hosts []string
		edit  func(*provider.CreateRequest)
		want  string
	}{
		{"unknown host", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Zone = "nas9" }, "not in kvm.hosts"},
		{"empty zone with two hosts", []string{"nas1", "nas2"}, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "placement.zone is required"},
		{"empty zone with no hosts", nil, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "kvm.hosts is empty"},
		{"unknown machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "huge" }, "not in kvm.machineTypes"},
		{"empty machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "" }, "not in kvm.machineTypes"},
		{"unknown image", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Image = "windows" }, "neither a key"},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fakes := hostFakes{}
			for _, h := range tc.hosts {
				fakes[h] = readyHost()
			}
			p := newTestProvider(fakes, tc.hosts...)
			req := testRequest()
			tc.edit(&req)

			_, err := p.Create(context.Background(), req)
			wantClass(t, err, provider.ErrPermanent)
			if !strings.Contains(err.Error(), tc.want) {
				t.Errorf("error = %q, want it to mention %q", err, tc.want)
			}
			for _, api := range fakes {
				if len(api.calls) != 0 {
					t.Errorf("host was called: %v", api.calls)
				}
			}
		})
	}
}

func TestCreate_hostFailures(t *testing.T) {
	t.Parallel()
	cases := []struct {
		name  string
		setup func() (hostFakes, string)
		class error
	}{
		{
			name:  "host unreachable",
			setup: func() (hostFakes, string) { return hostFakes{}, "nas1" },
			class: provider.ErrTransient,
		},
		{
			name: "pool missing",
			setup: func() (hostFakes, string) {
				api := newFakeAPI("other")
				return hostFakes{"nas1": api}, "nas1"
			},
			class: provider.ErrPermanent,
		},
		{
			name: "domain xml rejected",
			setup: func() (hostFakes, string) {
				api := readyHost()
				api.fail["DomainDefineXML"] = lvError(libvirt.ErrConfigUnsupported, "no q35 here")
				return hostFakes{"nas1": api}, "nas1"
			},
			class: provider.ErrPermanent,
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fakes, host := tc.setup()
			p := newTestProvider(fakes, host)
			_, err := p.Create(context.Background(), testRequest())
			wantClass(t, err, tc.class)
		})
	}
}

func TestCreate_cancelledContext(t *testing.T) {
	t.Parallel()
	api := readyHost()
	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
	ctx, cancel := context.WithCancel(context.Background())
	cancel()

	_, err := p.Create(ctx, testRequest())
	wantClass(t, err, provider.ErrTransient)
	if !errors.Is(err, context.Canceled) {
		t.Errorf("error = %v, want it to unwrap to context.Canceled", err)
	}
}

func TestGet(t *testing.T) {
	t.Parallel()
	arp := map[uint32][]libvirt.DomainInterface{
		uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Name: "vnet0", Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.50")}}},
	}
	agentOnly := map[uint32][]libvirt.DomainInterface{
		uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
			{Name: "lo", Addrs: []libvirt.DomainIPAddr{ipv4("127.0.0.1"), ipv6("::1")}},
			{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv6("fe80::1"), ipv4("192.168.1.51")}},
		},
	}
	linkLocalOnly := map[uint32][]libvirt.DomainInterface{
		uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
			{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv4("169.254.10.10")}},
		},
	}
	cases := []struct {
		name    string
		running bool
		addrs   map[uint32][]libvirt.DomainInterface
		state   provider.InstanceState
		ip      string
	}{
		{"running with arp address", true, arp, provider.StateRunning, "192.168.1.50"},
		{"running, agent only, skips loopback and v6", true, agentOnly, provider.StateRunning, "192.168.1.51"},
		{"running, no address yet", true, nil, provider.StateProvisioning, ""},
		{"running, only link-local", true, linkLocalOnly, provider.StateProvisioning, ""},
		{"shut off", false, arp, provider.StateStopped, ""},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			api := readyHost()
			managedDomain(t, api, testName, testUID, tc.running, tc.addrs)
			p := newTestProvider(hostFakes{"nas1": api}, "nas1")

			inst, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
			if err != nil {
				t.Fatalf("Get: %v", err)
			}
			want := provider.Instance{ID: "hosts/nas1/domains/" + testName, IP: tc.ip, State: tc.state, UID: testUID, CreatedAt: testNow}
			if *inst != want {
				t.Errorf("instance = %+v, want %+v", *inst, want)
			}
		})
	}
}

func TestGet_unmanagedDomainHasNoUID(t *testing.T) {
	t.Parallel()
	api := readyHost()
	api.putDomain(&fakeDomain{name: "hand-made", running: true})
	p := newTestProvider(hostFakes{"nas1": api}, "nas1")

	inst, err := p.Get(context.Background(), "hosts/nas1/domains/hand-made")
	if err != nil {
		t.Fatalf("Get: %v", err)
	}
	if inst.UID != "" || !inst.CreatedAt.IsZero() {
		t.Errorf("instance = %+v, want empty UID and zero CreatedAt", inst)
	}
}

func TestGet_errors(t *testing.T) {
	t.Parallel()
	cases := []struct {
		name  string
		id    string
		setup func(t *testing.T, api *fakeAPI)
		class error
	}{
		{"missing domain", "hosts/nas1/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrNotFound},
		{"malformed id", "zones/x/instances/y", func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
		{"host not configured", "hosts/nas9/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
		{
			"state read fails", "hosts/nas1/domains/" + testName,
			func(t *testing.T, api *fakeAPI) {
				managedDomain(t, api, testName, testUID, true, nil)
				api.fail["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
			},
			provider.ErrTransient,
		},
		{
			"corrupt metadata", "hosts/nas1/domains/" + testName,
			func(_ *testing.T, api *fakeAPI) {
				api.putDomain(&fakeDomain{name: testName, metadata: "<instance", running: true})
			},
			provider.ErrPermanent,
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			api := readyHost()
			tc.setup(t, api)
			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
			_, err := p.Get(context.Background(), tc.id)
			wantClass(t, err, tc.class)
		})
	}
}

func TestGet_hostUnreachable(t *testing.T) {
	t.Parallel()
	p := newTestProvider(hostFakes{}, "nas1")
	_, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
	wantClass(t, err, provider.ErrTransient)
}

func assertGone(t *testing.T, api *fakeAPI) {
	t.Helper()
	if api.domain(testName) != nil {
		t.Error("domain still defined")
	}
	if names := api.volNames("default"); len(names) != 2 {
		t.Errorf("pool holds %v, want only the cached base image and its marker", names)
	}
}

func TestDelete(t *testing.T) {
	t.Parallel()
	cases := []struct {
		name  string
		setup func(t *testing.T, api *fakeAPI)
		check func(t *testing.T, api *fakeAPI)
	}{
		{
			name:  "running domain",
			setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, true, nil) },
			check: func(t *testing.T, api *fakeAPI) {
				if !api.hasCall("DomainDestroy") || !api.hasCall("DomainUndefineFlags") {
					t.Errorf("calls = %v, want destroy and undefine", api.calls)
				}
			},
		},
		{
			name:  "stopped domain",
			setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, false, nil) },
			check: func(t *testing.T, api *fakeAPI) {
				if !api.hasCall("DomainUndefineFlags") {
					t.Errorf("calls = %v, want undefine despite destroy being invalid", api.calls)
				}
			},
		},
		{
			name: "domain gone, volumes left",
			setup: func(_ *testing.T, api *fakeAPI) {
				api.putVol("default", overlayVolumeName(testName), formatQcow2, nil)
				api.putVol("default", seedVolumeName(testName), formatRaw, nil)
			},
			check: func(t *testing.T, api *fakeAPI) {
				if api.hasCall("DomainDestroy") || api.hasCall("DomainUndefineFlags") {
					t.Errorf("calls = %v, want no domain operations", api.calls)
				}
			},
		},
		{
			name:  "nothing exists",
			setup: func(*testing.T, *fakeAPI) {},
			check: func(t *testing.T, api *fakeAPI) {
				if api.hasCall("StorageVolDelete") {
					t.Errorf("calls = %v, want no deletes", api.calls)
				}
			},
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			api := readyHost()
			tc.setup(t, api)
			p := newTestProvider(hostFakes{"nas1": api}, "nas1")

			if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
				t.Fatalf("Delete: %v", err)
			}
			assertGone(t, api)
			tc.check(t, api)
			if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
				t.Fatalf("second Delete: %v", err)
			}
		})
	}
}

func TestDelete_missingPoolIsNotAnError(t *testing.T) {
	t.Parallel()
	api := newFakeAPI("other")
	api.putDomain(&fakeDomain{name: testName, metadata: metadataXML(t, testUID, testNow), running: true})
	p := newTestProvider(hostFakes{"nas1": api}, "nas1")

	if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
		t.Fatalf("Delete: %v", err)
	}
	if api.domain(testName) != nil {
		t.Error("domain still defined")
	}
}

func TestDelete_errors(t *testing.T) {
	t.Parallel()
	cases := []struct {
		name  string
		id    string
		fakes func(t *testing.T) hostFakes
		class error
	}{
		{"malformed id", "nope", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
		{"host not configured", "hosts/nas9/domains/x", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
		{"host unreachable", "hosts/nas1/domains/x", func(*testing.T) hostFakes { return hostFakes{} }, provider.ErrTransient},
		{
			"undefine fails", "hosts/nas1/domains/" + testName,
			func(t *testing.T) hostFakes {
				api := readyHost()
				managedDomain(t, api, testName, testUID, true, nil)
				api.fail["DomainUndefineFlags"] = lvError(libvirt.ErrInternalError, "boom")
				return hostFakes{"nas1": api}
			},
			provider.ErrTransient,
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			p := newTestProvider(tc.fakes(t), "nas1")
			wantClass(t, p.Delete(context.Background(), tc.id), tc.class)
		})
	}
}

func TestListByTag_acrossHosts(t *testing.T) {
	t.Parallel()
	nas1, nas2 := readyHost(), readyHost()
	managedDomain(t, nas1, "proxy-one", "uid-1", true, map[uint32][]libvirt.DomainInterface{
		uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.10")}}},
	})
	nas1.putDomain(&fakeDomain{name: "hand-made", running: true})
	managedDomain(t, nas2, "proxy-two", "uid-2", false, nil)
	nas2.putDomain(&fakeDomain{name: "foreign", metadata: `<instance xmlns="` + metadataURI + `"><managed>false</managed></instance>`})
	p := newTestProvider(hostFakes{"nas1": nas1, "nas2": nas2}, "nas1", "nas2")

	got, err := p.ListByTag(context.Background())
	if err != nil {
		t.Fatalf("ListByTag: %v", err)
	}
	want := []provider.Instance{
		{ID: "hosts/nas1/domains/proxy-one", IP: "192.168.1.10", State: provider.StateRunning, UID: "uid-1", CreatedAt: testNow},
		{ID: "hosts/nas2/domains/proxy-two", State: provider.StateStopped, UID: "uid-2", CreatedAt: testNow},
	}
	if len(got) != len(want) {
		t.Fatalf("got %d instances %+v, want %d", len(got), got, len(want))
	}
	for i := range want {
		if got[i] != want[i] {
			t.Errorf("instance[%d] = %+v, want %+v", i, got[i], want[i])
		}
	}
}

func TestListByTag_oneHostDownFailsSweep(t *testing.T) {
	t.Parallel()
	nas1 := readyHost()
	managedDomain(t, nas1, "proxy-one", "uid-1", true, nil)
	p := newTestProvider(hostFakes{"nas1": nas1}, "nas1", "nas2")

	got, err := p.ListByTag(context.Background())
	wantClass(t, err, provider.ErrTransient)
	if got != nil {
		t.Errorf("got %+v alongside the error, want nil (no partial view)", got)
	}
}

// A domain that cannot be read is left out of the sweep rather than
// failing it: GC never deletes what it does not see, so skipping is the
// safe direction, while one bad domain blocking GC for all others is not.
func TestListByTag_skipsUnreadableDomain(t *testing.T) {
	t.Parallel()
	cases := []struct {
		name  string
		setup func(t *testing.T, api *fakeAPI)
	}{
		{
			"deleted mid-sweep",
			func(t *testing.T, api *fakeAPI) {
				managedDomain(t, api, "proxy-one", "uid-1", true, nil)
				api.failOnce["DomainGetMetadata"] = lvError(libvirt.ErrNoDomain, "gone")
			},
		},
		{
			"corrupt metadata",
			func(_ *testing.T, api *fakeAPI) {
				api.putDomain(&fakeDomain{name: "proxy-one", metadata: "<instance", running: true})
			},
		},
		{
			"state unreadable",
			func(t *testing.T, api *fakeAPI) {
				managedDomain(t, api, "proxy-one", "uid-1", true, nil)
				api.failOnce["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
			},
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			nas1 := readyHost()
			tc.setup(t, nas1)
			managedDomain(t, nas1, "proxy-two", "uid-2", false, nil)
			p := newTestProvider(hostFakes{"nas1": nas1}, "nas1")

			got, err := p.ListByTag(context.Background())
			if err != nil {
				t.Fatalf("ListByTag: %v", err)
			}
			if len(got) != 1 || got[0].UID != "uid-2" {
				t.Errorf("got %+v, want only proxy-two", got)
			}
		})
	}
}

func TestListByTag_noHosts(t *testing.T) {
	t.Parallel()
	p := newTestProvider(hostFakes{})
	got, err := p.ListByTag(context.Background())
	if err != nil || len(got) != 0 {
		t.Errorf("got %v, %v; want empty, nil", got, err)
	}
}

func TestMapState(t *testing.T) {
	t.Parallel()
	cases := []struct {
		state libvirt.DomainState
		ip    string
		want  provider.InstanceState
	}{
		{libvirt.DomainRunning, "10.0.0.1", provider.StateRunning},
		{libvirt.DomainRunning, "", provider.StateProvisioning},
		{libvirt.DomainBlocked, "10.0.0.1", provider.StateRunning},
		{libvirt.DomainPaused, "10.0.0.1", provider.StateStopped},
		{libvirt.DomainShutdown, "", provider.StateStopped},
		{libvirt.DomainShutoff, "", provider.StateStopped},
		{libvirt.DomainCrashed, "", provider.StateStopped},
		{libvirt.DomainPmsuspended, "", provider.StateStopped},
		{libvirt.DomainNostate, "", provider.StateProvisioning},
	}
	for _, tc := range cases {
		t.Run(fmt.Sprintf("%s/%q", tc.state, tc.ip), func(t *testing.T) {
			t.Parallel()
			if got := mapState(tc.state, tc.ip); got != tc.want {
				t.Errorf("mapState(%s, %q) = %s, want %s", tc.state, tc.ip, got, tc.want)
			}
		})
	}
}

func TestNew_badKeyPathFailsFast(t *testing.T) {
	t.Parallel()
	cfg := testConfig("nas1")
	cfg.KVM.SSHKeyFile = t.TempDir() + "/missing"
	if _, err := New(context.Background(), cfg); err == nil {
		t.Fatal("New succeeded with a missing key file")
	}
}

// runAllOps drives one full lifecycle over a ready fake host so logging
// tests see every provider method and every libvirt call.
func runAllOps(t *testing.T, ctx context.Context, req provider.CreateRequest) {
	t.Helper()
	p := newTestProvider(hostFakes{"nas1": readyHost()}, "nas1")
	id, err := p.Create(ctx, req)
	if err != nil {
		t.Fatalf("Create: %v", err)
	}
	if _, err := p.Get(ctx, id); err != nil {
		t.Fatalf("Get: %v", err)
	}
	if _, err := p.ListByTag(ctx); err != nil {
		t.Fatalf("ListByTag: %v", err)
	}
	if err := p.Delete(ctx, id); err != nil {
		t.Fatalf("Delete: %v", err)
	}
}

func TestLogging_verbosityTiers(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name        string
		verbosity   int
		wantLines   []string
		absentLines []string
	}{
		{
			name:        "v0 stays silent",
			verbosity:   0,
			absentLines: []string{"libvirt call", "KVM domain ensured running"},
		},
		{
			name:      "v1 logs one line per libvirt call plus the operation outcomes",
			verbosity: 1,
			wantLines: []string{
				`kvm/libvirt "level"=1 "msg"="libvirt host connected"`,
				`"msg"="libvirt call"`,
				`"rpc"="virDomainLookupByName"`,
				`"libvirtError"="ErrNoDomain"`,
				`"rpc"="virStorageVolUpload"`,
				`"rpc"="virDomainDefineXML"`,
				`"rpc"="virDomainCreate"`,
				`"rpc"="virDomainInterfaceAddresses"`,
				`"rpc"="virConnectListAllDomains"`,
				`"rpc"="virDomainUndefineFlags"`,
				`"rpc"="virStorageVolDelete"`,
				`"provider"="kvm-home"`,
				`"host"="nas1"`,
				`"name"="` + testName + `"`,
				`"msg"="KVM domain ensured running"`,
			},
			absentLines: []string{`"xml"=`, `"metadata"=`},
		},
		{
			name:      "v2 adds xml, metadata and addresses on the same lines",
			verbosity: 2,
			wantLines: []string{`"xml"="<domain`, `"xml"="<volume`, `"metadata"="<instance`, `"names"=[`},
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			ctx, snapshot := captureContext(tc.verbosity)
			req := testRequest()
			const sentinel = "SENTINEL-cloud-init-must-never-be-logged"
			req.CloudInit = "#cloud-config\n" + sentinel + "\n"

			runAllOps(t, ctx, req)

			lines := snapshot()
			joined := strings.Join(lines, "\n")
			if tc.verbosity == 0 && len(lines) != 0 {
				t.Errorf("verbosity 0 logged %d lines:\n%s", len(lines), joined)
			}
			for _, want := range tc.wantLines {
				if !strings.Contains(joined, want) {
					t.Errorf("output missing %s:\n%s", want, joined)
				}
			}
			for _, absent := range tc.absentLines {
				if strings.Contains(joined, absent) {
					t.Errorf("output unexpectedly contains %s:\n%s", absent, joined)
				}
			}
			if strings.Contains(joined, sentinel) {
				t.Errorf("cloud-init content leaked into logs:\n%s", joined)
			}
		})
	}
}