egress-proxies-operator/internal/provider/proxmox/get_test.go

package proxmox

import (
	"errors"
	"slices"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

// mutateOn changes the fake's state under its lock at the start of every
// call to method: a VM changing between two of Get's calls.
func mutateOn(f *fakeAPI, method string, fn func()) {
	f.hookOn(method, func() error {
		f.mu.Lock()
		defer f.mu.Unlock()
		fn()
		return nil
	})
}

// capturedIfaces is the agent's answer from the Phase 0 guest: lo and ens18
// with 192.168.0.150.
func capturedIfaces(t *testing.T) []agentInterface {
	t.Helper()
	return decodeData[struct {
		Result []agentInterface `json:"result"`
	}](t, "agent_network_get_interfaces").Result
}

// assertGetFails runs Get on fx.id(), checks the error's class and returns
// its text.
func assertGetFails(t *testing.T, fx *fixture, class error) string {
	t.Helper()
	inst, err := fx.p.Get(fx.ctx, fx.id())
	if inst != nil {
		t.Errorf("Get = %+v, want nil", inst)
	}
	assertProviderError(t, err, "get", class)
	return err.Error()
}

func TestGet_missingVMIsNotFound(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID+1, testName, "running")
	assertGetFails(t, fx, provider.ErrNotFound)
	lines := fx.logs.matching(`"msg"="proxmox vm not found"`)
	if len(lines) != 1 || !strings.Contains(lines[0], `"vmid"=150042`) || !strings.Contains(lines[0], `"name"="`+testName+`"`) {
		t.Errorf("not-found lines = %v, want one with vmid and name", lines)
	}
	if got := fx.fake.callLog(); !slices.Equal(got, []string{"ClusterResources"}) {
		t.Errorf("calls = %v, want only the resources listing", got)
	}
}

func TestGet_containerAtVMIDIsNotFound(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running").kind = "lxc"
	assertGetFails(t, fx, provider.ErrNotFound)
}

func TestGet_nameMismatchIsNotFound(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, "proxy-someoneelse00000", "running")
	msg := assertGetFails(t, fx, provider.ErrNotFound)
	if !strings.Contains(msg, "reused") {
		t.Errorf("error = %s, want it to say the VMID was reused", msg)
	}
	if n := fx.fake.callCount("VMStatus"); n != 0 {
		t.Errorf("VMStatus calls = %d, want 0 for a VM that is not ours", n)
	}
	if lines := fx.logs.matching(`"msg"="proxmox vm not found"`); len(lines) != 1 {
		t.Errorf("not-found lines = %v, want one", lines)
	}
}

func TestGet_unmarkedIsNotFound(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name        string
		description string
	}{
		{name: "no description"},
		{name: "managed false", description: "proxy-operator-managed=false\nproxy-operator-uid=" + testUID},
		{name: "admin notes only", description: "hand made, do not touch"},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			vm := fx.fake.addVM(testVMID, testName, "running")
			if tc.description != "" {
				vm.config["description"] = tc.description
			}
			msg := assertGetFails(t, fx, provider.ErrNotFound)
			if !strings.Contains(msg, "no proxy-operator mark") {
				t.Errorf("error = %s, want it to name the missing mark", msg)
			}
		})
	}
}

// TestGet_otherVMWithOurNameIsIgnored replaces the plan's
// TestGet_twoVMsWithOurNameIsPermanent: spec entries 2 and 9 address Get by
// VMID, and only Create treats a duplicate name as Permanent.
func TestGet_otherVMWithOurNameIsIgnored(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "stopped")
	fx.addOurVM(testVMID+7, testName, "running")
	if inst := fx.get(fx.id()); inst.State != provider.StateStopped || inst.ID != fx.id() {
		t.Errorf("Get = %+v, want the VM at the ID's VMID, Stopped", inst)
	}
}

func TestGet_vmOnAnotherNodeIsFound(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	vm := fx.addOurVM(testVMID, testName, "running")
	vm.node = "pv1-2"
	vm.ifaces = capturedIfaces(t)
	inst := fx.get(fx.id())
	if inst.State != provider.StateRunning || inst.IP != "192.168.0.150" {
		t.Errorf("Get = %+v, want Running with 192.168.0.150 read from pv1-2", inst)
	}
	if lines := fx.logs.matching(`"msg"="proxmox vm fetched"`); len(lines) != 1 || !strings.Contains(lines[0], `"node"="pv1-2"`) {
		t.Errorf("fetched lines = %v, want the resolved node", lines)
	}
}

func TestGet_runningWithoutIPIsProvisioning(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running")
	if inst := fx.get(fx.id()); inst.State != provider.StateProvisioning || inst.IP != "" {
		t.Errorf("Get = %+v, want Provisioning with no IP", inst)
	}
}

func TestGet_runningWithIPIsRunning(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running").ifaces = capturedIfaces(t)
	inst := fx.get(fx.id())
	if inst.State != provider.StateRunning || inst.IP != "192.168.0.150" || inst.ID != fx.id() {
		t.Errorf("Get = %+v, want Running with 192.168.0.150", inst)
	}
	lines := fx.logs.matching(`"msg"="proxmox vm fetched"`)
	if len(lines) != 1 {
		t.Fatalf("fetched lines = %v, want one", lines)
	}
	for _, kv := range []string{`"provider"="proxmox-home"`, `"vmid"=150042`, `"name"="` + testName + `"`, `"node"="pv1-1"`, `"state"="Running"`, `"ip"="192.168.0.150"`, `"uid"="` + testUID + `"`} {
		if !strings.Contains(lines[0], kv) {
			t.Errorf("fetched line lacks %s: %s", kv, lines[0])
		}
	}
}

func TestGet_runningMakesFourCalls(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running").ifaces = capturedIfaces(t)
	fx.get(fx.id())
	want := []string{"ClusterResources", "VMConfig", "VMStatus", "AgentNetworkInterfaces"}
	if got := fx.fake.callLog(); !slices.Equal(got, want) {
		t.Errorf("calls = %v, want %v", got, want)
	}
}

func TestGet_agentNotRunningIsProvisioning(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running").ifaces = capturedIfaces(t)
	fx.fake.agentNotRunning = true
	if inst := fx.get(fx.id()); inst.State != provider.StateProvisioning || inst.IP != "" {
		t.Errorf("Get = %+v, want Provisioning while the agent is not running", inst)
	}
}

// TestGet_agentMessagesTable is spec entry 7's table, with the agent's
// answers taken from the captures.
func TestGet_agentMessagesTable(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name      string
		agentErr  error
		ifaces    []agentInterface
		wantState provider.InstanceState
		wantClass error
	}{
		{name: "200 with a routable IPv4", ifaces: capturedIfaces(t), wantState: provider.StateRunning},
		{
			name:      "200 without an IPv4",
			ifaces:    []agentInterface{{Name: "lo", IPAddresses: []agentIP{{Address: "127.0.0.1", Type: "ipv4"}}}, {Name: "ens18", IPAddresses: []agentIP{{Address: "fe80::1", Type: "ipv6"}}}},
			wantState: provider.StateProvisioning,
		},
		{name: "agent not running", agentErr: loadAPIError(t, "err_agent_not_running"), wantState: provider.StateProvisioning},
		{name: "no agent configured", agentErr: loadAPIError(t, "err_agent_vm_stopped"), wantState: provider.StateStopped},
		{name: "VM stopped since the status read", agentErr: loadAPIError(t, "err_agent_vm_stopped_agent_enabled"), wantClass: provider.ErrTransient},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			fx.addOurVM(testVMID, testName, "running").ifaces = tc.ifaces
			if tc.agentErr != nil {
				fx.fake.failWith("AgentNetworkInterfaces", tc.agentErr)
			}
			if tc.wantClass != nil {
				assertGetFails(t, fx, tc.wantClass)
				return
			}
			inst := fx.get(fx.id())
			if inst.State != tc.wantState {
				t.Errorf("Get state = %s, want %s", inst.State, tc.wantState)
			}
			if (inst.State == provider.StateRunning) != (inst.IP != "") {
				t.Errorf("Get = %+v: an IP goes with Running and only with Running", inst)
			}
		})
	}
}

func TestGet_agentOtherErrorIsTransient(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name string
		err  error
	}{
		{name: "forbidden", err: loadAPIError(t, "err_forbidden")},
		{name: "VM vanished since the status read", err: loadAPIError(t, "err_vm_missing")},
		{name: "pveproxy 596", err: fakeErr(596, "Broken pipe")},
		{name: "other 500", err: fakeErr(500, "fake: guest-network-get-interfaces timed out")},
		{name: "not an api error", err: errors.New("read tcp: connection reset by peer")},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			fx.addOurVM(testVMID, testName, "running")
			fx.fake.failWith("AgentNetworkInterfaces", tc.err)
			assertGetFails(t, fx, provider.ErrTransient)
		})
	}
}

func TestGet_ipFilterSkipsLoopbackAndLinkLocal(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running").ifaces = []agentInterface{
		{Name: "lo", IPAddresses: []agentIP{{Address: "127.0.0.1", Type: "ipv4"}}},
		{Name: "ens18", IPAddresses: []agentIP{{Address: "169.254.10.2", Type: "ipv4"}, {Address: "fe80::be24:11ff:fe48:1146", Type: "ipv6"}}},
		{Name: "ens19", IPAddresses: []agentIP{{Address: "2001:db8::5", Type: "ipv6"}, {Address: "192.168.4.147", Type: "ipv4"}}},
	}
	if inst := fx.get(fx.id()); inst.State != provider.StateRunning || inst.IP != "192.168.4.147" {
		t.Errorf("Get = %+v, want Running with 192.168.4.147", inst)
	}
}

func TestGet_stoppedIsStopped(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "stopped")
	if inst := fx.get(fx.id()); inst.State != provider.StateStopped || inst.IP != "" || inst.UID != testUID {
		t.Errorf("Get = %+v, want Stopped with uid %s", inst, testUID)
	}
	if n := fx.fake.callCount("AgentNetworkInterfaces"); n != 0 {
		t.Errorf("agent calls = %d, want 0 for a stopped VM", n)
	}
}

func TestGet_pausedIsStopped(t *testing.T) {
	t.Parallel()
	for _, qmp := range []string{"paused", "suspended", "io-error", "postmigrate", "internal-error"} {
		t.Run(qmp, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			fx.addOurVM(testVMID, testName, "running").qmpstatus = qmp
			if inst := fx.get(fx.id()); inst.State != provider.StateStopped {
				t.Errorf("Get state with qmpstatus %s = %s, want Stopped", qmp, inst.State)
			}
			if n := fx.fake.callCount("AgentNetworkInterfaces"); n != 0 {
				t.Errorf("agent calls = %d, want 0", n)
			}
		})
	}
}

// TestGet_pausedIsStoppedFromStatusCurrent uses F14's paused VM: the
// resources listing shows plain running, only status/current tells.
func TestGet_pausedIsStoppedFromStatusCurrent(t *testing.T) {
	t.Parallel()
	st := decodeData[vmStatus](t, "status_current_paused")
	if st.Status != "running" || st.QMPStatus != "paused" {
		t.Fatalf("capture status_current_paused = %+v, want running and paused", st)
	}
	fx := newFixture(t, 1)
	vm := fx.addOurVM(testVMID, testName, st.Status)
	vm.qmpstatus = st.QMPStatus
	vm.ifaces = capturedIfaces(t)
	if r, _ := resourceByID(t, fx.fake, testVMID); r.Status != "running" {
		t.Fatalf("resources status = %q, want running as F14 saw", r.Status)
	}
	if inst := fx.get(fx.id()); inst.State != provider.StateStopped || inst.IP != "" {
		t.Errorf("Get = %+v, want Stopped with no IP", inst)
	}
}

func TestGet_prelaunchIsProvisioning(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	vm := fx.addOurVM(testVMID, testName, "running")
	vm.qmpstatus = "prelaunch"
	vm.ifaces = capturedIfaces(t)
	if inst := fx.get(fx.id()); inst.State != provider.StateProvisioning || inst.IP != "" {
		t.Errorf("Get = %+v, want Provisioning", inst)
	}
}

func TestGet_suspendedLockIsStopped(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "stopped").lock = lockSuspended
	inst := fx.get(fx.id())
	if inst.State != provider.StateStopped || inst.UID != testUID {
		t.Errorf("Get = %+v, want Stopped with uid", inst)
	}
	if got := fx.fake.callLog(); !slices.Equal(got, []string{"ClusterResources", "VMConfig"}) {
		t.Errorf("calls = %v, want resources and config only", got)
	}
}

func TestGet_unknownStatusIsTransient(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running").status = "unknown"
	msg := assertGetFails(t, fx, provider.ErrTransient)
	// On pv1-1 the first Get after Create met this with the node up: the
	// listing had not reported the new VM yet (Phase 10 run, finding P6).
	if !strings.Contains(msg, "pv1-1") || !strings.Contains(msg, "too new to be reported yet") {
		t.Errorf("error = %s, want it to name the node and both causes", msg)
	}
	if n := fx.fake.callCount("VMConfig"); n != 0 {
		t.Errorf("VMConfig calls = %d, want 0 while the node is unknown", n)
	}
}

func TestGet_lockedIsTransientNeverNotFound(t *testing.T) {
	t.Parallel()
	for _, lock := range []string{"backup", "migrate", "snapshot", "rollback", "create"} {
		t.Run("resources lock "+lock, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			fx.addOurVM(testVMID, testName, "running").lock = lock
			msg := assertGetFails(t, fx, provider.ErrTransient)
			if !strings.Contains(msg, "locked ("+lock+")") {
				t.Errorf("error = %s, want it to name the lock", msg)
			}
		})
	}
	t.Run("lock taken between the resources and config reads", func(t *testing.T) {
		t.Parallel()
		fx := newFixture(t, 1)
		vm := fx.addOurVM(testVMID, testName, "running")
		mutateOn(fx.fake, "VMConfig", func() { vm.lock = "backup" })
		assertGetFails(t, fx, provider.ErrTransient)
		if n := fx.fake.callCount("VMStatus"); n != 0 {
			t.Errorf("VMStatus calls = %d, want 0", n)
		}
	})
}

// TestGet_midCloneNamelessEntryIsTransient drives a real fake clone: while
// it runs the entry has no name (F8), which must not read as NotFound.
func TestGet_midCloneNamelessEntryIsTransient(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.fake.addTemplate(9001, "debian-13-template")
	upid, err := fx.fake.Clone(fx.ctx, fakeNode, 9001, cloneParams{NewID: testVMID, Name: testName, Description: formatDescription(testUID, testCreatedAt)})
	if err != nil {
		t.Fatalf("Clone error = %v", err)
	}
	if r, _ := resourceByID(t, fx.fake, testVMID); r.Name != "" || r.Lock != "clone" {
		t.Fatalf("mid-clone entry = %+v, want nameless and locked", r)
	}
	assertGetFails(t, fx, provider.ErrTransient)
	pollTask(t, fx.fake, upid)
	if inst := fx.get(fx.id()); inst.State != provider.StateStopped || inst.UID != testUID {
		t.Errorf("Get after the clone = %+v, want Stopped with uid", inst)
	}
}

func TestGet_vanishedBetweenCallsIsNotFound(t *testing.T) {
	t.Parallel()
	for _, method := range []string{"VMConfig", "VMStatus"} {
		t.Run("before "+method, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			fx.addOurVM(testVMID, testName, "running")
			mutateOn(fx.fake, method, func() { delete(fx.fake.vms, testVMID) })
			assertGetFails(t, fx, provider.ErrNotFound)
		})
	}
}

func TestGet_unknownStateIsStopped(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	vm := fx.addOurVM(testVMID, testName, "running")
	vm.status, vm.qmpstatus = "hibernating-v2", "hibernating-v2"
	if inst := fx.get(fx.id()); inst.State != provider.StateStopped {
		t.Errorf("Get state = %s, want Stopped for an unrecognised status", inst.State)
	}
}

func TestGet_readsUIDAndCreatedAtFromBackend(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	created := time.Date(2026, 9, 1, 8, 30, 0, 0, time.UTC)
	fx.addOurVM(testVMID, testName, "stopped").config["description"] = formatDescription("uid-from-backend", created)
	inst := fx.get(fx.id())
	if inst.UID != "uid-from-backend" || !inst.CreatedAt.Equal(created) {
		t.Errorf("Get = uid %q createdAt %v, want uid-from-backend and %v", inst.UID, inst.CreatedAt, created)
	}
}

// TestGet_unmanagedDescriptionHasEmptyUID: a mark without a uid line is
// still ours, with an empty UID, which callers must tolerate.
func TestGet_unmanagedDescriptionHasEmptyUID(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "stopped").config["description"] = "proxy-operator-managed=true\r\nadmin note\r\n"
	inst := fx.get(fx.id())
	if inst.UID != "" || !inst.CreatedAt.IsZero() || inst.State != provider.StateStopped {
		t.Errorf("Get = %+v, want empty uid, zero createdAt, Stopped", inst)
	}
}

func TestGet_malformedIDIsPermanent(t *testing.T) {
	t.Parallel()
	for _, id := range []string{"", "vms/150042", "vms/0150042/" + testName, "vm/150042/" + testName, "vms/abc/" + testName, "vms/150042/a/b"} {
		t.Run(id, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			_, err := fx.p.Get(fx.ctx, id)
			assertProviderError(t, err, "get", provider.ErrPermanent)
			if calls := fx.fake.callLog(); len(calls) != 0 {
				t.Errorf("calls = %v, want none", calls)
			}
		})
	}
}

func TestGet_resourcesFailureIsTransient(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name string
		err  error
	}{
		{name: "pveproxy 595", err: fakeErr(595, "Connection refused")},
		{name: "other 500", err: fakeErr(500, "fake: cluster filesystem not ready")},
		{name: "network", err: errors.New("dial tcp 192.168.0.8:8006: connect: no route to host")},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			fx.addOurVM(testVMID, testName, "running")
			fx.fake.failWith("ClusterResources", tc.err)
			assertGetFails(t, fx, provider.ErrTransient)
		})
	}
}

func TestGet_afterDeleteReturnsNotFound(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running").ifaces = capturedIfaces(t)
	if inst := fx.get(fx.id()); inst.State != provider.StateRunning {
		t.Fatalf("Get before Delete = %+v, want Running", inst)
	}
	if err := fx.p.Delete(fx.ctx, fx.id()); err != nil {
		t.Fatalf("Delete error = %v", err)
	}
	assertGetFails(t, fx, provider.ErrNotFound)
}

// TestGet_duringDeleteStillReturnsInstance holds the destroy task running:
// destroying sets no lock and keeps the VM listed until the task ends, as
// Phase 0's mid-destroy captures show, so Get sees it Stopped, not gone.
func TestGet_duringDeleteStillReturnsInstance(t *testing.T) {
	t.Parallel()
	if _, locked := capturedConfig(t, "config_during_destroy")["lock"]; locked {
		t.Fatal("capture config_during_destroy has a lock; the fake models none")
	}
	listed := decodeData[[]vmResource](t, "cluster_resources_during_destroy")
	if !slices.ContainsFunc(listed, func(r vmResource) bool { return r.VMID == 199999 && r.Status == "stopped" && r.Lock == "" }) {
		t.Fatal("capture cluster_resources_during_destroy does not list VM 199999 stopped and unlocked")
	}

	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running")
	inTask, release := make(chan struct{}), make(chan struct{})
	var once sync.Once
	fx.fake.hookOn("TaskStatus", func() error {
		if fx.fake.callCount("Destroy") > 0 {
			once.Do(func() {
				close(inTask)
				<-release
			})
		}
		return nil
	})
	done := make(chan error, 1)
	go func() { done <- fx.p.Delete(fx.ctx, fx.id()) }()
	select {
	case <-inTask:
	case err := <-done:
		t.Fatalf("Delete returned %v before its destroy task was polled", err)
	}
	inst := fx.get(fx.id())
	if inst.State != provider.StateStopped || inst.UID != testUID || inst.ID != fx.id() {
		t.Errorf("Get during the destroy = %+v, want the instance Stopped with its uid", inst)
	}
	close(release)
	if err := <-done; err != nil {
		t.Fatalf("Delete error = %v", err)
	}
	assertGetFails(t, fx, provider.ErrNotFound)
}