egress-proxies-operator/internal/provider/proxmox/delete_test.go

package proxmox

import (
	"context"
	"errors"
	"net/http"
	"slices"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

// putSeed uploads testName's seed to the fake's ISO storage, as Create
// leaves it.
func (fx *fixture) putSeed() {
	fx.t.Helper()
	upid, err := fx.fake.UploadISO(context.Background(), fakeNode, fakeISOStorage, seedISOName(testName), strings.NewReader("seed"), 4)
	if err != nil {
		fx.t.Fatalf("UploadISO error = %v", err)
	}
	pollTask(fx.t, fx.fake, upid)
}

func (fx *fixture) seedPresent() bool {
	_, ok := fx.fake.iso(fakeISOStorage, seedISOName(testName))
	return ok
}

// deleteFails runs Delete on fx.id(), checks the error's class and returns
// its text.
func (fx *fixture) deleteFails(class error) string {
	fx.t.Helper()
	err := fx.p.Delete(fx.ctx, fx.id())
	assertProviderError(fx.t, err, "delete", class)
	return err.Error()
}

func (fx *fixture) delete() {
	fx.t.Helper()
	if err := fx.p.Delete(fx.ctx, fx.id()); err != nil {
		fx.t.Fatalf("Delete error = %v", err)
	}
}

// seedDeleteNodes returns the node of every DeleteVolume call, read from
// the decorator's lines; the fixture's api must be observed.
func (fx *fixture) seedDeleteNodes() []string {
	lines := fx.logs.matching(`"call"="DeleteVolume"`)
	nodes := make([]string, 0, len(lines))
	for _, l := range lines {
		_, rest, _ := strings.Cut(l, `"node"="`)
		node, _, _ := strings.Cut(rest, `"`)
		nodes = append(nodes, node)
	}
	return nodes
}

// twoNodeFixture is an observed fixture at V(1) with nodes pv1-1 and pv1-2.
func twoNodeFixture(t *testing.T) *fixture {
	t.Helper()
	fx := newFixture(t, 1)
	fx.p.api = observe(fx.fake, testProvider)
	fx.p.cfg.Nodes = []string{fakeNode, "pv1-2"}
	return fx
}

func TestDelete_malformedIDIsPermanent(t *testing.T) {
	t.Parallel()
	for _, id := range []string{"", "vms/150042", "vms/0150042/" + testName, "vm/150042/" + testName, "vms/abc/" + testName, "vms/150042/a/b", "vms/99/" + testName} {
		t.Run(id, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			assertProviderError(t, fx.p.Delete(fx.ctx, id), "delete", provider.ErrPermanent)
			if calls := fx.fake.callLog(); len(calls) != 0 {
				t.Errorf("calls = %v, want none", calls)
			}
		})
	}
}

func TestDelete_twiceSucceeds(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running")
	fx.putSeed()
	fx.delete()
	fx.delete()
	if _, ok := fx.fake.vmSnapshot(testVMID); ok {
		t.Error("VM still exists")
	}
	if fx.seedPresent() {
		t.Error("seed still exists")
	}
	for method, want := range map[string]int{"Stop": 1, "Destroy": 1, "DeleteVolume": 2} {
		if got := fx.fake.callCount(method); got != want {
			t.Errorf("%s calls = %d, want %d", method, got, want)
		}
	}
}

// TestDelete_absentStillDeletesSeedOnEveryNode replaces the plan's
// TestDelete_absentIsNoOp: spec entry 11 deletes the seed even when the VM
// is gone, on every node, because the VM's node is no longer known.
func TestDelete_absentStillDeletesSeedOnEveryNode(t *testing.T) {
	t.Parallel()
	fx := twoNodeFixture(t)
	fx.putSeed()
	fx.delete()
	if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{fakeNode, "pv1-2"}) {
		t.Errorf("seed deleted on %v, want every node in config order", got)
	}
	if fx.seedPresent() {
		t.Error("seed still exists")
	}
	for _, method := range []string{"VMConfig", "Stop", "Destroy"} {
		if n := fx.fake.callCount(method); n != 0 {
			t.Errorf("%s calls = %d, want 0 for an absent VM", method, n)
		}
	}
	if lines := fx.logs.matching(`"msg"="proxmox vm already gone"`); len(lines) != 1 {
		t.Errorf("already-gone lines = %v, want one", lines)
	}
}

func TestDelete_containerAtVMIDIsLeftAlone(t *testing.T) {
	t.Parallel()
	fx := twoNodeFixture(t)
	fx.addOurVM(testVMID, testName, "running").kind = "lxc"
	fx.delete()
	if vm, ok := fx.fake.vmSnapshot(testVMID); !ok || vm.status != "running" {
		t.Errorf("container = %+v, %v; want it untouched", vm, ok)
	}
	if got := fx.seedDeleteNodes(); len(got) != 2 {
		t.Errorf("seed deleted on %v, want every node", got)
	}
}

// TestDelete_foreignNameLeavesVMAlone replaces the plan's
// TestDelete_foreignNameIsNoOp: the VM is not touched, but our seed is
// still deleted, on every node since ours is gone.
func TestDelete_foreignNameLeavesVMAlone(t *testing.T) {
	t.Parallel()
	fx := twoNodeFixture(t)
	fx.addOurVM(testVMID, "proxy-someoneelse00000", "running")
	fx.putSeed()
	fx.delete()
	if vm, ok := fx.fake.vmSnapshot(testVMID); !ok || vm.status != "running" {
		t.Errorf("foreign VM = %+v, %v; want it running and untouched", vm, ok)
	}
	for _, method := range []string{"Stop", "Destroy", "SetVMConfig"} {
		if n := fx.fake.callCount(method); n != 0 {
			t.Errorf("%s calls = %d, want 0", method, n)
		}
	}
	if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{fakeNode, "pv1-2"}) {
		t.Errorf("seed deleted on %v, want every node", got)
	}
	lines := fx.logs.matching(`"msg"="proxmox vm at this vmid is not ours, left alone"`)
	if len(lines) != 1 || !strings.Contains(lines[0], `"vmName"="proxy-someoneelse00000"`) {
		t.Errorf("no-op lines = %v, want one naming the other VM", lines)
	}
}

// TestDelete_unmarkedLeavesVMAlone replaces the plan's
// TestDelete_unmarkedIsNoOp, for the same reason.
func TestDelete_unmarkedLeavesVMAlone(t *testing.T) {
	t.Parallel()
	for name, desc := range map[string]string{
		"no description": "",
		"managed false":  "proxy-operator-managed=false\nproxy-operator-uid=" + testUID,
	} {
		t.Run(name, func(t *testing.T) {
			t.Parallel()
			fx := twoNodeFixture(t)
			vm := fx.fake.addVM(testVMID, testName, "stopped")
			if desc != "" {
				vm.config["description"] = desc
			}
			fx.delete()
			if _, ok := fx.fake.vmSnapshot(testVMID); !ok {
				t.Error("unmarked VM was destroyed")
			}
			if n := fx.fake.callCount("Destroy"); n != 0 {
				t.Errorf("Destroy calls = %d, want 0", n)
			}
			if got := fx.seedDeleteNodes(); len(got) != 2 {
				t.Errorf("seed deleted on %v, want every node", got)
			}
			if lines := fx.logs.matching(`"msg"="proxmox vm carries no proxy-operator mark, left alone"`); len(lines) != 1 {
				t.Errorf("no-op lines = %v, want one", lines)
			}
		})
	}
}

// TestDelete_runningVMStoppedFirst covers a paused VM too: the listing
// shows it as running (F14), and Stop ends it. The fake refuses to destroy
// a running VM, and each task takes two polls, so a Destroy that does not
// wait for the stop fails.
func TestDelete_runningVMStoppedFirst(t *testing.T) {
	t.Parallel()
	// pv1-1 took a stop on a paused VM as a task that ended OK with the VM
	// stopped (Phase 10 captures), which is what the fake does for all three.
	accepted := readCapture(t, "stop_paused_response")
	done := decodeData[taskStatus](t, "task_status_stop_paused")
	after := decodeData[vmStatus](t, "status_current_after_stop_paused")
	if accepted.Status != http.StatusOK || done.Type != "qmstop" || done.ExitStatus != "OK" || after.Status != "stopped" || after.QMPStatus != "stopped" {
		t.Fatalf("captures = %d, %+v, %+v; want the stop accepted, OK, and the VM stopped", accepted.Status, done, after)
	}
	for _, qmp := range []string{"running", "paused", "prelaunch"} {
		t.Run(qmp, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			fx.fake.taskPolls = 2
			fx.addOurVM(testVMID, testName, "running").qmpstatus = qmp
			fx.delete()
			log := fx.fake.callLog()
			stop, destroy := slices.Index(log, "Stop"), slices.Index(log, "Destroy")
			if stop < 0 || destroy < stop {
				t.Fatalf("calls = %v, want Stop before Destroy", log)
			}
			if n := callsBefore(log[stop:], "TaskStatus", "DeleteVolume"); n != fx.fake.taskPolls+1 {
				t.Errorf("stop task polled %d times before the seed delete, want %d", n, fx.fake.taskPolls+1)
			}
			if _, ok := fx.fake.vmSnapshot(testVMID); ok {
				t.Error("VM still exists")
			}
		})
	}
}

func TestDelete_stoppedVMSkipsStop(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "stopped")
	fx.delete()
	want := []string{"ClusterResources", "VMConfig", "DeleteVolume", "TaskStatus", "Destroy", "TaskStatus"}
	if got := fx.fake.callLog(); !slices.Equal(got, want) {
		t.Errorf("calls = %v, want %v", got, want)
	}
}

func TestDelete_lockedIsTransient(t *testing.T) {
	t.Parallel()
	for _, lock := range []string{"backup", "migrate", "snapshot", "rollback", "create"} {
		t.Run("resources lock "+lock, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			fx.addOurVM(testVMID, testName, "running").lock = lock
			fx.putSeed()
			if msg := fx.deleteFails(provider.ErrTransient); !strings.Contains(msg, "locked ("+lock+")") {
				t.Errorf("error = %s, want it to name the lock", msg)
			}
			if got := fx.fake.callLog(); !slices.Equal(got, []string{"UploadISO", "TaskStatus", "ClusterResources"}) {
				t.Errorf("calls = %v, want only the listing after the seed setup", got)
			}
			if !fx.seedPresent() {
				t.Error("seed deleted while the VM is locked")
			}
		})
	}
	t.Run("mid-clone nameless entry", func(t *testing.T) {
		t.Parallel()
		fx := newFixture(t, 1)
		fx.fake.addTemplate(9001, "debian-13-template")
		if _, err := fx.fake.Clone(fx.ctx, fakeNode, 9001, cloneParams{NewID: testVMID, Name: testName, Description: formatDescription(testUID, testCreatedAt)}); err != nil {
			t.Fatalf("Clone error = %v", err)
		}
		fx.deleteFails(provider.ErrTransient)
		if _, ok := fx.fake.vmSnapshot(testVMID); !ok {
			t.Error("mid-clone VM is gone")
		}
	})
	t.Run("lock taken between the resources and config reads", func(t *testing.T) {
		t.Parallel()
		fx := newFixture(t, 1)
		vm := fx.addOurVM(testVMID, testName, "running")
		mutateOn(fx.fake, "VMConfig", func() { vm.lock = "backup" })
		fx.deleteFails(provider.ErrTransient)
		if n := fx.fake.callCount("Stop"); n != 0 {
			t.Errorf("Stop calls = %d, want 0", n)
		}
	})
}

func TestDelete_unknownStatusIsTransient(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running").status = "unknown"
	if msg := fx.deleteFails(provider.ErrTransient); !strings.Contains(msg, fakeNode) {
		t.Errorf("error = %s, want it to name the node", msg)
	}
	if n := fx.fake.callCount("VMConfig"); n != 0 {
		t.Errorf("VMConfig calls = %d, want 0 while the node is unknown", n)
	}
}

// TestDelete_hibernatedVMIsDestroyedIfOurs: on pv1-1 the destroy took a
// hibernated VM, suspend state volume included, and its task ended OK.
// Delete used to refuse it as Permanent on the belief that the destroy
// would. A hibernated VM that is not ours is still left alone.
func TestDelete_hibernatedVMIsDestroyedIfOurs(t *testing.T) {
	t.Parallel()
	t.Run("ours", func(t *testing.T) {
		t.Parallel()
		accepted := readCapture(t, "destroy_hibernated_response")
		done := decodeData[taskStatus](t, "task_status_destroy_hibernated")
		if accepted.Status != http.StatusOK || done.Type != "qmdestroy" || done.ExitStatus != "OK" {
			t.Fatalf("captures = %d, %+v; want the destroy accepted and its task OK", accepted.Status, done)
		}
		fx := newFixture(t, 1)
		fx.addOurVM(testVMID, testName, "stopped").lock = lockSuspended
		fx.putSeed()
		fx.delete()
		want := []string{"ClusterResources", "VMConfig", "DeleteVolume", "TaskStatus", "Destroy", "TaskStatus"}
		if got := fx.fake.callLog()[2:]; !slices.Equal(got, want) {
			t.Errorf("calls = %v, want %v: no stop, since the VM is not running", got, want)
		}
		if _, ok := fx.fake.vmSnapshot(testVMID); ok {
			t.Error("hibernated VM still exists")
		}
		if fx.seedPresent() {
			t.Error("seed still exists")
		}
	})
	t.Run("someone else's", func(t *testing.T) {
		t.Parallel()
		fx := newFixture(t, 1)
		fx.addOurVM(testVMID, "proxy-someoneelse00000", "stopped").lock = lockSuspended
		fx.delete()
		if vm, ok := fx.fake.vmSnapshot(testVMID); !ok || vm.lock != lockSuspended {
			t.Errorf("foreign hibernated VM = %+v, %v; want it untouched", vm, ok)
		}
	})
}

func TestDelete_removesSeedISO(t *testing.T) {
	t.Parallel()
	fx := twoNodeFixture(t)
	fx.addOurVM(testVMID, testName, "stopped").node = "pv1-2"
	fx.putSeed()
	fx.delete()
	if fx.seedPresent() {
		t.Error("seed still exists")
	}
	if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{"pv1-2"}) {
		t.Errorf("seed deleted on %v, want only the VM's node pv1-2", got)
	}
	lines := fx.logs.matching(`"call"="DeleteVolume"`)
	if len(lines) != 1 || !strings.Contains(lines[0], `"volid"="`+seedVolID(fakeISOStorage, testName)+`"`) || !strings.Contains(lines[0], `"storage"="`+fakeISOStorage+`"`) {
		t.Errorf("DeleteVolume lines = %v, want the seed's volid on the ISO storage", lines)
	}
}

// TestDelete_seedDeleteIsTaskAndIdempotent replaces the plan's
// TestDelete_toleratesMissingISO: Proxmox ends a delete of a missing volume
// with a successful imgdel task (F13), so nothing is matched, but the task
// is still waited on.
func TestDelete_seedDeleteIsTaskAndIdempotent(t *testing.T) {
	t.Parallel()
	missing := decodeData[taskStatus](t, "task_status_volume_missing")
	if missing.Type != "imgdel" || missing.ExitStatus != "OK" {
		t.Fatalf("capture task_status_volume_missing = %+v, want an imgdel task ending OK", missing)
	}
	for _, withSeed := range []bool{true, false} {
		t.Run("seed present "+strconv.FormatBool(withSeed), func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			fx.addOurVM(testVMID, testName, "stopped")
			if withSeed {
				fx.putSeed()
			}
			fx.fake.taskPolls = 2
			fx.delete()
			log := fx.fake.callLog()
			last := len(log) - 1
			for log[last] != "DeleteVolume" {
				last--
			}
			if n := callsBefore(log[last:], "TaskStatus", "Destroy"); n != fx.fake.taskPolls+1 {
				t.Errorf("seed delete task polled %d times, want %d: %v", n, fx.fake.taskPolls+1, log)
			}
			if fx.seedPresent() {
				t.Error("seed still exists")
			}
		})
	}
}

// TestDelete_seedDeleteFailureOnVMNodeFails: the VM outlives a failed seed
// delete, so Get keeps finding it and the next Delete finishes the job with
// the same class of answer (review findings CR1, CR2, P1).
func TestDelete_seedDeleteFailureOnVMNodeFails(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name  string
		setup func(fx *fixture)
		class error
	}{
		{name: "token lacks Datastore.Allocate", setup: func(fx *fixture) {
			fx.fake.failWith("DeleteVolume", loadAPIError(t, "err_forbidden_volume_delete"))
		}, class: provider.ErrPermanent},
		{name: "pveproxy 596", setup: func(fx *fixture) { fx.fake.failWith("DeleteVolume", fakeErr(596, "Broken pipe")) }, class: provider.ErrTransient},
		{name: "task failed", setup: func(fx *fixture) {
			fx.fake.hookOn("TaskStatus", func() error {
				if fx.fake.callCount("DeleteVolume") > 0 {
					return fakeErr(500, "fake: task log unreadable")
				}
				return nil
			})
		}, class: provider.ErrTransient},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			fx.addOurVM(testVMID, testName, "stopped")
			tc.setup(fx)
			fx.putSeed()
			fx.deleteFails(tc.class)
			if n := fx.fake.callCount("Destroy"); n != 0 {
				t.Errorf("Destroy calls = %d, want 0 while the seed is still there", n)
			}
			if _, err := fx.p.Get(fx.ctx, fx.id()); errors.Is(err, provider.ErrNotFound) {
				t.Fatalf("Get = %v, want the VM still found so Delete runs again", err)
			}
			fx.deleteFails(tc.class)
			fx.fake.clearHooks()
			fx.delete()
			if _, ok := fx.fake.vmSnapshot(testVMID); ok {
				t.Error("VM still exists after the retry")
			}
			if fx.seedPresent() {
				t.Error("seed still exists after the retry")
			}
		})
	}
}

func TestDelete_seedDeleteNodeFailureSkippedWhenNodeUnknown(t *testing.T) {
	t.Parallel()
	t.Run("one node refuses", func(t *testing.T) {
		t.Parallel()
		fx := twoNodeFixture(t)
		fx.putSeed()
		var calls int
		fx.fake.hookOn("DeleteVolume", func() error {
			if calls++; calls == 1 {
				return fakeErr(500, "fake: storage 'proxy-seeds' is not available on node 'pv1-1'")
			}
			return nil
		})
		fx.delete()
		if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{fakeNode, "pv1-2"}) {
			t.Errorf("seed deleted on %v, want both nodes tried", got)
		}
		lines := fx.logs.matching(`"msg"="proxmox seed delete failed on a node, skipped"`)
		if len(lines) != 1 || !strings.Contains(lines[0], `"node"="pv1-1"`) || !strings.Contains(lines[0], "not available") {
			t.Errorf("skip lines = %v, want one naming pv1-1 and the reason", lines)
		}
		if fx.seedPresent() {
			t.Error("seed still exists after pv1-2's delete")
		}
	})
	t.Run("a refused privilege fails the call", func(t *testing.T) {
		t.Parallel()
		fx := twoNodeFixture(t)
		fx.putSeed()
		fx.fake.failWith("DeleteVolume", loadAPIError(t, "err_forbidden_volume_delete"))
		if msg := fx.deleteFails(provider.ErrPermanent); !strings.Contains(msg, "Datastore.Allocate") {
			t.Errorf("error = %s, want the missing privilege named", msg)
		}
		if n := fx.fake.callCount("DeleteVolume"); n != 1 {
			t.Errorf("DeleteVolume calls = %d, want 1: every node would refuse the same", n)
		}
	})
	t.Run("ended ctx fails the call", func(t *testing.T) {
		t.Parallel()
		fx := twoNodeFixture(t)
		ctx, cancel := context.WithCancel(fx.ctx)
		defer cancel()
		fx.fake.hookOn("DeleteVolume", func() error {
			cancel()
			return context.Canceled
		})
		err := fx.p.Delete(ctx, fx.id())
		assertProviderError(t, err, "delete", provider.ErrTransient)
		if n := fx.fake.callCount("DeleteVolume"); n != 1 {
			t.Errorf("DeleteVolume calls = %d, want 1: the ended ctx stops the loop", n)
		}
	})
}

func TestDelete_destroyTaskFailureIsTransient(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "stopped")
	fx.putSeed()
	fx.fake.destroyFailsWith = "fake: unable to remove disk"
	if msg := fx.deleteFails(provider.ErrTransient); !strings.Contains(msg, "unable to remove disk") {
		t.Errorf("error = %s, want the task's exitstatus", msg)
	}
	if _, ok := fx.fake.vmSnapshot(testVMID); !ok {
		t.Error("VM gone after a failed destroy task")
	}
	if fx.seedPresent() {
		t.Error("seed still exists: it goes before the destroy")
	}
	if lines := fx.logs.matching(`"msg"="proxmox task failed"`); len(lines) != 1 {
		t.Errorf("task failed lines = %v, want one", lines)
	}
}

// TestDelete_goneMidwayIsSuccess: the VM vanishing at any step counts as
// done, and F2's does-not-exist answer is what each step then sees.
func TestDelete_goneMidwayIsSuccess(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name        string
		status      string
		method      string
		wantDestroy int
		wantNodes   []string
	}{
		{name: "before the config read", status: "stopped", method: "VMConfig", wantNodes: []string{fakeNode, "pv1-2"}},
		{name: "before the stop", status: "running", method: "Stop", wantDestroy: 1, wantNodes: []string{fakeNode}},
		{name: "before the destroy", status: "stopped", method: "Destroy", wantDestroy: 1, wantNodes: []string{fakeNode}},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fx := twoNodeFixture(t)
			fx.addOurVM(testVMID, testName, tc.status)
			mutateOn(fx.fake, tc.method, func() { delete(fx.fake.vms, testVMID) })
			fx.delete()
			if n := fx.fake.callCount("Destroy"); n != tc.wantDestroy {
				t.Errorf("Destroy calls = %d, want %d", n, tc.wantDestroy)
			}
			if got := fx.seedDeleteNodes(); !slices.Equal(got, tc.wantNodes) {
				t.Errorf("seed deleted on %v, want %v", got, tc.wantNodes)
			}
		})
	}
}

func TestDelete_resourcesFailureIsTransient(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running")
	fx.fake.failWith("ClusterResources", fakeErr(595, "Connection refused"))
	fx.deleteFails(provider.ErrTransient)
	if n := fx.fake.callCount("DeleteVolume"); n != 0 {
		t.Errorf("DeleteVolume calls = %d, want 0 when the VM's state is unknown", n)
	}
}

// TestDelete_usesPurgeAndDestroyUnreferencedDisks runs Delete over the real
// client against Phase 0's destroy captures: the DELETE carries both flags
// and the seed delete sends the volid as one escaped segment.
func TestDelete_usesPurgeAndDestroyUnreferencedDisks(t *testing.T) {
	t.Parallel()
	const (
		vmid = 199999
		name = "proxy-reconaaaaaaaaaaa"
	)
	var (
		mu   sync.Mutex
		seen []string
	)
	cfx := newClientFixture(t, func(w http.ResponseWriter, r *http.Request) {
		path := strings.TrimPrefix(r.URL.EscapedPath(), "/api2/json/")
		mu.Lock()
		seen = append(seen, r.Method+" "+path+"?"+r.URL.RawQuery)
		mu.Unlock()
		capture := ""
		switch {
		case r.Method == http.MethodGet && path == "cluster/resources":
			capture = "cluster_resources_during_destroy"
		case r.Method == http.MethodGet && path == "nodes/pv1-1/qemu/199999/config":
			capture = "config_during_destroy"
		case r.Method == http.MethodDelete && path == "nodes/pv1-1/qemu/199999":
			capture = "destroy_response"
		case r.Method == http.MethodDelete && path == "nodes/pv1-1/storage/proxy-seeds/content/proxy-seeds:iso%2F"+name+"-seed.iso":
			capture = "content_delete_response"
		case r.Method == http.MethodGet && strings.Contains(path, ":imgdel:"):
			capture = "task_status_volume_missing"
		case r.Method == http.MethodGet && strings.Contains(path, ":qmdestroy:"):
			capture = "task_status_stopped_ok"
		default:
			t.Errorf("unexpected request %s %s", r.Method, path)
			http.Error(w, "unexpected", http.StatusNotImplemented)
			return
		}
		serveJSON(w, readCapture(t, capture).Body)
	})
	cfg := testConfig()
	cfg.URL, cfg.TokenFile, cfg.CAFile = cfx.cfg.URL, cfx.cfg.TokenFile, cfx.cfg.CAFile
	p := newProvider(testProvider, cfg, cfx.api(t))
	p.budgets = testBudgets()
	if err := p.Delete(context.Background(), formatProviderID(vmid, name)); err != nil {
		t.Fatalf("Delete error = %v", err)
	}
	mu.Lock()
	defer mu.Unlock()
	if !slices.Contains(seen, "DELETE nodes/pv1-1/qemu/199999?destroy-unreferenced-disks=1&purge=1") {
		t.Errorf("requests = %v, want the destroy with purge=1 and destroy-unreferenced-disks=1", seen)
	}
	if !slices.ContainsFunc(seen, func(s string) bool { return strings.HasPrefix(s, "DELETE nodes/pv1-1/storage/proxy-seeds/content/") }) {
		t.Errorf("requests = %v, want the seed delete", seen)
	}
	if slices.ContainsFunc(seen, func(s string) bool { return strings.Contains(s, "status/stop") }) {
		t.Errorf("requests = %v, want no stop for the captured stopped VM", seen)
	}
}

func TestDelete_concurrentSameNameSafe(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running")
	release := fx.fake.blockOn("Stop")
	errs := make([]error, 2)
	first := make(chan struct{})
	var wg sync.WaitGroup
	wg.Go(func() {
		defer close(first)
		errs[0] = fx.p.Delete(fx.ctx, fx.id())
	})
	for fx.fake.callCount("Stop") == 0 {
		select {
		case <-first:
			t.Fatalf("first Delete returned %v without stopping the VM", errs[0])
		case <-time.After(time.Millisecond):
		}
	}
	wg.Go(func() { errs[1] = fx.p.Delete(fx.ctx, fx.id()) })
	close(release)
	wg.Wait()
	for i, err := range errs {
		if err != nil {
			t.Errorf("Delete %d error = %v", i, err)
		}
	}
	for method, want := range map[string]int{"Stop": 1, "Destroy": 1} {
		if got := fx.fake.callCount(method); got != want {
			t.Errorf("%s calls = %d, want %d", method, got, want)
		}
	}
}

// TestDelete_waitsOnTheNameLock: Delete takes the per-name lock before any
// call, so it cannot interleave with a Create or another Delete of that
// name (spec entry 19).
func TestDelete_waitsOnTheNameLock(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.addOurVM(testVMID, testName, "running")
	release, err := fx.p.lockName(fx.ctx, testName)
	if err != nil {
		t.Fatalf("lockName error = %v", err)
	}
	defer release()
	ctx, cancel := context.WithTimeout(fx.ctx, 20*time.Millisecond)
	defer cancel()
	within(t, 5*time.Second, func() { err = fx.p.Delete(ctx, fx.id()) })
	assertProviderError(t, err, "delete", provider.ErrTransient)
	if !errors.Is(err, context.DeadlineExceeded) {
		t.Errorf("error = %v, want it to wrap DeadlineExceeded", err)
	}
	if calls := fx.fake.callLog(); len(calls) != 0 {
		t.Errorf("calls = %v, want none while the name is locked", calls)
	}
}

func TestDelete_cancelledMidWaitReturnsPromptly(t *testing.T) {
	t.Parallel()
	fx := newFixture(t, 1)
	fx.p.budgets = defaultBudgets()
	fx.p.budgets.poll = time.Millisecond
	fx.fake.taskPolls = 1 << 30
	fx.addOurVM(testVMID, testName, "running")
	ctx, cancel := context.WithCancel(fx.ctx)
	defer cancel()
	fx.fake.hookOn("TaskStatus", func() error {
		cancel()
		return nil
	})
	start := time.Now()
	var err error
	within(t, 10*time.Second, func() { err = fx.p.Delete(ctx, fx.id()) })
	if elapsed := time.Since(start); elapsed > 5*time.Second {
		t.Errorf("Delete took %v after cancellation", elapsed)
	}
	assertProviderError(t, err, "delete", provider.ErrTransient)
	if !errors.Is(err, context.Canceled) {
		t.Errorf("error = %v, want it to wrap context.Canceled", err)
	}
}

func TestDelete_outcomeLine(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name  string
		setup func(fx *fixture)
		want  []string
		never string
	}{
		{
			name:  "destroyed",
			setup: func(fx *fixture) { fx.addOurVM(testVMID, testName, "stopped") },
			want:  []string{`"node"="` + fakeNode + `"`, `"destroyed"=true`},
		},
		{name: "already gone", setup: func(*fixture) {}, want: []string{`"destroyed"=false`}, never: `"node"=`},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 1)
			tc.setup(fx)
			fx.delete()
			lines := fx.logs.matching(`"msg"="proxmox vm deleted"`)
			if len(lines) != 1 {
				t.Fatalf("outcome lines = %d, want 1:\n%s", len(lines), strings.Join(fx.logs.all(), "\n"))
			}
			for _, want := range append(tc.want, `"vmid"=150042`, `"name"="`+testName+`"`, `"provider"="`+testProvider+`"`) {
				if !strings.Contains(lines[0], want) {
					t.Errorf("outcome line lacks %s: %s", want, lines[0])
				}
			}
			if tc.never != "" && strings.Contains(lines[0], tc.never) {
				t.Errorf("outcome line has %s: %s", tc.never, lines[0])
			}
		})
	}
}

func TestDelete_noOpLinesAreSilentAtV0(t *testing.T) {
	t.Parallel()
	tests := map[string]func(fx *fixture){
		"absent":   func(*fixture) {},
		"foreign":  func(fx *fixture) { fx.addOurVM(testVMID, "proxy-someoneelse00000", "running") },
		"unmarked": func(fx *fixture) { fx.fake.addVM(testVMID, testName, "running") },
		"ours":     func(fx *fixture) { fx.addOurVM(testVMID, testName, "running") },
	}
	for name, setup := range tests {
		t.Run(name, func(t *testing.T) {
			t.Parallel()
			fx := newFixture(t, 0)
			fx.p.api = observe(fx.fake, testProvider)
			setup(fx)
			fx.delete()
			if lines := fx.logs.all(); len(lines) != 0 {
				t.Errorf("V(0) lines = %v, want none", lines)
			}
		})
	}
}