proxmox: Delete with seed cleanup, health Checker, CRD type comment
Delete stops and destroys a VM of ours under the per-name lock, then deletes its seed on the VM's node, or on every node when no VM of ours was left. A hibernated VM of ours is Permanent, as in Create. The Checker verifies each node concurrently (version, node status, ISO storage, templates) and adds CertificateInvalid, CredentialsUnavailable, StorageNotFound and TemplateNotFound to the shared reasons. The ProviderHealth type comment gains proxmox. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,717 @@
|
||||
package proxmox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// putSeed uploads testName's seed to the fake's ISO storage, as Create
|
||||
// leaves it.
|
||||
func (fx *fixture) putSeed() {
|
||||
fx.t.Helper()
|
||||
upid, err := fx.fake.UploadISO(context.Background(), fakeNode, fakeISOStorage, seedISOName(testName), strings.NewReader("seed"), 4)
|
||||
if err != nil {
|
||||
fx.t.Fatalf("UploadISO error = %v", err)
|
||||
}
|
||||
pollTask(fx.t, fx.fake, upid)
|
||||
}
|
||||
|
||||
func (fx *fixture) seedPresent() bool {
|
||||
_, ok := fx.fake.iso(fakeISOStorage, seedISOName(testName))
|
||||
return ok
|
||||
}
|
||||
|
||||
// deleteFails runs Delete on fx.id(), checks the error's class and returns
|
||||
// its text.
|
||||
func (fx *fixture) deleteFails(class error) string {
|
||||
fx.t.Helper()
|
||||
err := fx.p.Delete(fx.ctx, fx.id())
|
||||
assertProviderError(fx.t, err, "delete", class)
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
func (fx *fixture) delete() {
|
||||
fx.t.Helper()
|
||||
if err := fx.p.Delete(fx.ctx, fx.id()); err != nil {
|
||||
fx.t.Fatalf("Delete error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// seedDeleteNodes returns the node of every DeleteVolume call, read from
|
||||
// the decorator's lines; the fixture's api must be observed.
|
||||
func (fx *fixture) seedDeleteNodes() []string {
|
||||
lines := fx.logs.matching(`"call"="DeleteVolume"`)
|
||||
nodes := make([]string, 0, len(lines))
|
||||
for _, l := range lines {
|
||||
_, rest, _ := strings.Cut(l, `"node"="`)
|
||||
node, _, _ := strings.Cut(rest, `"`)
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
// twoNodeFixture is an observed fixture at V(1) with nodes pv1-1 and pv1-2.
|
||||
func twoNodeFixture(t *testing.T) *fixture {
|
||||
t.Helper()
|
||||
fx := newFixture(t, 1)
|
||||
fx.p.api = observe(fx.fake, testProvider)
|
||||
fx.p.cfg.Nodes = []string{fakeNode, "pv1-2"}
|
||||
return fx
|
||||
}
|
||||
|
||||
func TestDelete_malformedIDIsPermanent(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, id := range []string{"", "vms/150042", "vms/0150042/" + testName, "vm/150042/" + testName, "vms/abc/" + testName, "vms/150042/a/b", "vms/99/" + testName} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
assertProviderError(t, fx.p.Delete(fx.ctx, id), "delete", provider.ErrPermanent)
|
||||
if calls := fx.fake.callLog(); len(calls) != 0 {
|
||||
t.Errorf("calls = %v, want none", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_twiceSucceeds(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
fx.putSeed()
|
||||
fx.delete()
|
||||
fx.delete()
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); ok {
|
||||
t.Error("VM still exists")
|
||||
}
|
||||
if fx.seedPresent() {
|
||||
t.Error("seed still exists")
|
||||
}
|
||||
for method, want := range map[string]int{"Stop": 1, "Destroy": 1, "DeleteVolume": 2} {
|
||||
if got := fx.fake.callCount(method); got != want {
|
||||
t.Errorf("%s calls = %d, want %d", method, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_absentStillDeletesSeedOnEveryNode replaces the plan's
|
||||
// TestDelete_absentIsNoOp: spec entry 11 deletes the seed even when the VM
|
||||
// is gone, on every node, because the VM's node is no longer known.
|
||||
func TestDelete_absentStillDeletesSeedOnEveryNode(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.putSeed()
|
||||
fx.delete()
|
||||
if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{fakeNode, "pv1-2"}) {
|
||||
t.Errorf("seed deleted on %v, want every node in config order", got)
|
||||
}
|
||||
if fx.seedPresent() {
|
||||
t.Error("seed still exists")
|
||||
}
|
||||
for _, method := range []string{"VMConfig", "Stop", "Destroy"} {
|
||||
if n := fx.fake.callCount(method); n != 0 {
|
||||
t.Errorf("%s calls = %d, want 0 for an absent VM", method, n)
|
||||
}
|
||||
}
|
||||
if lines := fx.logs.matching(`"msg"="proxmox vm already gone"`); len(lines) != 1 {
|
||||
t.Errorf("already-gone lines = %v, want one", lines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_containerAtVMIDIsLeftAlone(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.addOurVM(testVMID, testName, "running").kind = "lxc"
|
||||
fx.delete()
|
||||
if vm, ok := fx.fake.vmSnapshot(testVMID); !ok || vm.status != "running" {
|
||||
t.Errorf("container = %+v, %v; want it untouched", vm, ok)
|
||||
}
|
||||
if got := fx.seedDeleteNodes(); len(got) != 2 {
|
||||
t.Errorf("seed deleted on %v, want every node", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_foreignNameLeavesVMAlone replaces the plan's
|
||||
// TestDelete_foreignNameIsNoOp: the VM is not touched, but our seed is
|
||||
// still deleted, on every node since ours is gone.
|
||||
func TestDelete_foreignNameLeavesVMAlone(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.addOurVM(testVMID, "proxy-someoneelse00000", "running")
|
||||
fx.putSeed()
|
||||
fx.delete()
|
||||
if vm, ok := fx.fake.vmSnapshot(testVMID); !ok || vm.status != "running" {
|
||||
t.Errorf("foreign VM = %+v, %v; want it running and untouched", vm, ok)
|
||||
}
|
||||
for _, method := range []string{"Stop", "Destroy", "SetVMConfig"} {
|
||||
if n := fx.fake.callCount(method); n != 0 {
|
||||
t.Errorf("%s calls = %d, want 0", method, n)
|
||||
}
|
||||
}
|
||||
if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{fakeNode, "pv1-2"}) {
|
||||
t.Errorf("seed deleted on %v, want every node", got)
|
||||
}
|
||||
lines := fx.logs.matching(`"msg"="proxmox vm at this vmid is not ours, left alone"`)
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], `"vmName"="proxy-someoneelse00000"`) {
|
||||
t.Errorf("no-op lines = %v, want one naming the other VM", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_unmarkedLeavesVMAlone replaces the plan's
|
||||
// TestDelete_unmarkedIsNoOp, for the same reason.
|
||||
func TestDelete_unmarkedLeavesVMAlone(t *testing.T) {
|
||||
t.Parallel()
|
||||
for name, desc := range map[string]string{
|
||||
"no description": "",
|
||||
"managed false": "proxy-operator-managed=false\nproxy-operator-uid=" + testUID,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
vm := fx.fake.addVM(testVMID, testName, "stopped")
|
||||
if desc != "" {
|
||||
vm.config["description"] = desc
|
||||
}
|
||||
fx.delete()
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); !ok {
|
||||
t.Error("unmarked VM was destroyed")
|
||||
}
|
||||
if n := fx.fake.callCount("Destroy"); n != 0 {
|
||||
t.Errorf("Destroy calls = %d, want 0", n)
|
||||
}
|
||||
if got := fx.seedDeleteNodes(); len(got) != 2 {
|
||||
t.Errorf("seed deleted on %v, want every node", got)
|
||||
}
|
||||
if lines := fx.logs.matching(`"msg"="proxmox vm carries no proxy-operator mark, left alone"`); len(lines) != 1 {
|
||||
t.Errorf("no-op lines = %v, want one", lines)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_runningVMStoppedFirst covers a paused VM too: the listing
|
||||
// shows it as running (F14), and Stop ends it. The fake refuses to destroy
|
||||
// a running VM, and each task takes two polls, so a Destroy that does not
|
||||
// wait for the stop fails.
|
||||
func TestDelete_runningVMStoppedFirst(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, qmp := range []string{"running", "paused", "prelaunch"} {
|
||||
t.Run(qmp, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.fake.taskPolls = 2
|
||||
fx.addOurVM(testVMID, testName, "running").qmpstatus = qmp
|
||||
fx.delete()
|
||||
log := fx.fake.callLog()
|
||||
stop, destroy := slices.Index(log, "Stop"), slices.Index(log, "Destroy")
|
||||
if stop < 0 || destroy < stop {
|
||||
t.Fatalf("calls = %v, want Stop before Destroy", log)
|
||||
}
|
||||
if n := callsBefore(log[stop:], "TaskStatus", "Destroy"); n != fx.fake.taskPolls+1 {
|
||||
t.Errorf("stop task polled %d times before Destroy, want %d", n, fx.fake.taskPolls+1)
|
||||
}
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); ok {
|
||||
t.Error("VM still exists")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_stoppedVMSkipsStop(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "stopped")
|
||||
fx.delete()
|
||||
want := []string{"ClusterResources", "VMConfig", "Destroy", "TaskStatus", "DeleteVolume", "TaskStatus"}
|
||||
if got := fx.fake.callLog(); !slices.Equal(got, want) {
|
||||
t.Errorf("calls = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_lockedIsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, lock := range []string{"backup", "migrate", "snapshot", "rollback", "create"} {
|
||||
t.Run("resources lock "+lock, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running").lock = lock
|
||||
fx.putSeed()
|
||||
if msg := fx.deleteFails(provider.ErrTransient); !strings.Contains(msg, "locked ("+lock+")") {
|
||||
t.Errorf("error = %s, want it to name the lock", msg)
|
||||
}
|
||||
if got := fx.fake.callLog(); !slices.Equal(got, []string{"UploadISO", "TaskStatus", "ClusterResources"}) {
|
||||
t.Errorf("calls = %v, want only the listing after the seed setup", got)
|
||||
}
|
||||
if !fx.seedPresent() {
|
||||
t.Error("seed deleted while the VM is locked")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("mid-clone nameless entry", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.fake.addTemplate(9001, "debian-13-template")
|
||||
if _, err := fx.fake.Clone(fx.ctx, fakeNode, 9001, cloneParams{NewID: testVMID, Name: testName, Description: formatDescription(testUID, testCreatedAt)}); err != nil {
|
||||
t.Fatalf("Clone error = %v", err)
|
||||
}
|
||||
fx.deleteFails(provider.ErrTransient)
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); !ok {
|
||||
t.Error("mid-clone VM is gone")
|
||||
}
|
||||
})
|
||||
t.Run("lock taken between the resources and config reads", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
vm := fx.addOurVM(testVMID, testName, "running")
|
||||
mutateOn(fx.fake, "VMConfig", func() { vm.lock = "backup" })
|
||||
fx.deleteFails(provider.ErrTransient)
|
||||
if n := fx.fake.callCount("Stop"); n != 0 {
|
||||
t.Errorf("Stop calls = %d, want 0", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDelete_unknownStatusIsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running").status = "unknown"
|
||||
if msg := fx.deleteFails(provider.ErrTransient); !strings.Contains(msg, fakeNode) {
|
||||
t.Errorf("error = %s, want it to name the node", msg)
|
||||
}
|
||||
if n := fx.fake.callCount("VMConfig"); n != 0 {
|
||||
t.Errorf("VMConfig calls = %d, want 0 while the node is unknown", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_hibernatedVMIsPermanentIfOurs matches Create's ownership
|
||||
// check: Destroy refuses the suspended lock, which never clears on its own,
|
||||
// so an admin must act. A hibernated VM that is not ours is left alone.
|
||||
func TestDelete_hibernatedVMIsPermanentIfOurs(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("ours", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "stopped").lock = lockSuspended
|
||||
fx.putSeed()
|
||||
if msg := fx.deleteFails(provider.ErrPermanent); !strings.Contains(msg, "resume or destroy it in Proxmox") {
|
||||
t.Errorf("error = %s, want the admin action", msg)
|
||||
}
|
||||
for _, method := range []string{"Stop", "Destroy", "DeleteVolume"} {
|
||||
if n := fx.fake.callCount(method); n != 0 {
|
||||
t.Errorf("%s calls = %d, want 0", method, n)
|
||||
}
|
||||
}
|
||||
if _, err := fx.fake.Destroy(fx.ctx, fakeNode, testVMID); !isLocked(err) {
|
||||
t.Errorf("fake Destroy on the hibernated VM error = %v, want the lock answer", err)
|
||||
}
|
||||
})
|
||||
t.Run("someone else's", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, "proxy-someoneelse00000", "stopped").lock = lockSuspended
|
||||
fx.delete()
|
||||
if vm, ok := fx.fake.vmSnapshot(testVMID); !ok || vm.lock != lockSuspended {
|
||||
t.Errorf("foreign hibernated VM = %+v, %v; want it untouched", vm, ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDelete_removesSeedISO(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.addOurVM(testVMID, testName, "stopped").node = "pv1-2"
|
||||
fx.putSeed()
|
||||
fx.delete()
|
||||
if fx.seedPresent() {
|
||||
t.Error("seed still exists")
|
||||
}
|
||||
if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{"pv1-2"}) {
|
||||
t.Errorf("seed deleted on %v, want only the VM's node pv1-2", got)
|
||||
}
|
||||
lines := fx.logs.matching(`"call"="DeleteVolume"`)
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], `"volid"="`+seedVolID(fakeISOStorage, testName)+`"`) || !strings.Contains(lines[0], `"storage"="`+fakeISOStorage+`"`) {
|
||||
t.Errorf("DeleteVolume lines = %v, want the seed's volid on the ISO storage", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_seedDeleteIsTaskAndIdempotent replaces the plan's
|
||||
// TestDelete_toleratesMissingISO: Proxmox ends a delete of a missing volume
|
||||
// with a successful imgdel task (F13), so nothing is matched, but the task
|
||||
// is still waited on.
|
||||
func TestDelete_seedDeleteIsTaskAndIdempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
missing := decodeData[taskStatus](t, "task_status_volume_missing")
|
||||
if missing.Type != "imgdel" || missing.ExitStatus != "OK" {
|
||||
t.Fatalf("capture task_status_volume_missing = %+v, want an imgdel task ending OK", missing)
|
||||
}
|
||||
for _, withSeed := range []bool{true, false} {
|
||||
t.Run("seed present "+strconv.FormatBool(withSeed), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "stopped")
|
||||
if withSeed {
|
||||
fx.putSeed()
|
||||
}
|
||||
fx.fake.taskPolls = 2
|
||||
fx.delete()
|
||||
log := fx.fake.callLog()
|
||||
last := len(log) - 1
|
||||
for log[last] != "DeleteVolume" {
|
||||
last--
|
||||
}
|
||||
if n := callsBefore(log[last:], "TaskStatus", ""); n != fx.fake.taskPolls+1 {
|
||||
t.Errorf("seed delete task polled %d times, want %d: %v", n, fx.fake.taskPolls+1, log)
|
||||
}
|
||||
if fx.seedPresent() {
|
||||
t.Error("seed still exists")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_seedDeleteFailureOnVMNodeFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(fx *fixture)
|
||||
class error
|
||||
}{
|
||||
{name: "token lacks Datastore.Allocate", setup: func(fx *fixture) {
|
||||
fx.fake.failWith("DeleteVolume", loadAPIError(t, "err_forbidden_volume_delete"))
|
||||
}, class: provider.ErrPermanent},
|
||||
{name: "pveproxy 596", setup: func(fx *fixture) { fx.fake.failWith("DeleteVolume", fakeErr(596, "Broken pipe")) }, class: provider.ErrTransient},
|
||||
{name: "task failed", setup: func(fx *fixture) {
|
||||
fx.fake.hookOn("TaskStatus", func() error {
|
||||
if fx.fake.callCount("DeleteVolume") > 0 {
|
||||
return fakeErr(500, "fake: task log unreadable")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}, class: provider.ErrTransient},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "stopped")
|
||||
tc.setup(fx)
|
||||
fx.deleteFails(tc.class)
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); ok {
|
||||
t.Error("VM still exists: the destroy should have run before the seed delete")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_seedDeleteNodeFailureSkippedWhenNodeUnknown(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("one node refuses", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.putSeed()
|
||||
var calls int
|
||||
fx.fake.hookOn("DeleteVolume", func() error {
|
||||
if calls++; calls == 1 {
|
||||
return fakeErr(500, "fake: storage 'proxy-seeds' is not available on node 'pv1-1'")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
fx.delete()
|
||||
if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{fakeNode, "pv1-2"}) {
|
||||
t.Errorf("seed deleted on %v, want both nodes tried", got)
|
||||
}
|
||||
lines := fx.logs.matching(`"msg"="proxmox seed delete failed on a node, skipped"`)
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], `"node"="pv1-1"`) || !strings.Contains(lines[0], "not available") {
|
||||
t.Errorf("skip lines = %v, want one naming pv1-1 and the reason", lines)
|
||||
}
|
||||
if fx.seedPresent() {
|
||||
t.Error("seed still exists after pv1-2's delete")
|
||||
}
|
||||
})
|
||||
t.Run("ended ctx fails the call", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
ctx, cancel := context.WithCancel(fx.ctx)
|
||||
defer cancel()
|
||||
fx.fake.hookOn("DeleteVolume", func() error {
|
||||
cancel()
|
||||
return context.Canceled
|
||||
})
|
||||
err := fx.p.Delete(ctx, fx.id())
|
||||
assertProviderError(t, err, "delete", provider.ErrTransient)
|
||||
if n := fx.fake.callCount("DeleteVolume"); n != 1 {
|
||||
t.Errorf("DeleteVolume calls = %d, want 1: the ended ctx stops the loop", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDelete_destroyTaskFailureIsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "stopped")
|
||||
fx.putSeed()
|
||||
fx.fake.destroyFailsWith = "fake: unable to remove disk"
|
||||
if msg := fx.deleteFails(provider.ErrTransient); !strings.Contains(msg, "unable to remove disk") {
|
||||
t.Errorf("error = %s, want the task's exitstatus", msg)
|
||||
}
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); !ok {
|
||||
t.Error("VM gone after a failed destroy task")
|
||||
}
|
||||
if !fx.seedPresent() {
|
||||
t.Error("seed deleted although the VM is still there")
|
||||
}
|
||||
if lines := fx.logs.matching(`"msg"="proxmox task failed"`); len(lines) != 1 {
|
||||
t.Errorf("task failed lines = %v, want one", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_goneMidwayIsSuccess: the VM vanishing at any step counts as
|
||||
// done, and F2's does-not-exist answer is what each step then sees.
|
||||
func TestDelete_goneMidwayIsSuccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
status string
|
||||
method string
|
||||
wantDestroy int
|
||||
wantNodes []string
|
||||
}{
|
||||
{name: "before the config read", status: "stopped", method: "VMConfig", wantNodes: []string{fakeNode, "pv1-2"}},
|
||||
{name: "before the stop", status: "running", method: "Stop", wantNodes: []string{fakeNode}},
|
||||
{name: "before the destroy", status: "stopped", method: "Destroy", wantDestroy: 1, wantNodes: []string{fakeNode}},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.addOurVM(testVMID, testName, tc.status)
|
||||
mutateOn(fx.fake, tc.method, func() { delete(fx.fake.vms, testVMID) })
|
||||
fx.delete()
|
||||
if n := fx.fake.callCount("Destroy"); n != tc.wantDestroy {
|
||||
t.Errorf("Destroy calls = %d, want %d", n, tc.wantDestroy)
|
||||
}
|
||||
if got := fx.seedDeleteNodes(); !slices.Equal(got, tc.wantNodes) {
|
||||
t.Errorf("seed deleted on %v, want %v", got, tc.wantNodes)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_resourcesFailureIsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
fx.fake.failWith("ClusterResources", fakeErr(595, "Connection refused"))
|
||||
fx.deleteFails(provider.ErrTransient)
|
||||
if n := fx.fake.callCount("DeleteVolume"); n != 0 {
|
||||
t.Errorf("DeleteVolume calls = %d, want 0 when the VM's state is unknown", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_usesPurgeAndDestroyUnreferencedDisks runs Delete over the real
|
||||
// client against Phase 0's destroy captures: the DELETE carries both flags
|
||||
// and the seed delete sends the volid as one escaped segment.
|
||||
func TestDelete_usesPurgeAndDestroyUnreferencedDisks(t *testing.T) {
|
||||
t.Parallel()
|
||||
const (
|
||||
vmid = 199999
|
||||
name = "proxy-reconaaaaaaaaaaa"
|
||||
)
|
||||
var (
|
||||
mu sync.Mutex
|
||||
seen []string
|
||||
)
|
||||
cfx := newClientFixture(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
path := strings.TrimPrefix(r.URL.EscapedPath(), "/api2/json/")
|
||||
mu.Lock()
|
||||
seen = append(seen, r.Method+" "+path+"?"+r.URL.RawQuery)
|
||||
mu.Unlock()
|
||||
capture := ""
|
||||
switch {
|
||||
case r.Method == http.MethodGet && path == "cluster/resources":
|
||||
capture = "cluster_resources_during_destroy"
|
||||
case r.Method == http.MethodGet && path == "nodes/pv1-1/qemu/199999/config":
|
||||
capture = "config_during_destroy"
|
||||
case r.Method == http.MethodDelete && path == "nodes/pv1-1/qemu/199999":
|
||||
capture = "destroy_response"
|
||||
case r.Method == http.MethodDelete && path == "nodes/pv1-1/storage/proxy-seeds/content/proxy-seeds:iso%2F"+name+"-seed.iso":
|
||||
capture = "content_delete_response"
|
||||
case r.Method == http.MethodGet && strings.Contains(path, ":imgdel:"):
|
||||
capture = "task_status_volume_missing"
|
||||
case r.Method == http.MethodGet && strings.Contains(path, ":qmdestroy:"):
|
||||
capture = "task_status_stopped_ok"
|
||||
default:
|
||||
t.Errorf("unexpected request %s %s", r.Method, path)
|
||||
http.Error(w, "unexpected", http.StatusNotImplemented)
|
||||
return
|
||||
}
|
||||
serveJSON(w, readCapture(t, capture).Body)
|
||||
})
|
||||
cfg := testConfig()
|
||||
cfg.URL, cfg.TokenFile, cfg.CAFile = cfx.cfg.URL, cfx.cfg.TokenFile, cfx.cfg.CAFile
|
||||
p := newProvider(testProvider, cfg, cfx.api(t))
|
||||
p.budgets = testBudgets()
|
||||
if err := p.Delete(context.Background(), formatProviderID(vmid, name)); err != nil {
|
||||
t.Fatalf("Delete error = %v", err)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !slices.Contains(seen, "DELETE nodes/pv1-1/qemu/199999?destroy-unreferenced-disks=1&purge=1") {
|
||||
t.Errorf("requests = %v, want the destroy with purge=1 and destroy-unreferenced-disks=1", seen)
|
||||
}
|
||||
if !slices.ContainsFunc(seen, func(s string) bool { return strings.HasPrefix(s, "DELETE nodes/pv1-1/storage/proxy-seeds/content/") }) {
|
||||
t.Errorf("requests = %v, want the seed delete", seen)
|
||||
}
|
||||
if slices.ContainsFunc(seen, func(s string) bool { return strings.Contains(s, "status/stop") }) {
|
||||
t.Errorf("requests = %v, want no stop for the captured stopped VM", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_concurrentSameNameSafe(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
release := fx.fake.blockOn("Stop")
|
||||
errs := make([]error, 2)
|
||||
first := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
wg.Go(func() {
|
||||
defer close(first)
|
||||
errs[0] = fx.p.Delete(fx.ctx, fx.id())
|
||||
})
|
||||
for fx.fake.callCount("Stop") == 0 {
|
||||
select {
|
||||
case <-first:
|
||||
t.Fatalf("first Delete returned %v without stopping the VM", errs[0])
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
wg.Go(func() { errs[1] = fx.p.Delete(fx.ctx, fx.id()) })
|
||||
close(release)
|
||||
wg.Wait()
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
t.Errorf("Delete %d error = %v", i, err)
|
||||
}
|
||||
}
|
||||
for method, want := range map[string]int{"Stop": 1, "Destroy": 1} {
|
||||
if got := fx.fake.callCount(method); got != want {
|
||||
t.Errorf("%s calls = %d, want %d", method, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_waitsOnTheNameLock: Delete takes the per-name lock before any
|
||||
// call, so it cannot interleave with a Create or another Delete of that
|
||||
// name (spec entry 19).
|
||||
func TestDelete_waitsOnTheNameLock(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
release, err := fx.p.lockName(fx.ctx, testName)
|
||||
if err != nil {
|
||||
t.Fatalf("lockName error = %v", err)
|
||||
}
|
||||
defer release()
|
||||
ctx, cancel := context.WithTimeout(fx.ctx, 20*time.Millisecond)
|
||||
defer cancel()
|
||||
err = fx.p.Delete(ctx, fx.id())
|
||||
assertProviderError(t, err, "delete", provider.ErrTransient)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Errorf("error = %v, want it to wrap DeadlineExceeded", err)
|
||||
}
|
||||
if calls := fx.fake.callLog(); len(calls) != 0 {
|
||||
t.Errorf("calls = %v, want none while the name is locked", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_cancelledMidWaitReturnsPromptly(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.p.budgets = defaultBudgets()
|
||||
fx.p.budgets.poll = time.Millisecond
|
||||
fx.fake.taskPolls = 1 << 30
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
ctx, cancel := context.WithCancel(fx.ctx)
|
||||
defer cancel()
|
||||
fx.fake.hookOn("TaskStatus", func() error {
|
||||
cancel()
|
||||
return nil
|
||||
})
|
||||
start := time.Now()
|
||||
err := fx.p.Delete(ctx, fx.id())
|
||||
if elapsed := time.Since(start); elapsed > 5*time.Second {
|
||||
t.Errorf("Delete took %v after cancellation", elapsed)
|
||||
}
|
||||
assertProviderError(t, err, "delete", provider.ErrTransient)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("error = %v, want it to wrap context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_outcomeLine(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(fx *fixture)
|
||||
want []string
|
||||
never string
|
||||
}{
|
||||
{
|
||||
name: "destroyed",
|
||||
setup: func(fx *fixture) { fx.addOurVM(testVMID, testName, "stopped") },
|
||||
want: []string{`"node"="` + fakeNode + `"`, `"destroyed"=true`},
|
||||
},
|
||||
{name: "already gone", setup: func(*fixture) {}, want: []string{`"destroyed"=false`}, never: `"node"=`},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
tc.setup(fx)
|
||||
fx.delete()
|
||||
lines := fx.logs.matching(`"msg"="proxmox vm deleted"`)
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("outcome lines = %d, want 1:\n%s", len(lines), strings.Join(fx.logs.all(), "\n"))
|
||||
}
|
||||
for _, want := range append(tc.want, `"vmid"=150042`, `"name"="`+testName+`"`, `"provider"="`+testProvider+`"`) {
|
||||
if !strings.Contains(lines[0], want) {
|
||||
t.Errorf("outcome line lacks %s: %s", want, lines[0])
|
||||
}
|
||||
}
|
||||
if tc.never != "" && strings.Contains(lines[0], tc.never) {
|
||||
t.Errorf("outcome line has %s: %s", tc.never, lines[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_noOpLinesAreSilentAtV0(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := map[string]func(fx *fixture){
|
||||
"absent": func(*fixture) {},
|
||||
"foreign": func(fx *fixture) { fx.addOurVM(testVMID, "proxy-someoneelse00000", "running") },
|
||||
"unmarked": func(fx *fixture) { fx.fake.addVM(testVMID, testName, "running") },
|
||||
"ours": func(fx *fixture) { fx.addOurVM(testVMID, testName, "running") },
|
||||
}
|
||||
for name, setup := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 0)
|
||||
fx.p.api = observe(fx.fake, testProvider)
|
||||
setup(fx)
|
||||
fx.delete()
|
||||
if lines := fx.logs.all(); len(lines) != 0 {
|
||||
t.Errorf("V(0) lines = %v, want none", lines)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user