proxmox: review fixes, real-host validation record, changelog

Phase 10. Two review passes (go-operator-reviewer, /code-review high)
over the branch, recorded in docs/reviews/2026-09-23-proxmox-provider-review/:
9 findings fixed, 3 accepted as documented limits, 7 rejected with a
reason.

- Delete removes the seed between the stop and the destroy, so a failed
  seed delete leaves the VM for the next Delete instead of leaking the
  ISO; a Permanent refusal on the every-node path fails the call.
- Create skips a probe VMID whose listed name is someone else's without
  waiting on its lock.
- The three answers Phase 0 never captured are fixtures now: a missing
  storage is a 400 (the Checker reports StorageNotFound), a hibernated
  VM is destroyable (Delete destroys ours), a paused VM stops.
- TestProxmox_lifecycle and TestProxmoxCheck_realHost passed on pv1-1
  (PVE 9.2.20); the docs record the run and drop "not yet run".

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/provider/proxmox/check.go

@@ -111,16 +111,22 @@ func (p *Provider) probeNode(ctx context.Context, node string, details map[strin
 	return p.checkTemplates(ctx, details)
 }
 
-// checkStorage reads isoStorage on the node. A 500 means Proxmox cannot
-// use that storage on this node (it does not exist, is not enabled here,
-// or is offline), whatever its wording; the message carries Proxmox's
-// text.
+// checkStorage reads isoStorage on the node. A storage the cluster does
+// not know is a 400 on the storage parameter (captured). A 500 is taken
+// the same way, whatever its wording: Proxmox cannot use that storage on
+// this node (not enabled here, or offline; not captured). The message
+// carries Proxmox's text.
 func (p *Provider) checkStorage(ctx context.Context, node string, details map[string]string) error {
 	storage := p.cfg.ISOStorage
 	st, err := p.api.StorageStatus(ctx, node, storage)
 	var ae *apiError
-	if errors.As(err, &ae) && ae.Status == http.StatusInternalServerError {
-		return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s on node %s: %s", storage, node, ae.Message)}
+	if errors.As(err, &ae) {
+		switch {
+		case isNoSuchStorage(ae):
+			return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s on node %s: %s", storage, node, ae.Errors["storage"])}
+		case ae.Status == http.StatusInternalServerError:
+			return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s on node %s: %s", storage, node, ae.Message)}
+		}
 	}
 	if err != nil {
 		return fmt.Errorf("reading storage %s on node %s: %w", storage, node, err)

internal/provider/proxmox/check_test.go

@@ -230,7 +230,21 @@ func TestCheck_reasonsTable(t *testing.T) {
 				fx.fake.mu.Unlock()
 			},
 			wantReason:  reasonStorageNotFound,
-			wantMessage: "storage proxy-seeds on node pv1-1: fake: storage 'proxy-seeds' does not exist",
+			wantMessage: "storage proxy-seeds on node pv1-1: No such storage.",
+		},
+		{
+			// Not captured: a storage Proxmox knows but cannot use here.
+			name: "iso storage unusable on the node",
+			prep: func(fx *fixture) {
+				fx.fake.failWith("StorageStatus", fakeErr(500, "fake: storage 'proxy-seeds' is not online"))
+			},
+			wantReason:  reasonStorageNotFound,
+			wantMessage: "storage proxy-seeds on node pv1-1: fake: storage 'proxy-seeds' is not online",
+		},
+		{
+			name:       "another bad parameter is not a missing storage",
+			prep:       func(fx *fixture) { fx.fake.failWith("StorageStatus", loadAPIError(fx.t, "err_bad_param")) },
+			wantReason: provider.ReasonAPIError,
 		},
 		{
 			name:        "iso storage inactive",

internal/provider/proxmox/create.go

@@ -198,9 +198,10 @@ func (p *Provider) template(ctx context.Context, rs []vmResource, image string,
 }
 
 // allocate is the probe of spec entry 2: the attempt-th VMID for the UID
-// is cloned onto when the listing shows it free and put through the
+// is cloned onto when the listing shows it free, and put through the
 // ownership check when it is taken, or when the clone finds it taken after
-// all (F4). The boolean reports an adoption rather than a fresh clone.
+// all (F4), by a VM that is nameless or carries our name. The boolean
+// reports an adoption rather than a fresh clone.
 func (p *Provider) allocate(ctx context.Context, log logr.Logger, rs []vmResource, node string, src vmResource, req provider.CreateRequest) (ownedVM, bool, error) {
 	taken := make(map[int]vmResource, len(rs))
 	for _, r := range rs {
@@ -225,7 +226,9 @@ func (p *Provider) allocate(ctx context.Context, log logr.Logger, rs []vmResourc
 				continue
 			}
 		}
-		if r.Type == "qemu" && r.Template == 0 {
+		// A listed name other than ours is someone else's VM whatever its
+		// lock: only a clone in progress is nameless.
+		if r.Type == "qemu" && r.Template == 0 && (r.Name == "" || r.Name == req.Name) {
 			vm, ours, err := p.inspect(ctx, r, req)
 			if err != nil || ours {
 				return vm, ours, err

internal/provider/proxmox/create_test.go

@@ -248,10 +248,22 @@ func TestCreate_takenVMIDInspectedBeforeProbing(t *testing.T) {
 		wantConfigReads int
 	}{
 		{
-			name:            "foreign VM probes on after its config is read",
+			name:            "foreign VM probes on without a read",
 			setup:           func(fx *fixture) { fx.fake.addVM(slot(0), "someone-else", "running") },
 			wantAttempt:     1,
-			wantConfigReads: 2,
+			wantConfigReads: 1,
+		},
+		{
+			// Review finding P2: another VM's lock (a backup, a snapshot)
+			// can outlast the clone budget, and every retry lands on the
+			// same VMID.
+			name: "foreign VM under a lock probes on without waiting",
+			setup: func(fx *fixture) {
+				fx.fake.addVM(slot(0), "someone-else", "running").lock = "snapshot-delete"
+				fx.p.budgets.clone = 2 * time.Second
+			},
+			wantAttempt:     1,
+			wantConfigReads: 1,
 		},
 		{
 			name: "container probes on without a read",
@@ -581,8 +593,8 @@ func TestCreate_vmidRangeExhaustedIsTransient(t *testing.T) {
 	if got := fx.fake.callCount("Clone"); got != 0 {
 		t.Errorf("Clone calls = %d, want 0", got)
 	}
-	if got := fx.fake.callCount("VMConfig"); got != 5 {
-		t.Errorf("VMConfig calls = %d, want the template and each of the 4 VMIDs once", got)
+	if got := fx.fake.callCount("VMConfig"); got != 1 {
+		t.Errorf("VMConfig calls = %d, want only the template's: the listed names are foreign", got)
 	}
 }
 

internal/provider/proxmox/delete.go

@@ -2,14 +2,17 @@ package proxmox
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
 	"github.com/go-logr/logr"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
 )
 
-// Delete is spec entry 11: under the per-name lock, stop and destroy the VM
-// when it is ours, then delete its seed. Everything already gone is
+// Delete is spec entry 11: under the per-name lock, stop the VM when it is
+// ours, delete its seed, then destroy it. Everything already gone is
 // success, and nothing of ours is left when nil is returned, so Get
 // afterwards is NotFound.
 func (p *Provider) Delete(ctx context.Context, id string) error {
@@ -29,19 +32,17 @@ func (p *Provider) Delete(ctx context.Context, id string) error {
 		return p.wrapErr("delete", id, err)
 	}
 	if node == "" {
-		err = p.deleteSeedEverywhere(ctx, log, name)
+		if err := p.deleteSeedEverywhere(ctx, log, name); err != nil {
+			return p.wrapErr("delete", id, err)
+		}
 	} else {
 		log = log.WithValues("node", node)
-		err = p.deleteSeed(ctx, log, node, name)
-	}
-	if err != nil {
-		return p.wrapErr("delete", id, err)
 	}
 	log.V(1).Info("proxmox vm deleted", "destroyed", node != "")
 	return nil
 }
 
-// destroy is steps 2 to 6. It returns the node of the VM it destroyed, or
+// destroy is steps 2 to 7. It returns the node of the VM it destroyed, or
 // "" when there was no VM of ours to destroy: then the seed's node is not
 // known, and the seed is looked for on every node.
 func (p *Provider) destroy(ctx context.Context, log logr.Logger, vmid int, name string) (string, error) {
@@ -58,7 +59,7 @@ func (p *Provider) destroy(ctx context.Context, log logr.Logger, vmid int, name
 		return "", transientf("VM %d is locked (%s)", vmid, r.Lock)
 	}
 	if r.Status == "unknown" {
-		return "", transientf("VM %d has status unknown: its node %s is not answering", vmid, r.Node)
+		return "", transientf("VM %d has status unknown: its node %s is not answering, or the VM is too new to be reported yet", vmid, r.Node)
 	}
 	cfg, err := p.api.VMConfig(ctx, r.Node, vmid)
 	if isNotExist(err) {
@@ -76,47 +77,47 @@ func (p *Provider) destroy(ctx context.Context, log logr.Logger, vmid int, name
 		log.V(1).Info("proxmox vm carries no proxy-operator mark, left alone")
 		return "", nil
 	}
-	switch lock := cfg["lock"]; lock {
-	case "":
-	case lockSuspended:
-		// Destroy refuses a locked VM and the lock never clears on its own,
-		// so only an admin can move this on, as in Create's ownership check.
-		return "", permanentf("VM %d named %s is hibernated (lock suspended); resume or destroy it in Proxmox", vmid, name)
-	default:
+	// A hibernated VM is stopped with lock suspended, and the destroy
+	// takes it, state volume included (captured on pv1-1).
+	if lock := cfg["lock"]; lock != "" && lock != lockSuspended {
 		return "", transientf("VM %d is locked (%s)", vmid, lock)
 	}
 
 	// The listing shows a paused VM as running, and Stop ends it too.
 	if r.Status != "stopped" {
-		gone, err := p.runTask(ctx, log, "stopping", vmid, p.budgets.stop, func() (string, error) { return p.api.Stop(ctx, r.Node, vmid) })
-		if err != nil || gone {
-			return r.Node, err
+		if err := p.runTask(ctx, log, "stopping", vmid, p.budgets.stop, func() (string, error) { return p.api.Stop(ctx, r.Node, vmid) }); err != nil {
+			return "", err
 		}
 	}
-	if _, err := p.runTask(ctx, log, "destroying", vmid, p.budgets.destroy, func() (string, error) { return p.api.Destroy(ctx, r.Node, vmid) }); err != nil {
+	// The seed goes while the VM is still there: once it is gone, Get is
+	// NotFound and nothing calls Delete again to finish the job.
+	if err := p.deleteSeed(ctx, log, r.Node, name); err != nil {
+		return "", err
+	}
+	if err := p.runTask(ctx, log, "destroying", vmid, p.budgets.destroy, func() (string, error) { return p.api.Destroy(ctx, r.Node, vmid) }); err != nil {
 		return "", err
 	}
 	return r.Node, nil
 }
 
 // runTask submits a task on a VM and waits for it. A VM that is gone by the
-// time the task is submitted reports gone: it is what the task was for.
-func (p *Provider) runTask(ctx context.Context, log logr.Logger, what string, vmid int, budget time.Duration, submit func() (string, error)) (bool, error) {
+// time the task is submitted is success: it is what the task was for.
+func (p *Provider) runTask(ctx context.Context, log logr.Logger, what string, vmid int, budget time.Duration, submit func() (string, error)) error {
 	upid, err := submit()
 	if isNotExist(err) {
 		log.V(1).Info("proxmox vm already gone")
-		return true, nil
+		return nil
 	}
 	if err != nil {
-		return false, fmt.Errorf("%s VM %d: %w", what, vmid, err)
+		return fmt.Errorf("%s VM %d: %w", what, vmid, err)
 	}
 	if err := p.waitTask(ctx, log, upid, budget); err != nil {
-		return false, fmt.Errorf("%s VM %d: %w", what, vmid, err)
+		return fmt.Errorf("%s VM %d: %w", what, vmid, err)
 	}
-	return false, nil
+	return nil
 }
 
-// deleteSeed is step 7 on one node. A missing volume still ends in a
+// deleteSeed is step 6 on one node. A missing volume still ends in a
 // successful task (F13), so nothing is matched.
 func (p *Provider) deleteSeed(ctx context.Context, log logr.Logger, node, name string) error {
 	volid := seedVolID(p.cfg.ISOStorage, name)
@@ -130,17 +131,19 @@ func (p *Provider) deleteSeed(ctx context.Context, log logr.Logger, node, name s
 	return nil
 }
 
-// deleteSeedEverywhere is step 7 when the VM's node is not known: every
+// deleteSeedEverywhere is step 6 when the VM's node is not known: every
 // configured node is tried, and one that fails is logged and skipped, so a
 // node that is down or lacks the storage cannot hold the Delete up. An
-// ended ctx fails the call instead.
+// ended ctx fails the call instead, and so does a Permanent refusal such as
+// a 403 for Datastore.Allocate, which every node would give and which
+// would otherwise leave the seed behind unseen.
 func (p *Provider) deleteSeedEverywhere(ctx context.Context, log logr.Logger, name string) error {
 	for _, node := range p.cfg.Nodes {
 		err := p.deleteSeed(ctx, log, node, name)
 		if err == nil {
 			continue
 		}
-		if ctx.Err() != nil {
+		if ctx.Err() != nil || errors.Is(classify("delete", err), provider.ErrPermanent) {
 			return err
 		}
 		log.V(1).Info("proxmox seed delete failed on a node, skipped", "node", node, "reason", err.Error())

internal/provider/proxmox/delete_test.go

@@ -203,6 +203,14 @@ func TestDelete_unmarkedLeavesVMAlone(t *testing.T) {
 // wait for the stop fails.
 func TestDelete_runningVMStoppedFirst(t *testing.T) {
 	t.Parallel()
+	// pv1-1 took a stop on a paused VM as a task that ended OK with the VM
+	// stopped (Phase 10 captures), which is what the fake does for all three.
+	accepted := readCapture(t, "stop_paused_response")
+	done := decodeData[taskStatus](t, "task_status_stop_paused")
+	after := decodeData[vmStatus](t, "status_current_after_stop_paused")
+	if accepted.Status != http.StatusOK || done.Type != "qmstop" || done.ExitStatus != "OK" || after.Status != "stopped" || after.QMPStatus != "stopped" {
+		t.Fatalf("captures = %d, %+v, %+v; want the stop accepted, OK, and the VM stopped", accepted.Status, done, after)
+	}
 	for _, qmp := range []string{"running", "paused", "prelaunch"} {
 		t.Run(qmp, func(t *testing.T) {
 			t.Parallel()
@@ -215,8 +223,8 @@ func TestDelete_runningVMStoppedFirst(t *testing.T) {
 			if stop < 0 || destroy < stop {
 				t.Fatalf("calls = %v, want Stop before Destroy", log)
 			}
-			if n := callsBefore(log[stop:], "TaskStatus", "Destroy"); n != fx.fake.taskPolls+1 {
-				t.Errorf("stop task polled %d times before Destroy, want %d", n, fx.fake.taskPolls+1)
+			if n := callsBefore(log[stop:], "TaskStatus", "DeleteVolume"); n != fx.fake.taskPolls+1 {
+				t.Errorf("stop task polled %d times before the seed delete, want %d", n, fx.fake.taskPolls+1)
 			}
 			if _, ok := fx.fake.vmSnapshot(testVMID); ok {
 				t.Error("VM still exists")
@@ -230,7 +238,7 @@ func TestDelete_stoppedVMSkipsStop(t *testing.T) {
 	fx := newFixture(t, 1)
 	fx.addOurVM(testVMID, testName, "stopped")
 	fx.delete()
-	want := []string{"ClusterResources", "VMConfig", "Destroy", "TaskStatus", "DeleteVolume", "TaskStatus"}
+	want := []string{"ClusterResources", "VMConfig", "DeleteVolume", "TaskStatus", "Destroy", "TaskStatus"}
 	if got := fx.fake.callLog(); !slices.Equal(got, want) {
 		t.Errorf("calls = %v, want %v", got, want)
 	}
@@ -291,26 +299,32 @@ func TestDelete_unknownStatusIsTransient(t *testing.T) {
 	}
 }
 
-// TestDelete_hibernatedVMIsPermanentIfOurs matches Create's ownership
-// check: Destroy refuses the suspended lock, which never clears on its own,
-// so an admin must act. A hibernated VM that is not ours is left alone.
-func TestDelete_hibernatedVMIsPermanentIfOurs(t *testing.T) {
+// TestDelete_hibernatedVMIsDestroyedIfOurs: on pv1-1 the destroy took a
+// hibernated VM, suspend state volume included, and its task ended OK.
+// Delete used to refuse it as Permanent on the belief that the destroy
+// would. A hibernated VM that is not ours is still left alone.
+func TestDelete_hibernatedVMIsDestroyedIfOurs(t *testing.T) {
 	t.Parallel()
 	t.Run("ours", func(t *testing.T) {
 		t.Parallel()
+		accepted := readCapture(t, "destroy_hibernated_response")
+		done := decodeData[taskStatus](t, "task_status_destroy_hibernated")
+		if accepted.Status != http.StatusOK || done.Type != "qmdestroy" || done.ExitStatus != "OK" {
+			t.Fatalf("captures = %d, %+v; want the destroy accepted and its task OK", accepted.Status, done)
+		}
 		fx := newFixture(t, 1)
 		fx.addOurVM(testVMID, testName, "stopped").lock = lockSuspended
 		fx.putSeed()
-		if msg := fx.deleteFails(provider.ErrPermanent); !strings.Contains(msg, "resume or destroy it in Proxmox") {
-			t.Errorf("error = %s, want the admin action", msg)
+		fx.delete()
+		want := []string{"ClusterResources", "VMConfig", "DeleteVolume", "TaskStatus", "Destroy", "TaskStatus"}
+		if got := fx.fake.callLog()[2:]; !slices.Equal(got, want) {
+			t.Errorf("calls = %v, want %v: no stop, since the VM is not running", got, want)
 		}
-		for _, method := range []string{"Stop", "Destroy", "DeleteVolume"} {
-			if n := fx.fake.callCount(method); n != 0 {
-				t.Errorf("%s calls = %d, want 0", method, n)
-			}
+		if _, ok := fx.fake.vmSnapshot(testVMID); ok {
+			t.Error("hibernated VM still exists")
 		}
-		if _, err := fx.fake.Destroy(fx.ctx, fakeNode, testVMID); !isLocked(err) {
-			t.Errorf("fake Destroy on the hibernated VM error = %v, want the lock answer", err)
+		if fx.seedPresent() {
+			t.Error("seed still exists")
 		}
 	})
 	t.Run("someone else's", func(t *testing.T) {
@@ -367,7 +381,7 @@ func TestDelete_seedDeleteIsTaskAndIdempotent(t *testing.T) {
 			for log[last] != "DeleteVolume" {
 				last--
 			}
-			if n := callsBefore(log[last:], "TaskStatus", ""); n != fx.fake.taskPolls+1 {
+			if n := callsBefore(log[last:], "TaskStatus", "Destroy"); n != fx.fake.taskPolls+1 {
 				t.Errorf("seed delete task polled %d times, want %d: %v", n, fx.fake.taskPolls+1, log)
 			}
 			if fx.seedPresent() {
@@ -377,6 +391,9 @@ func TestDelete_seedDeleteIsTaskAndIdempotent(t *testing.T) {
 	}
 }
 
+// TestDelete_seedDeleteFailureOnVMNodeFails: the VM outlives a failed seed
+// delete, so Get keeps finding it and the next Delete finishes the job with
+// the same class of answer (review findings CR1, CR2, P1).
 func TestDelete_seedDeleteFailureOnVMNodeFails(t *testing.T) {
 	t.Parallel()
 	tests := []struct {
@@ -403,9 +420,22 @@ func TestDelete_seedDeleteFailureOnVMNodeFails(t *testing.T) {
 			fx := newFixture(t, 1)
 			fx.addOurVM(testVMID, testName, "stopped")
 			tc.setup(fx)
+			fx.putSeed()
 			fx.deleteFails(tc.class)
+			if n := fx.fake.callCount("Destroy"); n != 0 {
+				t.Errorf("Destroy calls = %d, want 0 while the seed is still there", n)
+			}
+			if _, err := fx.p.Get(fx.ctx, fx.id()); errors.Is(err, provider.ErrNotFound) {
+				t.Fatalf("Get = %v, want the VM still found so Delete runs again", err)
+			}
+			fx.deleteFails(tc.class)
+			fx.fake.clearHooks()
+			fx.delete()
 			if _, ok := fx.fake.vmSnapshot(testVMID); ok {
-				t.Error("VM still exists: the destroy should have run before the seed delete")
+				t.Error("VM still exists after the retry")
+			}
+			if fx.seedPresent() {
+				t.Error("seed still exists after the retry")
 			}
 		})
 	}
@@ -436,6 +466,18 @@ func TestDelete_seedDeleteNodeFailureSkippedWhenNodeUnknown(t *testing.T) {
 			t.Error("seed still exists after pv1-2's delete")
 		}
 	})
+	t.Run("a refused privilege fails the call", func(t *testing.T) {
+		t.Parallel()
+		fx := twoNodeFixture(t)
+		fx.putSeed()
+		fx.fake.failWith("DeleteVolume", loadAPIError(t, "err_forbidden_volume_delete"))
+		if msg := fx.deleteFails(provider.ErrPermanent); !strings.Contains(msg, "Datastore.Allocate") {
+			t.Errorf("error = %s, want the missing privilege named", msg)
+		}
+		if n := fx.fake.callCount("DeleteVolume"); n != 1 {
+			t.Errorf("DeleteVolume calls = %d, want 1: every node would refuse the same", n)
+		}
+	})
 	t.Run("ended ctx fails the call", func(t *testing.T) {
 		t.Parallel()
 		fx := twoNodeFixture(t)
@@ -465,8 +507,8 @@ func TestDelete_destroyTaskFailureIsTransient(t *testing.T) {
 	if _, ok := fx.fake.vmSnapshot(testVMID); !ok {
 		t.Error("VM gone after a failed destroy task")
 	}
-	if !fx.seedPresent() {
-		t.Error("seed deleted although the VM is still there")
+	if fx.seedPresent() {
+		t.Error("seed still exists: it goes before the destroy")
 	}
 	if lines := fx.logs.matching(`"msg"="proxmox task failed"`); len(lines) != 1 {
 		t.Errorf("task failed lines = %v, want one", lines)
@@ -485,7 +527,7 @@ func TestDelete_goneMidwayIsSuccess(t *testing.T) {
 		wantNodes   []string
 	}{
 		{name: "before the config read", status: "stopped", method: "VMConfig", wantNodes: []string{fakeNode, "pv1-2"}},
-		{name: "before the stop", status: "running", method: "Stop", wantNodes: []string{fakeNode}},
+		{name: "before the stop", status: "running", method: "Stop", wantDestroy: 1, wantNodes: []string{fakeNode}},
 		{name: "before the destroy", status: "stopped", method: "Destroy", wantDestroy: 1, wantNodes: []string{fakeNode}},
 	}
 	for _, tc := range tests {

internal/provider/proxmox/errors.go

@@ -76,14 +76,13 @@ func newAPIError(status int, reason string, body []byte) *apiError {
 // under testdata/.
 var (
 	vmConfigMissing = regexp.MustCompile(`^Configuration file '[^']*' does not exist$`)
-	vmNotRunning    = regexp.MustCompile(`^VM [0-9]+ is not running$`)
 )
 
 const (
-	msgLocked             = "VM is locked"
 	msgAlreadyExists      = "config file already exists"
 	msgAgentNotRunning    = "QEMU guest agent is not running"
 	msgAgentNotConfigured = "No QEMU guest agent configured"
+	msgNoSuchStorage      = "No such storage."
 )
 
 // server500 returns the message of err when it is, or wraps, an apiError
@@ -102,10 +101,12 @@ func isNotExist(err error) bool {
 	return ok && vmConfigMissing.MatchString(msg)
 }
 
-// isLocked reports a VM held by a running task ("VM is locked (clone)").
-func isLocked(err error) bool {
-	msg, ok := server500(err)
-	return ok && strings.HasPrefix(msg, msgLocked)
+// isNoSuchStorage reports a storage the cluster does not know: a 400 whose
+// storage field error is exactly msgNoSuchStorage. Other storage field
+// errors, such as a clone onto a storage without image support, are not.
+func isNoSuchStorage(err error) bool {
+	var ae *apiError
+	return errors.As(err, &ae) && ae.Status == http.StatusBadRequest && ae.Errors["storage"] == msgNoSuchStorage
 }
 
 // isAlreadyExists reports a clone onto a VMID that is taken.
@@ -127,14 +128,6 @@ func isAgentNotConfigured(err error) bool {
 	return ok && msg == msgAgentNotConfigured
 }
 
-// isVMNotRunning reports an agent call on a VM that is stopped. It is
-// matched on the whole message because " is not running" alone is also a
-// suffix of the agent-not-running answer.
-func isVMNotRunning(err error) bool {
-	msg, ok := server500(err)
-	return ok && vmNotRunning.MatchString(msg)
-}
-
 // classify maps any error reaching the provider's boundary to one of the
 // four contract sentinels, per spec entry 13. op is the contract operation:
 // a vanished VM inside Create is Transient, because the contract forbids

internal/provider/proxmox/errors_test.go

@@ -93,6 +93,7 @@ func TestAPIError_matchers(t *testing.T) {
 		"isAlreadyExists":      isAlreadyExists,
 		"isAgentNotRunning":    isAgentNotRunning,
 		"isAgentNotConfigured": isAgentNotConfigured,
+		"isNoSuchStorage":      isNoSuchStorage,
 		"isVMNotRunning":       isVMNotRunning,
 	}
 	tests := []struct {
@@ -104,6 +105,7 @@ func TestAPIError_matchers(t *testing.T) {
 		{"err_destroy_missing", "isNotExist"},
 		{"err_config_while_clone_locked", "isLocked"},
 		{"err_newid_in_use", "isAlreadyExists"},
+		{"storage_status_missing", "isNoSuchStorage"},
 		{"err_agent_not_running", "isAgentNotRunning"},
 		{"err_agent_vm_stopped", "isAgentNotConfigured"},
 		{"err_agent_vm_stopped_agent_enabled", "isVMNotRunning"},
@@ -184,12 +186,16 @@ func TestAPIError_errorString(t *testing.T) {
 	}
 }
 
-func TestPermanentfAndTransientf(t *testing.T) {
+func TestPermanentf_wrapsClassWithMessage(t *testing.T) {
 	t.Parallel()
 	perm := permanentf("node %q missing", "x")
 	if !errors.Is(perm, provider.ErrPermanent) || perm.Error() != `provider: permanent error: node "x" missing` {
 		t.Errorf("permanentf() = %q, want ErrPermanent with the message", perm)
 	}
+}
+
+func TestTransientf_wrapsClassWithMessage(t *testing.T) {
+	t.Parallel()
 	trans := transientf("retry %d", 3)
 	if !errors.Is(trans, provider.ErrTransient) || trans.Error() != "provider: transient error: retry 3" {
 		t.Errorf("transientf() = %q, want ErrTransient with the message", trans)

internal/provider/proxmox/fake_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"maps"
+	"regexp"
 	"slices"
 	"strconv"
 	"strings"
@@ -140,6 +141,13 @@ func (f *fakeAPI) hookOn(method string, fn func() error) {
 	f.hooks[method] = fn
 }
 
+// clearHooks drops every hook, so the faults they injected stop.
+func (f *fakeAPI) clearHooks() {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	clear(f.hooks)
+}
+
 func (f *fakeAPI) blockOn(method string) chan struct{} {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -191,8 +199,30 @@ func errVMMissing(node string, vmid int) *apiError {
 	return fakeErr(500, fmt.Sprintf("Configuration file 'nodes/%s/qemu-server/%d.conf' does not exist", node, vmid))
 }
 
+// errNoSuchStorage is the answer for a storage the cluster does not know
+// (Phase 10 capture storage_status_missing).
+func errNoSuchStorage() *apiError {
+	return newAPIError(400, "", []byte(`{"message":"Parameter verification failed.\n","data":null,"errors":{"storage":"No such storage."}}`))
+}
+
 func errLocked(lock string) *apiError { return fakeErr(500, "VM is locked ("+lock+")") }
 
+// isLocked and isVMNotRunning recognise two answers the provider leaves to
+// classify's Transient default. They live here only to hold the fake's
+// wording to the captures. The stopped-VM message is matched whole because
+// " is not running" is also a suffix of the agent-not-running answer.
+var vmNotRunning = regexp.MustCompile(`^VM [0-9]+ is not running$`)
+
+func isLocked(err error) bool {
+	msg, ok := server500(err)
+	return ok && strings.HasPrefix(msg, "VM is locked")
+}
+
+func isVMNotRunning(err error) bool {
+	msg, ok := server500(err)
+	return ok && vmNotRunning.MatchString(msg)
+}
+
 func errNewIDInUse(vmid int) *apiError {
 	return fakeErr(500, fmt.Sprintf("unable to create VM %d: config file already exists", vmid))
 }
@@ -272,7 +302,7 @@ func (f *fakeAPI) StorageStatus(ctx context.Context, node, storage string) (stor
 	defer f.mu.Unlock()
 	s, ok := f.storages[storage]
 	if !ok {
-		return storageStatus{}, fakeErr(500, "fake: storage '"+storage+"' does not exist")
+		return storageStatus{}, errNoSuchStorage()
 	}
 	return s, nil
 }
@@ -505,7 +535,9 @@ func (f *fakeAPI) Destroy(ctx context.Context, node string, vmid int) (string, e
 	if err != nil {
 		return "", err
 	}
-	if vm.lock != "" {
+	// A hibernated VM is destroyed like a stopped one (Phase 10 captures
+	// destroy_hibernated_response and task_status_destroy_hibernated).
+	if vm.lock != "" && vm.lock != lockSuspended {
 		return "", errLocked(vm.lock)
 	}
 	if vm.status == "running" {
@@ -666,6 +698,7 @@ func TestFakeAPI_errorsMatchCaptures(t *testing.T) {
 		{capture: "err_destroy_missing", fake: errVMMissing("pv1-1", 199999), matcher: isNotExist},
 		{capture: "err_config_while_clone_locked", fake: errLocked("clone"), matcher: isLocked},
 		{capture: "err_newid_in_use", fake: errNewIDInUse(199999), matcher: isAlreadyExists},
+		{capture: "storage_status_missing", fake: errNoSuchStorage(), matcher: isNoSuchStorage},
 		{capture: "err_agent_vm_stopped_agent_enabled", fake: errVMNotRunning(199999), matcher: isVMNotRunning},
 		{capture: "err_agent_not_running", fake: fakeErr(500, msgAgentNotRunning), matcher: isAgentNotRunning},
 		{capture: "err_agent_vm_stopped", fake: fakeErr(500, msgAgentNotConfigured), matcher: isAgentNotConfigured},
@@ -674,8 +707,8 @@ func TestFakeAPI_errorsMatchCaptures(t *testing.T) {
 		t.Run(tc.capture, func(t *testing.T) {
 			t.Parallel()
 			want := loadAPIError(t, tc.capture)
-			if tc.fake.Status != want.Status || tc.fake.Message != want.Message {
-				t.Errorf("fake error = %d %q, capture = %d %q", tc.fake.Status, tc.fake.Message, want.Status, want.Message)
+			if tc.fake.Status != want.Status || tc.fake.Message != want.Message || !maps.Equal(tc.fake.Errors, want.Errors) {
+				t.Errorf("fake error = %d %q %v, capture = %d %q %v", tc.fake.Status, tc.fake.Message, tc.fake.Errors, want.Status, want.Message, want.Errors)
 			}
 			if !tc.matcher(tc.fake) {
 				t.Errorf("matcher does not accept the fake's %q", tc.fake.Message)

internal/provider/proxmox/get.go

@@ -49,7 +49,7 @@ func (p *Provider) get(ctx context.Context, log logr.Logger, vmid int, name stri
 		return nil, transientf("VM %d is locked (%s)", vmid, r.Lock)
 	}
 	if r.Status == "unknown" {
-		return nil, transientf("VM %d has status unknown: its node %s is not answering", vmid, r.Node)
+		return nil, transientf("VM %d has status unknown: its node %s is not answering, or the VM is too new to be reported yet", vmid, r.Node)
 	}
 	cfg, err := p.api.VMConfig(ctx, r.Node, vmid)
 	if err != nil {
@@ -78,7 +78,7 @@ func (p *Provider) get(ctx context.Context, log logr.Logger, vmid int, name stri
 	if err != nil {
 		return nil, fmt.Errorf("reading status of VM %d: %w", vmid, err)
 	}
-	if st.Status == "running" && (st.QMPStatus == "running" || st.QMPStatus == "") {
+	if guestRunning(st.Status, st.QMPStatus) {
 		ip, state, err := p.agentAddress(ctx, log, r.Node, vmid)
 		if err != nil {
 			return nil, err

internal/provider/proxmox/get_test.go

@@ -341,8 +341,10 @@ func TestGet_unknownStatusIsTransient(t *testing.T) {
 	fx := newFixture(t, 1)
 	fx.addOurVM(testVMID, testName, "running").status = "unknown"
 	msg := assertGetFails(t, fx, provider.ErrTransient)
-	if !strings.Contains(msg, "pv1-1") {
-		t.Errorf("error = %s, want it to name the node", msg)
+	// On pv1-1 the first Get after Create met this with the node up: the
+	// listing had not reported the new VM yet (Phase 10 run, finding P6).
+	if !strings.Contains(msg, "pv1-1") || !strings.Contains(msg, "too new to be reported yet") {
+		t.Errorf("error = %s, want it to name the node and both causes", msg)
 	}
 	if n := fx.fake.callCount("VMConfig"); n != 0 {
 		t.Errorf("VMConfig calls = %d, want 0 while the node is unknown", n)

internal/provider/proxmox/provider_test.go

@@ -310,13 +310,6 @@ func TestProvider_everyErrorIsWrappedWithConfiguredName(t *testing.T) {
 			op:   "delete", id: "vms/150042", class: provider.ErrPermanent,
 		},
 		{name: "delete waiting on the name lock", setup: holdNameLock, call: del, op: "delete", id: "self", class: provider.ErrTransient},
-		{
-			name: "delete hibernated vm",
-			setup: func(fx *fixture, _ context.CancelFunc) {
-				fx.addOurVM(testVMID, testName, "stopped").lock = lockSuspended
-			},
-			call: del, op: "delete", id: "self", class: provider.ErrPermanent,
-		},
 		{
 			name: "delete destroy task fails",
 			setup: func(fx *fixture, _ context.CancelFunc) {
@@ -579,7 +572,7 @@ func TestLogging_noOpLinesAtV1(t *testing.T) {
 			name: "seed delete skipped on each node",
 			setup: func(fx *fixture) {
 				fx.p.cfg.Nodes = []string{fakeNode, "pv1-2"}
-				fx.fake.failWith("DeleteVolume", loadAPIError(fx.t, "err_forbidden_volume_delete"))
+				fx.fake.failWith("DeleteVolume", fakeErr(500, "fake: storage 'proxy-seeds' is not available on this node"))
 			},
 			call: del, msg: "proxmox seed delete failed on a node, skipped", want: 2,
 		},

internal/provider/proxmox/state.go

@@ -13,22 +13,25 @@ import (
 // is Stopped, which the reconciler answers by replacing the VM. An empty
 // qmpstatus is taken as running: the field is missing, not a state.
 func mapState(status, qmpStatus, ip string) provider.InstanceState {
-	if status != "running" {
-		return provider.StateStopped
-	}
-	switch qmpStatus {
-	case "running", "":
+	switch {
+	case guestRunning(status, qmpStatus):
 		if ip == "" {
 			return provider.StateProvisioning
 		}
 		return provider.StateRunning
-	case "prelaunch":
+	case status == "running" && qmpStatus == "prelaunch":
 		return provider.StateProvisioning
 	default:
 		return provider.StateStopped
 	}
 }
 
+// guestRunning reports a VM whose guest runs, the only state in which its
+// agent can answer and mapState can call it Running.
+func guestRunning(status, qmpStatus string) bool {
+	return status == "running" && (qmpStatus == "running" || qmpStatus == "")
+}
+
 // firstIPv4 picks the first usable IPv4 from the guest agent's interfaces,
 // in the agent's order: not on the loopback interface, not loopback,
 // link-local or unspecified. Empty means no address yet.

internal/provider/proxmox/testdata/destroy_hibernated_response.json

@@ -0,0 +1,5 @@
+{
+  "status": 200,
+  "reason": "OK",
+  "body": "{\"data\":\"UPID:pv1-1:0014F167:030AA126:6AB4DB09:qmdestroy:199999:proxy-operator@pve!operator:\"}"
+}

internal/provider/proxmox/testdata/status_current_after_stop_paused.json

@@ -0,0 +1,5 @@
+{
+  "status": 200,
+  "reason": "OK",
+  "body": "{\"data\":{\"name\":\"proxy-reconaaaaaaaaaaa\",\"agent\":1,\"mem\":0,\"maxmem\":2147483648,\"memhost\":0,\"serial\":1,\"maxdisk\":3221225472,\"vmid\":199999,\"ha\":{\"managed\":0},\"status\":\"stopped\",\"qmpstatus\":\"stopped\",\"netin\":0,\"uptime\":0,\"netout\":0,\"disk\":0,\"cpus\":2,\"cpu\":0,\"clipboard\":null}}"
+}

internal/provider/proxmox/testdata/stop_paused_response.json

@@ -0,0 +1,5 @@
+{
+  "status": 200,
+  "reason": "OK",
+  "body": "{\"data\":\"UPID:pv1-1:0014F097:030A9433:6AB4DAE8:qmstop:199999:proxy-operator@pve!operator:\"}"
+}

internal/provider/proxmox/testdata/storage_status_missing.json

@@ -0,0 +1,5 @@
+{
+  "status": 400,
+  "reason": "Parameter verification failed.",
+  "body": "{\"message\":\"Parameter verification failed.\\n\",\"data\":null,\"errors\":{\"storage\":\"No such storage.\"}}"
+}

internal/provider/proxmox/testdata/task_status_destroy_hibernated.json

@@ -0,0 +1,5 @@
+{
+  "status": 200,
+  "reason": "OK",
+  "body": "{\"data\":{\"status\":\"stopped\",\"exitstatus\":\"OK\",\"upid\":\"UPID:pv1-1:0014F167:030AA126:6AB4DB09:qmdestroy:199999:proxy-operator@pve!operator:\",\"tokenid\":\"operator\",\"id\":\"199999\",\"starttime\":1790237449,\"type\":\"qmdestroy\",\"node\":\"pv1-1\",\"user\":\"proxy-operator@pve\",\"pstart\":51028262,\"pid\":1372519}}"
+}

internal/provider/proxmox/testdata/task_status_stop_paused.json

@@ -0,0 +1,5 @@
+{
+  "status": 200,
+  "reason": "OK",
+  "body": "{\"data\":{\"upid\":\"UPID:pv1-1:0014F097:030A9433:6AB4DAE8:qmstop:199999:proxy-operator@pve!operator:\",\"status\":\"stopped\",\"exitstatus\":\"OK\",\"node\":\"pv1-1\",\"user\":\"proxy-operator@pve\",\"pstart\":51024947,\"pid\":1372311,\"id\":\"199999\",\"tokenid\":\"operator\",\"starttime\":1790237416,\"type\":\"qmstop\"}}"
+}