kvm2: apply the milestone review

Shorter start wait with immediate verdicts for dead or paused domains,
ListByTag without state or address RPCs, Timeout for an expired check
round, no adoption of unmarked domains, reference-counted keyed locks,
dead options removed, HEAD fallback on any non-2xx.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

internal/provider/kvm2/check.go

@@ -62,9 +62,15 @@ func (p *Provider) checkHost(ctx context.Context, host provider.KVMHost) provide
 		ts.Reason = provider.ReasonReachable
 	default:
 		var inactive *poolNotActiveError
-		if errors.As(err, &inactive) {
+		switch {
+		case ctx.Err() != nil:
+			// The round ran out of time, whichever stage was running: the
+			// error's shape (a dial that saw the ctx end, an abandoned
+			// call) says nothing the admin should act on.
+			ts.Reason = provider.ReasonTimeout
+		case errors.As(err, &inactive):
 			ts.Reason = reasonPoolNotActive
-		} else {
+		default:
 			ts.Reason = checkReason(err)
 		}
 		ts.Message = err.Error()
@@ -121,10 +127,11 @@ func (e *poolNotActiveError) Error() string {
 	return fmt.Sprintf("storage pool %q is %s", e.pool, e.state)
 }
 
-// checkReason maps a failed probe to a reason. A dial failure is judged by
-// its stage, so a TCP connect or an SSH handshake that ran out of time is
-// Unreachable, not Timeout: the host did not answer. Timeout is for ctx
-// ending or a budget expiring once the connection was up.
+// checkReason maps a failed probe to a reason once checkHost has ruled out
+// the round's ctx ending. A dial failure is judged by its stage, so a TCP
+// connect or an SSH handshake that ran out of the dial budget on its own is
+// Unreachable, not Timeout: the host did not answer. Timeout is for a
+// budget expiring once the connection was up.
 func checkReason(err error) string {
 	var de *dialError
 	if errors.As(err, &de) {

internal/provider/kvm2/check_test.go

@@ -33,7 +33,9 @@ func checkContext() (context.Context, *logCapture) {
 }
 
 // checkDialer hands out a distinct fake per host name, fails the hosts in
-// errs, counts every dial, and, with block set, waits for ctx to end.
+// errs, counts every dial, and, with block set, waits for ctx to end and
+// then fails the way dialSSHSocket does when the TCP connect sees the ctx
+// end: a dialError at the tcp stage wrapping the ctx's error.
 type checkDialer struct {
 	backends map[string]*fakeBackend
 	errs     map[string]error
@@ -46,7 +48,7 @@ func (d *checkDialer) dial(ctx context.Context, host provider.KVMHost) (backend,
 	d.dials.Add(1)
 	if d.block {
 		<-ctx.Done()
-		return nil, fmt.Errorf("%w: dialing %s: %w", provider.ErrTransient, host.Name, ctx.Err())
+		return nil, &dialError{stage: stageTCP, host: host.Name, err: fmt.Errorf("dial tcp %s: %w", host.Address, ctx.Err())}
 	}
 	if err := d.errs[host.Name]; err != nil {
 		return nil, err
@@ -140,13 +142,18 @@ func TestCheck_hostsCheckedConcurrently(t *testing.T) {
 	}
 }
 
+// TestCheck_expiredContextReturnsPromptly drives the production shape of a
+// round that ran out of time during the dial: the dialer fails at the tcp
+// stage wrapping context.DeadlineExceeded, which checkReason alone would
+// call Unreachable; the verdict must still be Timeout because the round's
+// ctx ended.
 func TestCheck_expiredContextReturnsPromptly(t *testing.T) {
 	t.Parallel()
 	cfg := twoHosts()
 	p := newProvider(testProvider, cfg, &checkDialer{block: true})
 	ctx, _ := checkContext()
-	ctx, cancel := context.WithCancel(ctx)
-	cancel()
+	ctx, cancel := context.WithTimeout(ctx, 0)
+	defer cancel()
 	start := time.Now()
 	got := p.Check(ctx)
 	if elapsed := time.Since(start); elapsed > time.Second {
@@ -159,8 +166,8 @@ func TestCheck_expiredContextReturnsPromptly(t *testing.T) {
 		if ts.Ready || ts.Reason != provider.ReasonTimeout {
 			t.Errorf("target %s = ready %v reason %s, want not ready, Timeout", ts.Name, ts.Ready, ts.Reason)
 		}
-		if ts.Message == "" {
-			t.Errorf("target %s has no message", ts.Name)
+		if !strings.Contains(ts.Message, context.DeadlineExceeded.Error()) || !strings.Contains(ts.Message, string(stageTCP)) {
+			t.Errorf("target %s message = %q, want the tcp stage and the deadline named", ts.Name, ts.Message)
 		}
 	}
 }
@@ -217,7 +224,7 @@ func TestCheck_reasonsTable(t *testing.T) {
 			wantReason: provider.ReasonUnreachable,
 		},
 		{
-			name: "tcp timed out",
+			name: "tcp timed out on the dial budget but ctx is fine",
 			dialErr: func(*testing.T) error {
 				return &dialError{stage: stageTCP, host: testHost, err: fmt.Errorf("dial tcp 192.0.2.1:22: %w", context.DeadlineExceeded)}
 			},
@@ -248,7 +255,7 @@ func TestCheck_reasonsTable(t *testing.T) {
 			wantReason: provider.ReasonUnreachable,
 		},
 		{
-			name:       "ssh handshake timed out",
+			name:       "ssh handshake timed out on the dial budget but ctx is fine",
 			dialErr:    func(*testing.T) error { return sshHandshake(context.DeadlineExceeded) },
 			wantReason: provider.ReasonUnreachable,
 		},

internal/provider/kvm2/create.go

@@ -15,9 +15,11 @@ import (
 
 // The wait after DomainCreate (spec entry 18): Create returns only once the
 // domain is running, so Get never sees a domain kvm2 defined but did not
-// start (spec entry 9).
+// start (spec entry 9). DomainCreate is synchronous, so the first read
+// normally succeeds; the cap only covers state settling and must not pin a
+// reconcile worker for long.
 const (
-	defaultRunningWait = 2 * time.Minute
+	defaultRunningWait = 15 * time.Second
 	defaultRunningPoll = 500 * time.Millisecond
 )
 
@@ -75,6 +77,9 @@ func (p *Provider) create(ctx context.Context, log logr.Logger, be backend, host
 		if err != nil {
 			return err
 		}
+		if !meta.managed() {
+			return permanentf("domain %s exists but carries no kvm2 mark; it was not created by this provider", req.Name)
+		}
 		if meta.UID != "" && meta.UID != req.UID {
 			return permanentf("domain %s belongs to proxy uid %s, not %s", req.Name, meta.UID, req.UID)
 		}
@@ -211,14 +216,19 @@ func (p *Provider) writeSeed(ctx context.Context, log logr.Logger, be backend, p
 
 // ensureRunning starts the domain unless it is active already, then waits
 // until libvirt reports it running. ErrOperationInvalid from DomainCreate
-// means another caller started it between the state read and the call.
+// means another caller started it between the state read and the call. A
+// paused or pmsuspended domain is an admin's doing and DomainCreate would
+// refuse it; the verdict is Permanent so the reconciler does not retry it.
 func (p *Provider) ensureRunning(ctx context.Context, log logr.Logger, be backend, dom libvirt.Domain) error {
 	state, _, err := be.DomainGetState(ctx, dom, 0)
 	if err != nil {
 		return fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
 	}
-	if isActive(libvirt.DomainState(state)) {
+	switch s := libvirt.DomainState(state); {
+	case isActive(s):
 		return nil
+	case s == libvirt.DomainPaused || s == libvirt.DomainPmsuspended:
+		return permanentf("domain %s is %s and cannot be started by the operator; resume or destroy it on the host", dom.Name, s)
 	}
 	switch err := be.DomainCreate(ctx, dom); {
 	case err == nil:
@@ -233,7 +243,9 @@ func (p *Provider) ensureRunning(ctx context.Context, log logr.Logger, be backen
 
 // waitRunning polls DomainGetState until the domain is active, the cap
 // passes (Transient: the next Create attempt finds the domain defined and
-// tries again) or ctx ends.
+// tries again) or ctx ends. Only nostate is worth another poll; a domain
+// that is shut off, crashed or paused right after its start is not going
+// to come up on its own, so that is Transient at once.
 func (p *Provider) waitRunning(ctx context.Context, be backend, dom libvirt.Domain) error {
 	deadline := time.NewTimer(p.runningWait)
 	defer deadline.Stop()
@@ -244,8 +256,11 @@ func (p *Provider) waitRunning(ctx context.Context, be backend, dom libvirt.Doma
 		if err != nil {
 			return fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
 		}
-		if isActive(libvirt.DomainState(state)) {
+		switch s := libvirt.DomainState(state); {
+		case isActive(s):
 			return nil
+		case s != libvirt.DomainNostate:
+			return transientf("domain %s went %s right after start", dom.Name, s)
 		}
 		select {
 		case <-ticker.C:

internal/provider/kvm2/create_test.go

@@ -182,6 +182,64 @@ func TestCreate_returnsOnlyWhenRunning(t *testing.T) {
 	})
 }
 
+func TestCreate_domainThatDiesAfterStartIsTransientAtOnce(t *testing.T) {
+	t.Parallel()
+	for _, state := range []libvirt.DomainState{libvirt.DomainCrashed, libvirt.DomainShutoff, libvirt.DomainPaused} {
+		t.Run(state.String(), func(t *testing.T) {
+			t.Parallel()
+			// The cap is well over the bound asserted below, so a poll that
+			// ran to the cap would fail the test.
+			fx := newFixture(t, 1, withRunningWait(time.Millisecond, 10*time.Second))
+			fx.fake.stateAfterStart = state
+			start := time.Now()
+			id, err := fx.p.Create(fx.ctx, fx.req(testName))
+			elapsed := time.Since(start)
+			if id != "" {
+				t.Errorf("Create = %q, want no ID", id)
+			}
+			assertProviderError(t, err, "create", provider.ErrTransient)
+			if !strings.Contains(err.Error(), "right after start") {
+				t.Errorf("error = %v, want the state after start named", err)
+			}
+			if elapsed > time.Second {
+				t.Errorf("Create took %v, want it to give up at once, not poll to the cap", elapsed)
+			}
+			if got := fx.fake.callCount("DomainCreate"); got != 1 {
+				t.Errorf("DomainCreate calls = %d, want 1", got)
+			}
+		})
+	}
+}
+
+func TestCreate_pausedDomainIsPermanent(t *testing.T) {
+	t.Parallel()
+	for _, state := range []libvirt.DomainState{libvirt.DomainPaused, libvirt.DomainPmsuspended} {
+		t.Run(state.String(), func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			fx.fake.addDomain(testName, state, managedMetadata(testUID, time.Now()))
+			id, err := fx.p.Create(fx.ctx, fx.req(testName))
+			if id != "" {
+				t.Errorf("Create = %q, want no ID", id)
+			}
+			assertProviderError(t, err, "create", provider.ErrPermanent)
+			if !strings.Contains(err.Error(), "cannot be started by the operator") {
+				t.Errorf("error = %v, want the admin action named", err)
+			}
+			for _, rpc := range []string{"DomainCreate", "DomainDefineXML", "DomainDestroy"} {
+				if got := fx.fake.callCount(rpc); got != 0 {
+					t.Errorf("%s calls = %d, want 0", rpc, got)
+				}
+			}
+			fx.withDomain(testName, func(d *fakeDomain) {
+				if d.state != state {
+					t.Errorf("domain state = %s, want it left %s", d.state, state)
+				}
+			})
+		})
+	}
+}
+
 func TestCreate_rejectsUnknownPlacement(t *testing.T) {
 	t.Parallel()
 	tests := []struct {
@@ -248,24 +306,42 @@ func TestCreate_adoptionWithForeignUIDIsPermanent(t *testing.T) {
 	}
 }
 
-func TestCreate_adoptsMatchingOrEmptyUID(t *testing.T) {
+func TestCreate_adoptsOnlyMarkedDomains(t *testing.T) {
 	t.Parallel()
 	tests := []struct {
-		name     string
-		metadata string
+		name      string
+		metadata  string
+		wantAdopt bool
 	}{
-		{name: "matching uid", metadata: managedMetadata(testUID, time.Now())},
-		{name: "empty uid", metadata: managedMetadata("", time.Now())},
+		{name: "matching uid", metadata: managedMetadata(testUID, time.Now()), wantAdopt: true},
+		{name: "managed, empty uid", metadata: managedMetadata("", time.Now()), wantAdopt: true},
 		{name: "no metadata element", metadata: ""},
+		{name: "element without the managed mark", metadata: `<instance><uid>` + testUID + `</uid></instance>`},
+		{name: "managed false", metadata: `<instance><managed>false</managed><uid>` + testUID + `</uid></instance>`},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 			fx := newFixture(t, 1)
 			fx.fake.addDomain(testName, libvirt.DomainRunning, tc.metadata)
-			fx.create(testName)
-			if got := fx.fake.callCount("DomainDefineXML"); got != 0 {
-				t.Errorf("DomainDefineXML calls = %d, want 0 (adopted)", got)
+			id, err := fx.p.Create(fx.ctx, fx.req(testName))
+			if tc.wantAdopt {
+				if err != nil || id != formatProviderID(testHost, testName) {
+					t.Fatalf("Create = %q, %v; want the existing domain adopted", id, err)
+				}
+			} else {
+				if id != "" {
+					t.Errorf("Create = %q, want no ID", id)
+				}
+				assertProviderError(t, err, "create", provider.ErrPermanent)
+				if !strings.Contains(err.Error(), "carries no kvm2 mark") {
+					t.Errorf("error = %v, want the missing mark named", err)
+				}
+			}
+			for _, rpc := range []string{"DomainDefineXML", "DomainCreate", "StorageVolCreateXML", "DomainDestroy"} {
+				if got := fx.fake.callCount(rpc); got != 0 {
+					t.Errorf("%s calls = %d, want 0", rpc, got)
+				}
 			}
 		})
 	}

internal/provider/kvm2/delete.go

@@ -26,7 +26,7 @@ func (p *Provider) Delete(ctx context.Context, providerID string) error {
 	log := p.logger(ctx).WithValues("host", hostName, "name", name)
 	host, err := p.host(hostName)
 	if err != nil {
-		log.Info("host is not configured; delete has nothing to clean up", "id", providerID)
+		log.V(1).Info("host is not configured; delete has nothing to clean up", "id", providerID)
 		return nil
 	}
 	unlock, err := p.lockName(ctx, name)
@@ -82,7 +82,7 @@ func removeDomain(ctx context.Context, log logr.Logger, be backend, name string)
 func removeVolumes(ctx context.Context, log logr.Logger, be backend, poolName, name string) error {
 	pool, err := be.StoragePoolLookupByName(ctx, poolName)
 	if hasCode(err, libvirt.ErrNoStoragePool) {
-		log.Info("pool not found, no volumes to delete", "pool", poolName)
+		log.V(1).Info("pool not found, no volumes to delete", "pool", poolName)
 		return nil
 	}
 	if err != nil {

internal/provider/kvm2/delete_test.go

@@ -128,7 +128,7 @@ func TestDelete_toleratesMissingVolumesAndPool(t *testing.T) {
 	})
 	t.Run("pool gone", func(t *testing.T) {
 		t.Parallel()
-		fx := newFixture(t, 0)
+		fx := newFixture(t, 1)
 		fx.p.cfg.Hosts[0].Pool = "retired"
 		fx.fake.addDomain(testName, libvirt.DomainShutoff, managedMetadata(testUID, time.Now()))
 		if err := fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)); err != nil {
@@ -138,7 +138,7 @@ func TestDelete_toleratesMissingVolumesAndPool(t *testing.T) {
 			t.Error("domain still defined after Delete")
 		}
 		if got := fx.logs.matching("pool not found, no volumes to delete"); len(got) != 1 || !strings.Contains(got[0], `"pool"="retired"`) {
-			t.Errorf("'pool not found' Info line = %v", got)
+			t.Errorf("'pool not found' V(1) line = %v", got)
 		}
 	})
 	t.Run("nothing at all", func(t *testing.T) {
@@ -152,7 +152,7 @@ func TestDelete_toleratesMissingVolumesAndPool(t *testing.T) {
 
 func TestDelete_unconfiguredHostIsNoOp(t *testing.T) {
 	t.Parallel()
-	fx := newFixture(t, 0)
+	fx := newFixture(t, 1)
 	id := formatProviderID("nas9", testName)
 	if err := fx.p.Delete(fx.ctx, id); err != nil {
 		t.Fatalf("Delete() error = %v, want nil", err)
@@ -164,7 +164,21 @@ func TestDelete_unconfiguredHostIsNoOp(t *testing.T) {
 		t.Errorf("backend calls = %v, want none", got)
 	}
 	if got := fx.logs.matching("host is not configured; delete has nothing to clean up"); len(got) != 1 || !strings.Contains(got[0], `"host"="nas9"`) {
-		t.Errorf("Info line = %v, want one naming the host", got)
+		t.Errorf("V(1) line = %v, want one naming the host", got)
+	}
+}
+
+func TestDelete_noOpLinesAreSilentAtV0(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 0)
+	fx.p.cfg.Hosts[0].Pool = "retired"
+	for _, id := range []string{formatProviderID("nas9", testName), formatProviderID(testHost, testName)} {
+		if err := fx.p.Delete(fx.ctx, id); err != nil {
+			t.Fatalf("Delete(%s) error = %v, want nil", id, err)
+		}
+	}
+	if lines := fx.logs.all(); len(lines) != 0 {
+		t.Errorf("Delete logged %d lines at V(0), want none:\n%s", len(lines), strings.Join(lines, "\n"))
 	}
 }
 

internal/provider/kvm2/fake_test.go

@@ -36,10 +36,13 @@ type fakeBackend struct {
 	// sha256 in fixed chunks instead of holding the bytes, for tests that
 	// push more through the fake than they want in memory.
 	discardUploads bool
-	// startDelayPolls, when set, makes DomainCreate leave the domain shut
-	// off; it turns running on the startDelayPolls-th DomainGetState after
-	// that, as a real host that is still booting QEMU would.
+	// startDelayPolls, when set, makes DomainCreate leave the domain in
+	// nostate; it turns running on the startDelayPolls-th DomainGetState
+	// after that, as a host whose QEMU is still settling would.
 	startDelayPolls int
+	// stateAfterStart, when not nostate, is what DomainCreate leaves the
+	// domain in instead of running, as a guest that dies at once would.
+	stateAfterStart libvirt.DomainState
 	// undefineFlags records the flags of the last DomainUndefineFlags per
 	// domain name.
 	undefineFlags map[string]libvirt.DomainUndefineFlagsValues
@@ -291,9 +294,14 @@ func (f *fakeBackend) DomainCreate(ctx context.Context, dom libvirt.Domain) erro
 	if isActive(d.state) {
 		return fakeErr(libvirt.ErrOperationInvalid, "Requested operation is not valid: domain is already running")
 	}
-	if f.startDelayPolls > 0 {
+	switch {
+	case f.startDelayPolls > 0:
+		d.state = libvirt.DomainNostate
 		d.pendingStart = f.startDelayPolls
 		return nil
+	case f.stateAfterStart != libvirt.DomainNostate:
+		d.state = f.stateAfterStart
+		return nil
 	}
 	d.state = libvirt.DomainRunning
 	d.id = int32(len(f.domains))

internal/provider/kvm2/get.go

@@ -42,6 +42,8 @@ func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instan
 	return inst, nil
 }
 
+// get builds the Instance from the domain's metadata, its state and, for an
+// active domain, its discovered address.
 func get(ctx context.Context, log logr.Logger, be backend, hostName, name string) (*provider.Instance, error) {
 	dom, err := be.DomainLookupByName(ctx, name)
 	if err != nil {
@@ -51,11 +53,24 @@ func get(ctx context.Context, log logr.Logger, be backend, hostName, name string
 	if err != nil {
 		return nil, err
 	}
-	inst, err := inspect(ctx, log, be, hostName, dom, meta)
+	state, _, err := be.DomainGetState(ctx, dom, 0)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
 	}
-	return &inst, nil
+	s := libvirt.DomainState(state)
+	var ip string
+	if isActive(s) {
+		if ip, err = address(ctx, log, be, dom); err != nil {
+			return nil, err
+		}
+	}
+	return &provider.Instance{
+		ID:        formatProviderID(hostName, dom.Name),
+		IP:        ip,
+		State:     mapState(s, ip),
+		UID:       meta.UID,
+		CreatedAt: meta.createdAt(),
+	}, nil
 }
 
 // readMetadata returns the kvm2 mark of a domain, or the zero value (not
@@ -72,29 +87,6 @@ func readMetadata(ctx context.Context, be backend, dom libvirt.Domain) (instance
 	return parseInstanceMetadata(raw)
 }
 
-// inspect builds the Instance from the backend's state and, for an active
-// domain, its discovered address.
-func inspect(ctx context.Context, log logr.Logger, be backend, hostName string, dom libvirt.Domain, meta instanceMetadata) (provider.Instance, error) {
-	state, _, err := be.DomainGetState(ctx, dom, 0)
-	if err != nil {
-		return provider.Instance{}, fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
-	}
-	s := libvirt.DomainState(state)
-	var ip string
-	if isActive(s) {
-		if ip, err = address(ctx, log, be, dom); err != nil {
-			return provider.Instance{}, err
-		}
-	}
-	return provider.Instance{
-		ID:        formatProviderID(hostName, dom.Name),
-		IP:        ip,
-		State:     mapState(s, ip),
-		UID:       meta.UID,
-		CreatedAt: meta.createdAt(),
-	}, nil
-}
-
 // addressSources is the discovery order of spec entry 7: the guest agent
 // answers where the host's ARP table never sees guest traffic.
 var addressSources = []struct {

internal/provider/kvm2/hosts.go

@@ -2,7 +2,6 @@ package kvm2
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -93,26 +92,6 @@ func (t *hostTable) dialFresh(ctx context.Context, host provider.KVMHost) (backe
 	return t.dialer.dial(ctx, host)
 }
 
-// closeAll closes every pooled connection, waiting for in-flight dials.
-func (t *hostTable) closeAll() error {
-	t.mu.Lock()
-	entries := make([]*hostEntry, 0, len(t.hosts))
-	for _, e := range t.hosts {
-		entries = append(entries, e)
-	}
-	t.mu.Unlock()
-	var errs []error
-	for _, e := range entries {
-		e.sem <- struct{}{}
-		if e.backend != nil {
-			errs = append(errs, e.backend.Close())
-			e.backend = nil
-		}
-		<-e.sem
-	}
-	return errors.Join(errs...)
-}
-
 // libvirtDialer is the production dialer: SSH transport, go-libvirt over
 // the forwarded socket, the connect handshake, then the observe decorator,
 // so provider code never holds an undecorated backend.

internal/provider/kvm2/hosts_test.go

@@ -176,23 +176,6 @@ func TestHosts_dialErrorIsNotCached(t *testing.T) {
 	}
 }
 
-func TestHosts_closeAll(t *testing.T) {
-	t.Parallel()
-	table := newHostTable(&stubDialer{fresh: func() backend { return newFakeBackend() }})
-	a, _ := table.get(context.Background(), hostA)
-	b, _ := table.get(context.Background(), hostB)
-	if err := table.closeAll(); err != nil {
-		t.Fatalf("closeAll error = %v", err)
-	}
-	if a.(*fakeBackend).closed != 1 || b.(*fakeBackend).closed != 1 {
-		t.Errorf("closed counts = %d, %d, want 1, 1", a.(*fakeBackend).closed, b.(*fakeBackend).closed)
-	}
-	again, _ := table.get(context.Background(), hostA)
-	if again == a {
-		t.Error("get after closeAll returned the closed backend")
-	}
-}
-
 // perHostDialer routes each host to its own stub.
 type perHostDialer map[string]*stubDialer
 

internal/provider/kvm2/image.go

@@ -66,24 +66,15 @@ func resolveImage(cfg provider.KVMConfig, image string) (imageSource, error) {
 }
 
 // lockImage takes the per-(host, volume) lock of spec entry 19, or gives up
-// with ctx. The lock is a one-slot channel rather than a mutex so a caller
-// whose ctx ends while another caller's download runs returns promptly
-// (contract §1.5) instead of waiting out a 20-minute transfer.
+// with ctx: a caller whose ctx ends while another caller's download runs
+// returns promptly (contract §1.5) instead of waiting out a 20-minute
+// transfer.
 func (p *Provider) lockImage(ctx context.Context, hostName, volume string) (func(), error) {
-	key := hostName + "/" + volume
-	p.imageMu.Lock()
-	sem, ok := p.imageLocks[key]
-	if !ok {
-		sem = make(chan struct{}, 1)
-		p.imageLocks[key] = sem
-	}
-	p.imageMu.Unlock()
-	select {
-	case sem <- struct{}{}:
-		return func() { <-sem }, nil
-	case <-ctx.Done():
-		return nil, fmt.Errorf("%w: waiting for image %s on host %s: %w", provider.ErrTransient, volume, hostName, ctx.Err())
+	release, err := p.imageLocks.acquire(ctx, hostName+"/"+volume)
+	if err != nil {
+		return nil, fmt.Errorf("%w: waiting for image %s on host %s: %w", provider.ErrTransient, volume, hostName, err)
 	}
+	return release, nil
 }
 
 // ensureImage makes the base image present and complete in the host's pool
@@ -208,7 +199,7 @@ func deleteVolumeIfPresent(ctx context.Context, be backend, pool libvirt.Storage
 // finding no marker, and removing it now frees the space at once.
 func (p *Provider) fetchImage(ctx context.Context, host provider.KVMHost, pool libvirt.StoragePool, be backend, src imageSource) (libvirt.StorageVol, error) {
 	name := src.volumeName()
-	ctx, cancel := context.WithTimeout(ctx, p.budgets.transfer)
+	ctx, cancel := context.WithTimeout(ctx, defaultBudgets.transfer)
 	defer cancel()
 	ctx, span := tracing.StartSpan(ctx, tracing.Tracer(p.tracing...), "kvm2.download",
 		trace.WithSpanKind(trace.SpanKindClient),
@@ -337,7 +328,7 @@ func (c *countingReader) state() (int64, error) {
 // budget bounds it on its own. Failure is logged, not returned: the next
 // attempt finds the volume without a marker and deletes it then.
 func (p *Provider) discardVolume(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, name string) {
-	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), p.budgets.rpc)
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultBudgets.rpc)
 	defer cancel()
 	if err := deleteVolumeIfPresent(ctx, be, pool, name); err != nil {
 		log.V(1).Info("incomplete base volume left behind; the next attempt removes it", "error", err.Error())
@@ -421,8 +412,10 @@ func urlHost(rawURL string) string {
 // imageSizeIfUncached returns the image's Content-Length when the base is
 // not cached on the host, and 0 when the marker is present, for the quota
 // precheck of spec entry 13. It never downloads: HEAD first, and a GET whose
-// body is closed at once when the server does not allow HEAD. A marker is
-// taken at face value here; ensureImage decides whether it still matches.
+// body is closed at once when HEAD answers anything but 2xx, since servers
+// refuse HEAD with 405, 403 or 501 alike; the GET's status is what gets
+// classified. A marker is taken at face value here; ensureImage decides
+// whether it still matches.
 func (p *Provider) imageSizeIfUncached(ctx context.Context, be backend, pool libvirt.StoragePool, src imageSource) (int64, error) {
 	_, err := be.StorageVolLookupByName(ctx, pool, src.markerName())
 	switch {
@@ -431,14 +424,14 @@ func (p *Provider) imageSizeIfUncached(ctx context.Context, be backend, pool lib
 	case !hasCode(err, libvirt.ErrNoStorageVol):
 		return 0, fmt.Errorf("looking up marker %s: %w", src.markerName(), err)
 	}
-	ctx, cancel := context.WithTimeout(ctx, p.budgets.rpc)
+	ctx, cancel := context.WithTimeout(ctx, defaultBudgets.rpc)
 	defer cancel()
 	resp, err := p.do(ctx, http.MethodHead, src.URL)
 	if err != nil {
 		return 0, err
 	}
 	_ = resp.Body.Close()
-	if resp.StatusCode == http.StatusMethodNotAllowed {
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		if resp, err = p.do(ctx, http.MethodGet, src.URL); err != nil {
 			return 0, err
 		}

internal/provider/kvm2/image_test.go

@@ -152,6 +152,17 @@ func serveStatus(code int) http.HandlerFunc {
 	return func(w http.ResponseWriter, _ *http.Request) { http.Error(w, http.StatusText(code), code) }
 }
 
+// headFails answers HEAD with code and hands every other method to next.
+func headFails(code int, next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodHead {
+			http.Error(w, http.StatusText(code), code)
+			return
+		}
+		next(w, r)
+	}
+}
+
 func TestResolveImage_table(t *testing.T) {
 	t.Parallel()
 	cfg := withDefaults(provider.KVMConfig{Images: map[string]provider.KVMImage{
@@ -883,17 +894,30 @@ func TestImageSizeIfUncached_headThenGetFallback(t *testing.T) {
 			wantMethods: []string{http.MethodHead},
 		},
 		{
-			name: "HEAD not allowed falls back to GET with the body closed",
-			handler: func(w http.ResponseWriter, r *http.Request) {
-				if r.Method == http.MethodHead {
-					http.Error(w, "nope", http.StatusMethodNotAllowed)
-					return
-				}
-				serveBytes(body)(w, r)
-			},
+			name:        "HEAD not allowed falls back to GET with the body closed",
+			handler:     headFails(http.StatusMethodNotAllowed, serveBytes(body)),
 			wantSize:    int64(len(body)),
 			wantMethods: []string{http.MethodHead, http.MethodGet},
 		},
+		{
+			name:        "HEAD forbidden falls back to GET",
+			handler:     headFails(http.StatusForbidden, serveBytes(body)),
+			wantSize:    int64(len(body)),
+			wantMethods: []string{http.MethodHead, http.MethodGet},
+		},
+		{
+			name:        "HEAD not implemented falls back to GET",
+			handler:     headFails(http.StatusNotImplemented, serveBytes(body)),
+			wantSize:    int64(len(body)),
+			wantMethods: []string{http.MethodHead, http.MethodGet},
+		},
+		{
+			name:        "HEAD refused and GET 404 is permanent from the GET",
+			handler:     headFails(http.StatusMethodNotAllowed, serveStatus(http.StatusNotFound)),
+			wantMethods: []string{http.MethodHead, http.MethodGet},
+			wantClass:   provider.ErrPermanent,
+			wantErr:     "404",
+		},
 		{
 			name:        "marker present means nothing to download",
 			handler:     serveBytes(body),
@@ -902,16 +926,16 @@ func TestImageSizeIfUncached_headThenGetFallback(t *testing.T) {
 			wantMethods: nil,
 		},
 		{
-			name:        "404 is permanent",
+			name:        "404 on both is permanent",
 			handler:     serveStatus(http.StatusNotFound),
-			wantMethods: []string{http.MethodHead},
+			wantMethods: []string{http.MethodHead, http.MethodGet},
 			wantClass:   provider.ErrPermanent,
 			wantErr:     "404",
 		},
 		{
-			name:        "503 is transient",
+			name:        "503 on both is transient",
 			handler:     serveStatus(http.StatusServiceUnavailable),
-			wantMethods: []string{http.MethodHead},
+			wantMethods: []string{http.MethodHead, http.MethodGet},
 			wantClass:   provider.ErrTransient,
 		},
 		{

internal/provider/kvm2/kvm2.go

@@ -33,7 +33,6 @@ type Provider struct {
 	now     func() time.Time
 	hosts   *hostTable
 	http    *http.Client
-	budgets budgets
 	tracing []tracing.Option
 
 	// runningWait caps the wait for a started domain to report running,
@@ -42,28 +41,23 @@ type Provider struct {
 	runningPoll time.Duration
 
 	// imageLocks serialises image fetches per (host, volume), spec entry 19.
-	imageMu    sync.Mutex
-	imageLocks map[string]chan struct{}
+	imageLocks *keyedLocks
 
 	// nameLocks serialises Create and Delete per instance name, spec entry
 	// 19, so two Deletes never race on destroy and undefine and a Delete
 	// cannot interleave with a Create's steps.
-	nameMu    sync.Mutex
-	nameLocks map[string]chan struct{}
+	nameLocks *keyedLocks
 }
 
 // providerOption adjusts what newProvider builds; tests inject an HTTP
-// client bound to a test server, a span recorder and shorter budgets.
+// client bound to a test server, a span recorder and a shorter running
+// wait.
 type providerOption func(*Provider)
 
 func withHTTPClient(c *http.Client) providerOption {
 	return func(p *Provider) { p.http = c }
 }
 
-func withBudgets(b budgets) providerOption {
-	return func(p *Provider) { p.budgets = b }
-}
-
 func withTracing(opts ...tracing.Option) providerOption {
 	return func(p *Provider) { p.tracing = opts }
 }
@@ -104,7 +98,7 @@ func New(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, er
 // newProvider is the constructor tests use: d hands out the backends, the
 // production one being newLibvirtDialer. Without options the image
 // downloads use http.DefaultClient (system CA roots, no client timeout: the
-// per-request ctx carries the transfer budget) and the default budgets.
+// per-request ctx carries the transfer budget).
 func newProvider(name string, cfg provider.KVMConfig, d dialer, opts ...providerOption) *Provider {
 	p := &Provider{
 		name:        name,
@@ -112,11 +106,10 @@ func newProvider(name string, cfg provider.KVMConfig, d dialer, opts ...provider
 		now:         time.Now,
 		hosts:       newHostTable(d),
 		http:        http.DefaultClient,
-		budgets:     defaultBudgets,
 		runningWait: defaultRunningWait,
 		runningPoll: defaultRunningPoll,
-		imageLocks:  make(map[string]chan struct{}),
-		nameLocks:   make(map[string]chan struct{}),
+		imageLocks:  newKeyedLocks(),
+		nameLocks:   newKeyedLocks(),
 	}
 	for _, opt := range opts {
 		opt(p)
@@ -187,21 +180,64 @@ func (p *Provider) wrapErr(op, id string, err error) error {
 	return provider.Wrap(classify(op, err), op, p.name, id, err)
 }
 
-// lockName takes the per-name lock of spec entry 19, or gives up with ctx.
-// A one-slot channel rather than a mutex, so a caller whose ctx ends while
-// another caller's Create is downloading an image returns promptly.
+// lockName takes the per-name lock of spec entry 19, or gives up with ctx,
+// so a caller whose ctx ends while another caller's Create is downloading
+// an image returns promptly.
 func (p *Provider) lockName(ctx context.Context, name string) (func(), error) {
-	p.nameMu.Lock()
-	sem, ok := p.nameLocks[name]
-	if !ok {
-		sem = make(chan struct{}, 1)
-		p.nameLocks[name] = sem
+	release, err := p.nameLocks.acquire(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("%w: waiting for the lock on %s: %w", provider.ErrTransient, name, err)
 	}
-	p.nameMu.Unlock()
+	return release, nil
+}
+
+// keyedLocks is a set of one-slot semaphores by key (spec entry 19): a
+// semaphore rather than a mutex so a waiter can give up with its ctx. Each
+// entry counts the callers holding or waiting for it and is dropped from
+// the map at zero, so the map does not grow with every name ever seen while
+// two waiters on one key still always share one semaphore.
+type keyedLocks struct {
+	mu    sync.Mutex
+	locks map[string]*keyedLock
+}
+
+type keyedLock struct {
+	sem  chan struct{}
+	refs int
+}
+
+func newKeyedLocks() *keyedLocks {
+	return &keyedLocks{locks: make(map[string]*keyedLock)}
+}
+
+// acquire takes the lock for key, or gives up with ctx's error. The
+// returned func releases it and must be called exactly once.
+func (k *keyedLocks) acquire(ctx context.Context, key string) (func(), error) {
+	k.mu.Lock()
+	l, ok := k.locks[key]
+	if !ok {
+		l = &keyedLock{sem: make(chan struct{}, 1)}
+		k.locks[key] = l
+	}
+	l.refs++
+	k.mu.Unlock()
 	select {
-	case sem <- struct{}{}:
-		return func() { <-sem }, nil
+	case l.sem <- struct{}{}:
+		return func() {
+			<-l.sem
+			k.unref(key, l)
+		}, nil
 	case <-ctx.Done():
-		return nil, fmt.Errorf("%w: waiting for the lock on %s: %w", provider.ErrTransient, name, ctx.Err())
+		k.unref(key, l)
+		return nil, ctx.Err()
+	}
+}
+
+func (k *keyedLocks) unref(key string, l *keyedLock) {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	l.refs--
+	if l.refs == 0 {
+		delete(k.locks, key)
 	}
 }

internal/provider/kvm2/kvm2_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -254,6 +255,86 @@ func TestWithDefaults_doesNotAliasInput(t *testing.T) {
 	}
 }
 
+func lockCount(k *keyedLocks) int {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	return len(k.locks)
+}
+
+func TestKeyedLocks_entryRemovedWhenUncontended(t *testing.T) {
+	t.Parallel()
+	k := newKeyedLocks()
+	release, err := k.acquire(context.Background(), "a")
+	if err != nil {
+		t.Fatalf("acquire error = %v", err)
+	}
+	if lockCount(k) != 1 {
+		t.Errorf("entries while held = %d, want 1", lockCount(k))
+	}
+	release()
+	if lockCount(k) != 0 {
+		t.Errorf("entries after release = %d, want 0", lockCount(k))
+	}
+	if _, err := k.acquire(context.Background(), "a"); err != nil {
+		t.Fatalf("acquire after release error = %v", err)
+	}
+}
+
+func TestKeyedLocks_waitersShareOneLock(t *testing.T) {
+	t.Parallel()
+	k := newKeyedLocks()
+	const n = 20
+	var inside, overlaps atomic.Int32
+	var wg sync.WaitGroup
+	for range n {
+		wg.Go(func() {
+			release, err := k.acquire(context.Background(), "shared")
+			if err != nil {
+				t.Errorf("acquire error = %v", err)
+				return
+			}
+			if inside.Add(1) > 1 {
+				overlaps.Add(1)
+			}
+			time.Sleep(time.Millisecond)
+			inside.Add(-1)
+			release()
+		})
+	}
+	wg.Wait()
+	if overlaps.Load() != 0 {
+		t.Errorf("%d holders overlapped on one key", overlaps.Load())
+	}
+	if lockCount(k) != 0 {
+		t.Errorf("entries after every holder released = %d, want 0", lockCount(k))
+	}
+}
+
+func TestKeyedLocks_waiterGivesUpWithContext(t *testing.T) {
+	t.Parallel()
+	k := newKeyedLocks()
+	release, err := k.acquire(context.Background(), "a")
+	if err != nil {
+		t.Fatalf("acquire error = %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	if _, err := k.acquire(ctx, "a"); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("waiter error = %v, want DeadlineExceeded", err)
+	}
+	if time.Since(start) > time.Second {
+		t.Errorf("waiter took %v, want to give up with its ctx", time.Since(start))
+	}
+	if lockCount(k) != 1 {
+		t.Errorf("entries with the holder still in = %d, want 1: the waiter's exit must not drop the holder's entry", lockCount(k))
+	}
+	release()
+	if lockCount(k) != 0 {
+		t.Errorf("entries after the holder released = %d, want 0", lockCount(k))
+	}
+}
+
 func TestWrapErr_carriesProviderNameOpAndID(t *testing.T) {
 	t.Parallel()
 	p := newProvider("kvm2-home", twoHosts(), &stubDialer{})

internal/provider/kvm2/list.go

@@ -10,10 +10,13 @@ import (
 )
 
 // ListByTag is spec entry 12: every domain on every configured host, in
-// config order, filtered client-side on the kvm2 mark. A host that cannot
-// be reached or listed fails the whole call (fail-whole); a single domain
-// that cannot be read is skipped, unless ctx ended, which fails the call
-// rather than pass off an empty view as the truth.
+// config order, filtered client-side on the kvm2 mark. GC uses only ID, UID
+// and CreatedAt (contract §2.4), all of which the name and the metadata
+// element give, so no state or address RPC is made and State and IP stay
+// empty. A host that cannot be reached or listed fails the whole call
+// (fail-whole); a single domain whose metadata cannot be read is skipped,
+// unless ctx ended, which fails the call rather than pass off an empty view
+// as the truth.
 func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
 	log := p.logger(ctx)
 	var out []provider.Instance
@@ -43,15 +46,8 @@ func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
 			if !meta.managed() {
 				continue
 			}
-			inst, err := inspect(ctx, dlog, be, host.Name, dom, meta)
-			if err != nil {
-				if ctx.Err() != nil {
-					return nil, p.wrapErr("list", "", err)
-				}
-				dlog.V(1).Info("skipping domain that could not be inspected", "error", err.Error())
-				continue
-			}
-			dlog.V(2).Info("kvm2 listed instance", "id", inst.ID, "state", inst.State, "uid", inst.UID, "createdAt", inst.CreatedAt)
+			inst := provider.Instance{ID: formatProviderID(host.Name, dom.Name), UID: meta.UID, CreatedAt: meta.createdAt()}
+			dlog.V(2).Info("kvm2 listed instance", "id", inst.ID, "uid", inst.UID, "createdAt", inst.CreatedAt)
 			out = append(out, inst)
 		}
 	}

internal/provider/kvm2/list_test.go

@@ -72,8 +72,43 @@ func TestListByTag_acrossHostsInConfigOrder(t *testing.T) {
 	if got := listIDs(list); strings.Join(got, ",") != strings.Join(want, ",") {
 		t.Errorf("ListByTag = %v, want %v (config order, then name order)", got, want)
 	}
-	if list[0].State != provider.StateStopped || list[1].State != provider.StateProvisioning {
-		t.Errorf("states = %s, %s, want Stopped, Provisioning", list[0].State, list[1].State)
+	if list[0].UID != "uid-a" || list[1].UID != "uid-z" || list[2].UID != "uid-b" {
+		t.Errorf("uids = %s, %s, %s, want uid-a, uid-z, uid-b", list[0].UID, list[1].UID, list[2].UID)
+	}
+}
+
+func TestListByTag_doesNotReadStateOrAddresses(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 2)
+	id := fx.create(testName)
+	fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcAgent, "10.0.0.5")
+	fx.fake.addDomain("proxy-off", libvirt.DomainShutoff, managedMetadata("uid-off", time.Now()))
+	before := len(fx.fake.callLog())
+
+	list, err := fx.p.ListByTag(fx.ctx)
+	if err != nil {
+		t.Fatalf("ListByTag() error = %v", err)
+	}
+	if len(list) != 2 || list[0].ID != id || list[1].ID != formatProviderID(testHost, "proxy-off") {
+		t.Fatalf("ListByTag = %v, want the two managed domains", listIDs(list))
+	}
+	for _, inst := range list {
+		if inst.State != "" || inst.IP != "" {
+			t.Errorf("listed %s = state %q ip %q, want both empty: GC does not use them", inst.ID, inst.State, inst.IP)
+		}
+		if inst.UID == "" || inst.CreatedAt.IsZero() {
+			t.Errorf("listed %s lacks uid or createdAt: %+v", inst.ID, inst)
+		}
+	}
+	for _, rpc := range fx.fake.callLog()[before:] {
+		switch rpc {
+		case "ConnectListAllDomains", "DomainGetMetadata":
+		default:
+			t.Errorf("ListByTag made %s; want only the listing and one metadata read per domain", rpc)
+		}
+	}
+	if got := fx.logs.matching("kvm2 listed instance"); len(got) != 2 || strings.Contains(got[0], `"state"=`) {
+		t.Errorf("listed instance lines = %v, want two without a state", got)
 	}
 }
 
@@ -126,12 +161,12 @@ func TestListByTag_unreadableDomainSkipped(t *testing.T) {
 			line: "skipping domain with unreadable metadata",
 		},
 		{
-			name: "state rpc fails for the first domain only",
+			name: "domain gone between listing and metadata read",
 			setup: func(fx *fixture) {
 				fx.fake.addDomain("proxy-bad", libvirt.DomainRunning, managedMetadata("uid-bad", time.Now()))
 				var calls int
 				fx.fake.mu.Lock()
-				fx.fake.hooks["DomainGetState"] = func() error {
+				fx.fake.hooks["DomainGetMetadata"] = func() error {
 					calls++
 					if calls == 1 {
 						return fakeErr(libvirt.ErrNoDomain, "Domain not found")
@@ -140,7 +175,7 @@ func TestListByTag_unreadableDomainSkipped(t *testing.T) {
 				}
 				fx.fake.mu.Unlock()
 			},
-			line: "skipping domain that could not be inspected",
+			line: "skipping domain with unreadable metadata",
 		},
 	}
 	for _, tc := range tests {
@@ -172,7 +207,7 @@ func TestListByTag_includesInactiveDomains(t *testing.T) {
 	if err != nil {
 		t.Fatalf("ListByTag() error = %v", err)
 	}
-	if len(list) != 1 || list[0].ID != id || list[0].State != provider.StateStopped {
-		t.Errorf("ListByTag = %+v, want the shut-off domain as Stopped", list)
+	if len(list) != 1 || list[0].ID != id || list[0].UID != testUID {
+		t.Errorf("ListByTag = %+v, want the shut-off domain listed with its uid", list)
 	}
 }