Probe hosts for health instead of risking the shared connection

The health check shares each host's libvirt connection with the reconciles,
so it must not be able to take that connection away from them.

- A timed-out check no longer drops the link. The check goes through the new
  hostPool.probe, which abandons its RPCs when the context ends and leaves
  the connection alone, so an image upload or a Create running on the same
  link is not cut off. A link that really died is closed by the keepalive.
  One abandoned probe per host may stay waiting; until it returns, the next
  check reports Timeout at once (errProbeBusy).
- Waiting for a host's connection slot now ends with the caller's context.
  hostEntry used a mutex, so a check could wait out a reconcile redialing a
  dead host — minutes — past its own timeout. The slot is a one-slot channel
  and acquire bounds only the wait, never the uncontended take.
- --provider-check-timeout defaults to 0 and derives min(15s, interval).
  The flat 15s default contradicted the validation that the timeout may not
  exceed the interval, so --provider-check-interval=12s alone failed to
  start.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

internal/provider/kvm/check_test.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"maps"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -174,14 +175,18 @@ func TestCheck_hostsCheckedConcurrently(t *testing.T) {
 	}
 }
 
-func TestCheck_timeout(t *testing.T) {
+// A daemon that stopped answering times the check out, but the connection
+// the reconciles share stays up, and the next check does not stack a
+// second hung RPC on top of the first.
+func TestCheck_timeoutKeepsTheConnection(t *testing.T) {
 	t.Parallel()
 	api := newFakeAPI("default")
 	block := make(chan struct{})
 	t.Cleanup(func() { close(block) })
 	blocked := &blockingAPI{fakeAPI: api, block: block}
+	var closed atomic.Bool
 	dial := func(context.Context, provider.KVMHost) (*hostClient, error) {
-		return &hostClient{api: blocked, close: func() {}}, nil
+		return &hostClient{api: blocked, close: func() { closed.Store(true) }}, nil
 	}
 	pc := testConfig("nas1")
 	p := newWithDialer(pc.Name, *pc.KVM, dial, nil)
@@ -193,6 +198,49 @@ func TestCheck_timeout(t *testing.T) {
 	if got[0].Ready || got[0].Reason != provider.ReasonTimeout {
 		t.Errorf("Check() = %+v, want Timeout", got[0])
 	}
+	if closed.Load() {
+		t.Error("the timed-out check closed the connection the reconciles share")
+	}
+	again := p.Check(context.Background())
+	if again[0].Reason != provider.ReasonTimeout || !strings.Contains(again[0].Message, "still waiting") {
+		t.Errorf("second Check() = %+v, want an immediate Timeout while the first still waits", again[0])
+	}
+}
+
+// A reconcile redialing a dead host holds the host's slot for minutes; the
+// check must give up at its own deadline rather than wait that out.
+func TestCheck_doesNotWaitBehindAReconcileDial(t *testing.T) {
+	t.Parallel()
+	release := make(chan struct{})
+	dialing := make(chan struct{})
+	var once sync.Once
+	dial := func(context.Context, provider.KVMHost) (*hostClient, error) {
+		once.Do(func() { close(dialing) })
+		<-release
+		return nil, errHostOff
+	}
+	pc := testConfig("nas1")
+	p := newWithDialer(pc.Name, *pc.KVM, dial, nil)
+	reconcile := make(chan error, 1)
+	go func() {
+		_, err := p.Get(context.Background(), formatProviderID("nas1", testName))
+		reconcile <- err
+	}()
+	<-dialing
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+
+	got := p.Check(ctx)
+
+	if waited := time.Since(start); waited > time.Second {
+		t.Errorf("Check() took %s behind the reconcile's dial, want about 50ms", waited)
+	}
+	if got[0].Reason != provider.ReasonTimeout {
+		t.Errorf("Check() = %+v, want Timeout", got[0])
+	}
+	close(release)
+	<-reconcile
 }
 
 // blockingAPI hangs in ConnectGetLibVersion until block closes, like a
@@ -222,6 +270,7 @@ func TestCheckReason(t *testing.T) {
 			&dialError{stage: stageSSHHandshake, err: context.DeadlineExceeded}), reasonSSHHandshakeFailed},
 		{"deadline", context.DeadlineExceeded, provider.ReasonTimeout},
 		{"cancelled", context.Canceled, provider.ReasonTimeout},
+		{"previous check still waiting", errProbeBusy, provider.ReasonTimeout},
 		{"no pool", lvError(libvirt.ErrNoStoragePool, "no pool"), reasonPoolNotFound},
 		{"other libvirt error", lvError(libvirt.ErrInternalError, "boom"), reasonLibvirtError},
 		{"link died mid-call", io.EOF, provider.ReasonUnreachable},