kvm2: do not let a stalled write delay the connection teardown (H1)

sshSocket.Close was errors.Join(s.Conn.Close(), s.client.Close()). Go
evaluates those left to right, so the blocking close ran before the
unblocking one. s.Conn is the forwarded libvirt socket, and closing an SSH
channel writes a close message under ch.writeMu — which go-libvirt already
holds while streaming a base image through that same channel. Against a
host that vanished without RST or FIN, that write blocks until the kernel
stops retransmitting (tcp_retries2, ~15 min), and s.client.Close(), a bare
net.Conn.Close() that takes no lock and unblocks the stalled writer through
netpoll, was never reached. Close the client only; nothing wants the polite
channel-close message, and reordering instead would leave an error to
filter in a memoised closeErr.

link.markDead compounded it by running kill() before close(l.closed), both
inside a sync.Once. Reordering alone was not enough: sync.Once.Do blocks
concurrent callers until the first body returns, so every other markDead —
including the one in call is own ctx-cancel branch — queued behind the
stalled kill, and cancelling the reconcile did not help. The once field is
gone; the mutex the struct already had guards the close-exactly-once
invariant, reason is set before the signal, and kill runs outside the lock.

Blast radius was the whole operator, not one host: MaxConcurrentReconciles
is 3 and ensureImage holds a per-image lock whose acquire is ctx-aware but
reconcile contexts carry no deadline, so three Proxies on one host with an
uncached image parked every worker.

The same edits close a second route to the same pin: go-libvirt is
Disconnect closes the net.Conn it was handed (libvirt.go:331 ->
socket.go:164), and that conn is our sshSocket, so libvirtBackend.Close is
Disconnect goroutine entered closeOnce and hung; the 5 s closeBudget
released Close, which called markDead -> kill -> sshSocket.Close and
blocked in the other goroutine is sync.Once.

Verified against the pinned x/crypto v0.54.0 rather than assumed: the
flow-control park (remoteWin.reserve, channel.go:268) is outside writeMu
and only writePacket holds it across the TCP write, so a writer parked in
SSH flow control never blocks the close and a writer inside the TCP write
always does. To reproduce by hand, shrink SO_SNDBUF well under the 2 MiB
channel window (channelWindowSize = 64 * channelMaxPacket) so the socket
buffer fills before the window does.

The kvm provider does not have this bug: it closes client only and never
touches the forwarded channel.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

internal/provider/kvm2/backend.go

@@ -106,7 +106,6 @@ type link struct {
 	label string
 	kill  func()
 
-	once   sync.Once
 	closed chan struct{}
 	mu     sync.Mutex
 	reason string
@@ -133,18 +132,23 @@ func (l *link) deadReason() string {
 	return l.reason
 }
 
-// markDead records why the connection is unusable, closes the transport and
-// signals dead. Later calls are no-ops.
+// markDead records why the connection is unusable, signals dead and then
+// closes the transport. Later calls are no-ops. Signalling before killing,
+// and killing outside the lock, is what keeps a kill that takes its time
+// from holding up the callers waiting on dead or the next markDead: a
+// transport close can block for as long as the kernel retransmits.
 func (l *link) markDead(reason string) {
-	l.once.Do(func() {
-		l.mu.Lock()
-		l.reason = reason
+	l.mu.Lock()
+	if l.isDead() {
 		l.mu.Unlock()
-		if l.kill != nil {
-			l.kill()
-		}
-		close(l.closed)
-	})
+		return
+	}
+	l.reason = reason
+	close(l.closed)
+	l.mu.Unlock()
+	if l.kill != nil {
+		l.kill()
+	}
 }
 
 // call executes a blocking go-libvirt call with a budget and returns its

internal/provider/kvm2/backend_test.go

@@ -148,6 +148,71 @@ func TestCall_killedByAnotherCallerFailsTransient(t *testing.T) {
 	}
 }
 
+// TestMarkDead_slowKillDoesNotBlockCallers pins the guarantee the pooled
+// connection rests on: a kill that takes its time — closing a transport can
+// block for as long as the kernel retransmits to a host that went away —
+// holds up neither the callers waiting on dead nor the next markDead. The
+// call's own fn signals when it is in flight, so the parked caller is
+// genuinely parked rather than refused by the dead check on the way in.
+func TestMarkDead_slowKillDoesNotBlockCallers(t *testing.T) {
+	t.Parallel()
+	release := make(chan struct{})
+	t.Cleanup(func() { close(release) })
+	killing := make(chan struct{})
+	var killed atomic.Int32
+	l := newLink("beelink", func() {
+		killed.Add(1)
+		close(killing)
+		<-release
+	})
+
+	inflight := make(chan struct{})
+	waiting := make(chan error, 1)
+	go func() {
+		waiting <- l.run(context.Background(), "StorageVolUpload", time.Minute, func() error {
+			close(inflight)
+			<-release
+			return nil
+		})
+	}()
+	<-inflight
+
+	go l.markDead("keepalive failed")
+	<-killing
+
+	select {
+	case <-l.dead():
+	case <-time.After(time.Second):
+		t.Fatal("link not signalled dead while its kill is still running")
+	}
+	select {
+	case err := <-waiting:
+		if !errors.Is(err, provider.ErrTransient) || !strings.Contains(err.Error(), "keepalive failed") {
+			t.Errorf("parked call error = %v, want ErrTransient with the death reason", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("parked call not released while the kill is still running")
+	}
+
+	second := make(chan struct{})
+	go func() {
+		l.markDead("closed")
+		close(second)
+	}()
+	select {
+	case <-second:
+	case <-time.After(time.Second):
+		t.Fatal("second markDead blocked behind the running kill")
+	}
+
+	if killed.Load() != 1 {
+		t.Errorf("kill hook called %d times, want 1", killed.Load())
+	}
+	if !strings.Contains(l.deadReason(), "keepalive failed") {
+		t.Errorf("deadReason() = %q, want the first reason", l.deadReason())
+	}
+}
+
 func TestBudgets_upload(t *testing.T) {
 	t.Parallel()
 	tests := []struct {

internal/provider/kvm2/ssh.go

@@ -96,8 +96,12 @@ func loadCredentials(cfg provider.KVMConfig) (credentials, error) {
 }
 
 // sshSocket is the libvirt socket forwarded through one SSH connection. The
-// embedded Conn is the channel; Close also closes the SSH client, which is
-// what unblocks anything reading the channel.
+// embedded Conn is the channel; Close closes the SSH client, whose TCP
+// connection carries it, and that alone unblocks the channel's readers and
+// writers. Closing the channel as well would be worse than useless: its
+// Close writes a close message under the channel's write lock, so it would
+// queue behind a transfer stuck in a TCP write, which is the case Close
+// exists to survive.
 type sshSocket struct {
 	net.Conn
 	client *ssh.Client
@@ -107,9 +111,7 @@ type sshSocket struct {
 }
 
 func (s *sshSocket) Close() error {
-	s.closeOnce.Do(func() {
-		s.closeErr = errors.Join(s.Conn.Close(), s.client.Close())
-	})
+	s.closeOnce.Do(func() { s.closeErr = s.client.Close() })
 	return s.closeErr
 }