Add the KVM health check: reachability and storage pool per host

Each configured host is checked concurrently over the pooled SSH+libvirt
connection: libvirt version and hostname prove the daemon answers, then
the configured pool must exist and be running, with its free space
reported. Dial failures are now tagged with their stage (SSH material,
TCP, SSH handshake, libvirt connect) by a dialError that keeps the
message and unwraps to the cause, so classification is unchanged and the
check can say why a host is unreachable. A pool verdict is decided after
the call returns so it never drops a working connection.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-09-11 20:17:35 +00:00
co-authored by Claude Opus 5
parent 1962365f44
commit 04dd2c0284
9 changed files with 694 additions and 5 deletions
+28 -4
View File
@@ -44,6 +44,9 @@ type libvirtAPI interface {
StorageVolUpload(vol libvirt.StorageVol, r io.Reader, offset, length uint64, flags libvirt.StorageVolUploadFlags) error
StorageVolDownload(vol libvirt.StorageVol, w io.Writer, offset, length uint64, flags libvirt.StorageVolDownloadFlags) error
StorageVolDelete(vol libvirt.StorageVol, flags libvirt.StorageVolDeleteFlags) error
ConnectGetLibVersion() (uint64, error)
ConnectGetHostname() (string, error)
StoragePoolGetInfo(pool libvirt.StoragePool) (state uint8, capacity, allocation, available uint64, err error)
}
// hostClient is one live connection to a hypervisor.
@@ -68,6 +71,27 @@ func (c *hostClient) alive() bool {
// real one; tests inject their own.
type hostDialer func(ctx context.Context, host provider.KVMHost) (*hostClient, error)
// dialStage names the step of a connection attempt that failed, so the
// health check can say why a host is unreachable without parsing messages.
type dialStage string
const (
stageSSHMaterial dialStage = "ssh-material"
stageTCP dialStage = "tcp"
stageSSHHandshake dialStage = "ssh-handshake"
stageLibvirtConnect dialStage = "libvirt-connect"
)
// dialError tags a dial failure with its stage. It keeps the message and
// unwraps to the cause, so classify sees exactly what it saw before.
type dialError struct {
stage dialStage
err error
}
func (e *dialError) Error() string { return e.err.Error() }
func (e *dialError) Unwrap() error { return e.err }
// hostPool caches one connection per host, dialing lazily and redialing
// after the connection drops. Dials for different hosts don't block each
// other; concurrent callers for the same host share a single dial.
@@ -254,7 +278,7 @@ func (l *lazySSH) load() (*sshAuth, error) {
func (l *lazySSH) dial(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
a, err := l.load()
if err != nil {
return nil, err
return nil, &dialError{stage: stageSSHMaterial, err: err}
}
return a.dial(ctx, host)
}
@@ -285,7 +309,7 @@ func (a *sshAuth) dialSSH(ctx context.Context, host provider.KVMHost) (*ssh.Clie
var d net.Dialer
nc, err := d.DialContext(ctx, "tcp", host.Address)
if err != nil {
return nil, fmt.Errorf("ssh dial: %w", err)
return nil, &dialError{stage: stageTCP, err: fmt.Errorf("ssh dial: %w", err)}
}
_ = nc.SetDeadline(deadline)
conn, chans, reqs, err := ssh.NewClientConn(nc, host.Address, &ssh.ClientConfig{
@@ -295,7 +319,7 @@ func (a *sshAuth) dialSSH(ctx context.Context, host provider.KVMHost) (*ssh.Clie
})
if err != nil {
nc.Close()
return nil, fmt.Errorf("ssh handshake: %w", err)
return nil, &dialError{stage: stageSSHHandshake, err: fmt.Errorf("ssh handshake: %w", err)}
}
_ = nc.SetDeadline(time.Time{})
return ssh.NewClient(conn, chans, reqs), nil
@@ -319,7 +343,7 @@ func (u unixForward) Dial() (net.Conn, error) {
func connectLibvirt(ctx context.Context, client *ssh.Client, socket string) (*hostClient, error) {
lv := libvirt.NewWithDialer(unixForward{client: client, path: socket})
if err := awaitOrCancel(ctx, lv.Connect, func() { client.Close() }); err != nil {
return nil, fmt.Errorf("libvirt connect via %s: %w", socket, err)
return nil, &dialError{stage: stageLibvirtConnect, err: fmt.Errorf("libvirt connect via %s: %w", socket, err)}
}
stop := make(chan struct{})
var once sync.Once