Add the KVM health check: reachability and storage pool per host
Each configured host is checked concurrently over the pooled SSH+libvirt connection: libvirt version and hostname prove the daemon answers, then the configured pool must exist and be running, with its free space reported. Dial failures are now tagged with their stage (SSH material, TCP, SSH handshake, libvirt connect) by a dialError that keeps the message and unwraps to the cause, so classification is unchanged and the check can say why a host is unreachable. A pool verdict is decided after the call returns so it never drops a working connection. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -44,6 +44,9 @@ type libvirtAPI interface {
|
||||
StorageVolUpload(vol libvirt.StorageVol, r io.Reader, offset, length uint64, flags libvirt.StorageVolUploadFlags) error
|
||||
StorageVolDownload(vol libvirt.StorageVol, w io.Writer, offset, length uint64, flags libvirt.StorageVolDownloadFlags) error
|
||||
StorageVolDelete(vol libvirt.StorageVol, flags libvirt.StorageVolDeleteFlags) error
|
||||
ConnectGetLibVersion() (uint64, error)
|
||||
ConnectGetHostname() (string, error)
|
||||
StoragePoolGetInfo(pool libvirt.StoragePool) (state uint8, capacity, allocation, available uint64, err error)
|
||||
}
|
||||
|
||||
// hostClient is one live connection to a hypervisor.
|
||||
@@ -68,6 +71,27 @@ func (c *hostClient) alive() bool {
|
||||
// real one; tests inject their own.
|
||||
type hostDialer func(ctx context.Context, host provider.KVMHost) (*hostClient, error)
|
||||
|
||||
// dialStage names the step of a connection attempt that failed, so the
|
||||
// health check can say why a host is unreachable without parsing messages.
|
||||
type dialStage string
|
||||
|
||||
const (
|
||||
stageSSHMaterial dialStage = "ssh-material"
|
||||
stageTCP dialStage = "tcp"
|
||||
stageSSHHandshake dialStage = "ssh-handshake"
|
||||
stageLibvirtConnect dialStage = "libvirt-connect"
|
||||
)
|
||||
|
||||
// dialError tags a dial failure with its stage. It keeps the message and
|
||||
// unwraps to the cause, so classify sees exactly what it saw before.
|
||||
type dialError struct {
|
||||
stage dialStage
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *dialError) Error() string { return e.err.Error() }
|
||||
func (e *dialError) Unwrap() error { return e.err }
|
||||
|
||||
// hostPool caches one connection per host, dialing lazily and redialing
|
||||
// after the connection drops. Dials for different hosts don't block each
|
||||
// other; concurrent callers for the same host share a single dial.
|
||||
@@ -254,7 +278,7 @@ func (l *lazySSH) load() (*sshAuth, error) {
|
||||
func (l *lazySSH) dial(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
|
||||
a, err := l.load()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, &dialError{stage: stageSSHMaterial, err: err}
|
||||
}
|
||||
return a.dial(ctx, host)
|
||||
}
|
||||
@@ -285,7 +309,7 @@ func (a *sshAuth) dialSSH(ctx context.Context, host provider.KVMHost) (*ssh.Clie
|
||||
var d net.Dialer
|
||||
nc, err := d.DialContext(ctx, "tcp", host.Address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ssh dial: %w", err)
|
||||
return nil, &dialError{stage: stageTCP, err: fmt.Errorf("ssh dial: %w", err)}
|
||||
}
|
||||
_ = nc.SetDeadline(deadline)
|
||||
conn, chans, reqs, err := ssh.NewClientConn(nc, host.Address, &ssh.ClientConfig{
|
||||
@@ -295,7 +319,7 @@ func (a *sshAuth) dialSSH(ctx context.Context, host provider.KVMHost) (*ssh.Clie
|
||||
})
|
||||
if err != nil {
|
||||
nc.Close()
|
||||
return nil, fmt.Errorf("ssh handshake: %w", err)
|
||||
return nil, &dialError{stage: stageSSHHandshake, err: fmt.Errorf("ssh handshake: %w", err)}
|
||||
}
|
||||
_ = nc.SetDeadline(time.Time{})
|
||||
return ssh.NewClient(conn, chans, reqs), nil
|
||||
@@ -319,7 +343,7 @@ func (u unixForward) Dial() (net.Conn, error) {
|
||||
func connectLibvirt(ctx context.Context, client *ssh.Client, socket string) (*hostClient, error) {
|
||||
lv := libvirt.NewWithDialer(unixForward{client: client, path: socket})
|
||||
if err := awaitOrCancel(ctx, lv.Connect, func() { client.Close() }); err != nil {
|
||||
return nil, fmt.Errorf("libvirt connect via %s: %w", socket, err)
|
||||
return nil, &dialError{stage: stageLibvirtConnect, err: fmt.Errorf("libvirt connect via %s: %w", socket, err)}
|
||||
}
|
||||
stop := make(chan struct{})
|
||||
var once sync.Once
|
||||
|
||||
Reference in New Issue
Block a user