Add the KVM health check: reachability and storage pool per host

Each configured host is checked concurrently over the pooled SSH+libvirt
connection: libvirt version and hostname prove the daemon answers, then
the configured pool must exist and be running, with its free space
reported. Dial failures are now tagged with their stage (SSH material,
TCP, SSH handshake, libvirt connect) by a dialError that keeps the
message and unwraps to the cause, so classification is unchanged and the
check can say why a host is unreachable. A pool verdict is decided after
the call returns so it never drops a working connection.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-09-11 20:17:35 +00:00
co-authored by Claude Opus 5
parent 1962365f44
commit 04dd2c0284
9 changed files with 694 additions and 5 deletions
+80
View File
@@ -614,6 +614,86 @@ func TestNew_missingSSHMaterialStillBuilds(t *testing.T) {
}
}
// The health check reports where a connection broke, so every stage of a
// real dial must come back tagged.
func TestLazySSH_dial_tagsTheFailingStage(t *testing.T) {
t.Parallel()
tests := []struct {
name string
mutate func(t *testing.T, f *sshFixture)
want dialStage
wantReason string
}{
{
name: "key material missing",
mutate: func(t *testing.T, f *sshFixture) { f.cfg.SSHKeyFile = filepath.Join(t.TempDir(), "absent") },
want: stageSSHMaterial,
wantReason: reasonSSHMaterialUnavailable,
},
{
name: "nothing listening",
mutate: func(t *testing.T, f *sshFixture) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
f.host.Address = ln.Addr().String()
ln.Close()
},
want: stageTCP,
wantReason: provider.ReasonUnreachable,
},
{
name: "host key not in known_hosts",
mutate: func(t *testing.T, f *sshFixture) {
impostor, _ := newKeyPair(t)
line := knownhosts.Line([]string{f.server.addr}, impostor.PublicKey())
if err := os.WriteFile(f.cfg.KnownHostsFile, []byte(line+"\n"), 0o600); err != nil {
t.Fatal(err)
}
},
want: stageSSHHandshake,
wantReason: reasonSSHHandshakeFailed,
},
{
name: "client key rejected",
mutate: func(t *testing.T, f *sshFixture) {
_, otherPEM := newKeyPair(t)
if err := os.WriteFile(f.keyFile, otherPEM, 0o600); err != nil {
t.Fatal(err)
}
},
want: stageSSHHandshake,
wantReason: reasonSSHHandshakeFailed,
},
{
name: "libvirt socket missing",
mutate: func(_ *testing.T, f *sshFixture) { f.host.Socket = "/nonexistent/libvirt-sock" },
want: stageLibvirtConnect,
wantReason: reasonLibvirtConnectFailed,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
f := newSSHFixture(t, true)
tc.mutate(t, f)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := newLazySSH(f.cfg).dial(ctx, f.host)
var de *dialError
if !errors.As(err, &de) || de.stage != tc.want {
t.Fatalf("dial() error = %v, want a %s dialError", err, tc.want)
}
if got := checkReason(err); got != tc.wantReason {
t.Errorf("checkReason() = %s, want %s", got, tc.wantReason)
}
})
}
}
func TestKeepalive_closesClientWhenServerStalls(t *testing.T) {
t.Parallel()
f := newSSHFixture(t, false)