kvm2: SSH transport, ctx-first libvirt seam, host table and span decorator
Every go-libvirt call runs under its own budget and a cancelled or overlong call kills the connection; credentials are read on every dial; one span and one log line per RPC come from a decorator applied where connections are made. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,216 @@
|
||||
package kvm2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// dialStage names the step of a dial that failed, so the Checker (spec
|
||||
// entry 16) can pick a reason without parsing messages.
|
||||
type dialStage string
|
||||
|
||||
const (
|
||||
// stageCredentials: the key or known_hosts file is missing or unparsable.
|
||||
stageCredentials dialStage = "credentials"
|
||||
// stageTCP: the TCP connection to host.Address failed.
|
||||
stageTCP dialStage = "tcp"
|
||||
// stageSSH: the SSH handshake failed, including host key and auth.
|
||||
stageSSH dialStage = "ssh"
|
||||
// stageSocket: the forwarded channel to the libvirt socket was refused.
|
||||
stageSocket dialStage = "socket"
|
||||
)
|
||||
|
||||
// dialError tags a dial failure with its stage and host. Unwrap keeps the
|
||||
// cause visible, so classify still sees *knownhosts.KeyError, fs.ErrNotExist,
|
||||
// the "unable to authenticate" text and *net.OpError.
|
||||
type dialError struct {
|
||||
stage dialStage
|
||||
host string
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *dialError) Error() string {
|
||||
return fmt.Sprintf("dialing host %s (%s): %v", e.host, e.stage, e.err)
|
||||
}
|
||||
|
||||
func (e *dialError) Unwrap() error { return e.err }
|
||||
|
||||
// stageOf returns the failed stage of a dial error, if err is one.
|
||||
func stageOf(err error) (dialStage, bool) {
|
||||
var de *dialError
|
||||
if !errors.As(err, &de) {
|
||||
return "", false
|
||||
}
|
||||
return de.stage, true
|
||||
}
|
||||
|
||||
const (
|
||||
defaultKeepaliveInterval = 15 * time.Second
|
||||
keepaliveRequest = "keepalive@openssh.com"
|
||||
)
|
||||
|
||||
// credentials is what one dial needs from the mounted files. They are read
|
||||
// on every dial and never cached (spec entry 14), so rotating the Secret
|
||||
// takes effect on the next connection.
|
||||
type credentials struct {
|
||||
signer ssh.Signer
|
||||
hostKey ssh.HostKeyCallback
|
||||
}
|
||||
|
||||
// loadCredentials reads and parses the key and known_hosts files. A missing
|
||||
// file keeps fs.ErrNotExist (Transient: the Secret has not landed); a file
|
||||
// that does not parse is Permanent. Messages name the path and carry the
|
||||
// parser's own error, never the file's content.
|
||||
func loadCredentials(cfg provider.KVMConfig) (credentials, error) {
|
||||
keyBytes, err := os.ReadFile(cfg.SSHKeyFile)
|
||||
if err != nil {
|
||||
return credentials{}, fmt.Errorf("reading ssh key file %s: %w", cfg.SSHKeyFile, err)
|
||||
}
|
||||
signer, err := ssh.ParsePrivateKey(keyBytes)
|
||||
if err != nil {
|
||||
return credentials{}, permanentf("parsing ssh key file %s: %v", cfg.SSHKeyFile, err)
|
||||
}
|
||||
if cfg.InsecureIgnoreHostKey {
|
||||
return credentials{signer: signer, hostKey: ssh.InsecureIgnoreHostKey()}, nil
|
||||
}
|
||||
hostKey, err := knownhosts.New(cfg.KnownHostsFile)
|
||||
if err != nil {
|
||||
var pathErr *fs.PathError
|
||||
if errors.As(err, &pathErr) {
|
||||
return credentials{}, fmt.Errorf("reading known_hosts file %s: %w", cfg.KnownHostsFile, err)
|
||||
}
|
||||
return credentials{}, permanentf("parsing known_hosts file %s: %v", cfg.KnownHostsFile, err)
|
||||
}
|
||||
return credentials{signer: signer, hostKey: hostKey}, nil
|
||||
}
|
||||
|
||||
// sshSocket is the libvirt socket forwarded through one SSH connection. The
|
||||
// embedded Conn is the channel; Close also closes the SSH client, which is
|
||||
// what unblocks anything reading the channel.
|
||||
type sshSocket struct {
|
||||
net.Conn
|
||||
client *ssh.Client
|
||||
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
}
|
||||
|
||||
func (s *sshSocket) Close() error {
|
||||
s.closeOnce.Do(func() {
|
||||
s.closeErr = errors.Join(s.Conn.Close(), s.client.Close())
|
||||
})
|
||||
return s.closeErr
|
||||
}
|
||||
|
||||
// keepalive sends keepalive@openssh.com every interval until stop is closed
|
||||
// or a request fails or takes longer than the interval, and returns why it
|
||||
// stopped (nil for stop). The caller marks its link dead on an error.
|
||||
func (s *sshSocket) keepalive(interval time.Duration, stop <-chan struct{}) error {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, err := s.client.SendRequest(keepaliveRequest, true, nil)
|
||||
done <- err
|
||||
}()
|
||||
timer := time.NewTimer(interval)
|
||||
select {
|
||||
case err := <-done:
|
||||
timer.Stop()
|
||||
if err != nil {
|
||||
return fmt.Errorf("keepalive: %w", err)
|
||||
}
|
||||
case <-timer.C:
|
||||
return fmt.Errorf("keepalive: no reply within %s", interval)
|
||||
case <-stop:
|
||||
timer.Stop()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dialSSHSocket opens host's libvirt socket through SSH: credentials, TCP,
|
||||
// handshake, channel. timeout bounds the whole sequence on top of ctx. Each
|
||||
// failure is a *dialError naming its stage. Nothing here logs.
|
||||
func dialSSHSocket(ctx context.Context, host provider.KVMHost, cfg provider.KVMConfig, timeout time.Duration) (*sshSocket, error) {
|
||||
if timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
}
|
||||
creds, err := loadCredentials(cfg)
|
||||
if err != nil {
|
||||
return nil, &dialError{stage: stageCredentials, host: host.Name, err: err}
|
||||
}
|
||||
var nd net.Dialer
|
||||
tcp, err := nd.DialContext(ctx, "tcp", host.Address)
|
||||
if err != nil {
|
||||
return nil, &dialError{stage: stageTCP, host: host.Name, err: err}
|
||||
}
|
||||
clientCfg := &ssh.ClientConfig{
|
||||
User: host.User,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(creds.signer)},
|
||||
HostKeyCallback: creds.hostKey,
|
||||
Timeout: timeout,
|
||||
}
|
||||
client, err := await(ctx, tcp.Close, func() (*ssh.Client, error) {
|
||||
c, chans, reqs, err := ssh.NewClientConn(tcp, host.Address, clientCfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ssh.NewClient(c, chans, reqs), nil
|
||||
})
|
||||
if err != nil {
|
||||
_ = tcp.Close()
|
||||
return nil, &dialError{stage: stageSSH, host: host.Name, err: err}
|
||||
}
|
||||
conn, err := await(ctx, func() error { return client.Close() }, func() (net.Conn, error) {
|
||||
return client.Dial("unix", host.Socket)
|
||||
})
|
||||
if err != nil {
|
||||
_ = client.Close()
|
||||
return nil, &dialError{stage: stageSocket, host: host.Name, err: fmt.Errorf("opening %s: %w", host.Socket, err)}
|
||||
}
|
||||
return &sshSocket{Conn: conn, client: client}, nil
|
||||
}
|
||||
|
||||
// await runs a blocking step in a goroutine. When ctx ends first it calls
|
||||
// abort, which closes the transport the step is blocked on, waits for the
|
||||
// step to give up, and returns ctx's error instead.
|
||||
func await[T any](ctx context.Context, abort func() error, step func() (T, error)) (T, error) {
|
||||
type result struct {
|
||||
v T
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
v, err := step()
|
||||
done <- result{v, err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
return r.v, r.err
|
||||
case <-ctx.Done():
|
||||
_ = abort()
|
||||
<-done
|
||||
var zero T
|
||||
return zero, ctx.Err()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user