address() tried the guest agent then the host's ARP table and skipped any
error that was not a context error, returning ("", nil) after both. An
active domain with no IP maps to StateProvisioning; the controller clears
status.IP on that, and the health engine prunes the proxy's counters and
calls ForgetProxy in one tick once EffectiveHost() goes empty.
The damaging trigger was not the guest. The libvirt connection is pooled
per host, so when another reconcile's cancel or budget overrun killed it,
call returned "connection is closed" -- carrying no context sentinel, so
the check did not recognise it -- and Get returned a *successful*
instance in Provisioning with an empty IP. An unrelated proxy's timeout
dropped a healthy proxy out of rotation.
The discriminator was already in the spec without the conclusion being
drawn: during a normal boot on beelink exactly one source fails. The
agent errors because it is not up yet, while ARP answers without error
and with nothing, since the host never sees the guest's traffic. So one
source failing stays "no address yet"; every source failing, or a
connection already dead, is ErrTransient.
The explicit provider.ErrTransient wrap is load-bearing, not decoration.
Without it classify picks the first libvirt code out of the joined error,
and ErrInvalidArg, ErrAccessDenied and ErrNoSupport are Permanent -- an
address read failing would latch a running proxy as Failed, a worse
version of the harm being fixed.
Half of the finding is deliberately still open, and it is the half that
fires more often: a guest agent restarting inside a running VM still
demotes it, because that is indistinguishable from a VM still booting.
The discriminator is the previous IP, which the provider does not hold by
design and the controller already has. Follow-up finding to file against
proxy_controller.go:208-211 -- on StateProvisioning, keep an existing IP
and Provisioned condition for a few polls instead of clearing on the
first. The kvm2-local alternative (use the mark's createdAt as a
provisioning window) was rejected: createdAt is creation, not boot, and
it invents a deadline that belongs to the controller.
TestGet_addressErrorIsProvisioning failed against the new rule, correctly
-- it fails both sources with ErrAgentUnresponsive, and code 86 is
agent-specific, so the ARP source cannot produce it. Re-pointed as
TestGet_agentFailsAndARPEmptyIsProvisioning, the beelink boot it was
reaching for, which nothing covered before.
TestGet_everyAddressSourceFailingIsTransient covers the other side, both
sources erroring and the pooled link dying under the read; against the
unfixed code both return a Provisioning instance instead of an error.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
296 lines
9.7 KiB
Go
296 lines
9.7 KiB
Go
package kvm2
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/digitalocean/go-libvirt"
|
|
|
|
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
|
)
|
|
|
|
func TestGet_afterDeleteReturnsNotFound(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
id := fx.create(testName)
|
|
if err := fx.p.Delete(fx.ctx, id); err != nil {
|
|
t.Fatalf("Delete() error = %v", err)
|
|
}
|
|
inst, err := fx.p.Get(fx.ctx, id)
|
|
if inst != nil {
|
|
t.Errorf("Get after Delete = %+v, want nil", inst)
|
|
}
|
|
assertProviderError(t, err, "get", provider.ErrNotFound)
|
|
}
|
|
|
|
func TestGet_duringDeleteStillReturnsInstance(t *testing.T) {
|
|
t.Parallel()
|
|
t.Run("destroyed but still defined is stopped", func(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
id := fx.create(testName)
|
|
fx.withDomain(testName, func(d *fakeDomain) { d.state = libvirt.DomainShutoff })
|
|
inst := fx.get(id)
|
|
if inst.State != provider.StateStopped || inst.UID != testUID {
|
|
t.Errorf("Get = %+v, want Stopped with uid %s", inst, testUID)
|
|
}
|
|
})
|
|
t.Run("undefine in flight still answers", func(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
id := fx.create(testName)
|
|
release := fx.fake.blockOn("DomainUndefineFlags")
|
|
done := make(chan error, 1)
|
|
go func() { done <- fx.p.Delete(fx.ctx, id) }()
|
|
waitFor(t, "the undefine to block", func() bool { return fx.fake.callCount("DomainUndefineFlags") == 1 })
|
|
|
|
inst, err := fx.p.Get(fx.ctx, id)
|
|
if err != nil {
|
|
t.Fatalf("Get during Delete error = %v, want the instance", err)
|
|
}
|
|
if inst.State != provider.StateStopped || inst.ID != id {
|
|
t.Errorf("Get during Delete = %+v, want Stopped %s", inst, id)
|
|
}
|
|
close(release)
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Delete() error = %v", err)
|
|
}
|
|
_, err = fx.p.Get(fx.ctx, id)
|
|
assertProviderError(t, err, "get", provider.ErrNotFound)
|
|
})
|
|
}
|
|
|
|
func TestGet_backendBusyIsTransientNeverNotFound(t *testing.T) {
|
|
t.Parallel()
|
|
tests := []struct {
|
|
name string
|
|
err error
|
|
}{
|
|
{name: "operation invalid", err: fakeErr(libvirt.ErrOperationInvalid, "Requested operation is not valid: domain is being migrated")},
|
|
{name: "call timed out", err: fmt.Errorf("%w: libvirt call DomainGetState on beelink exceeded its 30s budget: %w", provider.ErrTransient, context.DeadlineExceeded)},
|
|
{name: "unknown libvirt code", err: fakeErr(libvirt.ErrInternalError, "internal error: qemu monitor unresponsive")},
|
|
{name: "not a libvirt error", err: errors.New("read tcp: connection reset by peer")},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
id := fx.create(testName)
|
|
fx.fake.failWith("DomainGetState", tc.err)
|
|
inst, err := fx.p.Get(fx.ctx, id)
|
|
if inst != nil {
|
|
t.Errorf("Get = %+v, want nil", inst)
|
|
}
|
|
assertProviderError(t, err, "get", provider.ErrTransient)
|
|
if errors.Is(err, provider.ErrNotFound) {
|
|
t.Errorf("Get error = %v, must never be NotFound", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGet_runningWithoutIPIsProvisioning(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
inst := fx.get(fx.create(testName))
|
|
if inst.State != provider.StateProvisioning || inst.IP != "" {
|
|
t.Errorf("Get = %+v, want Provisioning with no IP", inst)
|
|
}
|
|
}
|
|
|
|
func TestGet_runningWithIPIsRunning(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
id := fx.create(testName)
|
|
fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcAgent, "127.0.0.1", "169.254.1.2", "10.0.0.5")
|
|
inst := fx.get(id)
|
|
if inst.State != provider.StateRunning || inst.IP != "10.0.0.5" {
|
|
t.Errorf("Get = %+v, want Running with 10.0.0.5", inst)
|
|
}
|
|
if got := fx.logs.matching("kvm2 domain fetched"); len(got) != 1 || !strings.Contains(got[0], `"ip"="10.0.0.5"`) || !strings.Contains(got[0], `"state"="Running"`) {
|
|
t.Errorf("fetched line = %v, want state and ip", got)
|
|
}
|
|
}
|
|
|
|
func TestGet_agentPreferredOverARP(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
id := fx.create(testName)
|
|
fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcAgent, "10.0.0.5")
|
|
fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcArp, "10.0.0.6")
|
|
before := fx.fake.callCount("DomainInterfaceAddresses")
|
|
inst := fx.get(id)
|
|
if inst.IP != "10.0.0.5" {
|
|
t.Errorf("IP = %q, want the agent's 10.0.0.5", inst.IP)
|
|
}
|
|
if got := fx.fake.callCount("DomainInterfaceAddresses") - before; got != 1 {
|
|
t.Errorf("DomainInterfaceAddresses calls = %d, want 1 (ARP not consulted)", got)
|
|
}
|
|
}
|
|
|
|
func TestGet_arpFallbackWhenAgentFails(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 2)
|
|
id := fx.create(testName)
|
|
fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcArp, "10.0.0.6")
|
|
var calls int
|
|
fx.fake.mu.Lock()
|
|
fx.fake.hooks["DomainInterfaceAddresses"] = func() error {
|
|
calls++
|
|
if calls%2 == 1 {
|
|
return fakeErr(libvirt.ErrAgentUnresponsive, "Guest agent is not responding")
|
|
}
|
|
return nil
|
|
}
|
|
fx.fake.mu.Unlock()
|
|
inst := fx.get(id)
|
|
if inst.State != provider.StateRunning || inst.IP != "10.0.0.6" {
|
|
t.Errorf("Get = %+v, want Running with ARP's 10.0.0.6", inst)
|
|
}
|
|
if got := fx.logs.matching("no addresses from source"); len(got) != 1 || !strings.Contains(got[0], `"source"="agent"`) {
|
|
t.Errorf("agent failure line = %v", got)
|
|
}
|
|
}
|
|
|
|
// TestGet_agentFailsAndARPEmptyIsProvisioning is a VM part way through
|
|
// booting on a plain bridge: the agent is not up yet, and the host's ARP
|
|
// table answers without error because it has nothing to report. One
|
|
// source failing must stay "no address yet", not an error.
|
|
func TestGet_agentFailsAndARPEmptyIsProvisioning(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 2)
|
|
id := fx.create(testName)
|
|
var calls int
|
|
fx.fake.mu.Lock()
|
|
fx.fake.hooks["DomainInterfaceAddresses"] = func() error {
|
|
calls++
|
|
if calls%2 == 1 {
|
|
return fakeErr(libvirt.ErrAgentUnresponsive, "Guest agent is not responding")
|
|
}
|
|
return nil
|
|
}
|
|
fx.fake.mu.Unlock()
|
|
inst := fx.get(id)
|
|
if inst.State != provider.StateProvisioning || inst.IP != "" {
|
|
t.Errorf("Get = %+v, want Provisioning with no IP", inst)
|
|
}
|
|
}
|
|
|
|
// TestGet_everyAddressSourceFailingIsTransient covers the other side of
|
|
// that line. A running VM whose address cannot be read is not a VM
|
|
// without an address: reporting Provisioning would make the controller
|
|
// clear its IP and the health engine drop its counters.
|
|
func TestGet_everyAddressSourceFailingIsTransient(t *testing.T) {
|
|
t.Parallel()
|
|
tests := []struct {
|
|
name string
|
|
setup func(fx *fixture)
|
|
}{
|
|
{
|
|
name: "both sources error",
|
|
setup: func(fx *fixture) {
|
|
fx.fake.failWith("DomainInterfaceAddresses", fakeErr(libvirt.ErrOperationInvalid, "domain is not running"))
|
|
},
|
|
},
|
|
{
|
|
name: "the pooled connection died under the read",
|
|
setup: func(fx *fixture) {
|
|
fx.fake.failWith("DomainInterfaceAddresses", transientf("libvirt call DomainInterfaceAddresses on beelink: connection is closed: keepalive failed"))
|
|
fx.fake.markDead()
|
|
},
|
|
},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
id := fx.create(testName)
|
|
tc.setup(fx)
|
|
inst, err := fx.p.Get(fx.ctx, id)
|
|
if inst != nil {
|
|
t.Errorf("Get = %+v, want no instance", inst)
|
|
}
|
|
assertProviderError(t, err, "get", provider.ErrTransient)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGet_readsUIDAndCreatedAtFromBackend(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
created := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)
|
|
fx.p.now = func() time.Time { return created }
|
|
id := fx.create(testName)
|
|
inst := fx.get(id)
|
|
if inst.UID != testUID || !inst.CreatedAt.Equal(created) {
|
|
t.Errorf("Get = %+v, want uid %s created %s", inst, testUID, created)
|
|
}
|
|
|
|
later := created.Add(48 * time.Hour)
|
|
fx.withDomain(testName, func(d *fakeDomain) { d.metadata = managedMetadata("rewritten-uid", later) })
|
|
inst = fx.get(id)
|
|
if inst.UID != "rewritten-uid" || !inst.CreatedAt.Equal(later) {
|
|
t.Errorf("Get after metadata change = %+v, want uid rewritten-uid created %s (read back, not remembered)", inst, later)
|
|
}
|
|
}
|
|
|
|
func TestGet_unmanagedDomainHasEmptyUID(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
fx.fake.addDomain("legacy-vm", libvirt.DomainRunning, "")
|
|
inst := fx.get(formatProviderID(testHost, "legacy-vm"))
|
|
if inst.UID != "" || !inst.CreatedAt.IsZero() || inst.State != provider.StateProvisioning {
|
|
t.Errorf("Get = %+v, want empty uid, zero createdAt, Provisioning", inst)
|
|
}
|
|
}
|
|
|
|
func TestGet_malformedIDIsPermanent(t *testing.T) {
|
|
t.Parallel()
|
|
ids := []string{"", "proxy-x", "beelink/proxy-x", "hosts/beelink/proxy-x", "hosts//domains/proxy-x", "hosts/beelink/domains/", "hosts/beelink/domains/a/b", "nodes/beelink/domains/proxy-x"}
|
|
for _, id := range ids {
|
|
t.Run(id, func(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
inst, err := fx.p.Get(fx.ctx, id)
|
|
if inst != nil {
|
|
t.Errorf("Get(%q) = %+v, want nil", id, inst)
|
|
}
|
|
assertProviderError(t, err, "get", provider.ErrPermanent)
|
|
var pe *provider.Error
|
|
if errors.As(err, &pe) && pe.ID != id {
|
|
t.Errorf("error.ID = %q, want %q", pe.ID, id)
|
|
}
|
|
if got := fx.dialer.dials.Load(); got != 0 {
|
|
t.Errorf("dials = %d, want 0", got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGet_unconfiguredHostIsPermanent(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
_, err := fx.p.Get(fx.ctx, formatProviderID("nas9", testName))
|
|
assertProviderError(t, err, "get", provider.ErrPermanent)
|
|
if !strings.Contains(err.Error(), `"nas9"`) {
|
|
t.Errorf("error = %v, want the host named", err)
|
|
}
|
|
if got := fx.dialer.dials.Load(); got != 0 {
|
|
t.Errorf("dials = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestGet_unknownStateIsStopped(t *testing.T) {
|
|
t.Parallel()
|
|
fx := newFixture(t, 1)
|
|
fx.fake.addDomain(testName, libvirt.DomainState(42), managedMetadata(testUID, time.Now()))
|
|
inst := fx.get(formatProviderID(testHost, testName))
|
|
if inst.State != provider.StateStopped {
|
|
t.Errorf("State = %s, want Stopped for an unknown backend state", inst.State)
|
|
}
|
|
}
|