kacerr/egress-proxies-operator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kvm2: Create, Get, Delete and ListByTag with the contract tests

Browse Source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
kacerr
2026-09-16 19:23:49 +00:00
co-authored by Claude Fable 5.1
parent 6b3c985b4a
commit 794a2aef3d
14 changed files with 2120 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/kvm2-provider-spec.md
+1 -1
View File
@@ -163,7 +163,7 @@ element `<instance xmlns="https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxie
with children `<managed>true</managed>`, `<uid>req.UID</uid>` and
`<createdAt>RFC3339 UTC</createdAt>`. The mark lands in the same call that
defines the domain, so there is no mark window. Read back with
`DomainGetMetadata(dom, DomainMetadataElement, uri, DomainAffectConfig)`;
`DomainGetMetadata(dom, DomainMetadataElement, uri, DomainAffectCurrent)`;
libvirt 11.6 returns the element without its namespace declaration, so the
parser matches on the local name `instance` only (the URI was already used to
select it). A domain without the element is not managed and reports an empty
docs/plans-executions/2026-09-16-1333-kvm2-provider.md
+41 -1
View File
@@ -8,7 +8,7 @@ Plan: [docs/plans/2026-09-16-1333-kvm2-provider.md](../plans/2026-09-16-1333-kvm
- [x] Step 3 — Pure builders: domain and volume XML, metadata, seed
- [x] Step 4 — Transport, seam, decorator, fakes
- [x] Step 5 — Image cache and pool-space precheck
- [ ] Step 6 — The four contract methods
- [x] Step 6 — The four contract methods
- [ ] Step 7 — Checker and the CRD comment
- [ ] Step 8 — Deployment files, Makefile, integration test
- [ ] Step 9 — Documentation
@@ -210,3 +210,43 @@ grep -n 'context.Background()' internal/provider/kvm2/*.go | grep -v _test.go #
```
All `ok`. Attestation: "files read outside the allowed set: none".
## Step 6 — The four contract methods
2026-09-16 18:36. Fifth subagent step: `create.go`, `get.go`, `delete.go`,
`list.go` and five test files (the package now has 150 top-level tests).
Create follows spec entry 10 step for step under a per-name lock, with the
adoption guard, the quota precheck, the image cache, an overlay that is
reused and a seed that is always rebuilt, file-type disks from resolved
paths, the metadata element in the define call, and a wait for running
capped at two minutes. Get maps only `ErrNoDomain` to NotFound, prefers the
guest agent over ARP, and reads UID and CreatedAt back every time. Delete is
idempotent at every step and a no-op success for an unconfigured host.
ListByTag fails whole per host and skips single unreadable domains.
Decisions the agent made, accepted: the adoption path also waits for
running, so a repeat Create on a shut-off domain blocks like a first one;
`address` and `ListByTag` propagate ctx cancellation instead of reading it as
"no address" or "skip", so a cancelled Get is Transient rather than a false
Provisioning; NotFound from Get gets its own V(1) line because polling to
NotFound is expected; the domain XML rides on the `domain defined` line at
V(2). Flagged, not solved: adopting a paused domain fails Transient on every
retry (`DomainCreate` refuses and the wait expires), which the spec does not
cover; recorded as a known gap for the docs step.
The agent found a one-word conflict between the brief and the spec
(`DomainAffectCurrent` versus `DomainAffectConfig` for `DomainGetMetadata`);
the code uses `DomainAffectCurrent`, and spec entry 8 was corrected to
match in this commit.
Commands (agent and orchestrator):
```sh
gofmt -l ./internal/provider/kvm2 ; go vet ./internal/provider/kvm2/ && go test -race -count=2 ./internal/provider/kvm2/
go build ./... && go test -short ./...
grep -n 'CloudInit' internal/provider/kvm2/create.go # only len(req.CloudInit) and the seed builder call
grep -n 'not implemented' internal/provider/kvm2/*.go # empty
grep -n 'provider.Wrap(' internal/provider/kvm2/*.go | grep -v _test.go | grep -v 'p.name' # empty
```
All `ok`. Attestation: "files read outside the allowed set: none".
internal/provider/kvm2/create.go
+265
View File
@@ -0,0 +1,265 @@
package kvm2
import (
"bytes"
"fmt"
"time"
"context"
"github.com/digitalocean/go-libvirt"
"github.com/go-logr/logr"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
// The wait after DomainCreate (spec entry 18): Create returns only once the
// domain is running, so Get never sees a domain kvm2 defined but did not
// start (spec entry 9).
const (
defaultRunningWait = 2 * time.Minute
defaultRunningPoll = 500 * time.Millisecond
)
// Create is spec entry 10: validate placement, then, under the per-name
// lock, find or build the domain step by step, every step idempotent, and
// return once it is running.
func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
host, err := p.host(req.Placement.Zone)
if err != nil {
if req.Placement.Zone != "" {
err = permanentf("placement.zone %q is not in kvm2.hosts", req.Placement.Zone)
}
return "", p.wrapErr("create", "", err)
}
mt, ok := p.cfg.MachineTypes[req.Placement.MachineType]
if !ok {
return "", p.wrapErr("create", "", permanentf("placement.machineType %q is not in kvm2.machineTypes", req.Placement.MachineType))
}
src, err := resolveImage(p.cfg, req.Placement.Image)
if err != nil {
return "", p.wrapErr("create", "", err)
}
id := formatProviderID(host.Name, req.Name)
log := p.logger(ctx).WithValues("host", host.Name, "name", req.Name)
log.V(2).Info("kvm2 create resolved",
"pool", host.Pool, "bridge", host.Bridge, "machineType", req.Placement.MachineType,
"vcpus", mt.VCPUs, "memoryMiB", mt.MemoryMiB, "diskSizeGb", p.cfg.DiskSizeGB,
"image", src.volumeName(), "firmware", src.Firmware, "cloudInitBytes", len(req.CloudInit))
unlock, err := p.lockName(ctx, req.Name)
if err != nil {
return "", p.wrapErr("create", id, err)
}
defer unlock()
be, err := p.hosts.get(ctx, host)
if err != nil {
logAPIError(log, "create", err)
return "", p.wrapErr("create", id, err)
}
if err := p.create(ctx, log, be, host, req, mt, src); err != nil {
logAPIError(log, "create", err)
return "", p.wrapErr("create", id, err)
}
log.V(1).Info("kvm2 domain ensured running", "id", id)
return id, nil
}
func (p *Provider) create(ctx context.Context, log logr.Logger, be backend, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType, src imageSource) error {
dom, err := be.DomainLookupByName(ctx, req.Name)
switch {
case err == nil:
meta, err := readMetadata(ctx, be, dom)
if err != nil {
return err
}
if meta.UID != "" && meta.UID != req.UID {
return permanentf("domain %s belongs to proxy uid %s, not %s", req.Name, meta.UID, req.UID)
}
log.V(1).Info("kvm2 domain already defined", "uid", meta.UID)
return p.ensureRunning(ctx, log, be, dom)
case !hasCode(err, libvirt.ErrNoDomain):
return fmt.Errorf("looking up domain %s: %w", req.Name, err)
}
pool, err := be.StoragePoolLookupByName(ctx, host.Pool)
if err != nil {
return fmt.Errorf("looking up storage pool %s: %w", host.Pool, err)
}
imageBytes, err := p.imageSizeIfUncached(ctx, be, pool, src)
if err != nil {
return err
}
if err := checkPoolSpace(ctx, be, pool, host.Pool, neededBytes(imageBytes, p.cfg.DiskSizeGB)); err != nil {
return err
}
basePath, err := p.ensureImage(ctx, host, pool, be, src)
if err != nil {
return err
}
overlay, err := ensureOverlay(ctx, log, be, pool, req.Name, basePath, src.Format, p.cfg.DiskSizeGB)
if err != nil {
return err
}
seed, err := p.writeSeed(ctx, log, be, pool, req)
if err != nil {
return err
}
diskPath, err := be.StorageVolGetPath(ctx, overlay)
if err != nil {
return fmt.Errorf("reading the path of overlay volume %s: %w", overlay.Name, err)
}
seedPath, err := be.StorageVolGetPath(ctx, seed)
if err != nil {
return fmt.Errorf("reading the path of seed volume %s: %w", seed.Name, err)
}
xmlDoc, err := buildDomainXML(domainParams{
Name: req.Name,
UID: req.UID,
CreatedAt: p.now(),
VCPUs: mt.VCPUs,
MemoryMiB: mt.MemoryMiB,
Firmware: src.Firmware,
Bridge: host.Bridge,
DiskPath: diskPath,
SeedPath: seedPath,
DiskFormat: provider.KVMFormatQcow2,
})
if err != nil {
return err
}
dom, err = be.DomainDefineXML(ctx, xmlDoc)
if err != nil {
return fmt.Errorf("defining domain %s: %w", req.Name, err)
}
kv := []any{"uid", req.UID, "firmware", src.Firmware}
if log.V(2).Enabled() {
kv = append(kv, "xml", xmlDoc)
}
log.V(1).Info("domain defined", kv...)
return p.ensureRunning(ctx, log, be, dom)
}
// ensureOverlay creates the per-VM qcow2 overlay backed by the base image,
// or reuses the one a previous attempt left: volume creation is atomic, so
// an existing overlay is complete.
func ensureOverlay(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, name, basePath, baseFormat string, diskSizeGB int64) (libvirt.StorageVol, error) {
volName := overlayVolumeName(name)
xmlDoc, err := buildVolumeXML(volumeSpec{
Name: volName,
CapacityBytes: diskSizeGB << 30,
Format: provider.KVMFormatQcow2,
BackingPath: basePath,
BackingFormat: baseFormat,
})
if err != nil {
return libvirt.StorageVol{}, err
}
vol, err := be.StorageVolCreateXML(ctx, pool, xmlDoc, 0)
switch {
case err == nil:
log.V(1).Info("overlay volume created", "volume", volName)
return vol, nil
case hasCode(err, libvirt.ErrStorageVolExist):
vol, err = be.StorageVolLookupByName(ctx, pool, volName)
if err != nil {
return libvirt.StorageVol{}, fmt.Errorf("looking up overlay volume %s: %w", volName, err)
}
log.V(1).Info("overlay volume already exists", "volume", volName)
return vol, nil
default:
return libvirt.StorageVol{}, fmt.Errorf("creating overlay volume %s: %w", volName, err)
}
}
// writeSeed builds the NoCloud seed and uploads it as a raw volume,
// replacing whatever a previous attempt left (spec entry 5): a cut-off
// upload would boot a VM without cloud-init. The ISO is tens of KiB, so it
// is built in memory. Nothing about its content is logged or put in an
// error.
func (p *Provider) writeSeed(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, req provider.CreateRequest) (libvirt.StorageVol, error) {
name := seedVolumeName(req.Name)
if err := deleteVolumeIfPresent(ctx, be, pool, name); err != nil {
return libvirt.StorageVol{}, err
}
installAgent := p.cfg.InstallGuestAgent == nil || *p.cfg.InstallGuestAgent
vendorData, err := buildVendorData(p.cfg.VendorData, installAgent)
if err != nil {
return libvirt.StorageVol{}, permanentf("building vendor-data for %s: %v", req.Name, err)
}
var iso bytes.Buffer
if err := buildSeedISO(req.Name, req.CloudInit, vendorData, &iso); err != nil {
return libvirt.StorageVol{}, permanentf("building seed ISO for %s: %v", req.Name, err)
}
xmlDoc, err := buildVolumeXML(volumeSpec{Name: name, CapacityBytes: int64(iso.Len()), Format: provider.KVMFormatRaw})
if err != nil {
return libvirt.StorageVol{}, err
}
vol, err := be.StorageVolCreateXML(ctx, pool, xmlDoc, 0)
if err != nil {
return libvirt.StorageVol{}, fmt.Errorf("creating seed volume %s: %w", name, err)
}
if err := be.StorageVolUpload(ctx, vol, bytes.NewReader(iso.Bytes()), 0, uint64(iso.Len()), 0); err != nil {
return libvirt.StorageVol{}, fmt.Errorf("uploading seed volume %s: %w", name, err)
}
log.V(1).Info("seed volume uploaded", "volume", name, "bytes", iso.Len())
return vol, nil
}
// ensureRunning starts the domain unless it is active already, then waits
// until libvirt reports it running. ErrOperationInvalid from DomainCreate
// means another caller started it between the state read and the call.
func (p *Provider) ensureRunning(ctx context.Context, log logr.Logger, be backend, dom libvirt.Domain) error {
state, _, err := be.DomainGetState(ctx, dom, 0)
if err != nil {
return fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
}
if isActive(libvirt.DomainState(state)) {
return nil
}
switch err := be.DomainCreate(ctx, dom); {
case err == nil:
log.V(1).Info("domain started")
case hasCode(err, libvirt.ErrOperationInvalid):
log.V(1).Info("domain start reported it already running")
default:
return fmt.Errorf("starting domain %s: %w", dom.Name, err)
}
return p.waitRunning(ctx, be, dom)
}
// waitRunning polls DomainGetState until the domain is active, the cap
// passes (Transient: the next Create attempt finds the domain defined and
// tries again) or ctx ends.
func (p *Provider) waitRunning(ctx context.Context, be backend, dom libvirt.Domain) error {
deadline := time.NewTimer(p.runningWait)
defer deadline.Stop()
ticker := time.NewTicker(p.runningPoll)
defer ticker.Stop()
for {
state, _, err := be.DomainGetState(ctx, dom, 0)
if err != nil {
return fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
}
if isActive(libvirt.DomainState(state)) {
return nil
}
select {
case <-ticker.C:
case <-deadline.C:
return transientf("domain %s did not reach running within %s (state %s)", dom.Name, p.runningWait, libvirt.DomainState(state))
case <-ctx.Done():
return fmt.Errorf("%w: waiting for domain %s to run: %w", provider.ErrTransient, dom.Name, ctx.Err())
}
}
}
// logAPIError is the V(1) line per failed backend call of spec entry 20.
func logAPIError(log logr.Logger, op string, err error) {
kv := []any{"op", op, "error", err.Error()}
kv = append(kv, libvirtErrorFields(err)...)
log.V(1).Info("kvm2 backend call failed", kv...)
}
internal/provider/kvm2/create_test.go
+391
View File
@@ -0,0 +1,391 @@
package kvm2
import (
"bytes"
"errors"
"math"
"strconv"
"strings"
"testing"
"time"
"github.com/digitalocean/go-libvirt"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
func TestCreate_repeatReturnsSameIDAndCreatesNothing(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
first := fx.create(testName)
before := len(fx.fake.callLog())
defines, vols, uploads := fx.fake.callCount("DomainDefineXML"), fx.fake.callCount("StorageVolCreateXML"), fx.fake.callCount("StorageVolUpload")
second := fx.create(testName)
if second != first {
t.Errorf("second Create = %q, want %q", second, first)
}
if got := fx.fake.callCount("DomainDefineXML"); got != defines {
t.Errorf("DomainDefineXML calls after repeat = %d, want %d", got, defines)
}
if got := fx.fake.callCount("StorageVolCreateXML"); got != vols {
t.Errorf("StorageVolCreateXML calls after repeat = %d, want %d", got, vols)
}
if got := fx.fake.callCount("StorageVolUpload"); got != uploads {
t.Errorf("StorageVolUpload calls after repeat = %d, want %d", got, uploads)
}
if got := fx.fake.callCount("DomainCreate"); got != 1 {
t.Errorf("DomainCreate calls = %d, want 1", got)
}
repeat := fx.fake.callLog()[before:]
for _, rpc := range repeat {
switch rpc {
case "DomainLookupByName", "DomainGetMetadata", "DomainGetState":
default:
t.Errorf("repeat Create made %s; want only lookup, metadata and state reads: %v", rpc, repeat)
}
}
if len(fx.logs.matching("kvm2 domain already defined")) != 1 {
t.Errorf("no 'already defined' line:\n%s", strings.Join(fx.logs.all(), "\n"))
}
}
func TestCreate_resumesAfterEachPartialStep(t *testing.T) {
t.Parallel()
overlay, seed := overlayVolumeName(testName), seedVolumeName(testName)
tests := []struct {
name string
setup func(fx *fixture)
check func(t *testing.T, fx *fixture)
}{
{
name: "base without marker is refetched",
setup: func(fx *fixture) {
fx.putVol("base-debian-13.qcow2", []byte("truncated"))
},
check: func(t *testing.T, fx *fixture) {
if got := fx.fake.vol("base-debian-13.qcow2"); got == nil || !bytes.Equal(got.content, fx.image) {
t.Errorf("base volume not refetched")
}
if fx.fake.vol("base-debian-13.qcow2.sha256") == nil {
t.Errorf("marker not written")
}
},
},
{
name: "overlay exists is reused",
setup: func(fx *fixture) {
fx.cacheImage()
fx.putVol(overlay, []byte("overlay"))
},
check: func(t *testing.T, fx *fixture) {
if got := fx.fake.vol(overlay); got == nil || string(got.content) != "overlay" {
t.Errorf("overlay volume was replaced")
}
if len(fx.logs.matching("overlay volume already exists")) != 1 {
t.Errorf("no reuse line:\n%s", strings.Join(fx.logs.all(), "\n"))
}
},
},
{
name: "truncated seed is replaced",
setup: func(fx *fixture) {
fx.cacheImage()
fx.putVol(seed, []byte("cut"))
},
check: func(t *testing.T, fx *fixture) {
got := fx.fake.vol(seed)
if got == nil || len(got.content) <= 3 || !bytes.Contains(got.content, []byte(seedLabel)) {
t.Errorf("seed volume not rebuilt")
}
},
},
{
name: "domain defined but shutoff is started",
setup: func(fx *fixture) {
fx.cacheImage()
fx.putVol(overlay, []byte("overlay"))
fx.putVol(seed, []byte("seed"))
fx.fake.addDomain(testName, libvirt.DomainShutoff, managedMetadata(testUID, time.Now()))
},
check: func(t *testing.T, fx *fixture) {
if got := fx.fake.callCount("DomainDefineXML"); got != 0 {
t.Errorf("DomainDefineXML calls = %d, want 0", got)
}
if got := fx.fake.callCount("DomainCreate"); got != 1 {
t.Errorf("DomainCreate calls = %d, want 1", got)
}
},
},
{
name: "domain running is returned as is",
setup: func(fx *fixture) {
fx.fake.addDomain(testName, libvirt.DomainRunning, managedMetadata(testUID, time.Now()))
},
check: func(t *testing.T, fx *fixture) {
for _, rpc := range []string{"DomainDefineXML", "DomainCreate", "StorageVolCreateXML"} {
if got := fx.fake.callCount(rpc); got != 0 {
t.Errorf("%s calls = %d, want 0", rpc, got)
}
}
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
tc.setup(fx)
fx.create(testName)
fx.withDomain(testName, func(d *fakeDomain) {
if d.state != libvirt.DomainRunning {
t.Errorf("domain state = %s, want running", d.state)
}
})
tc.check(t, fx)
})
}
}
func TestCreate_returnsOnlyWhenRunning(t *testing.T) {
t.Parallel()
t.Run("returns after the domain reports running", func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.fake.startDelayPolls = 3
fx.create(testName)
fx.withDomain(testName, func(d *fakeDomain) {
if d.state != libvirt.DomainRunning {
t.Errorf("Create returned with domain state %s", d.state)
}
})
if got := fx.fake.callCount("DomainGetState"); got < 4 {
t.Errorf("DomainGetState calls = %d, want at least 4 (one before start, three polls)", got)
}
})
t.Run("never running is transient after the cap", func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1, withRunningWait(time.Millisecond, 30*time.Millisecond))
fx.fake.startDelayPolls = math.MaxInt32
start := time.Now()
id, err := fx.p.Create(fx.ctx, fx.req(testName))
if id != "" {
t.Errorf("Create = %q, want no ID", id)
}
assertProviderError(t, err, "create", provider.ErrTransient)
if !strings.Contains(err.Error(), "did not reach running") {
t.Errorf("error = %v, want the wait cap named", err)
}
if elapsed := time.Since(start); elapsed > 2*time.Second {
t.Errorf("Create took %v, want the shrunk cap to end it", elapsed)
}
})
}
func TestCreate_rejectsUnknownPlacement(t *testing.T) {
t.Parallel()
tests := []struct {
name string
mutate func(req *provider.CreateRequest)
wantMsg string
}{
{
name: "zone",
mutate: func(req *provider.CreateRequest) { req.Placement.Zone = "nas9" },
wantMsg: `placement.zone "nas9" is not in kvm2.hosts`,
},
{
name: "machineType",
mutate: func(req *provider.CreateRequest) { req.Placement.MachineType = "xlarge" },
wantMsg: `placement.machineType "xlarge" is not in kvm2.machineTypes`,
},
{
name: "image",
mutate: func(req *provider.CreateRequest) { req.Placement.Image = "ubuntu-24" },
wantMsg: `placement.image "ubuntu-24" is neither a key in kvm2.images nor an http(s) URL`,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
req := fx.req(testName)
tc.mutate(&req)
id, err := fx.p.Create(fx.ctx, req)
if id != "" {
t.Errorf("Create = %q, want no ID", id)
}
assertProviderError(t, err, "create", provider.ErrPermanent)
if !strings.Contains(err.Error(), tc.wantMsg) {
t.Errorf("error = %q, want substring %q", err, tc.wantMsg)
}
if got := fx.fake.callLog(); len(got) != 0 {
t.Errorf("backend calls = %v, want none before validation", got)
}
if got := fx.dialer.dials.Load(); got != 0 {
t.Errorf("dials = %d, want 0", got)
}
})
}
}
func TestCreate_adoptionWithForeignUIDIsPermanent(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.fake.addDomain(testName, libvirt.DomainRunning, managedMetadata("other-uid", time.Now()))
id, err := fx.p.Create(fx.ctx, fx.req(testName))
if id != "" {
t.Errorf("Create = %q, want no ID", id)
}
assertProviderError(t, err, "create", provider.ErrPermanent)
if !strings.Contains(err.Error(), "other-uid") || !strings.Contains(err.Error(), testUID) {
t.Errorf("error = %q, want both UIDs named", err)
}
for _, rpc := range []string{"DomainCreate", "DomainDefineXML", "StorageVolCreateXML", "DomainDestroy"} {
if got := fx.fake.callCount(rpc); got != 0 {
t.Errorf("%s calls = %d, want 0", rpc, got)
}
}
}
func TestCreate_adoptsMatchingOrEmptyUID(t *testing.T) {
t.Parallel()
tests := []struct {
name string
metadata string
}{
{name: "matching uid", metadata: managedMetadata(testUID, time.Now())},
{name: "empty uid", metadata: managedMetadata("", time.Now())},
{name: "no metadata element", metadata: ""},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.fake.addDomain(testName, libvirt.DomainRunning, tc.metadata)
fx.create(testName)
if got := fx.fake.callCount("DomainDefineXML"); got != 0 {
t.Errorf("DomainDefineXML calls = %d, want 0 (adopted)", got)
}
})
}
}
func TestCreate_poolOutOfSpaceIsQuotaExceeded(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.fake.mu.Lock()
fx.fake.pools[fakePoolName].available = 1 << 30
fx.fake.mu.Unlock()
_, err := fx.p.Create(fx.ctx, fx.req(testName))
assertProviderError(t, err, "create", provider.ErrQuotaExceeded)
if !strings.Contains(err.Error(), fakePoolName) || !strings.Contains(err.Error(), "needed") {
t.Errorf("error = %q, want the pool and the needed bytes named", err)
}
for _, rpc := range []string{"StorageVolCreateXML", "StorageVolUpload", "DomainDefineXML"} {
if got := fx.fake.callCount(rpc); got != 0 {
t.Errorf("%s calls = %d, want 0", rpc, got)
}
}
}
func TestCreate_libvirtNotFoundIsTransientNotNotFound(t *testing.T) {
t.Parallel()
tests := []struct {
name string
rpc string
err error
}{
{name: "volume vanished", rpc: "StorageVolGetPath", err: fakeErr(libvirt.ErrNoStorageVol, "Storage volume not found")},
{name: "domain vanished", rpc: "DomainGetState", err: fakeErr(libvirt.ErrNoDomain, "Domain not found")},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.fake.failWith(tc.rpc, tc.err)
_, err := fx.p.Create(fx.ctx, fx.req(testName))
assertProviderError(t, err, "create", provider.ErrTransient)
if errors.Is(err, provider.ErrNotFound) {
t.Errorf("Create error = %v, must never be NotFound", err)
}
})
}
}
func TestCreate_cloudInitSentinelNeverLoggedOrInErrors(t *testing.T) {
t.Parallel()
const sentinel = "SENTINEL-s3cr3t-passw0rd-7f3a"
fx := newFixture(t, 2)
fx.dialer.backend = observe(fx.fake, testProvider, testHost)
fx.fake.failWith("StorageVolUpload", fakeErr(libvirt.ErrInternalError, "internal error: upload failed"))
req := fx.req(testName)
req.CloudInit = "#cloud-config\nwrite_files:\n - content: " + sentinel + "\n"
_, err := fx.p.Create(fx.ctx, req)
assertProviderError(t, err, "create", provider.ErrTransient)
if strings.Contains(err.Error(), sentinel) {
t.Errorf("error carries the cloud-init: %v", err)
}
lines := fx.logs.all()
if len(lines) == 0 {
t.Fatal("no log lines captured at V(2)")
}
for _, l := range lines {
if strings.Contains(l, sentinel) {
t.Errorf("log line carries the cloud-init: %s", l)
}
}
if got := fx.logs.matching(`"cloudInitBytes"=` + strconv.Itoa(len(req.CloudInit))); len(got) != 1 {
t.Errorf("cloudInitBytes line = %d, want 1:\n%s", len(got), strings.Join(lines, "\n"))
}
}
func TestCreate_metadataWrittenInDefineCall(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.p.now = func() time.Time { return time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) }
fx.create(testName)
fx.withDomain(testName, func(d *fakeDomain) {
for _, want := range []string{`xmlns="` + metadataURI + `"`, "<uid>" + testUID + "</uid>", "<managed>true</managed>", "<createdAt>2026-09-16T12:00:00Z</createdAt>"} {
if !strings.Contains(d.xml, want) {
t.Errorf("defined XML lacks %s:\n%s", want, d.xml)
}
}
meta, err := parseInstanceMetadata(d.metadata)
if err != nil {
t.Fatalf("metadata the define stored does not parse: %v", err)
}
if !meta.managed() || meta.UID != testUID || meta.createdAt().IsZero() {
t.Errorf("metadata = %+v, want managed, uid %s and a createdAt", meta, testUID)
}
})
}
func TestCreate_seedAlwaysRebuilt(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.create(testName)
seed := seedVolumeName(testName)
first := fx.fake.vol(seed)
if first == nil {
t.Fatal("no seed volume after Create")
}
uploads := fx.fake.callCount("StorageVolUpload")
// A crash between the seed upload and the define leaves the volumes
// without a domain; the next attempt must not trust the seed.
fx.fake.mu.Lock()
delete(fx.fake.domains, testName)
fx.fake.mu.Unlock()
fx.create(testName)
second := fx.fake.vol(seed)
if second == nil || second == first {
t.Errorf("seed volume reused across attempts")
}
if got := fx.fake.callCount("StorageVolUpload"); got != uploads+1 {
t.Errorf("StorageVolUpload calls = %d, want %d", got, uploads+1)
}
if len(second.content) != len(first.content) || !bytes.Contains(second.content, []byte(seedLabel)) {
t.Errorf("rebuilt seed is %d bytes, first was %d; want a complete seed of the same size", len(second.content), len(first.content))
}
}
internal/provider/kvm2/delete.go
+98
View File
@@ -0,0 +1,98 @@
package kvm2
import (
"context"
"fmt"
"github.com/digitalocean/go-libvirt"
"github.com/go-logr/logr"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
// undefineFlags removes everything libvirt keeps for a domain beside its
// definition; UEFI domains keep NVRAM and undefine refuses without the flag.
const undefineFlags = libvirt.DomainUndefineManagedSave | libvirt.DomainUndefineSnapshotsMetadata | libvirt.DomainUndefineNvram
// Delete is spec entry 11: under the per-name lock, destroy and undefine
// the domain, then remove its overlay and seed. Everything already gone is
// success, and nothing is left when nil is returned, so Get afterwards is
// NotFound. The base image stays: it is a shared cache.
func (p *Provider) Delete(ctx context.Context, providerID string) error {
hostName, name, err := parseProviderID(providerID)
if err != nil {
return provider.Wrap(provider.ErrPermanent, "delete", p.name, providerID, err)
}
log := p.logger(ctx).WithValues("host", hostName, "name", name)
host, err := p.host(hostName)
if err != nil {
log.Info("host is not configured; delete has nothing to clean up", "id", providerID)
return nil
}
unlock, err := p.lockName(ctx, name)
if err != nil {
return p.wrapErr("delete", providerID, err)
}
defer unlock()
be, err := p.hosts.get(ctx, host)
if err != nil {
logAPIError(log, "delete", err)
return p.wrapErr("delete", providerID, err)
}
if err := removeDomain(ctx, log, be, name); err != nil {
logAPIError(log, "delete", err)
return p.wrapErr("delete", providerID, err)
}
if err := removeVolumes(ctx, log, be, host.Pool, name); err != nil {
logAPIError(log, "delete", err)
return p.wrapErr("delete", providerID, err)
}
log.V(1).Info("kvm2 domain deleted")
return nil
}
func removeDomain(ctx context.Context, log logr.Logger, be backend, name string) error {
dom, err := be.DomainLookupByName(ctx, name)
if hasCode(err, libvirt.ErrNoDomain) {
log.V(1).Info("domain already gone")
return nil
}
if err != nil {
return fmt.Errorf("looking up domain %s: %w", name, err)
}
switch err := be.DomainDestroy(ctx, dom); {
case err == nil:
log.V(1).Info("domain destroyed")
case hasCode(err, libvirt.ErrOperationInvalid):
log.V(1).Info("domain not running, destroy skipped")
case hasCode(err, libvirt.ErrNoDomain):
log.V(1).Info("domain already gone")
return nil
default:
return fmt.Errorf("destroying domain %s: %w", name, err)
}
if err := be.DomainUndefineFlags(ctx, dom, undefineFlags); err != nil && !hasCode(err, libvirt.ErrNoDomain) {
return fmt.Errorf("undefining domain %s: %w", name, err)
}
log.V(1).Info("domain undefined")
return nil
}
func removeVolumes(ctx context.Context, log logr.Logger, be backend, poolName, name string) error {
pool, err := be.StoragePoolLookupByName(ctx, poolName)
if hasCode(err, libvirt.ErrNoStoragePool) {
log.Info("pool not found, no volumes to delete", "pool", poolName)
return nil
}
if err != nil {
return fmt.Errorf("looking up storage pool %s: %w", poolName, err)
}
for _, vol := range []string{overlayVolumeName(name), seedVolumeName(name)} {
if err := deleteVolumeIfPresent(ctx, be, pool, vol); err != nil {
return err
}
}
log.V(1).Info("per-VM volumes deleted")
return nil
}
internal/provider/kvm2/delete_test.go
+222
View File
@@ -0,0 +1,222 @@
package kvm2
import (
"errors"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/digitalocean/go-libvirt"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
func TestDelete_malformedIDIsPermanent(t *testing.T) {
t.Parallel()
for _, id := range []string{"", "proxy-x", "hosts/beelink/proxy-x", "hosts//domains/proxy-x", "hosts/beelink/domains/"} {
t.Run(id, func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
err := fx.p.Delete(fx.ctx, id)
assertProviderError(t, err, "delete", provider.ErrPermanent)
var pe *provider.Error
if errors.As(err, &pe) && pe.ID != id {
t.Errorf("error.ID = %q, want %q", pe.ID, id)
}
if got := fx.dialer.dials.Load(); got != 0 {
t.Errorf("dials = %d, want 0", got)
}
})
}
}
func TestDelete_twiceSucceeds(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
for i := range 2 {
if err := fx.p.Delete(fx.ctx, id); err != nil {
t.Fatalf("Delete #%d error = %v", i+1, err)
}
}
if fx.hasDomain(testName) {
t.Error("domain still defined after Delete")
}
if got := fx.logs.matching("domain already gone"); len(got) != 1 {
t.Errorf("'already gone' lines = %d, want 1 from the second Delete", len(got))
}
if got := fx.logs.matching("kvm2 domain deleted"); len(got) != 2 {
t.Errorf("'deleted' lines = %d, want 2", len(got))
}
}
func TestDelete_removesOverlayAndSeed(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
if err := fx.p.Delete(fx.ctx, id); err != nil {
t.Fatalf("Delete() error = %v", err)
}
for _, name := range []string{overlayVolumeName(testName), seedVolumeName(testName)} {
if fx.fake.vol(name) != nil {
t.Errorf("volume %s still present after Delete", name)
}
}
for _, name := range []string{"base-debian-13.qcow2", "base-debian-13.qcow2.sha256"} {
if fx.fake.vol(name) == nil {
t.Errorf("shared base volume %s removed by Delete", name)
}
}
}
func TestDelete_runningDomainDestroyedFirst(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
before := len(fx.fake.callLog())
if err := fx.p.Delete(fx.ctx, id); err != nil {
t.Fatalf("Delete() error = %v", err)
}
calls := fx.fake.callLog()[before:]
destroy := slices.Index(calls, "DomainDestroy")
undefine := slices.Index(calls, "DomainUndefineFlags")
volDelete := slices.Index(calls, "StorageVolDelete")
if destroy < 0 || undefine < 0 || volDelete < 0 || !(destroy < undefine && undefine < volDelete) {
t.Errorf("call order = %v, want destroy before undefine before volume deletes", calls)
}
if n := strings.Count(strings.Join(calls, " "), "StorageVolDelete"); n != 2 {
t.Errorf("StorageVolDelete calls = %d, want 2 (overlay and seed)", n)
}
}
func TestDelete_stoppedDomainSkipsDestroy(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
fx.withDomain(testName, func(d *fakeDomain) { d.state = libvirt.DomainShutoff })
if err := fx.p.Delete(fx.ctx, id); err != nil {
t.Fatalf("Delete() error = %v", err)
}
if fx.hasDomain(testName) {
t.Error("domain still defined after Delete")
}
if got := fx.logs.matching("domain not running, destroy skipped"); len(got) != 1 {
t.Errorf("'destroy skipped' lines = %d, want 1:\n%s", len(got), strings.Join(fx.logs.all(), "\n"))
}
if got := fx.fake.callCount("DomainUndefineFlags"); got != 1 {
t.Errorf("DomainUndefineFlags calls = %d, want 1", got)
}
}
func TestDelete_toleratesMissingVolumesAndPool(t *testing.T) {
t.Parallel()
t.Run("domain without volumes", func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.fake.addDomain(testName, libvirt.DomainRunning, managedMetadata(testUID, time.Now()))
if err := fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)); err != nil {
t.Fatalf("Delete() error = %v", err)
}
if fx.hasDomain(testName) {
t.Error("domain still defined after Delete")
}
if got := fx.fake.callCount("StorageVolDelete"); got != 0 {
t.Errorf("StorageVolDelete calls = %d, want 0 for absent volumes", got)
}
})
t.Run("pool gone", func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 0)
fx.p.cfg.Hosts[0].Pool = "retired"
fx.fake.addDomain(testName, libvirt.DomainShutoff, managedMetadata(testUID, time.Now()))
if err := fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)); err != nil {
t.Fatalf("Delete() error = %v", err)
}
if fx.hasDomain(testName) {
t.Error("domain still defined after Delete")
}
if got := fx.logs.matching("pool not found, no volumes to delete"); len(got) != 1 || !strings.Contains(got[0], `"pool"="retired"`) {
t.Errorf("'pool not found' Info line = %v", got)
}
})
t.Run("nothing at all", func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
if err := fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)); err != nil {
t.Fatalf("Delete() error = %v", err)
}
})
}
func TestDelete_unconfiguredHostIsNoOp(t *testing.T) {
t.Parallel()
fx := newFixture(t, 0)
id := formatProviderID("nas9", testName)
if err := fx.p.Delete(fx.ctx, id); err != nil {
t.Fatalf("Delete() error = %v, want nil", err)
}
if got := fx.dialer.dials.Load(); got != 0 {
t.Errorf("dials = %d, want 0", got)
}
if got := fx.fake.callLog(); len(got) != 0 {
t.Errorf("backend calls = %v, want none", got)
}
if got := fx.logs.matching("host is not configured; delete has nothing to clean up"); len(got) != 1 || !strings.Contains(got[0], `"host"="nas9"`) {
t.Errorf("Info line = %v, want one naming the host", got)
}
}
func TestDelete_concurrentSameNameSafe(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
const n = 10
errs := make([]error, n)
var wg sync.WaitGroup
for i := range n {
wg.Add(1)
go func() {
defer wg.Done()
errs[i] = fx.p.Delete(fx.ctx, id)
}()
}
wg.Wait()
for i, err := range errs {
if err != nil {
t.Errorf("Delete #%d error = %v, want nil", i, err)
}
}
if fx.hasDomain(testName) {
t.Error("domain still defined after concurrent Deletes")
}
for _, name := range []string{overlayVolumeName(testName), seedVolumeName(testName)} {
if fx.fake.vol(name) != nil {
t.Errorf("volume %s still present", name)
}
}
if got := fx.fake.callCount("DomainUndefineFlags"); got != 1 {
t.Errorf("DomainUndefineFlags calls = %d, want 1 (the lock serialises the rest)", got)
}
}
func TestDelete_undefineUsesNvramFlag(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
if err := fx.p.Delete(fx.ctx, id); err != nil {
t.Fatalf("Delete() error = %v", err)
}
fx.fake.mu.Lock()
flags, ok := fx.fake.undefineFlags[testName]
fx.fake.mu.Unlock()
if !ok {
t.Fatal("DomainUndefineFlags was not called")
}
for _, want := range []libvirt.DomainUndefineFlagsValues{libvirt.DomainUndefineNvram, libvirt.DomainUndefineManagedSave, libvirt.DomainUndefineSnapshotsMetadata} {
if flags&want == 0 {
t.Errorf("undefine flags = %d, lack %d", flags, want)
}
}
}
internal/provider/kvm2/fake_test.go
+34 -10
View File
@@ -36,6 +36,13 @@ type fakeBackend struct {
// sha256 in fixed chunks instead of holding the bytes, for tests that
// push more through the fake than they want in memory.
discardUploads bool
// startDelayPolls, when set, makes DomainCreate leave the domain shut
// off; it turns running on the startDelayPolls-th DomainGetState after
// that, as a real host that is still booting QEMU would.
startDelayPolls int
// undefineFlags records the flags of the last DomainUndefineFlags per
// domain name.
undefineFlags map[string]libvirt.DomainUndefineFlagsValues
calls map[string]int
log []string
@@ -51,6 +58,9 @@ type fakeDomain struct {
addrs map[uint32][]libvirt.DomainInterface
uuid libvirt.UUID
id int32
// pendingStart counts the DomainGetState calls left before a delayed
// start (see startDelayPolls) turns the domain running.
pendingStart int
}
type fakePool struct {
@@ -72,15 +82,16 @@ const fakePoolName = "default"
func newFakeBackend() *fakeBackend {
return &fakeBackend{
mu: &sync.Mutex{},
domains: make(map[string]*fakeDomain),
pools: map[string]*fakePool{fakePoolName: newFakePool()},
hostname: "fake-host",
version: 11006000,
hooks: make(map[string]func() error),
block: make(map[string]chan struct{}),
calls: make(map[string]int),
deadCh: make(chan struct{}),
mu: &sync.Mutex{},
domains: make(map[string]*fakeDomain),
pools: map[string]*fakePool{fakePoolName: newFakePool()},
hostname: "fake-host",
version: 11006000,
hooks: make(map[string]func() error),
block: make(map[string]chan struct{}),
undefineFlags: make(map[string]libvirt.DomainUndefineFlagsValues),
calls: make(map[string]int),
deadCh: make(chan struct{}),
}
}
@@ -97,6 +108,7 @@ func (f *fakeBackend) twin() *fakeBackend {
hooks: make(map[string]func() error),
block: make(map[string]chan struct{}),
discardUploads: f.discardUploads,
undefineFlags: make(map[string]libvirt.DomainUndefineFlagsValues),
calls: make(map[string]int),
deadCh: make(chan struct{}),
}
@@ -279,6 +291,10 @@ func (f *fakeBackend) DomainCreate(ctx context.Context, dom libvirt.Domain) erro
if isActive(d.state) {
return fakeErr(libvirt.ErrOperationInvalid, "Requested operation is not valid: domain is already running")
}
if f.startDelayPolls > 0 {
d.pendingStart = f.startDelayPolls
return nil
}
d.state = libvirt.DomainRunning
d.id = int32(len(f.domains))
return nil
@@ -301,7 +317,7 @@ func (f *fakeBackend) DomainDestroy(ctx context.Context, dom libvirt.Domain) err
return nil
}
func (f *fakeBackend) DomainUndefineFlags(ctx context.Context, dom libvirt.Domain, _ libvirt.DomainUndefineFlagsValues) error {
func (f *fakeBackend) DomainUndefineFlags(ctx context.Context, dom libvirt.Domain, flags libvirt.DomainUndefineFlagsValues) error {
if err := f.enter(ctx, "DomainUndefineFlags"); err != nil {
return err
}
@@ -310,6 +326,7 @@ func (f *fakeBackend) DomainUndefineFlags(ctx context.Context, dom libvirt.Domai
if _, err := f.domain(dom.Name); err != nil {
return err
}
f.undefineFlags[dom.Name] = flags
delete(f.domains, dom.Name)
return nil
}
@@ -324,6 +341,13 @@ func (f *fakeBackend) DomainGetState(ctx context.Context, dom libvirt.Domain, _
if err != nil {
return 0, 0, err
}
if d.pendingStart > 0 {
d.pendingStart--
if d.pendingStart == 0 {
d.state = libvirt.DomainRunning
d.id = int32(len(f.domains))
}
}
return int32(d.state), 0, nil
}
internal/provider/kvm2/get.go
+127
View File
@@ -0,0 +1,127 @@
package kvm2
import (
"context"
"errors"
"fmt"
"github.com/digitalocean/go-libvirt"
"github.com/go-logr/logr"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
// Get is spec entries 7 to 9: look the domain up by name, read the mark
// back, then state and address. Only ErrNoDomain becomes NotFound; every
// other failure is classified, never guessed.
func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instance, error) {
hostName, name, err := parseProviderID(providerID)
if err != nil {
return nil, provider.Wrap(provider.ErrPermanent, "get", p.name, providerID, err)
}
host, err := p.host(hostName)
if err != nil {
return nil, p.wrapErr("get", providerID, err)
}
log := p.logger(ctx).WithValues("host", host.Name, "name", name)
be, err := p.hosts.get(ctx, host)
if err != nil {
logAPIError(log, "get", err)
return nil, p.wrapErr("get", providerID, err)
}
inst, err := get(ctx, log, be, host.Name, name)
if err != nil {
if hasCode(err, libvirt.ErrNoDomain) {
log.V(1).Info("kvm2 domain not found")
} else {
logAPIError(log, "get", err)
}
return nil, p.wrapErr("get", providerID, err)
}
log.V(1).Info("kvm2 domain fetched", "state", inst.State, "ip", inst.IP, "uid", inst.UID)
return inst, nil
}
func get(ctx context.Context, log logr.Logger, be backend, hostName, name string) (*provider.Instance, error) {
dom, err := be.DomainLookupByName(ctx, name)
if err != nil {
return nil, fmt.Errorf("looking up domain %s: %w", name, err)
}
meta, err := readMetadata(ctx, be, dom)
if err != nil {
return nil, err
}
inst, err := inspect(ctx, log, be, hostName, dom, meta)
if err != nil {
return nil, err
}
return &inst, nil
}
// readMetadata returns the kvm2 mark of a domain, or the zero value (not
// managed, empty UID) when the domain carries none. A mark that does not
// parse is Permanent, from parseInstanceMetadata.
func readMetadata(ctx context.Context, be backend, dom libvirt.Domain) (instanceMetadata, error) {
raw, err := be.DomainGetMetadata(ctx, dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, libvirt.DomainAffectCurrent)
if hasCode(err, libvirt.ErrNoDomainMetadata) {
return instanceMetadata{}, nil
}
if err != nil {
return instanceMetadata{}, fmt.Errorf("reading metadata of domain %s: %w", dom.Name, err)
}
return parseInstanceMetadata(raw)
}
// inspect builds the Instance from the backend's state and, for an active
// domain, its discovered address.
func inspect(ctx context.Context, log logr.Logger, be backend, hostName string, dom libvirt.Domain, meta instanceMetadata) (provider.Instance, error) {
state, _, err := be.DomainGetState(ctx, dom, 0)
if err != nil {
return provider.Instance{}, fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
}
s := libvirt.DomainState(state)
var ip string
if isActive(s) {
if ip, err = address(ctx, log, be, dom); err != nil {
return provider.Instance{}, err
}
}
return provider.Instance{
ID: formatProviderID(hostName, dom.Name),
IP: ip,
State: mapState(s, ip),
UID: meta.UID,
CreatedAt: meta.createdAt(),
}, nil
}
// addressSources is the discovery order of spec entry 7: the guest agent
// answers where the host's ARP table never sees guest traffic.
var addressSources = []struct {
name string
src libvirt.DomainInterfaceAddressesSource
}{
{"agent", libvirt.DomainInterfaceAddressesSrcAgent},
{"arp", libvirt.DomainInterfaceAddressesSrcArp},
}
// address returns the first usable IPv4 from the first source that yields
// one, or "" for "no address yet". A source that fails is logged and
// skipped; only ctx ending is an error, so a cancelled Get does not read
// as Provisioning.
func address(ctx context.Context, log logr.Logger, be backend, dom libvirt.Domain) (string, error) {
for _, s := range addressSources {
ifaces, err := be.DomainInterfaceAddresses(ctx, dom, uint32(s.src), 0)
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return "", fmt.Errorf("reading addresses of domain %s: %w", dom.Name, err)
}
log.V(2).Info("no addresses from source", "source", s.name, "error", err.Error())
continue
}
if ip := firstIPv4(ifaces); ip != "" {
return ip, nil
}
}
return "", nil
}
internal/provider/kvm2/get_test.go
+243
View File
@@ -0,0 +1,243 @@
package kvm2
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/digitalocean/go-libvirt"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
func TestGet_afterDeleteReturnsNotFound(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
if err := fx.p.Delete(fx.ctx, id); err != nil {
t.Fatalf("Delete() error = %v", err)
}
inst, err := fx.p.Get(fx.ctx, id)
if inst != nil {
t.Errorf("Get after Delete = %+v, want nil", inst)
}
assertProviderError(t, err, "get", provider.ErrNotFound)
}
func TestGet_duringDeleteStillReturnsInstance(t *testing.T) {
t.Parallel()
t.Run("destroyed but still defined is stopped", func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
fx.withDomain(testName, func(d *fakeDomain) { d.state = libvirt.DomainShutoff })
inst := fx.get(id)
if inst.State != provider.StateStopped || inst.UID != testUID {
t.Errorf("Get = %+v, want Stopped with uid %s", inst, testUID)
}
})
t.Run("undefine in flight still answers", func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
release := fx.fake.blockOn("DomainUndefineFlags")
done := make(chan error, 1)
go func() { done <- fx.p.Delete(fx.ctx, id) }()
waitFor(t, "the undefine to block", func() bool { return fx.fake.callCount("DomainUndefineFlags") == 1 })
inst, err := fx.p.Get(fx.ctx, id)
if err != nil {
t.Fatalf("Get during Delete error = %v, want the instance", err)
}
if inst.State != provider.StateStopped || inst.ID != id {
t.Errorf("Get during Delete = %+v, want Stopped %s", inst, id)
}
close(release)
if err := <-done; err != nil {
t.Fatalf("Delete() error = %v", err)
}
_, err = fx.p.Get(fx.ctx, id)
assertProviderError(t, err, "get", provider.ErrNotFound)
})
}
func TestGet_backendBusyIsTransientNeverNotFound(t *testing.T) {
t.Parallel()
tests := []struct {
name string
err error
}{
{name: "operation invalid", err: fakeErr(libvirt.ErrOperationInvalid, "Requested operation is not valid: domain is being migrated")},
{name: "call timed out", err: fmt.Errorf("%w: libvirt call DomainGetState on beelink exceeded its 30s budget: %w", provider.ErrTransient, context.DeadlineExceeded)},
{name: "unknown libvirt code", err: fakeErr(libvirt.ErrInternalError, "internal error: qemu monitor unresponsive")},
{name: "not a libvirt error", err: errors.New("read tcp: connection reset by peer")},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
fx.fake.failWith("DomainGetState", tc.err)
inst, err := fx.p.Get(fx.ctx, id)
if inst != nil {
t.Errorf("Get = %+v, want nil", inst)
}
assertProviderError(t, err, "get", provider.ErrTransient)
if errors.Is(err, provider.ErrNotFound) {
t.Errorf("Get error = %v, must never be NotFound", err)
}
})
}
}
func TestGet_runningWithoutIPIsProvisioning(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
inst := fx.get(fx.create(testName))
if inst.State != provider.StateProvisioning || inst.IP != "" {
t.Errorf("Get = %+v, want Provisioning with no IP", inst)
}
}
func TestGet_runningWithIPIsRunning(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcAgent, "127.0.0.1", "169.254.1.2", "10.0.0.5")
inst := fx.get(id)
if inst.State != provider.StateRunning || inst.IP != "10.0.0.5" {
t.Errorf("Get = %+v, want Running with 10.0.0.5", inst)
}
if got := fx.logs.matching("kvm2 domain fetched"); len(got) != 1 || !strings.Contains(got[0], `"ip"="10.0.0.5"`) || !strings.Contains(got[0], `"state"="Running"`) {
t.Errorf("fetched line = %v, want state and ip", got)
}
}
func TestGet_agentPreferredOverARP(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcAgent, "10.0.0.5")
fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcArp, "10.0.0.6")
before := fx.fake.callCount("DomainInterfaceAddresses")
inst := fx.get(id)
if inst.IP != "10.0.0.5" {
t.Errorf("IP = %q, want the agent's 10.0.0.5", inst.IP)
}
if got := fx.fake.callCount("DomainInterfaceAddresses") - before; got != 1 {
t.Errorf("DomainInterfaceAddresses calls = %d, want 1 (ARP not consulted)", got)
}
}
func TestGet_arpFallbackWhenAgentFails(t *testing.T) {
t.Parallel()
fx := newFixture(t, 2)
id := fx.create(testName)
fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcArp, "10.0.0.6")
var calls int
fx.fake.mu.Lock()
fx.fake.hooks["DomainInterfaceAddresses"] = func() error {
calls++
if calls%2 == 1 {
return fakeErr(libvirt.ErrAgentUnresponsive, "Guest agent is not responding")
}
return nil
}
fx.fake.mu.Unlock()
inst := fx.get(id)
if inst.State != provider.StateRunning || inst.IP != "10.0.0.6" {
t.Errorf("Get = %+v, want Running with ARP's 10.0.0.6", inst)
}
if got := fx.logs.matching("no addresses from source"); len(got) != 1 || !strings.Contains(got[0], `"source"="agent"`) {
t.Errorf("agent failure line = %v", got)
}
}
func TestGet_addressErrorIsProvisioning(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
fx.fake.failWith("DomainInterfaceAddresses", fakeErr(libvirt.ErrAgentUnresponsive, "Guest agent is not responding"))
inst := fx.get(id)
if inst.State != provider.StateProvisioning || inst.IP != "" {
t.Errorf("Get = %+v, want Provisioning with no IP", inst)
}
}
func TestGet_readsUIDAndCreatedAtFromBackend(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
created := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)
fx.p.now = func() time.Time { return created }
id := fx.create(testName)
inst := fx.get(id)
if inst.UID != testUID || !inst.CreatedAt.Equal(created) {
t.Errorf("Get = %+v, want uid %s created %s", inst, testUID, created)
}
later := created.Add(48 * time.Hour)
fx.withDomain(testName, func(d *fakeDomain) { d.metadata = managedMetadata("rewritten-uid", later) })
inst = fx.get(id)
if inst.UID != "rewritten-uid" || !inst.CreatedAt.Equal(later) {
t.Errorf("Get after metadata change = %+v, want uid rewritten-uid created %s (read back, not remembered)", inst, later)
}
}
func TestGet_unmanagedDomainHasEmptyUID(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.fake.addDomain("legacy-vm", libvirt.DomainRunning, "")
inst := fx.get(formatProviderID(testHost, "legacy-vm"))
if inst.UID != "" || !inst.CreatedAt.IsZero() || inst.State != provider.StateProvisioning {
t.Errorf("Get = %+v, want empty uid, zero createdAt, Provisioning", inst)
}
}
func TestGet_malformedIDIsPermanent(t *testing.T) {
t.Parallel()
ids := []string{"", "proxy-x", "beelink/proxy-x", "hosts/beelink/proxy-x", "hosts//domains/proxy-x", "hosts/beelink/domains/", "hosts/beelink/domains/a/b", "nodes/beelink/domains/proxy-x"}
for _, id := range ids {
t.Run(id, func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
inst, err := fx.p.Get(fx.ctx, id)
if inst != nil {
t.Errorf("Get(%q) = %+v, want nil", id, inst)
}
assertProviderError(t, err, "get", provider.ErrPermanent)
var pe *provider.Error
if errors.As(err, &pe) && pe.ID != id {
t.Errorf("error.ID = %q, want %q", pe.ID, id)
}
if got := fx.dialer.dials.Load(); got != 0 {
t.Errorf("dials = %d, want 0", got)
}
})
}
}
func TestGet_unconfiguredHostIsPermanent(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
_, err := fx.p.Get(fx.ctx, formatProviderID("nas9", testName))
assertProviderError(t, err, "get", provider.ErrPermanent)
if !strings.Contains(err.Error(), `"nas9"`) {
t.Errorf("error = %v, want the host named", err)
}
if got := fx.dialer.dials.Load(); got != 0 {
t.Errorf("dials = %d, want 0", got)
}
}
func TestGet_unknownStateIsStopped(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.fake.addDomain(testName, libvirt.DomainState(42), managedMetadata(testUID, time.Now()))
inst := fx.get(formatProviderID(testHost, testName))
if inst.State != provider.StateStopped {
t.Errorf("State = %s, want Stopped for an unknown backend state", inst.State)
}
}
internal/provider/kvm2/kvm2.go
+45 -28
View File
@@ -6,7 +6,7 @@ package kvm2
import (
"context"
"errors"
"fmt"
"net/http"
"os"
"sync"
@@ -36,9 +36,20 @@ type Provider struct {
budgets budgets
tracing []tracing.Option
// runningWait caps the wait for a started domain to report running,
// polled every runningPoll (spec entry 18).
runningWait time.Duration
runningPoll time.Duration
// imageLocks serialises image fetches per (host, volume), spec entry 19.
imageMu sync.Mutex
imageLocks map[string]chan struct{}
// nameLocks serialises Create and Delete per instance name, spec entry
// 19, so two Deletes never race on destroy and undefine and a Delete
// cannot interleave with a Create's steps.
nameMu sync.Mutex
nameLocks map[string]chan struct{}
}
// providerOption adjusts what newProvider builds; tests inject an HTTP
@@ -57,6 +68,12 @@ func withTracing(opts ...tracing.Option) providerOption {
return func(p *Provider) { p.tracing = opts }
}
// withRunningWait shrinks the poll interval and the cap of the wait for a
// domain to run.
func withRunningWait(poll, cap time.Duration) providerOption {
return func(p *Provider) { p.runningPoll, p.runningWait = poll, cap }
}
var _ provider.Provider = (*Provider)(nil)
// New builds a Provider from pc.KVM2. It never dials a host: the credential
@@ -90,13 +107,16 @@ func New(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, er
// per-request ctx carries the transfer budget) and the default budgets.
func newProvider(name string, cfg provider.KVMConfig, d dialer, opts ...providerOption) *Provider {
p := &Provider{
name: name,
cfg: withDefaults(cfg),
now: time.Now,
hosts: newHostTable(d),
http: http.DefaultClient,
budgets: defaultBudgets,
imageLocks: make(map[string]chan struct{}),
name: name,
cfg: withDefaults(cfg),
now: time.Now,
hosts: newHostTable(d),
http: http.DefaultClient,
budgets: defaultBudgets,
runningWait: defaultRunningWait,
runningPoll: defaultRunningPoll,
imageLocks: make(map[string]chan struct{}),
nameLocks: make(map[string]chan struct{}),
}
for _, opt := range opts {
opt(p)
@@ -167,24 +187,21 @@ func (p *Provider) wrapErr(op, id string, err error) error {
return provider.Wrap(classify(op, err), op, p.name, id, err)
}
var errNotImplemented = errors.New("kvm2: not implemented")
// Create is not implemented yet.
func (p *Provider) Create(_ context.Context, _ provider.CreateRequest) (string, error) {
return "", provider.Wrap(provider.ErrPermanent, "create", p.name, "", errNotImplemented)
}
// Get is not implemented yet.
func (p *Provider) Get(_ context.Context, _ string) (*provider.Instance, error) {
return nil, provider.Wrap(provider.ErrPermanent, "get", p.name, "", errNotImplemented)
}
// Delete is not implemented yet.
func (p *Provider) Delete(_ context.Context, _ string) error {
return provider.Wrap(provider.ErrPermanent, "delete", p.name, "", errNotImplemented)
}
// ListByTag is not implemented yet.
func (p *Provider) ListByTag(_ context.Context) ([]provider.Instance, error) {
return nil, provider.Wrap(provider.ErrPermanent, "list", p.name, "", errNotImplemented)
// lockName takes the per-name lock of spec entry 19, or gives up with ctx.
// A one-slot channel rather than a mutex, so a caller whose ctx ends while
// another caller's Create is downloading an image returns promptly.
func (p *Provider) lockName(ctx context.Context, name string) (func(), error) {
p.nameMu.Lock()
sem, ok := p.nameLocks[name]
if !ok {
sem = make(chan struct{}, 1)
p.nameLocks[name] = sem
}
p.nameMu.Unlock()
select {
case sem <- struct{}{}:
return func() { <-sem }, nil
case <-ctx.Done():
return nil, fmt.Errorf("%w: waiting for the lock on %s: %w", provider.ErrTransient, name, ctx.Err())
}
}
internal/provider/kvm2/kvm2_test.go
-23
View File
@@ -260,26 +260,3 @@ func TestWrapErr_carriesProviderNameOpAndID(t *testing.T) {
t.Errorf("wrapErr() = %+v, want provider kvm2-home, op get, the id and ErrPermanent", pe)
}
}
func TestStubs_returnPermanentWithProviderName(t *testing.T) {
t.Parallel()
ctx := context.Background()
p := newProvider("kvm2-home", twoHosts(), &stubDialer{})
_, createErr := p.Create(ctx, provider.CreateRequest{Name: "proxy-x"})
inst, getErr := p.Get(ctx, "hosts/beelink/domains/proxy-x")
deleteErr := p.Delete(ctx, "hosts/beelink/domains/proxy-x")
list, listErr := p.ListByTag(ctx)
if inst != nil || list != nil {
t.Errorf("Get = %+v, ListByTag = %+v, want nil results from stubs", inst, list)
}
for op, err := range map[string]error{"create": createErr, "get": getErr, "delete": deleteErr, "list": listErr} {
var pe *provider.Error
if !errors.As(err, &pe) {
t.Errorf("%s: error = %T, want *provider.Error", op, err)
continue
}
if pe.Provider != "kvm2-home" || pe.Op != op || !errors.Is(err, provider.ErrPermanent) {
t.Errorf("%s: error = %+v, want provider kvm2-home, op %s, ErrPermanent", op, pe, op)
}
}
}
internal/provider/kvm2/list.go
+60
View File
@@ -0,0 +1,60 @@
package kvm2
import (
"context"
"fmt"
"github.com/digitalocean/go-libvirt"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
// ListByTag is spec entry 12: every domain on every configured host, in
// config order, filtered client-side on the kvm2 mark. A host that cannot
// be reached or listed fails the whole call (fail-whole); a single domain
// that cannot be read is skipped, unless ctx ended, which fails the call
// rather than pass off an empty view as the truth.
func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
log := p.logger(ctx)
var out []provider.Instance
for _, host := range p.cfg.Hosts {
hlog := log.WithValues("host", host.Name)
be, err := p.hosts.get(ctx, host)
if err != nil {
logAPIError(hlog, "list", err)
return nil, p.wrapErr("list", "", err)
}
doms, _, err := be.ConnectListAllDomains(ctx, 1, libvirt.ConnectListDomainsActive|libvirt.ConnectListDomainsInactive)
if err != nil {
err = fmt.Errorf("listing domains on host %s: %w", host.Name, err)
logAPIError(hlog, "list", err)
return nil, p.wrapErr("list", "", err)
}
for _, dom := range doms {
dlog := hlog.WithValues("name", dom.Name)
meta, err := readMetadata(ctx, be, dom)
if err != nil {
if ctx.Err() != nil {
return nil, p.wrapErr("list", "", err)
}
dlog.V(1).Info("skipping domain with unreadable metadata", "error", err.Error())
continue
}
if !meta.managed() {
continue
}
inst, err := inspect(ctx, dlog, be, host.Name, dom, meta)
if err != nil {
if ctx.Err() != nil {
return nil, p.wrapErr("list", "", err)
}
dlog.V(1).Info("skipping domain that could not be inspected", "error", err.Error())
continue
}
dlog.V(2).Info("kvm2 listed instance", "id", inst.ID, "state", inst.State, "uid", inst.UID, "createdAt", inst.CreatedAt)
out = append(out, inst)
}
}
log.V(1).Info("kvm2 domains listed", "hosts", len(p.cfg.Hosts), "managed", len(out))
return out, nil
}
internal/provider/kvm2/list_test.go
+178
View File
@@ -0,0 +1,178 @@
package kvm2
import (
"errors"
"strings"
"testing"
"time"
"github.com/digitalocean/go-libvirt"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
func listIDs(list []provider.Instance) []string {
ids := make([]string, len(list))
for i, inst := range list {
ids[i] = inst.ID
}
return ids
}
func TestListByTag_excludesUnmarkedDomains(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
fx.fake.addDomain("legacy-vm", libvirt.DomainRunning, "")
fx.fake.addDomain("unmarked", libvirt.DomainRunning, `<instance><managed>false</managed><uid>x</uid></instance>`)
fx.fake.addDomain("unrelated", libvirt.DomainShutoff, `<instance><note>hand made</note></instance>`)
list, err := fx.p.ListByTag(fx.ctx)
if err != nil {
t.Fatalf("ListByTag() error = %v", err)
}
if got := listIDs(list); len(got) != 1 || got[0] != id {
t.Errorf("ListByTag = %v, want only %s", got, id)
}
if list[0].UID != testUID || list[0].CreatedAt.IsZero() {
t.Errorf("listed = %+v, want uid and createdAt", list[0])
}
if got := fx.logs.matching("kvm2 domains listed"); len(got) != 1 || !strings.Contains(got[0], `"hosts"=1`) || !strings.Contains(got[0], `"managed"=1`) {
t.Errorf("listed line = %v", got)
}
}
func TestListByTag_emptyUIDStillListed(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.fake.addDomain("proxy-nouid", libvirt.DomainRunning, managedMetadata("", time.Now()))
list, err := fx.p.ListByTag(fx.ctx)
if err != nil {
t.Fatalf("ListByTag() error = %v", err)
}
if len(list) != 1 || list[0].UID != "" || list[0].ID != formatProviderID(testHost, "proxy-nouid") {
t.Errorf("ListByTag = %+v, want the domain listed with an empty UID", list)
}
}
func TestListByTag_acrossHostsInConfigOrder(t *testing.T) {
t.Parallel()
cfg := twoHosts()
cfg.Hosts[1].Pool = ""
first, second := newFakeBackend(), newFakeBackend()
first.addDomain("proxy-z", libvirt.DomainRunning, managedMetadata("uid-z", time.Now()))
first.addDomain("proxy-a", libvirt.DomainShutoff, managedMetadata("uid-a", time.Now()))
second.addDomain("proxy-b", libvirt.DomainRunning, managedMetadata("uid-b", time.Now()))
p := newProvider(testProvider, cfg, &hostDialer{backends: map[string]backend{"beelink": first, "lab2": second}})
ctx, _ := captureContext()
list, err := p.ListByTag(ctx)
if err != nil {
t.Fatalf("ListByTag() error = %v", err)
}
want := []string{formatProviderID("beelink", "proxy-a"), formatProviderID("beelink", "proxy-z"), formatProviderID("lab2", "proxy-b")}
if got := listIDs(list); strings.Join(got, ",") != strings.Join(want, ",") {
t.Errorf("ListByTag = %v, want %v (config order, then name order)", got, want)
}
if list[0].State != provider.StateStopped || list[1].State != provider.StateProvisioning {
t.Errorf("states = %s, %s, want Stopped, Provisioning", list[0].State, list[1].State)
}
}
func TestListByTag_unreachableHostFailsWhole(t *testing.T) {
t.Parallel()
first := newFakeBackend()
first.addDomain("proxy-a", libvirt.DomainRunning, managedMetadata("uid-a", time.Now()))
d := &hostDialer{
backends: map[string]backend{"beelink": first},
errs: map[string]error{"lab2": errors.New("dial tcp 192.0.2.2:22: i/o timeout")},
}
p := newProvider(testProvider, twoHosts(), d)
ctx, _ := captureContext()
list, err := p.ListByTag(ctx)
if list != nil {
t.Errorf("ListByTag = %v, want nil with the error", list)
}
assertProviderError(t, err, "list", provider.ErrTransient)
}
func TestListByTag_unreadableDomainSkipped(t *testing.T) {
t.Parallel()
tests := []struct {
name string
setup func(fx *fixture)
line string
}{
{
name: "corrupt metadata on one domain",
setup: func(fx *fixture) {
fx.fake.addDomain("proxy-bad", libvirt.DomainRunning, "<bogus>not an instance</bogus>")
},
line: "skipping domain with unreadable metadata",
},
{
name: "metadata rpc fails for the first domain only",
setup: func(fx *fixture) {
fx.fake.addDomain("proxy-bad", libvirt.DomainRunning, managedMetadata("uid-bad", time.Now()))
var calls int
fx.fake.mu.Lock()
fx.fake.hooks["DomainGetMetadata"] = func() error {
calls++
if calls == 1 {
return fakeErr(libvirt.ErrInternalError, "internal error: metadata read failed")
}
return nil
}
fx.fake.mu.Unlock()
},
line: "skipping domain with unreadable metadata",
},
{
name: "state rpc fails for the first domain only",
setup: func(fx *fixture) {
fx.fake.addDomain("proxy-bad", libvirt.DomainRunning, managedMetadata("uid-bad", time.Now()))
var calls int
fx.fake.mu.Lock()
fx.fake.hooks["DomainGetState"] = func() error {
calls++
if calls == 1 {
return fakeErr(libvirt.ErrNoDomain, "Domain not found")
}
return nil
}
fx.fake.mu.Unlock()
},
line: "skipping domain that could not be inspected",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.fake.addDomain("proxy-good", libvirt.DomainRunning, managedMetadata("uid-good", time.Now()))
tc.setup(fx)
list, err := fx.p.ListByTag(fx.ctx)
if err != nil {
t.Fatalf("ListByTag() error = %v", err)
}
if got := listIDs(list); len(got) != 1 || got[0] != formatProviderID(testHost, "proxy-good") {
t.Errorf("ListByTag = %v, want only proxy-good", got)
}
if got := fx.logs.matching(tc.line); len(got) != 1 || !strings.Contains(got[0], `"name"="proxy-bad"`) {
t.Errorf("skip line = %v, want one naming proxy-bad", got)
}
})
}
}
func TestListByTag_includesInactiveDomains(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
id := fx.create(testName)
fx.withDomain(testName, func(d *fakeDomain) { d.state = libvirt.DomainShutoff })
list, err := fx.p.ListByTag(fx.ctx)
if err != nil {
t.Fatalf("ListByTag() error = %v", err)
}
if len(list) != 1 || list[0].ID != id || list[0].State != provider.StateStopped {
t.Errorf("ListByTag = %+v, want the shut-off domain as Stopped", list)
}
}
internal/provider/kvm2/provider_test.go
+415
View File
@@ -0,0 +1,415 @@
package kvm2
import (
"context"
"errors"
"fmt"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/digitalocean/go-libvirt"
"github.com/go-logr/logr/funcr"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
const (
testProvider = "kvm2-home"
testHost = "beelink"
testUID = "0f1e2d3c-uid-1"
testName = "proxy-abcdefghijklmnop"
)
// fixture is one configured host backed by a fake, an image server the
// catalogue points at, a log capture at a chosen verbosity and a provider
// with a fast running-wait, so a Create runs end to end in memory.
type fixture struct {
t *testing.T
p *Provider
fake *fakeBackend
dialer *stubDialer
logs *logCapture
ctx context.Context
image []byte
}
func newFixture(t *testing.T, verbosity int, opts ...providerOption) *fixture {
t.Helper()
fx := &fixture{t: t, fake: newFakeBackend(), logs: &logCapture{}, image: []byte("QFI\xfb fake base image bytes")}
srv := httptest.NewServer(serveBytes(fx.image))
t.Cleanup(srv.Close)
fx.dialer = &stubDialer{backend: fx.fake}
cfg := provider.KVMConfig{
Images: map[string]provider.KVMImage{imageKey: {URL: srv.URL + "/debian-13-generic-amd64.qcow2"}},
MachineTypes: map[string]provider.KVMMachineType{"small": {VCPUs: 1, MemoryMiB: 1024}},
Hosts: []provider.KVMHost{{Name: testHost, Address: "beelink:22", User: "virt", Bridge: "br0"}},
}
opts = append([]providerOption{withHTTPClient(srv.Client()), withRunningWait(time.Millisecond, 500*time.Millisecond)}, opts...)
fx.p = newProvider(testProvider, cfg, fx.dialer, opts...)
log := funcr.New(func(prefix, args string) {
fx.logs.mu.Lock()
defer fx.logs.mu.Unlock()
fx.logs.lines = append(fx.logs.lines, prefix+" "+args)
}, funcr.Options{Verbosity: verbosity})
fx.ctx = logf.IntoContext(context.Background(), log)
return fx
}
func (fx *fixture) req(name string) provider.CreateRequest {
return provider.CreateRequest{
Name: name,
UID: testUID,
Namespace: "default",
ProxyName: "egress",
Placement: provider.Placement{Zone: testHost, MachineType: "small", Image: imageKey},
CloudInit: "#cloud-config\npackages: [squid]\n",
Port: 3128,
}
}
func (fx *fixture) create(name string) string {
fx.t.Helper()
id, err := fx.p.Create(fx.ctx, fx.req(name))
if err != nil {
fx.t.Fatalf("Create(%s) error = %v", name, err)
}
if want := formatProviderID(testHost, name); id != want {
fx.t.Fatalf("Create(%s) = %q, want %q", name, id, want)
}
return id
}
func (fx *fixture) get(id string) *provider.Instance {
fx.t.Helper()
inst, err := fx.p.Get(fx.ctx, id)
if err != nil {
fx.t.Fatalf("Get(%s) error = %v", id, err)
}
return inst
}
// withDomain runs fn on the fake's domain under its lock.
func (fx *fixture) withDomain(name string, fn func(d *fakeDomain)) {
fx.t.Helper()
fx.fake.mu.Lock()
defer fx.fake.mu.Unlock()
d, ok := fx.fake.domains[name]
if !ok {
fx.t.Fatalf("fake has no domain %s", name)
}
fn(d)
}
func (fx *fixture) hasDomain(name string) bool {
fx.fake.mu.Lock()
defer fx.fake.mu.Unlock()
_, ok := fx.fake.domains[name]
return ok
}
func (fx *fixture) setAddrs(name string, src libvirt.DomainInterfaceAddressesSource, ips ...string) {
fx.withDomain(name, func(d *fakeDomain) {
var addrs []libvirt.DomainIPAddr
for _, ip := range ips {
addrs = append(addrs, libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv4), Addr: ip, Prefix: 24})
}
d.addrs[uint32(src)] = []libvirt.DomainInterface{{Name: "enp1s0", Addrs: addrs}}
})
}
// putVol places a volume in the fake's pool directly.
func (fx *fixture) putVol(name string, content []byte) *fakeVol {
fx.fake.mu.Lock()
defer fx.fake.mu.Unlock()
v := &fakeVol{content: content, capacity: uint64(len(content))}
fx.fake.pools[fakePoolName].vols[name] = v
return v
}
// cacheImage places a complete base image and its marker, as a previous
// Create on the host would have.
func (fx *fixture) cacheImage() {
src, err := resolveImage(fx.p.cfg, imageKey)
if err != nil {
fx.t.Fatal(err)
}
fx.putVol(src.volumeName(), fx.image)
fx.putVol(src.markerName(), []byte(digestOf(fx.image)))
}
func managedMetadata(uid string, createdAt time.Time) string {
s, err := marshalInstanceMetadata(newInstanceMetadata(uid, createdAt))
if err != nil {
panic(err)
}
return s
}
// assertProviderError checks err is a *provider.Error from the configured
// provider with the given op and class.
func assertProviderError(t *testing.T, err error, op string, class error) {
t.Helper()
if err == nil {
t.Fatalf("error = nil, want op %s class %v", op, class)
}
var pe *provider.Error
if !errors.As(err, &pe) {
t.Fatalf("error = %T (%v), want *provider.Error", err, err)
}
if pe.Provider != testProvider {
t.Errorf("error.Provider = %q, want %q", pe.Provider, testProvider)
}
if pe.Op != op {
t.Errorf("error.Op = %q, want %q", pe.Op, op)
}
if !errors.Is(err, class) {
t.Errorf("error = %v, want class %v", err, class)
}
for _, other := range []error{provider.ErrNotFound, provider.ErrQuotaExceeded, provider.ErrPermanent, provider.ErrTransient} {
if other != class && errors.Is(err, other) {
t.Errorf("error = %v, also matches %v", err, other)
}
}
}
// hostDialer hands out one backend per host name and fails hosts in errs.
type hostDialer struct {
backends map[string]backend
errs map[string]error
}
func (d *hostDialer) dial(_ context.Context, host provider.KVMHost) (backend, error) {
if err := d.errs[host.Name]; err != nil {
return nil, err
}
b, ok := d.backends[host.Name]
if !ok {
return nil, fmt.Errorf("hostDialer: no backend for %s", host.Name)
}
return b, nil
}
// waitFor polls cond until it holds or the deadline passes.
func waitFor(t *testing.T, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for !cond() {
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for %s", what)
}
time.Sleep(time.Millisecond)
}
}
func TestProvider_cancelledContextReturnsPromptly(t *testing.T) {
t.Parallel()
tests := []struct {
name string
firstRPC string
op string
call func(ctx context.Context, fx *fixture) error
}{
{
name: "create", firstRPC: "DomainLookupByName", op: "create",
call: func(ctx context.Context, fx *fixture) error { _, err := fx.p.Create(ctx, fx.req(testName)); return err },
},
{
name: "get", firstRPC: "DomainLookupByName", op: "get",
call: func(ctx context.Context, fx *fixture) error {
_, err := fx.p.Get(ctx, formatProviderID(testHost, testName))
return err
},
},
{
name: "delete", firstRPC: "DomainLookupByName", op: "delete",
call: func(ctx context.Context, fx *fixture) error {
return fx.p.Delete(ctx, formatProviderID(testHost, testName))
},
},
{
name: "list", firstRPC: "ConnectListAllDomains", op: "list",
call: func(ctx context.Context, fx *fixture) error { _, err := fx.p.ListByTag(ctx); return err },
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
release := fx.fake.blockOn(tc.firstRPC)
defer close(release)
ctx, cancel := context.WithCancel(fx.ctx)
done := make(chan error, 1)
go func() { done <- tc.call(ctx, fx) }()
waitFor(t, "the first RPC to block", func() bool { return fx.fake.callCount(tc.firstRPC) == 1 })
cancel()
select {
case err := <-done:
assertProviderError(t, err, tc.op, provider.ErrTransient)
case <-time.After(time.Second):
t.Fatalf("%s did not return within 1s of cancellation", tc.name)
}
})
}
}
func TestProvider_everyErrorIsWrappedWithConfiguredName(t *testing.T) {
t.Parallel()
dialErr := errors.New("dial tcp 192.0.2.1:22: connect: connection refused")
tests := []struct {
name string
setup func(fx *fixture)
call func(fx *fixture) error
op string
class error
}{
{
name: "create unknown zone",
call: func(fx *fixture) error {
req := fx.req(testName)
req.Placement.Zone = "nas9"
_, err := fx.p.Create(fx.ctx, req)
return err
},
op: "create", class: provider.ErrPermanent,
},
{
name: "create dial failure",
setup: func(fx *fixture) { fx.dialer.err = dialErr },
call: func(fx *fixture) error { _, err := fx.p.Create(fx.ctx, fx.req(testName)); return err },
op: "create", class: provider.ErrTransient,
},
{
name: "create pool out of space",
setup: func(fx *fixture) {
fx.fake.mu.Lock()
fx.fake.pools[fakePoolName].available = 1
fx.fake.mu.Unlock()
},
call: func(fx *fixture) error { _, err := fx.p.Create(fx.ctx, fx.req(testName)); return err },
op: "create", class: provider.ErrQuotaExceeded,
},
{
name: "create vanished volume",
setup: func(fx *fixture) { fx.fake.failWith("StorageVolGetPath", fakeErr(libvirt.ErrNoStorageVol, "gone")) },
call: func(fx *fixture) error { _, err := fx.p.Create(fx.ctx, fx.req(testName)); return err },
op: "create", class: provider.ErrTransient,
},
{
name: "get malformed id",
call: func(fx *fixture) error { _, err := fx.p.Get(fx.ctx, "beelink/"+testName); return err },
op: "get", class: provider.ErrPermanent,
},
{
name: "get missing domain",
call: func(fx *fixture) error { _, err := fx.p.Get(fx.ctx, formatProviderID(testHost, testName)); return err },
op: "get", class: provider.ErrNotFound,
},
{
name: "get busy backend",
setup: func(fx *fixture) {
fx.create(testName)
fx.fake.failWith("DomainGetState", fakeErr(libvirt.ErrOperationInvalid, "busy"))
},
call: func(fx *fixture) error { _, err := fx.p.Get(fx.ctx, formatProviderID(testHost, testName)); return err },
op: "get", class: provider.ErrTransient,
},
{
name: "delete malformed id",
call: func(fx *fixture) error { return fx.p.Delete(fx.ctx, "hosts//domains/"+testName) },
op: "delete", class: provider.ErrPermanent,
},
{
name: "delete dial failure",
setup: func(fx *fixture) { fx.dialer.err = dialErr },
call: func(fx *fixture) error { return fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)) },
op: "delete", class: provider.ErrTransient,
},
{
name: "delete access denied",
setup: func(fx *fixture) { fx.fake.failWith("DomainLookupByName", fakeErr(libvirt.ErrAccessDenied, "denied")) },
call: func(fx *fixture) error { return fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)) },
op: "delete", class: provider.ErrPermanent,
},
{
name: "list dial failure",
setup: func(fx *fixture) { fx.dialer.err = dialErr },
call: func(fx *fixture) error { _, err := fx.p.ListByTag(fx.ctx); return err },
op: "list", class: provider.ErrTransient,
},
{
name: "list rpc failure",
setup: func(fx *fixture) {
fx.fake.failWith("ConnectListAllDomains", fakeErr(libvirt.ErrInternalError, "boom"))
},
call: func(fx *fixture) error { _, err := fx.p.ListByTag(fx.ctx); return err },
op: "list", class: provider.ErrTransient,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
if tc.setup != nil {
tc.setup(fx)
}
assertProviderError(t, tc.call(fx), tc.op, tc.class)
})
}
}
func TestLogging_v0SilentV1OutcomeV2Detail(t *testing.T) {
t.Parallel()
outcomes := []string{"overlay volume created", "seed volume uploaded", "domain defined", "kvm2 domain ensured running"}
t.Run("v0 is silent", func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 0)
fx.create(testName)
if lines := fx.logs.all(); len(lines) != 0 {
t.Errorf("Create logged %d lines at V(0), want none:\n%s", len(lines), strings.Join(lines, "\n"))
}
})
t.Run("v1 has outcomes without xml", func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 1)
fx.create(testName)
for _, want := range outcomes {
if len(fx.logs.matching(want)) != 1 {
t.Errorf("V(1) lines matching %q = %d, want 1:\n%s", want, len(fx.logs.matching(want)), strings.Join(fx.logs.all(), "\n"))
}
}
if got := fx.logs.matching("<domain"); len(got) != 0 {
t.Errorf("V(1) carries domain XML:\n%s", strings.Join(got, "\n"))
}
if got := fx.logs.matching("kvm2 create resolved"); len(got) != 0 {
t.Errorf("V(1) carries the V(2) detail line:\n%s", strings.Join(got, "\n"))
}
for _, l := range fx.logs.all() {
if !strings.Contains(l, `"provider"="kvm2-home"`) || !strings.Contains(l, `"host"="beelink"`) {
t.Errorf("line lacks provider or host: %s", l)
}
if !strings.Contains(l, `"name"="`+testName+`"`) && !strings.Contains(l, `"volume"="base-`) {
t.Errorf("instance line lacks the name: %s", l)
}
}
})
t.Run("v2 has detail", func(t *testing.T) {
t.Parallel()
fx := newFixture(t, 2)
fx.create(testName)
if got := fx.logs.matching("<domain"); len(got) == 0 {
t.Errorf("V(2) lacks domain XML:\n%s", strings.Join(fx.logs.all(), "\n"))
}
resolved := fx.logs.matching("kvm2 create resolved")
if len(resolved) != 1 {
t.Fatalf("V(2) lines matching resolved = %d, want 1", len(resolved))
}
for _, key := range []string{`"cloudInitBytes"=`, `"image"="base-debian-13.qcow2"`, `"firmware"="uefi"`, `"vcpus"=1`, `"memoryMiB"=1024`, `"machineType"="small"`, `"bridge"="br0"`, `"pool"="default"`} {
if !strings.Contains(resolved[0], key) {
t.Errorf("resolved line lacks %s: %s", key, resolved[0])
}
}
})
}