proxmox: contract and observability tests, integration test, testing docs
The contract sweep in provider_test.go: every method and Check returns within a second of a cancel, every wrapErr path carries the configured name, op and id, the V0/V1/V2 logging tiers and the no-op lines, and a Get over the real client whose HTTP spans are children of provider.get. proxmox_integration_test.go (-tags=integration) holds TestProxmox_lifecycle and TestProxmoxCheck_realHost; both compile and skip without PROXMOX_TEST_*, and have not run against a host yet. make test-integration-proxmox runs them. docs/testing.md gains the proxmox, nocloud and keyedlock rows, the Live Proxmox gaps and the integration section; spec entry 21's section 11 list is ticked. Waits that could hang the package under a mutation are bounded: the concurrent-Create loop and eight direct blocking calls in older tests. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -117,7 +117,8 @@ func TestCheck_expiredContextReturnsPromptly(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(fx.ctx, 0)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
got := fx.p.Check(ctx)
|
||||
var got []provider.TargetStatus
|
||||
within(t, 5*time.Second, func() { got = fx.p.Check(ctx) })
|
||||
if elapsed := time.Since(start); elapsed > time.Second {
|
||||
t.Errorf("Check() took %v with an expired ctx, want under 1s", elapsed)
|
||||
}
|
||||
@@ -321,7 +322,8 @@ func TestCheck_reasonsTable(t *testing.T) {
|
||||
ctx, cancel = context.WithCancel(ctx)
|
||||
time.AfterFunc(50*time.Millisecond, cancel)
|
||||
}
|
||||
got := fx.p.Check(ctx)
|
||||
var got []provider.TargetStatus
|
||||
within(t, 5*time.Second, func() { got = fx.p.Check(ctx) })
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("Check() returned %d targets, want 1", len(got))
|
||||
}
|
||||
|
||||
@@ -1196,15 +1196,20 @@ func TestCreate_concurrentSameNameSerialised(t *testing.T) {
|
||||
release := fx.fake.blockOn("Clone")
|
||||
ids := make([]string, 2)
|
||||
errs := make([]error, 2)
|
||||
first := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for i := range ids {
|
||||
wg.Go(func() { ids[i], errs[i] = fx.p.Create(fx.ctx, fx.req()) })
|
||||
if i == 0 {
|
||||
for fx.fake.callCount("Clone") == 0 {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
wg.Go(func() {
|
||||
defer close(first)
|
||||
ids[0], errs[0] = fx.p.Create(fx.ctx, fx.req())
|
||||
})
|
||||
for fx.fake.callCount("Clone") == 0 {
|
||||
select {
|
||||
case <-first:
|
||||
t.Fatalf("first Create returned %v without cloning", errs[0])
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
wg.Go(func() { ids[1], errs[1] = fx.p.Create(fx.ctx, fx.req()) })
|
||||
close(release)
|
||||
wg.Wait()
|
||||
for i, err := range errs {
|
||||
@@ -1233,7 +1238,8 @@ func TestCreate_cancelledMidWaitReturnsPromptly(t *testing.T) {
|
||||
return nil
|
||||
})
|
||||
start := time.Now()
|
||||
_, err := fx.p.Create(ctx, fx.req())
|
||||
var err error
|
||||
within(t, 10*time.Second, func() { _, err = fx.p.Create(ctx, fx.req()) })
|
||||
if elapsed := time.Since(start); elapsed > 5*time.Second {
|
||||
t.Errorf("Create took %v after cancellation", elapsed)
|
||||
}
|
||||
|
||||
@@ -623,7 +623,7 @@ func TestDelete_waitsOnTheNameLock(t *testing.T) {
|
||||
defer release()
|
||||
ctx, cancel := context.WithTimeout(fx.ctx, 20*time.Millisecond)
|
||||
defer cancel()
|
||||
err = fx.p.Delete(ctx, fx.id())
|
||||
within(t, 5*time.Second, func() { err = fx.p.Delete(ctx, fx.id()) })
|
||||
assertProviderError(t, err, "delete", provider.ErrTransient)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Errorf("error = %v, want it to wrap DeadlineExceeded", err)
|
||||
@@ -647,7 +647,8 @@ func TestDelete_cancelledMidWaitReturnsPromptly(t *testing.T) {
|
||||
return nil
|
||||
})
|
||||
start := time.Now()
|
||||
err := fx.p.Delete(ctx, fx.id())
|
||||
var err error
|
||||
within(t, 10*time.Second, func() { err = fx.p.Delete(ctx, fx.id()) })
|
||||
if elapsed := time.Since(start); elapsed > 5*time.Second {
|
||||
t.Errorf("Delete took %v after cancellation", elapsed)
|
||||
}
|
||||
|
||||
@@ -3,10 +3,19 @@ package proxmox
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -101,3 +110,589 @@ func assertProviderError(t *testing.T, err error, op string, class error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// within runs fn and fails the test when it has not returned after d, so
|
||||
// a call that ignores its ctx fails by name instead of hanging the package
|
||||
// until the test binary's timeout.
|
||||
func within(t *testing.T, d time.Duration, fn func()) {
|
||||
t.Helper()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
fn()
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(d):
|
||||
t.Fatalf("did not return within %v", d)
|
||||
}
|
||||
}
|
||||
|
||||
// cancelWhileBlocked runs call with a ctx it cancels once call has reached
|
||||
// method, which blocks until then, and fails unless call returns within a
|
||||
// second of the cancel. Every wait is bounded, so a call that never reaches
|
||||
// method or never returns fails instead of hanging the package.
|
||||
func cancelWhileBlocked(t *testing.T, fx *fixture, method string, call func(ctx context.Context)) {
|
||||
t.Helper()
|
||||
release := fx.fake.blockOn(method)
|
||||
defer close(release)
|
||||
ctx, cancel := context.WithCancel(fx.ctx)
|
||||
defer cancel()
|
||||
returned := make(chan struct{})
|
||||
go func() {
|
||||
defer close(returned)
|
||||
call(ctx)
|
||||
}()
|
||||
deadline := time.After(5 * time.Second)
|
||||
for fx.fake.callCount(method) == 0 {
|
||||
select {
|
||||
case <-returned:
|
||||
t.Fatalf("returned before calling %s; calls: %v", method, fx.fake.callLog())
|
||||
case <-deadline:
|
||||
t.Fatalf("%s was not called within 5s; calls: %v", method, fx.fake.callLog())
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-returned:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("did not return within 1s of the cancel while blocked in %s", method)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_cancelledContextReturnsPromptly(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
blockOn string
|
||||
// first: blockOn is the operation's first seam call.
|
||||
first bool
|
||||
setup func(fx *fixture)
|
||||
op string
|
||||
call func(ctx context.Context, fx *fixture) error
|
||||
}{
|
||||
{
|
||||
name: "create", blockOn: "ClusterResources", first: true, op: "create",
|
||||
call: func(ctx context.Context, fx *fixture) error { _, err := fx.p.Create(ctx, fx.req()); return err },
|
||||
},
|
||||
{
|
||||
name: "create mid upload", blockOn: "UploadISO", op: "create",
|
||||
call: func(ctx context.Context, fx *fixture) error { _, err := fx.p.Create(ctx, fx.req()); return err },
|
||||
},
|
||||
{
|
||||
name: "get", blockOn: "ClusterResources", first: true, op: "get",
|
||||
setup: func(fx *fixture) { fx.addOurVM(testVMID, testName, "running") },
|
||||
call: func(ctx context.Context, fx *fixture) error { _, err := fx.p.Get(ctx, fx.id()); return err },
|
||||
},
|
||||
{
|
||||
name: "delete", blockOn: "ClusterResources", first: true, op: "delete",
|
||||
setup: func(fx *fixture) { fx.addOurVM(testVMID, testName, "running") },
|
||||
call: func(ctx context.Context, fx *fixture) error { return fx.p.Delete(ctx, fx.id()) },
|
||||
},
|
||||
{
|
||||
name: "delete mid stop", blockOn: "Stop", op: "delete",
|
||||
setup: func(fx *fixture) { fx.addOurVM(testVMID, testName, "running") },
|
||||
call: func(ctx context.Context, fx *fixture) error { return fx.p.Delete(ctx, fx.id()) },
|
||||
},
|
||||
{
|
||||
name: "list", blockOn: "ClusterResources", first: true, op: "list",
|
||||
setup: func(fx *fixture) { fx.addOurVM(testVMID, testName, "running") },
|
||||
call: func(ctx context.Context, fx *fixture) error { _, err := fx.p.ListByTag(ctx); return err },
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newCreateFixture(t, 1)
|
||||
if tc.setup != nil {
|
||||
tc.setup(fx)
|
||||
}
|
||||
var err error
|
||||
cancelWhileBlocked(t, fx, tc.blockOn, func(ctx context.Context) { err = tc.call(ctx, fx) })
|
||||
if calls := fx.fake.callLog(); tc.first && calls[0] != tc.blockOn {
|
||||
t.Errorf("first call = %s, want %s", calls[0], tc.blockOn)
|
||||
}
|
||||
assertProviderError(t, err, tc.op, provider.ErrTransient)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("error = %v, want it to carry context.Canceled", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("check", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newCheckFixture(t)
|
||||
var got []provider.TargetStatus
|
||||
cancelWhileBlocked(t, fx, "Version", func(ctx context.Context) { got = fx.p.Check(ctx) })
|
||||
if calls := fx.fake.callLog(); calls[0] != "Version" {
|
||||
t.Errorf("first call = %s, want Version", calls[0])
|
||||
}
|
||||
if len(got) != 1 || got[0].Ready || got[0].Reason != provider.ReasonTimeout {
|
||||
t.Errorf("Check() = %+v, want one target not ready with reason %s", got, provider.ReasonTimeout)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestProvider_everyErrorIsWrappedWithConfiguredName(t *testing.T) {
|
||||
t.Parallel()
|
||||
createWith := func(mutate func(*provider.CreateRequest)) func(context.Context, *fixture) error {
|
||||
return func(ctx context.Context, fx *fixture) error {
|
||||
req := fx.req()
|
||||
if mutate != nil {
|
||||
mutate(&req)
|
||||
}
|
||||
_, err := fx.p.Create(ctx, req)
|
||||
return err
|
||||
}
|
||||
}
|
||||
get := func(ctx context.Context, fx *fixture) error { _, err := fx.p.Get(ctx, fx.id()); return err }
|
||||
del := func(ctx context.Context, fx *fixture) error { return fx.p.Delete(ctx, fx.id()) }
|
||||
list := func(ctx context.Context, fx *fixture) error { _, err := fx.p.ListByTag(ctx); return err }
|
||||
holdNameLock := func(fx *fixture, cancel context.CancelFunc) {
|
||||
release, err := fx.p.nameLocks.Acquire(context.Background(), testName)
|
||||
if err != nil {
|
||||
fx.t.Fatalf("Acquire error = %v", err)
|
||||
}
|
||||
fx.t.Cleanup(release)
|
||||
cancel()
|
||||
}
|
||||
broken := fakeErr(596, "Broken pipe")
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(fx *fixture, cancel context.CancelFunc)
|
||||
call func(ctx context.Context, fx *fixture) error
|
||||
op string
|
||||
id string
|
||||
class error
|
||||
}{
|
||||
{name: "create without a name", call: createWith(func(r *provider.CreateRequest) { r.Name = "" }), op: "create", class: provider.ErrPermanent},
|
||||
{name: "create unknown zone", call: createWith(func(r *provider.CreateRequest) { r.Placement.Zone = "pv9-9" }), op: "create", class: provider.ErrPermanent},
|
||||
{name: "create unknown image", call: createWith(func(r *provider.CreateRequest) { r.Placement.Image = "arch" }), op: "create", class: provider.ErrPermanent},
|
||||
{name: "create waiting on the name lock", setup: holdNameLock, call: createWith(nil), op: "create", class: provider.ErrTransient},
|
||||
{
|
||||
name: "create resources unreadable",
|
||||
setup: func(fx *fixture, _ context.CancelFunc) { fx.fake.failWith("ClusterResources", broken) },
|
||||
call: createWith(nil), op: "create", class: provider.ErrTransient,
|
||||
},
|
||||
{
|
||||
name: "create clone forbidden",
|
||||
setup: func(fx *fixture, _ context.CancelFunc) {
|
||||
fx.fake.failWith("Clone", loadAPIError(fx.t, "err_forbidden"))
|
||||
},
|
||||
call: createWith(nil), op: "create", class: provider.ErrPermanent,
|
||||
},
|
||||
{
|
||||
name: "create start task fails",
|
||||
setup: func(fx *fixture, _ context.CancelFunc) { fx.fake.startFailsWith = "fake: start failed" },
|
||||
call: createWith(nil), op: "create", class: provider.ErrTransient,
|
||||
},
|
||||
{
|
||||
name: "get malformed id",
|
||||
call: func(ctx context.Context, fx *fixture) error { _, err := fx.p.Get(ctx, "vms/abc/"+testName); return err },
|
||||
op: "get", id: "vms/abc/" + testName, class: provider.ErrPermanent,
|
||||
},
|
||||
{name: "get missing vm", call: get, op: "get", id: "self", class: provider.ErrNotFound},
|
||||
{
|
||||
name: "get locked vm",
|
||||
setup: func(fx *fixture, _ context.CancelFunc) { fx.addOurVM(testVMID, testName, "running").lock = "backup" },
|
||||
call: get, op: "get", id: "self", class: provider.ErrTransient,
|
||||
},
|
||||
{
|
||||
name: "get unauthorized",
|
||||
setup: func(fx *fixture, _ context.CancelFunc) {
|
||||
fx.fake.failWith("ClusterResources", loadAPIError(fx.t, "err_unauthorized"))
|
||||
},
|
||||
call: get, op: "get", id: "self", class: provider.ErrPermanent,
|
||||
},
|
||||
{
|
||||
name: "delete malformed id",
|
||||
call: func(ctx context.Context, fx *fixture) error { return fx.p.Delete(ctx, "vms/150042") },
|
||||
op: "delete", id: "vms/150042", class: provider.ErrPermanent,
|
||||
},
|
||||
{name: "delete waiting on the name lock", setup: holdNameLock, call: del, op: "delete", id: "self", class: provider.ErrTransient},
|
||||
{
|
||||
name: "delete hibernated vm",
|
||||
setup: func(fx *fixture, _ context.CancelFunc) {
|
||||
fx.addOurVM(testVMID, testName, "stopped").lock = lockSuspended
|
||||
},
|
||||
call: del, op: "delete", id: "self", class: provider.ErrPermanent,
|
||||
},
|
||||
{
|
||||
name: "delete destroy task fails",
|
||||
setup: func(fx *fixture, _ context.CancelFunc) {
|
||||
fx.addOurVM(testVMID, testName, "stopped")
|
||||
fx.fake.destroyFailsWith = "fake: destroy failed"
|
||||
},
|
||||
call: del, op: "delete", id: "self", class: provider.ErrTransient,
|
||||
},
|
||||
{
|
||||
name: "delete seed forbidden",
|
||||
setup: func(fx *fixture, _ context.CancelFunc) {
|
||||
fx.addOurVM(testVMID, testName, "stopped")
|
||||
fx.fake.failWith("DeleteVolume", loadAPIError(fx.t, "err_forbidden_volume_delete"))
|
||||
},
|
||||
call: del, op: "delete", id: "self", class: provider.ErrPermanent,
|
||||
},
|
||||
{
|
||||
name: "list resources unreadable",
|
||||
setup: func(fx *fixture, _ context.CancelFunc) { fx.fake.failWith("ClusterResources", broken) },
|
||||
call: list, op: "list", class: provider.ErrTransient,
|
||||
},
|
||||
{
|
||||
name: "list unauthorized",
|
||||
setup: func(fx *fixture, _ context.CancelFunc) {
|
||||
fx.fake.failWith("ClusterResources", loadAPIError(fx.t, "err_unauthorized"))
|
||||
},
|
||||
call: list, op: "list", class: provider.ErrPermanent,
|
||||
},
|
||||
{
|
||||
name: "list ended on a candidate",
|
||||
setup: func(fx *fixture, cancel context.CancelFunc) {
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
fx.fake.hookOn("VMConfig", func() error {
|
||||
cancel()
|
||||
return broken
|
||||
})
|
||||
},
|
||||
call: list, op: "list", class: provider.ErrTransient,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newCreateFixture(t, 1)
|
||||
ctx, cancel := context.WithCancel(fx.ctx)
|
||||
defer cancel()
|
||||
if tc.setup != nil {
|
||||
tc.setup(fx, cancel)
|
||||
}
|
||||
var err error
|
||||
within(t, 5*time.Second, func() { err = tc.call(ctx, fx) })
|
||||
assertProviderError(t, err, tc.op, tc.class)
|
||||
wantID := tc.id
|
||||
if wantID == "self" {
|
||||
wantID = fx.id()
|
||||
}
|
||||
var pe *provider.Error
|
||||
if errors.As(err, &pe) && pe.ID != wantID {
|
||||
t.Errorf("error.ID = %q, want %q", pe.ID, wantID)
|
||||
}
|
||||
if !strings.HasPrefix(err.Error(), "provider "+testProvider+": "+tc.op) {
|
||||
t.Errorf("error = %q, want it to start with the configured name and op", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// logSentinel is in the CloudInit the logging lifecycle creates with; no
|
||||
// log line at any level may carry it.
|
||||
const logSentinel = "SENTINEL-cloud-init-7e2a"
|
||||
|
||||
// newLoggingFixture is an observed create fixture with a token file, so
|
||||
// one lifecycle covers every operation, Check and the decorator's lines.
|
||||
func newLoggingFixture(t *testing.T, verbosity int) *fixture {
|
||||
t.Helper()
|
||||
fx := newCreateFixture(t, verbosity)
|
||||
fx.p.api = observe(fx.fake, testProvider)
|
||||
fx.p.cfg.TokenFile = filepath.Join(t.TempDir(), "token")
|
||||
writeFile(t, fx.p.cfg.TokenFile, testToken+"\n")
|
||||
return fx
|
||||
}
|
||||
|
||||
// sentinelReq is fx.req() with logSentinel in its CloudInit.
|
||||
func (fx *fixture) sentinelReq() provider.CreateRequest {
|
||||
req := fx.req()
|
||||
req.CloudInit = "#cloud-config\nwrite_files:\n - path: /etc/sentinel\n content: " + logSentinel + "\n"
|
||||
return req
|
||||
}
|
||||
|
||||
// lifecycle runs Create with sentinelReq, a Get that finds the address,
|
||||
// ListByTag, Delete, Delete of the gone VM, and Check.
|
||||
func (fx *fixture) lifecycle() {
|
||||
fx.t.Helper()
|
||||
id := fx.create(fx.sentinelReq())
|
||||
ifaces := capturedIfaces(fx.t)
|
||||
fx.fake.mu.Lock()
|
||||
fx.fake.vms[slot(0)].ifaces = ifaces
|
||||
fx.fake.mu.Unlock()
|
||||
if inst := fx.get(id); inst.State != provider.StateRunning {
|
||||
fx.t.Fatalf("Get state = %s, want Running", inst.State)
|
||||
}
|
||||
if _, err := fx.p.ListByTag(fx.ctx); err != nil {
|
||||
fx.t.Fatalf("ListByTag error = %v", err)
|
||||
}
|
||||
for range 2 {
|
||||
if err := fx.p.Delete(fx.ctx, id); err != nil {
|
||||
fx.t.Fatalf("Delete error = %v", err)
|
||||
}
|
||||
}
|
||||
if ts := fx.p.Check(fx.ctx); len(ts) != 1 || !ts[0].Ready {
|
||||
fx.t.Fatalf("Check = %+v, want one Ready target", ts)
|
||||
}
|
||||
}
|
||||
|
||||
// assertNeverLogged checks every line for the cloud-init text and the
|
||||
// token's secret.
|
||||
func assertNeverLogged(t *testing.T, lines []string) {
|
||||
t.Helper()
|
||||
secret := testToken[strings.Index(testToken, "=")+1:]
|
||||
for _, l := range lines {
|
||||
for _, never := range []string{logSentinel, "#cloud-config", secret} {
|
||||
if strings.Contains(l, never) {
|
||||
t.Errorf("line carries %q: %s", never, l)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func msgLine(msg string) string { return `"msg"="` + msg + `"` }
|
||||
|
||||
// assertEveryV1Line checks that each line is from the proxmox logger at
|
||||
// V(1) with the provider, and that each line about an instance, which is
|
||||
// all but the decorator's, the list's and the check's, names its vmid and
|
||||
// name.
|
||||
func assertEveryV1Line(t *testing.T, lines []string, vmid, name string) {
|
||||
t.Helper()
|
||||
notInstance := []string{msgLine("proxmox api call"), msgLine("proxmox vms listed"), msgLine("proxmox node checked")}
|
||||
for _, l := range lines {
|
||||
if !strings.HasPrefix(l, "proxmox ") || !strings.Contains(l, `"provider"="`+testProvider+`"`) {
|
||||
t.Errorf("line lacks the proxmox logger or the provider: %s", l)
|
||||
}
|
||||
if !strings.Contains(l, `"level"=1`) {
|
||||
t.Errorf("line is not at V(1): %s", l)
|
||||
}
|
||||
instance := !slices.ContainsFunc(notInstance, func(m string) bool { return strings.Contains(l, m) })
|
||||
if instance && (!strings.Contains(l, vmid) || !strings.Contains(l, name)) {
|
||||
t.Errorf("instance line lacks vmid or name: %s", l)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogging_v0SilentV1OutcomeV2Detail(t *testing.T) {
|
||||
t.Parallel()
|
||||
vmid := `"vmid"=` + strconv.Itoa(slot(0))
|
||||
name := `"name"="` + testName + `"`
|
||||
t.Run("v0 is silent", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newLoggingFixture(t, 0)
|
||||
fx.lifecycle()
|
||||
if lines := fx.logs.all(); len(lines) != 0 {
|
||||
t.Errorf("the lifecycle logged %d lines at V(0), want none:\n%s", len(lines), strings.Join(lines, "\n"))
|
||||
}
|
||||
})
|
||||
t.Run("v1 has one outcome per operation", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newLoggingFixture(t, 1)
|
||||
fx.lifecycle()
|
||||
all := fx.logs.all()
|
||||
dump := strings.Join(all, "\n")
|
||||
outcomes := map[string][]string{
|
||||
"proxmox vm ensured running": {vmid, name, `"node"="pv1-1"`},
|
||||
"proxmox vm fetched": {vmid, name, `"state"="Running"`, `"ip"="192.168.0.150"`, `"uid"="` + testUID + `"`},
|
||||
"proxmox vms listed": {`"candidates"=1`, `"managed"=1`, `"skipped"=0`},
|
||||
"proxmox node checked": {`"node"="pv1-1"`, `"ready"=true`, `"reason"="Reachable"`},
|
||||
}
|
||||
for msg, keys := range outcomes {
|
||||
lines := fx.logs.matching(msgLine(msg))
|
||||
if len(lines) != 1 {
|
||||
t.Errorf("lines %q = %d, want 1:\n%s", msg, len(lines), dump)
|
||||
continue
|
||||
}
|
||||
for _, k := range keys {
|
||||
if !strings.Contains(lines[0], k) {
|
||||
t.Errorf("%q line lacks %s: %s", msg, k, lines[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
deleted := fx.logs.matching(msgLine("proxmox vm deleted"))
|
||||
if len(deleted) != 2 {
|
||||
t.Fatalf("deleted lines = %d, want 2 (destroyed, then already gone):\n%s", len(deleted), dump)
|
||||
}
|
||||
if l := deleted[0]; !strings.Contains(l, `"destroyed"=true`) || !strings.Contains(l, `"node"="pv1-1"`) || !strings.Contains(l, vmid) || !strings.Contains(l, name) {
|
||||
t.Errorf("first deleted line = %s, want destroyed=true with node, vmid and name", l)
|
||||
}
|
||||
if l := deleted[1]; !strings.Contains(l, `"destroyed"=false`) || strings.Contains(l, `"node"=`) || !strings.Contains(l, vmid) || !strings.Contains(l, name) {
|
||||
t.Errorf("second deleted line = %s, want destroyed=false with vmid and name and no node", l)
|
||||
}
|
||||
assertEveryV1Line(t, all, vmid, name)
|
||||
for _, v2 := range []string{"cloudInitBytes", "proxmox create resolved", "proxmox guest agent answered", "proxmox listed instance", `"description"=`, `"config"=`, `"params"=`} {
|
||||
if got := fx.logs.matching(v2); len(got) != 0 {
|
||||
t.Errorf("V(1) carries the V(2) detail %s:\n%s", v2, strings.Join(got, "\n"))
|
||||
}
|
||||
}
|
||||
assertNeverLogged(t, all)
|
||||
})
|
||||
t.Run("v2 has detail and never the cloud-init", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newLoggingFixture(t, 2)
|
||||
fx.lifecycle()
|
||||
resolved := fx.logs.matching(msgLine("proxmox create resolved"))
|
||||
if len(resolved) != 1 {
|
||||
t.Fatalf("resolved lines = %d, want 1", len(resolved))
|
||||
}
|
||||
for _, k := range []string{name, `"cloudInitBytes"=` + strconv.Itoa(len(fx.sentinelReq().CloudInit)), `"machineType"="small"`, `"cores"=1`, `"memoryMiB"=1024`, `"templateVMID"=9001`, `"isoStorage"="proxy-seeds"`} {
|
||||
if !strings.Contains(resolved[0], k) {
|
||||
t.Errorf("resolved line lacks %s: %s", k, resolved[0])
|
||||
}
|
||||
}
|
||||
details := map[string]string{
|
||||
msgLine("proxmox guest agent answered"): `"ip"="192.168.0.150"`,
|
||||
msgLine("proxmox listed instance"): `"uid"="` + testUID + `"`,
|
||||
`"call"="Clone"`: `"description"="proxy-operator-managed=true`,
|
||||
`"call"="VMConfig"`: `"config"=`,
|
||||
`"call"="UploadISO"`: `"bytes"=`,
|
||||
}
|
||||
for line, k := range details {
|
||||
got := fx.logs.matching(line)
|
||||
if len(got) == 0 || !strings.Contains(got[0], k) {
|
||||
t.Errorf("V(2) %s line lacks %s: %v", line, k, got)
|
||||
}
|
||||
}
|
||||
assertNeverLogged(t, fx.logs.all())
|
||||
})
|
||||
}
|
||||
|
||||
// TestLogging_noOpLinesAtV1: spec entry 20's no-op lines of Delete and
|
||||
// ListByTag are there at V(1), each naming the instance.
|
||||
func TestLogging_noOpLinesAtV1(t *testing.T) {
|
||||
t.Parallel()
|
||||
del := func(fx *fixture) error { return fx.p.Delete(fx.ctx, fx.id()) }
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(fx *fixture)
|
||||
call func(fx *fixture) error
|
||||
msg string
|
||||
want int
|
||||
}{
|
||||
{name: "vm already gone", call: del, msg: "proxmox vm already gone", want: 1},
|
||||
{
|
||||
name: "foreign vm",
|
||||
setup: func(fx *fixture) { fx.addOurVM(testVMID, "proxy-someoneelse00000", "running") },
|
||||
call: del, msg: "proxmox vm at this vmid is not ours, left alone", want: 1,
|
||||
},
|
||||
{
|
||||
name: "unmarked vm",
|
||||
setup: func(fx *fixture) { fx.fake.addVM(testVMID, testName, "running") },
|
||||
call: del, msg: "proxmox vm carries no proxy-operator mark, left alone", want: 1,
|
||||
},
|
||||
{
|
||||
name: "seed delete skipped on each node",
|
||||
setup: func(fx *fixture) {
|
||||
fx.p.cfg.Nodes = []string{fakeNode, "pv1-2"}
|
||||
fx.fake.failWith("DeleteVolume", loadAPIError(fx.t, "err_forbidden_volume_delete"))
|
||||
},
|
||||
call: del, msg: "proxmox seed delete failed on a node, skipped", want: 2,
|
||||
},
|
||||
{
|
||||
name: "unreadable candidate",
|
||||
setup: func(fx *fixture) {
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
fx.fake.failWith("VMConfig", fakeErr(596, "Broken pipe"))
|
||||
},
|
||||
call: func(fx *fixture) error { _, err := fx.p.ListByTag(fx.ctx); return err },
|
||||
msg: "skipping unreadable proxmox vm", want: 1,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
if tc.setup != nil {
|
||||
tc.setup(fx)
|
||||
}
|
||||
if err := tc.call(fx); err != nil {
|
||||
t.Fatalf("error = %v, want nil", err)
|
||||
}
|
||||
lines := fx.logs.matching(msgLine(tc.msg))
|
||||
if len(lines) != tc.want {
|
||||
t.Fatalf("lines %q = %d, want %d:\n%s", tc.msg, len(lines), tc.want, strings.Join(fx.logs.all(), "\n"))
|
||||
}
|
||||
for _, l := range lines {
|
||||
for _, k := range []string{`"provider"="` + testProvider + `"`, `"vmid"=150042`, `"level"=1`} {
|
||||
if !strings.Contains(l, k) {
|
||||
t.Errorf("line lacks %s: %s", k, l)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestTracing_httpSpansUnderOperationSpan runs a Get through
|
||||
// provider.WithTracing over the real client against Phase 0's captures:
|
||||
// each of Get's four requests is an HTTP span under provider.get, and no
|
||||
// span attribute carries the token.
|
||||
func TestTracing_httpSpansUnderOperationSpan(t *testing.T) {
|
||||
t.Parallel()
|
||||
const (
|
||||
vmid = 199999
|
||||
name = "proxy-reconaaaaaaaaaaa"
|
||||
)
|
||||
cfx := newClientFixture(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
path := strings.TrimPrefix(r.URL.Path, "/api2/json/")
|
||||
capture := map[string]string{
|
||||
"cluster/resources": "cluster_resources_during_destroy",
|
||||
"nodes/pv1-1/qemu/199999/config": "config_during_destroy",
|
||||
"nodes/pv1-1/qemu/199999/status/current": "status_current_running",
|
||||
"nodes/pv1-1/qemu/199999/agent/network-get-interfaces": "agent_network_get_interfaces",
|
||||
}[path]
|
||||
if r.Method != http.MethodGet || capture == "" {
|
||||
t.Errorf("unexpected request %s %s", r.Method, path)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
serveJSON(w, readCapture(t, capture).Body)
|
||||
})
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
opt := tracing.WithTracerProvider(tp)
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.URL, cfg.TokenFile, cfg.CAFile = cfx.cfg.URL, cfx.cfg.TokenFile, cfx.cfg.CAFile
|
||||
p := newProvider(testProvider, cfg, observe(cfx.api(t, opt), testProvider))
|
||||
p.budgets = testBudgets()
|
||||
traced := provider.WithTracing(testProvider, p, opt)
|
||||
|
||||
inst, err := traced.Get(context.Background(), formatProviderID(vmid, name))
|
||||
if err != nil {
|
||||
t.Fatalf("Get error = %v", err)
|
||||
}
|
||||
if inst.State != provider.StateRunning || inst.IP != "192.168.0.150" {
|
||||
t.Errorf("Get = %+v, want Running at 192.168.0.150", inst)
|
||||
}
|
||||
|
||||
var (
|
||||
op sdktrace.ReadOnlySpan
|
||||
requests []sdktrace.ReadOnlySpan
|
||||
)
|
||||
for _, s := range sr.Ended() {
|
||||
if s.Name() == "provider.get" {
|
||||
op = s
|
||||
} else {
|
||||
requests = append(requests, s)
|
||||
}
|
||||
}
|
||||
if op == nil {
|
||||
t.Fatalf("no provider.get span among %d", len(sr.Ended()))
|
||||
}
|
||||
if len(requests) != 4 {
|
||||
t.Errorf("HTTP spans = %d, want one per request of Get (4)", len(requests))
|
||||
}
|
||||
for _, s := range requests {
|
||||
if s.Name() != "HTTP GET" {
|
||||
t.Errorf("span %q, want HTTP GET (spec entry 17)", s.Name())
|
||||
}
|
||||
if s.Parent().SpanID() != op.SpanContext().SpanID() || s.SpanContext().TraceID() != op.SpanContext().TraceID() {
|
||||
t.Errorf("span %q is not a child of provider.get", s.Name())
|
||||
}
|
||||
}
|
||||
secret := testToken[strings.Index(testToken, "=")+1:]
|
||||
for _, s := range sr.Ended() {
|
||||
for _, kv := range s.Attributes() {
|
||||
if v := kv.Value.Emit(); strings.Contains(v, secret) || strings.Contains(v, "PVEAPIToken") {
|
||||
t.Errorf("span %s attribute %s carries the token: %s", s.Name(), kv.Key, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,624 @@
|
||||
//go:build integration
|
||||
|
||||
package proxmox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/go-logr/logr/funcr"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
// The real-host test of spec entry 21. It runs only with -tags=integration
|
||||
// and skips itself unless PROXMOX_TEST_* names a host; see docs/testing.md
|
||||
// for the variables and what one run asserts.
|
||||
const (
|
||||
integrationProvider = "proxmox-integration"
|
||||
integrationImage = "test-template"
|
||||
integrationMachine = "small"
|
||||
integrationPort = 3128
|
||||
// integrationDiskGiB is above the 3 GiB of the recon template, so a run
|
||||
// on such a template exercises the resize task; a larger template skips
|
||||
// it.
|
||||
integrationDiskGiB = 4
|
||||
|
||||
// integrationCloudInit is the user-data the VM boots with. Nothing
|
||||
// asserts on the guest running it; it exists so the sentinel check can
|
||||
// prove that no span attribute carries the cloud-init text.
|
||||
integrationCloudInit = "#cloud-config\nruncmd:\n - echo proxmox-integration\n"
|
||||
|
||||
defaultIntegrationTimeout = 20 * time.Minute
|
||||
defaultIntegrationLogLvl = 1
|
||||
getPollInterval = 5 * time.Second
|
||||
cleanupBudget = 5 * time.Minute
|
||||
)
|
||||
|
||||
type integrationEnv struct {
|
||||
url, tokenFile, node, isoStorage string
|
||||
caFile, storage, bridge string
|
||||
templateVMID, vlanTag int
|
||||
vmidMin, vmidMax int
|
||||
insecure, keep bool
|
||||
timeout time.Duration
|
||||
logLevel int
|
||||
}
|
||||
|
||||
// loadIntegrationEnv reads the PROXMOX_TEST_* variables, skipping the test
|
||||
// under -short or when a required one is missing.
|
||||
func loadIntegrationEnv(t *testing.T) integrationEnv {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("real-host test skipped under -short")
|
||||
}
|
||||
env := integrationEnv{timeout: defaultIntegrationTimeout, logLevel: defaultIntegrationLogLvl}
|
||||
var templateVMID string
|
||||
required := []struct {
|
||||
name string
|
||||
dst *string
|
||||
}{
|
||||
{"PROXMOX_TEST_URL", &env.url},
|
||||
{"PROXMOX_TEST_TOKEN_FILE", &env.tokenFile},
|
||||
{"PROXMOX_TEST_NODE", &env.node},
|
||||
{"PROXMOX_TEST_TEMPLATE_VMID", &templateVMID},
|
||||
{"PROXMOX_TEST_ISO_STORAGE", &env.isoStorage},
|
||||
}
|
||||
for _, r := range required {
|
||||
v := os.Getenv(r.name)
|
||||
if v == "" {
|
||||
t.Skipf("%s is not set; the proxmox integration test needs a real Proxmox VE host (see docs/testing.md)", r.name)
|
||||
}
|
||||
*r.dst = v
|
||||
}
|
||||
env.templateVMID = envInt(t, "PROXMOX_TEST_TEMPLATE_VMID", 0)
|
||||
env.caFile = os.Getenv("PROXMOX_TEST_CA_FILE")
|
||||
env.storage = os.Getenv("PROXMOX_TEST_STORAGE")
|
||||
env.bridge = os.Getenv("PROXMOX_TEST_BRIDGE")
|
||||
env.vlanTag = envInt(t, "PROXMOX_TEST_VLAN_TAG", 0)
|
||||
env.vmidMin = envInt(t, "PROXMOX_TEST_VMID_MIN", defaultVMIDMin)
|
||||
env.vmidMax = envInt(t, "PROXMOX_TEST_VMID_MAX", defaultVMIDMax)
|
||||
if env.vmidMin > env.vmidMax {
|
||||
t.Fatalf("PROXMOX_TEST_VMID_MIN=%d is above PROXMOX_TEST_VMID_MAX=%d", env.vmidMin, env.vmidMax)
|
||||
}
|
||||
env.keep = os.Getenv("PROXMOX_TEST_KEEP") != ""
|
||||
if v := os.Getenv("PROXMOX_TEST_INSECURE"); v != "" {
|
||||
b, err := strconv.ParseBool(v)
|
||||
if err != nil {
|
||||
t.Fatalf("PROXMOX_TEST_INSECURE=%q: %v", v, err)
|
||||
}
|
||||
env.insecure = b
|
||||
}
|
||||
if v := os.Getenv("PROXMOX_TEST_TIMEOUT"); v != "" {
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil {
|
||||
t.Fatalf("PROXMOX_TEST_TIMEOUT=%q: %v", v, err)
|
||||
}
|
||||
env.timeout = d
|
||||
}
|
||||
env.logLevel = envInt(t, "PROXMOX_TEST_LOG_LEVEL", defaultIntegrationLogLvl)
|
||||
return env
|
||||
}
|
||||
|
||||
// envInt reads a non-negative integer variable, def when it is unset.
|
||||
func envInt(t *testing.T, name string, def int) int {
|
||||
t.Helper()
|
||||
v := os.Getenv(name)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 0 {
|
||||
t.Fatalf("%s=%q: want a non-negative integer", name, v)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// providerConfig is what the operator would load for this host: one node,
|
||||
// one template, one machine type.
|
||||
func (e integrationEnv) providerConfig() provider.ProviderConfig {
|
||||
return provider.ProviderConfig{
|
||||
Name: integrationProvider,
|
||||
Type: "proxmox",
|
||||
Proxmox: &provider.ProxmoxConfig{
|
||||
URL: e.url,
|
||||
TokenFile: e.tokenFile,
|
||||
CAFile: e.caFile,
|
||||
InsecureSkipTLSVerify: e.insecure,
|
||||
Nodes: []string{e.node},
|
||||
ISOStorage: e.isoStorage,
|
||||
Storage: e.storage,
|
||||
Bridge: e.bridge,
|
||||
VLANTag: e.vlanTag,
|
||||
VMIDRange: &provider.ProxmoxVMIDRange{Min: e.vmidMin, Max: e.vmidMax},
|
||||
Templates: map[string]provider.ProxmoxTemplate{integrationImage: {VMID: e.templateVMID}},
|
||||
MachineTypes: map[string]provider.ProxmoxMachineType{
|
||||
integrationMachine: {Cores: 1, MemoryMiB: 1024, DiskGiB: integrationDiskGiB},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// integrationLog routes the provider's logr output into t.Log through funcr
|
||||
// at the chosen verbosity, keeps every line for assertions, and strips the
|
||||
// key/value pairs that are constant for the run (stated once up front) so
|
||||
// each line stays short. Lines arriving after the test ended are dropped:
|
||||
// t.Log panics then.
|
||||
type integrationLog struct {
|
||||
t *testing.T
|
||||
|
||||
mu sync.Mutex
|
||||
drop []string
|
||||
lines []string
|
||||
done bool
|
||||
}
|
||||
|
||||
func newIntegrationLog(t *testing.T, level int, drop ...string) (*integrationLog, logr.Logger) {
|
||||
t.Helper()
|
||||
sink := &integrationLog{t: t, drop: drop}
|
||||
t.Cleanup(func() {
|
||||
sink.mu.Lock()
|
||||
defer sink.mu.Unlock()
|
||||
sink.done = true
|
||||
})
|
||||
log := funcr.New(func(prefix, args string) { sink.write(prefix, args) }, funcr.Options{Verbosity: level})
|
||||
return sink, log
|
||||
}
|
||||
|
||||
// dropValue adds a rendered pair (`"key"="value"`) to strip from later lines.
|
||||
func (l *integrationLog) dropValue(key, value string) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.drop = append(l.drop, fmt.Sprintf("%q=%q", key, value))
|
||||
}
|
||||
|
||||
func (l *integrationLog) write(prefix, args string) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.lines = append(l.lines, args)
|
||||
if l.done {
|
||||
return
|
||||
}
|
||||
for _, d := range l.drop {
|
||||
args = strings.ReplaceAll(args, " "+d, "")
|
||||
}
|
||||
l.t.Logf("%s %s %s", time.Now().Format("15:04:05.000"), prefix, args)
|
||||
}
|
||||
|
||||
func (l *integrationLog) matching(substr string) []string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
var out []string
|
||||
for _, line := range l.lines {
|
||||
if strings.Contains(line, substr) {
|
||||
out = append(out, line)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// newIntegrationProvider builds the provider the way New does, with the
|
||||
// span recorder's tracer on the transport, so the HTTP spans land in the
|
||||
// same recorder as provider.WithTracing's provider.* spans. The bare client
|
||||
// is returned too, for the test's own read-only inspection of the host.
|
||||
func newIntegrationProvider(t *testing.T, pc provider.ProviderConfig, tp *sdktrace.TracerProvider) (*Provider, *httpAPI) {
|
||||
t.Helper()
|
||||
cfg := withDefaults(*pc.Proxmox)
|
||||
client, err := newHTTPAPI(cfg, tracing.WithTracerProvider(tp))
|
||||
if err != nil {
|
||||
t.Fatalf("newHTTPAPI error = %v", err)
|
||||
}
|
||||
return newProvider(pc.Name, cfg, observe(client, pc.Name)), client
|
||||
}
|
||||
|
||||
// randomUID is a UUID-shaped string, so the run's VM never collides with a
|
||||
// previous run's leftovers or with a real Proxy's.
|
||||
func randomUID(t *testing.T) string {
|
||||
t.Helper()
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
t.Fatalf("reading random bytes: %v", err)
|
||||
}
|
||||
h := hex.EncodeToString(b[:])
|
||||
return h[0:8] + "-" + h[8:12] + "-" + h[12:16] + "-" + h[16:20] + "-" + h[20:32]
|
||||
}
|
||||
|
||||
// stepDone logs one lifecycle step with its duration; the log is the record
|
||||
// of how long a clone or a boot took on this host.
|
||||
func stepDone(t *testing.T, step string, start time.Time) {
|
||||
t.Helper()
|
||||
t.Logf("step %s: %s", step, time.Since(start).Round(time.Millisecond))
|
||||
}
|
||||
|
||||
// waitUntil polls cond every interval until it is true or ctx ends.
|
||||
func waitUntil(ctx context.Context, t *testing.T, interval time.Duration, what string, cond func() bool) {
|
||||
t.Helper()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for !cond() {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("gave up waiting for %s: %v", what, ctx.Err())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// qemuVMs lists the cluster's QEMU VMs by VMID.
|
||||
func qemuVMs(ctx context.Context, t *testing.T, client *httpAPI) map[int]vmResource {
|
||||
t.Helper()
|
||||
rs, err := client.ClusterResources(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("listing cluster resources: %v", err)
|
||||
}
|
||||
out := make(map[int]vmResource, len(rs))
|
||||
for _, r := range rs {
|
||||
if r.Type == "qemu" {
|
||||
out[r.VMID] = r
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// vmidByName finds the VM named name, as a crashed Create may have left it.
|
||||
func vmidByName(ctx context.Context, t *testing.T, client *httpAPI, name string) (int, bool) {
|
||||
t.Helper()
|
||||
for vmid, r := range qemuVMs(ctx, t, client) {
|
||||
if r.Name == name {
|
||||
return vmid, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// seedOnStorage reports whether the ISO storage on node holds name's seed.
|
||||
// It is one extra read-only GET of /nodes/{node}/storage/{storage}/content
|
||||
// (captured as storage_content_iso.json): the seam has no method to list
|
||||
// storage content, and gets none for a test.
|
||||
func seedOnStorage(ctx context.Context, t *testing.T, client *httpAPI, node, storage, name string) bool {
|
||||
t.Helper()
|
||||
var content []struct {
|
||||
VolID string `json:"volid"`
|
||||
}
|
||||
path := apiPath("nodes", node, "storage", storage, "content")
|
||||
if err := client.do(ctx, http.MethodGet, path, url.Values{"content": {"iso"}}, nil, &content); err != nil {
|
||||
t.Fatalf("listing iso content of %s on %s: %v", storage, node, err)
|
||||
}
|
||||
want := seedVolID(storage, name)
|
||||
for _, c := range content {
|
||||
if c.VolID == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// checkNode runs the health check and requires the one node Ready, with the
|
||||
// details an admin reads.
|
||||
func checkNode(ctx context.Context, t *testing.T, built provider.Provider, env integrationEnv) {
|
||||
t.Helper()
|
||||
checker, ok := built.(provider.Checker)
|
||||
if !ok {
|
||||
t.Fatalf("New() returned %T, which is not a provider.Checker", built)
|
||||
}
|
||||
targets := checker.Check(ctx)
|
||||
if len(targets) != 1 {
|
||||
t.Fatalf("Check() returned %d targets, want 1: %+v", len(targets), targets)
|
||||
}
|
||||
ts := targets[0]
|
||||
if !ts.Ready || ts.Reason != provider.ReasonReachable {
|
||||
t.Fatalf("Check() = ready %v reason %q message %q, want Ready/%s (details %v)", ts.Ready, ts.Reason, ts.Message, provider.ReasonReachable, ts.Details)
|
||||
}
|
||||
for _, key := range []string{"pveVersion", "kernel", "tokenID", "isoStorageAvailGiB", "templates"} {
|
||||
if ts.Details[key] == "" {
|
||||
t.Errorf("Check() details lack %q: %v", key, ts.Details)
|
||||
}
|
||||
}
|
||||
if ts.Details["templates"] != "1/1" {
|
||||
t.Errorf("Check() templates = %q, want 1/1", ts.Details["templates"])
|
||||
}
|
||||
t.Logf("node %s (%s): pve %s, kernel %s, token %s, %s free on %s, templates %s, latency %s",
|
||||
ts.Name, ts.Address, ts.Details["pveVersion"], ts.Details["kernel"], ts.Details["tokenID"],
|
||||
ts.Details["isoStorageAvailGiB"]+" GiB", env.isoStorage, ts.Details["templates"], ts.Latency.Round(time.Millisecond))
|
||||
}
|
||||
|
||||
func TestProxmox_lifecycle(t *testing.T) {
|
||||
env := loadIntegrationEnv(t)
|
||||
pc := env.providerConfig()
|
||||
sink, log := newIntegrationLog(t, env.logLevel,
|
||||
fmt.Sprintf("%q=%q", "provider", integrationProvider),
|
||||
fmt.Sprintf("%q=%q", "node", env.node))
|
||||
ctx, cancel := context.WithTimeout(logf.IntoContext(context.Background(), log), env.timeout)
|
||||
defer cancel()
|
||||
|
||||
t.Logf("provider=%s url=%s node=%s template=%d isoStorage=%s storage=%q bridge=%q vlanTag=%d vmids %d..%d budget=%s log level %d; provider, node and name are left off every provider log line",
|
||||
integrationProvider, env.url, env.node, env.templateVMID, env.isoStorage, env.storage, env.bridge, env.vlanTag,
|
||||
env.vmidMin, env.vmidMax, env.timeout, env.logLevel)
|
||||
if env.insecure {
|
||||
t.Log("PROXMOX_TEST_INSECURE is set: the API's certificate is NOT verified for this run")
|
||||
}
|
||||
|
||||
// 1. New returns without a request.
|
||||
start := time.Now()
|
||||
built, err := New(ctx, pc)
|
||||
elapsed := time.Since(start)
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
if elapsed > 100*time.Millisecond {
|
||||
t.Errorf("New() took %s, want under 100ms: it must not make a request", elapsed)
|
||||
}
|
||||
if lines := sink.matching(`"msg"="proxmox api call"`); len(lines) > 0 {
|
||||
t.Errorf("New() called the API before any operation: %q", lines)
|
||||
}
|
||||
stepDone(t, "1 New", start)
|
||||
|
||||
// 2. Check reports the node Ready with details.
|
||||
start = time.Now()
|
||||
checkNode(ctx, t, built, env)
|
||||
stepDone(t, "2 Check", start)
|
||||
|
||||
// 3. Every later call goes through the tracing decorator, with the
|
||||
// transport's HTTP spans recorded alongside.
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
p, client := newIntegrationProvider(t, pc, tp)
|
||||
traced := provider.WithTracing(pc.Name, p, tracing.WithTracerProvider(tp))
|
||||
|
||||
// 4. Create.
|
||||
uid := randomUID(t)
|
||||
name := provider.NameFromUID(types.UID(uid))
|
||||
sink.dropValue("name", name)
|
||||
req := provider.CreateRequest{
|
||||
Name: name,
|
||||
UID: uid,
|
||||
Namespace: "integration",
|
||||
ProxyName: "proxmox-lifecycle",
|
||||
Placement: provider.Placement{Zone: env.node, MachineType: integrationMachine, Image: integrationImage},
|
||||
CloudInit: integrationCloudInit,
|
||||
Port: integrationPort,
|
||||
}
|
||||
t.Logf("VM name %s, uid %s", name, uid)
|
||||
|
||||
deleted := false
|
||||
t.Cleanup(func() {
|
||||
if deleted {
|
||||
return
|
||||
}
|
||||
cctx, ccancel := context.WithTimeout(logf.IntoContext(context.Background(), log), cleanupBudget)
|
||||
defer ccancel()
|
||||
vmid, found := vmidByName(cctx, t, client, name)
|
||||
if env.keep {
|
||||
msg := "PROXMOX_TEST_KEEP is set: the VM and its seed stay. Clean up as root on " + env.node + " with:\n"
|
||||
if found {
|
||||
msg += fmt.Sprintf(" qm stop %d\n qm destroy %d --purge\n", vmid, vmid)
|
||||
}
|
||||
t.Log(msg + " pvesm free " + seedVolID(env.isoStorage, name))
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
// No VM, but a seed may be left: Delete of an absent VM still
|
||||
// deletes it.
|
||||
vmid = vmidFor(uid, env.vmidMin, env.vmidMax, 0)
|
||||
}
|
||||
if err := p.Delete(cctx, formatProviderID(vmid, name)); err != nil {
|
||||
t.Errorf("cleanup Delete of VM %d error = %v; remove it and %s by hand", vmid, err, seedVolID(env.isoStorage, name))
|
||||
}
|
||||
})
|
||||
|
||||
start = time.Now()
|
||||
id, err := traced.Create(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
vmid, gotName, err := parseProviderID(id)
|
||||
if err != nil || gotName != name {
|
||||
t.Fatalf("Create() = %q, want vms/<vmid>/%s (parse error %v)", id, name, err)
|
||||
}
|
||||
if vmid < env.vmidMin || vmid > env.vmidMax {
|
||||
t.Errorf("Create() VMID %d is outside %d..%d", vmid, env.vmidMin, env.vmidMax)
|
||||
}
|
||||
sink.dropValue("vmid", strconv.Itoa(vmid))
|
||||
createReturned := time.Now()
|
||||
t.Logf("VM %d, id %s", vmid, id)
|
||||
stepDone(t, "4 Create", start)
|
||||
|
||||
// 5. Create again: same ID, nothing new in the cluster.
|
||||
start = time.Now()
|
||||
before := len(qemuVMs(ctx, t, client))
|
||||
again, err := traced.Create(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("second Create() error = %v", err)
|
||||
}
|
||||
if again != id {
|
||||
t.Errorf("second Create() = %q, want %q", again, id)
|
||||
}
|
||||
if after := len(qemuVMs(ctx, t, client)); after != before {
|
||||
t.Errorf("second Create() changed the cluster's VM count from %d to %d", before, after)
|
||||
}
|
||||
stepDone(t, "5 Create again", start)
|
||||
|
||||
// 6. Get reaches Running with an IPv4, and reads back what Create wrote.
|
||||
start = time.Now()
|
||||
var inst *provider.Instance
|
||||
waitUntil(ctx, t, getPollInterval, "the VM to be Running with an IPv4", func() bool {
|
||||
var err error
|
||||
inst, err = traced.Get(ctx, id)
|
||||
if errors.Is(err, provider.ErrTransient) {
|
||||
t.Logf("Get(%s) is Transient while the guest boots: %v", id, err)
|
||||
return false
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Get(%s) error = %v", id, err)
|
||||
}
|
||||
return inst.State == provider.StateRunning && net.ParseIP(inst.IP).To4() != nil
|
||||
})
|
||||
assertMarkReadBack(ctx, t, client, env.node, vmid, inst, uid)
|
||||
t.Logf("VM running at %s, %s after Create returned (createdAt %s)", inst.IP, time.Since(createReturned).Round(time.Millisecond), inst.CreatedAt.Format(time.RFC3339))
|
||||
stepDone(t, "6 Get until Running", start)
|
||||
|
||||
// 7. ListByTag contains it.
|
||||
start = time.Now()
|
||||
list, err := traced.ListByTag(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListByTag() error = %v", err)
|
||||
}
|
||||
assertListed(t, list, inst)
|
||||
stepDone(t, "7 ListByTag", start)
|
||||
|
||||
// 8. Reachability, logged only: this machine may not route to the VM's
|
||||
// network.
|
||||
start = time.Now()
|
||||
addr := net.JoinHostPort(inst.IP, "22")
|
||||
if conn, err := net.DialTimeout("tcp", addr, 3*time.Second); err != nil {
|
||||
t.Logf("TCP connect to %s failed (informational; the test host may not route to the VM's network): %v", addr, err)
|
||||
} else {
|
||||
_ = conn.Close()
|
||||
t.Logf("TCP connect to %s succeeded", addr)
|
||||
}
|
||||
stepDone(t, "8 TCP connect", start)
|
||||
|
||||
if env.keep {
|
||||
t.Log("PROXMOX_TEST_KEEP is set: skipping Delete and the leftovers check")
|
||||
} else {
|
||||
// 9. Delete, Get is NotFound, Delete again.
|
||||
start = time.Now()
|
||||
if err := traced.Delete(ctx, id); err != nil {
|
||||
t.Fatalf("Delete() error = %v", err)
|
||||
}
|
||||
deleted = true
|
||||
if _, err := traced.Get(ctx, id); !errors.Is(err, provider.ErrNotFound) {
|
||||
t.Errorf("Get() after Delete error = %v, want ErrNotFound", err)
|
||||
}
|
||||
if err := traced.Delete(ctx, id); err != nil {
|
||||
t.Errorf("second Delete() error = %v, want nil", err)
|
||||
}
|
||||
stepDone(t, "9 Delete, Get, Delete", start)
|
||||
|
||||
// 10. No VM with the name and no seed remain.
|
||||
start = time.Now()
|
||||
if left, ok := vmidByName(ctx, t, client, name); ok {
|
||||
t.Errorf("VM %d named %s is still in the cluster after Delete", left, name)
|
||||
}
|
||||
if seedOnStorage(ctx, t, client, env.node, env.isoStorage, name) {
|
||||
t.Errorf("%s is still on %s after Delete", seedVolID(env.isoStorage, name), env.node)
|
||||
}
|
||||
stepDone(t, "10 leftovers", start)
|
||||
}
|
||||
|
||||
// 11. Spans: HTTP spans are children of the decorator's.
|
||||
assertLifecycleSpans(t, sr.Ended())
|
||||
}
|
||||
|
||||
// assertMarkReadBack checks that Get's UID and CreatedAt are the mark
|
||||
// Create wrote into the VM's description, read back from its config, and
|
||||
// that CreatedAt is recent.
|
||||
func assertMarkReadBack(ctx context.Context, t *testing.T, client *httpAPI, node string, vmid int, inst *provider.Instance, uid string) {
|
||||
t.Helper()
|
||||
cfg, err := client.VMConfig(ctx, node, vmid)
|
||||
if err != nil {
|
||||
t.Fatalf("reading the config of VM %d: %v", vmid, err)
|
||||
}
|
||||
written := parseDescription(cfg["description"])
|
||||
if inst.UID != uid || written.UID != uid {
|
||||
t.Errorf("Get() UID = %q, the VM's mark says %q, want %q", inst.UID, written.UID, uid)
|
||||
}
|
||||
if !inst.CreatedAt.Equal(written.CreatedAt) || inst.CreatedAt.IsZero() {
|
||||
t.Errorf("Get() CreatedAt = %s, the VM's mark says %s; want them equal and set", inst.CreatedAt, written.CreatedAt)
|
||||
} else if skew := time.Since(inst.CreatedAt).Abs(); skew > 5*time.Minute {
|
||||
t.Errorf("Get() CreatedAt = %s, %s from now; want within a few minutes", inst.CreatedAt, skew)
|
||||
}
|
||||
}
|
||||
|
||||
// assertListed checks that list holds inst's ID with its UID and
|
||||
// CreatedAt.
|
||||
func assertListed(t *testing.T, list []provider.Instance, inst *provider.Instance) {
|
||||
t.Helper()
|
||||
for _, li := range list {
|
||||
if li.ID != inst.ID {
|
||||
continue
|
||||
}
|
||||
if li.UID != inst.UID || !li.CreatedAt.Equal(inst.CreatedAt) {
|
||||
t.Errorf("ListByTag() entry = %+v, want UID %q and CreatedAt %s", li, inst.UID, inst.CreatedAt)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Errorf("ListByTag() = %d instances, none with ID %s", len(list), inst.ID)
|
||||
}
|
||||
|
||||
// assertLifecycleSpans checks that at least one provider.create span has an
|
||||
// HTTP child, and that no attribute of any span carries the cloud-init text.
|
||||
func assertLifecycleSpans(t *testing.T, spans []sdktrace.ReadOnlySpan) {
|
||||
t.Helper()
|
||||
satisfied := false
|
||||
for _, create := range spans {
|
||||
if create.Name() != "provider.create" {
|
||||
continue
|
||||
}
|
||||
parent := create.SpanContext().SpanID()
|
||||
children := 0
|
||||
for _, s := range spans {
|
||||
if s.Parent().SpanID() == parent && strings.HasPrefix(s.Name(), "HTTP ") {
|
||||
children++
|
||||
}
|
||||
}
|
||||
t.Logf("provider.create span %s: %d HTTP children", parent, children)
|
||||
if children > 0 {
|
||||
satisfied = true
|
||||
}
|
||||
}
|
||||
if !satisfied {
|
||||
names := make([]string, len(spans))
|
||||
for i, s := range spans {
|
||||
names[i] = s.Name()
|
||||
}
|
||||
t.Errorf("no provider.create span has an HTTP child; spans: %v", names)
|
||||
}
|
||||
for _, s := range spans {
|
||||
for _, a := range s.Attributes() {
|
||||
v := a.Value.Emit()
|
||||
if strings.Contains(v, integrationCloudInit) || strings.Contains(v, "#cloud-config") {
|
||||
t.Errorf("span %s attribute %s carries the cloud-init text", s.Name(), a.Key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProxmoxCheck_realHost runs the health check against the host and
|
||||
// creates nothing.
|
||||
func TestProxmoxCheck_realHost(t *testing.T) {
|
||||
env := loadIntegrationEnv(t)
|
||||
pc := env.providerConfig()
|
||||
_, log := newIntegrationLog(t, env.logLevel,
|
||||
fmt.Sprintf("%q=%q", "provider", integrationProvider),
|
||||
fmt.Sprintf("%q=%q", "node", env.node))
|
||||
ctx, cancel := context.WithTimeout(logf.IntoContext(context.Background(), log), env.timeout)
|
||||
defer cancel()
|
||||
if env.insecure {
|
||||
t.Log("PROXMOX_TEST_INSECURE is set: the API's certificate is NOT verified for this run")
|
||||
}
|
||||
built, err := New(ctx, pc)
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
checkNode(ctx, t, built, env)
|
||||
}
|
||||
@@ -230,7 +230,7 @@ func TestLockName_givesUpWithContextAsTransient(t *testing.T) {
|
||||
other()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err = p.lockName(ctx, "proxy-abc")
|
||||
within(t, 5*time.Second, func() { _, err = p.lockName(ctx, "proxy-abc") })
|
||||
if !errors.Is(err, provider.ErrTransient) || !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Errorf("lockName on a held name error = %v, want ErrTransient wrapping DeadlineExceeded", err)
|
||||
}
|
||||
|
||||
@@ -110,7 +110,8 @@ func TestWaitTask_table(t *testing.T) {
|
||||
cancel()
|
||||
}
|
||||
start := time.Now()
|
||||
err := fx.p.waitTask(ctx, fx.p.logger(ctx), upid, budget)
|
||||
var err error
|
||||
within(t, 10*time.Second, func() { err = fx.p.waitTask(ctx, fx.p.logger(ctx), upid, budget) })
|
||||
if elapsed := time.Since(start); elapsed > 5*time.Second {
|
||||
t.Errorf("waitTask took %v", elapsed)
|
||||
}
|
||||
@@ -219,7 +220,11 @@ func TestWaitUnlocked_table(t *testing.T) {
|
||||
ctx, cancel = context.WithCancel(ctx)
|
||||
cancel()
|
||||
}
|
||||
cfg, err := fx.p.waitUnlocked(ctx, fakeNode, testVMID, budget)
|
||||
var (
|
||||
cfg vmConfig
|
||||
err error
|
||||
)
|
||||
within(t, 10*time.Second, func() { cfg, err = fx.p.waitUnlocked(ctx, fakeNode, testVMID, budget) })
|
||||
if tc.wantErr == nil {
|
||||
if err != nil || cfg["name"] != testName || cfg["lock"] != tc.wantLock {
|
||||
t.Errorf("waitUnlocked = %v, %v; want the config with lock %q", cfg, err, tc.wantLock)
|
||||
|
||||
Reference in New Issue
Block a user