proxmox: Delete with seed cleanup, health Checker, CRD type comment
Delete stops and destroys a VM of ours under the per-name lock, then deletes its seed on the VM's node, or on every node when no VM of ours was left. A hibernated VM of ours is Permanent, as in Create. The Checker verifies each node concurrently (version, node status, ISO storage, templates) and adds CertificateInvalid, CredentialsUnavailable, StorageNotFound and TemplateNotFound to the shared reasons. The ProviderHealth type comment gains proxmox. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,218 @@
|
||||
package proxmox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
var _ provider.Checker = (*Provider)(nil)
|
||||
|
||||
// Reasons only this provider can tell apart (spec entry 16). Each names a
|
||||
// failure an admin fixes somewhere other than where the shared reasons
|
||||
// point.
|
||||
const (
|
||||
// reasonCertificateInvalid: TLS verification failed or the CA file is
|
||||
// unusable; fix url or caFile, not the host.
|
||||
reasonCertificateInvalid = "CertificateInvalid"
|
||||
// reasonCredentialsUnavailable: the token file is missing or malformed;
|
||||
// fix the Secret, not Proxmox.
|
||||
reasonCredentialsUnavailable = "CredentialsUnavailable"
|
||||
// reasonStorageNotFound: isoStorage is missing, inactive, or holds no
|
||||
// iso content on the node; fix the storage on the node.
|
||||
reasonStorageNotFound = "StorageNotFound"
|
||||
// reasonTemplateNotFound: a configured template VMID does not exist or
|
||||
// is not a template; fix the template or the catalogue.
|
||||
reasonTemplateNotFound = "TemplateNotFound"
|
||||
)
|
||||
|
||||
// pveproxy's own answers when it cannot reach the node behind it.
|
||||
const (
|
||||
statusProxyConnect = 595
|
||||
statusProxyBroken = 596
|
||||
)
|
||||
|
||||
// Check verifies every configured node concurrently and returns one verdict
|
||||
// per node in config order. The requests share the provider's HTTP client,
|
||||
// but each is its own request, so a round that runs out of time cancels
|
||||
// only its own requests and never a reconcile's.
|
||||
func (p *Provider) Check(ctx context.Context) []provider.TargetStatus {
|
||||
out := make([]provider.TargetStatus, len(p.cfg.Nodes))
|
||||
var wg sync.WaitGroup
|
||||
for i, node := range p.cfg.Nodes {
|
||||
wg.Go(func() { out[i] = p.checkNode(ctx, node) })
|
||||
}
|
||||
wg.Wait()
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *Provider) checkNode(ctx context.Context, node string) provider.TargetStatus {
|
||||
start := time.Now()
|
||||
log := p.logger(ctx).WithValues("node", node)
|
||||
ts := provider.TargetStatus{
|
||||
Name: node,
|
||||
Address: p.cfg.URL,
|
||||
Details: map[string]string{"isoStorage": p.cfg.ISOStorage},
|
||||
}
|
||||
err := p.probeNode(ctx, node, ts.Details)
|
||||
ts.Latency = time.Since(start)
|
||||
switch {
|
||||
case err == nil:
|
||||
ts.Ready = true
|
||||
ts.Reason = provider.ReasonReachable
|
||||
case ctx.Err() != nil:
|
||||
// The round ran out of time, whichever request was running.
|
||||
ts.Reason = provider.ReasonTimeout
|
||||
ts.Message = err.Error()
|
||||
default:
|
||||
ts.Reason = checkReason(err)
|
||||
ts.Message = err.Error()
|
||||
}
|
||||
log.V(1).Info("proxmox node checked", "ready", ts.Ready, "reason", ts.Reason, "latency", ts.Latency)
|
||||
return ts
|
||||
}
|
||||
|
||||
// probeNode runs spec entry 16's four requests and fills details as facts
|
||||
// arrive, so a failure late in the sequence still reports what came before.
|
||||
func (p *Provider) probeNode(ctx context.Context, node string, details map[string]string) error {
|
||||
// The token file is read here only for its id: the secret after "="
|
||||
// is dropped at once and never reaches details, logs or errors.
|
||||
token, err := readToken(p.cfg.TokenFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tokenID, _, _ := strings.Cut(token, "=")
|
||||
details["tokenID"] = tokenID
|
||||
|
||||
v, err := p.api.Version(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the API version: %w", err)
|
||||
}
|
||||
details["pveVersion"] = v.Version
|
||||
ns, err := p.api.NodeStatus(ctx, node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the status of node %s: %w", node, err)
|
||||
}
|
||||
details["kernel"] = ns.CurrentKernel.Release
|
||||
if err := p.checkStorage(ctx, node, details); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.checkTemplates(ctx, details)
|
||||
}
|
||||
|
||||
// checkStorage reads isoStorage on the node. A 500 means Proxmox cannot
|
||||
// use that storage on this node (it does not exist, is not enabled here,
|
||||
// or is offline), whatever its wording; the message carries Proxmox's
|
||||
// text.
|
||||
func (p *Provider) checkStorage(ctx context.Context, node string, details map[string]string) error {
|
||||
storage := p.cfg.ISOStorage
|
||||
st, err := p.api.StorageStatus(ctx, node, storage)
|
||||
var ae *apiError
|
||||
if errors.As(err, &ae) && ae.Status == http.StatusInternalServerError {
|
||||
return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s on node %s: %s", storage, node, ae.Message)}
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading storage %s on node %s: %w", storage, node, err)
|
||||
}
|
||||
details["isoStorageAvailGiB"] = formatGiB(st.Avail)
|
||||
if st.Active == 0 {
|
||||
return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s is not active on node %s", storage, node)}
|
||||
}
|
||||
if !slices.Contains(strings.Split(st.Content, ","), "iso") {
|
||||
return &checkFailure{reason: reasonStorageNotFound, msg: fmt.Sprintf("storage %s on node %s does not hold iso content (content: %s)", storage, node, st.Content)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkTemplates looks every catalogue entry up in the cluster's resources:
|
||||
// a template may sit on any node, since a clone can target another.
|
||||
func (p *Provider) checkTemplates(ctx context.Context, details map[string]string) error {
|
||||
rs, err := p.api.ClusterResources(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing cluster resources: %w", err)
|
||||
}
|
||||
isTemplate := make(map[int]bool)
|
||||
for _, r := range rs {
|
||||
if r.Type == "qemu" {
|
||||
isTemplate[r.VMID] = r.Template == 1
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
for _, key := range slices.Sorted(maps.Keys(p.cfg.Templates)) {
|
||||
vmid := p.cfg.Templates[key].VMID
|
||||
switch tmpl, found := isTemplate[vmid]; {
|
||||
case !found:
|
||||
problems = append(problems, fmt.Sprintf("template %q: VM %d does not exist", key, vmid))
|
||||
case !tmpl:
|
||||
problems = append(problems, fmt.Sprintf("template %q: VM %d is not a template", key, vmid))
|
||||
}
|
||||
}
|
||||
details["templates"] = fmt.Sprintf("%d/%d", len(p.cfg.Templates)-len(problems), len(p.cfg.Templates))
|
||||
if len(problems) > 0 {
|
||||
return &checkFailure{reason: reasonTemplateNotFound, msg: strings.Join(problems, "; ")}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkFailure is a verdict the backend answered but the check refuses: the
|
||||
// storage or a template is not usable.
|
||||
type checkFailure struct {
|
||||
reason string
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *checkFailure) Error() string { return e.msg }
|
||||
|
||||
// checkReason maps a failed probe to a reason once checkNode has ruled out
|
||||
// the round's ctx ending. A failed connect or a cut connection is
|
||||
// Unreachable; a request timeout with the round still running is Timeout.
|
||||
func checkReason(err error) string {
|
||||
var (
|
||||
cf *checkFailure
|
||||
ae *apiError
|
||||
op *net.OpError
|
||||
ne net.Error
|
||||
)
|
||||
switch {
|
||||
case errors.As(err, &cf):
|
||||
return cf.reason
|
||||
case errors.Is(err, errCredentials):
|
||||
return reasonCredentialsUnavailable
|
||||
case errors.Is(err, errCAFile), isTLSVerification(err):
|
||||
return reasonCertificateInvalid
|
||||
case errors.As(err, &ae):
|
||||
switch ae.Status {
|
||||
case http.StatusUnauthorized:
|
||||
return provider.ReasonUnauthorized
|
||||
case http.StatusForbidden:
|
||||
return provider.ReasonForbidden
|
||||
case statusProxyConnect, statusProxyBroken:
|
||||
return provider.ReasonUnreachable
|
||||
}
|
||||
return provider.ReasonAPIError
|
||||
case errors.As(err, &op):
|
||||
// Checked before the deadline: a dial that runs out of its own
|
||||
// budget is a node that did not answer.
|
||||
return provider.ReasonUnreachable
|
||||
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
|
||||
return provider.ReasonTimeout
|
||||
case errors.As(err, &ne), errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
|
||||
return provider.ReasonUnreachable
|
||||
}
|
||||
return provider.ReasonAPIError
|
||||
}
|
||||
|
||||
func formatGiB(bytes int64) string {
|
||||
return strconv.FormatFloat(float64(bytes)/(1<<30), 'f', 1, 64)
|
||||
}
|
||||
@@ -0,0 +1,577 @@
|
||||
package proxmox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// checkSecret is the secret half of the Checker's token file; it must never
|
||||
// reach a verdict or a log line.
|
||||
const checkSecret = "SENTINEL-token-secret-4c1d"
|
||||
|
||||
// newCheckFixture is an observed create fixture at V(2) with a token file
|
||||
// whose secret is checkSecret.
|
||||
func newCheckFixture(t *testing.T) *fixture {
|
||||
t.Helper()
|
||||
fx := newCreateFixture(t, 2)
|
||||
fx.p.api = observe(fx.fake, testProvider)
|
||||
fx.p.cfg.TokenFile = filepath.Join(t.TempDir(), "token")
|
||||
writeFile(t, fx.p.cfg.TokenFile, "proxy-operator@pve!operator="+checkSecret+"\n")
|
||||
return fx
|
||||
}
|
||||
|
||||
// assertNoSecret checks every verdict and every log line for checkSecret.
|
||||
func assertNoSecret(t *testing.T, fx *fixture, got []provider.TargetStatus) {
|
||||
t.Helper()
|
||||
for _, ts := range got {
|
||||
if strings.Contains(ts.Message, checkSecret) || slices.ContainsFunc(slices.Collect(maps.Values(ts.Details)), func(v string) bool { return strings.Contains(v, checkSecret) }) {
|
||||
t.Errorf("verdict carries the token secret: %+v", ts)
|
||||
}
|
||||
}
|
||||
for _, l := range fx.logs.all() {
|
||||
if strings.Contains(l, checkSecret) {
|
||||
t.Errorf("log line carries the token secret: %s", l)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheck_oneEntryPerNodeInConfigOrder(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newCheckFixture(t)
|
||||
fx.p.cfg.Nodes = []string{"pv1-3", fakeNode, "pv1-2"}
|
||||
fx.fake.nodes["pv1-3"] = fx.fake.nodes[fakeNode]
|
||||
got := fx.p.Check(fx.ctx)
|
||||
if len(got) != len(fx.p.cfg.Nodes) {
|
||||
t.Fatalf("Check() returned %d targets, want %d", len(got), len(fx.p.cfg.Nodes))
|
||||
}
|
||||
for i, node := range fx.p.cfg.Nodes {
|
||||
ts := got[i]
|
||||
if ts.Name != node || ts.Address != fx.p.cfg.URL {
|
||||
t.Errorf("target %d = %s (%s), want %s (%s)", i, ts.Name, ts.Address, node, fx.p.cfg.URL)
|
||||
}
|
||||
if wantReady := node != "pv1-2"; ts.Ready != wantReady {
|
||||
t.Errorf("target %s ready = %v, want %v (%s: %s)", node, ts.Ready, wantReady, ts.Reason, ts.Message)
|
||||
}
|
||||
}
|
||||
if got[2].Reason != provider.ReasonAPIError || !strings.Contains(got[2].Message, "pv1-2") {
|
||||
t.Errorf("pv1-2 verdict = %s: %s, want APIError naming the node", got[2].Reason, got[2].Message)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheck_nodesCheckedConcurrently holds every node's first request
|
||||
// until all of them have arrived; checked one at a time, the first would
|
||||
// give up and fail its verdict.
|
||||
func TestCheck_nodesCheckedConcurrently(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newCheckFixture(t)
|
||||
fx.p.cfg.Nodes = []string{fakeNode, "pv1-2", "pv1-3"}
|
||||
for _, n := range fx.p.cfg.Nodes[1:] {
|
||||
fx.fake.nodes[n] = fx.fake.nodes[fakeNode]
|
||||
}
|
||||
var (
|
||||
mu sync.Mutex
|
||||
arrived int
|
||||
)
|
||||
all := make(chan struct{})
|
||||
fx.fake.hookOn("Version", func() error {
|
||||
mu.Lock()
|
||||
if arrived++; arrived == len(fx.p.cfg.Nodes) {
|
||||
close(all)
|
||||
}
|
||||
mu.Unlock()
|
||||
select {
|
||||
case <-all:
|
||||
return nil
|
||||
case <-time.After(2 * time.Second):
|
||||
return errors.New("barrier: the nodes were not checked concurrently")
|
||||
}
|
||||
})
|
||||
for _, ts := range fx.p.Check(fx.ctx) {
|
||||
if !ts.Ready {
|
||||
t.Errorf("target %s = %s: %s, want Ready", ts.Name, ts.Reason, ts.Message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheck_expiredContextReturnsPromptly(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newCheckFixture(t)
|
||||
fx.p.cfg.Nodes = []string{fakeNode, "pv1-2"}
|
||||
fx.fake.blockOn("Version")
|
||||
ctx, cancel := context.WithTimeout(fx.ctx, 0)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
got := fx.p.Check(ctx)
|
||||
if elapsed := time.Since(start); elapsed > time.Second {
|
||||
t.Errorf("Check() took %v with an expired ctx, want under 1s", elapsed)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("Check() returned %d targets, want 2", len(got))
|
||||
}
|
||||
for _, ts := range got {
|
||||
if ts.Ready || ts.Reason != provider.ReasonTimeout || !strings.Contains(ts.Message, context.DeadlineExceeded.Error()) {
|
||||
t.Errorf("target %s = ready %v reason %s (%s), want Timeout naming the deadline", ts.Name, ts.Ready, ts.Reason, ts.Message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheck_reasonsTable(t *testing.T) {
|
||||
t.Parallel()
|
||||
urlErr := func(err error) error {
|
||||
return &url.Error{Op: "Get", URL: "https://192.0.2.8:8006/api2/json/version", Err: err}
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
prep func(fx *fixture)
|
||||
timeout time.Duration
|
||||
cancel bool
|
||||
|
||||
wantReady bool
|
||||
wantReason string
|
||||
wantMessage string
|
||||
}{
|
||||
{name: "ok", wantReady: true, wantReason: provider.ReasonReachable},
|
||||
{
|
||||
name: "token file missing",
|
||||
prep: func(fx *fixture) { fx.p.cfg.TokenFile = filepath.Join(fx.t.TempDir(), "absent") },
|
||||
wantReason: reasonCredentialsUnavailable,
|
||||
wantMessage: "does not exist yet",
|
||||
},
|
||||
{
|
||||
name: "token file malformed",
|
||||
prep: func(fx *fixture) { writeFile(fx.t, fx.p.cfg.TokenFile, checkSecret+"\n") },
|
||||
wantReason: reasonCredentialsUnavailable,
|
||||
wantMessage: "is not <user>@<realm>!<tokenid>=<secret>",
|
||||
},
|
||||
{
|
||||
name: "token file gone by the time the client reads it",
|
||||
prep: func(fx *fixture) {
|
||||
fx.fake.failWith("NodeStatus", fmt.Errorf("%w: %w: /etc/proxy-operator/proxmox/token does not exist yet", provider.ErrTransient, errCredentials))
|
||||
},
|
||||
wantReason: reasonCredentialsUnavailable,
|
||||
},
|
||||
{
|
||||
name: "unknown certificate authority",
|
||||
prep: func(fx *fixture) { fx.fake.failWith("Version", urlErr(x509.UnknownAuthorityError{})) },
|
||||
wantReason: reasonCertificateInvalid,
|
||||
},
|
||||
{
|
||||
name: "host not in the certificate",
|
||||
prep: func(fx *fixture) { fx.fake.failWith("Version", urlErr(x509.HostnameError{Host: "192.168.2.8"})) },
|
||||
wantReason: reasonCertificateInvalid,
|
||||
},
|
||||
{
|
||||
name: "CA file unusable",
|
||||
prep: func(fx *fixture) {
|
||||
fx.fake.failWith("Version", fmt.Errorf("%w: %w: reading CA file: %w", provider.ErrTransient, errCAFile, syscall.ENOENT))
|
||||
},
|
||||
wantReason: reasonCertificateInvalid,
|
||||
},
|
||||
{
|
||||
name: "wrong secret",
|
||||
prep: func(fx *fixture) { fx.fake.failWith("Version", loadAPIError(fx.t, "err_unauthorized")) },
|
||||
wantReason: provider.ReasonUnauthorized,
|
||||
},
|
||||
{
|
||||
name: "no credentials sent",
|
||||
prep: func(fx *fixture) { fx.fake.failWith("Version", loadAPIError(fx.t, "err_no_credentials")) },
|
||||
wantReason: provider.ReasonUnauthorized,
|
||||
},
|
||||
{
|
||||
name: "privilege missing",
|
||||
prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", loadAPIError(fx.t, "err_forbidden")) },
|
||||
wantReason: provider.ReasonForbidden,
|
||||
wantMessage: "Sys.Syslog",
|
||||
},
|
||||
{
|
||||
name: "pveproxy cannot reach the node",
|
||||
prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(595, "Connection refused")) },
|
||||
wantReason: provider.ReasonUnreachable,
|
||||
},
|
||||
{
|
||||
name: "pveproxy broken pipe",
|
||||
prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(596, "Broken pipe")) },
|
||||
wantReason: provider.ReasonUnreachable,
|
||||
},
|
||||
{
|
||||
name: "connection refused",
|
||||
prep: func(fx *fixture) {
|
||||
fx.fake.failWith("Version", urlErr(&net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}))
|
||||
},
|
||||
wantReason: provider.ReasonUnreachable,
|
||||
},
|
||||
{
|
||||
name: "node status 500",
|
||||
prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(500, "fake: pve-cluster not running")) },
|
||||
wantReason: provider.ReasonAPIError,
|
||||
},
|
||||
{
|
||||
name: "iso storage missing",
|
||||
prep: func(fx *fixture) {
|
||||
fx.fake.mu.Lock()
|
||||
delete(fx.fake.storages, fakeISOStorage)
|
||||
fx.fake.mu.Unlock()
|
||||
},
|
||||
wantReason: reasonStorageNotFound,
|
||||
wantMessage: "storage proxy-seeds on node pv1-1: fake: storage 'proxy-seeds' does not exist",
|
||||
},
|
||||
{
|
||||
name: "iso storage inactive",
|
||||
prep: func(fx *fixture) { setStorage(fx, func(s *storageStatus) { s.Active = 0 }) },
|
||||
wantReason: reasonStorageNotFound,
|
||||
wantMessage: "not active",
|
||||
},
|
||||
{
|
||||
name: "iso storage without iso content",
|
||||
prep: func(fx *fixture) { setStorage(fx, func(s *storageStatus) { s.Content = "images,rootdir,isoish" }) },
|
||||
wantReason: reasonStorageNotFound,
|
||||
wantMessage: "does not hold iso content (content: images,rootdir,isoish)",
|
||||
},
|
||||
{
|
||||
name: "iso storage forbidden",
|
||||
prep: func(fx *fixture) { fx.fake.failWith("StorageStatus", loadAPIError(fx.t, "err_forbidden")) },
|
||||
wantReason: provider.ReasonForbidden,
|
||||
},
|
||||
{
|
||||
name: "template missing",
|
||||
prep: func(fx *fixture) {
|
||||
fx.fake.mu.Lock()
|
||||
delete(fx.fake.vms, testTemplateVMID)
|
||||
fx.fake.mu.Unlock()
|
||||
},
|
||||
wantReason: reasonTemplateNotFound,
|
||||
wantMessage: `template "debian-13": VM 9001 does not exist`,
|
||||
},
|
||||
{
|
||||
name: "template VMID holds a plain VM",
|
||||
prep: func(fx *fixture) { fx.fake.vms[testTemplateVMID].template = false },
|
||||
wantReason: reasonTemplateNotFound,
|
||||
wantMessage: `template "debian-13": VM 9001 is not a template`,
|
||||
},
|
||||
{
|
||||
name: "template VMID holds a container",
|
||||
prep: func(fx *fixture) { fx.fake.vms[testTemplateVMID].kind = "lxc" },
|
||||
wantReason: reasonTemplateNotFound,
|
||||
wantMessage: "does not exist",
|
||||
},
|
||||
{
|
||||
name: "resources listing fails",
|
||||
prep: func(fx *fixture) {
|
||||
fx.fake.failWith("ClusterResources", fakeErr(500, "fake: cluster filesystem not ready"))
|
||||
},
|
||||
wantReason: provider.ReasonAPIError,
|
||||
},
|
||||
{
|
||||
name: "request outlives the deadline",
|
||||
prep: func(fx *fixture) { fx.fake.blockOn("StorageStatus") },
|
||||
timeout: 50 * time.Millisecond,
|
||||
wantReason: provider.ReasonTimeout,
|
||||
},
|
||||
{
|
||||
// The real client's dial sees the round's ctx end and fails as
|
||||
// a dial error, which checkReason alone would call Unreachable.
|
||||
name: "dial cut short by the round's deadline",
|
||||
prep: func(fx *fixture) {
|
||||
fx.fake.hookOn("Version", func() error {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
return urlErr(&net.OpError{Op: "dial", Net: "tcp", Err: context.DeadlineExceeded})
|
||||
})
|
||||
},
|
||||
timeout: 20 * time.Millisecond,
|
||||
wantReason: provider.ReasonTimeout,
|
||||
},
|
||||
{
|
||||
name: "ctx cancelled mid check",
|
||||
prep: func(fx *fixture) { fx.fake.blockOn("ClusterResources") },
|
||||
cancel: true,
|
||||
wantReason: provider.ReasonTimeout,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newCheckFixture(t)
|
||||
if tc.prep != nil {
|
||||
tc.prep(fx)
|
||||
}
|
||||
ctx := fx.ctx
|
||||
switch {
|
||||
case tc.timeout > 0:
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, tc.timeout)
|
||||
defer cancel()
|
||||
case tc.cancel:
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithCancel(ctx)
|
||||
time.AfterFunc(50*time.Millisecond, cancel)
|
||||
}
|
||||
got := fx.p.Check(ctx)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("Check() returned %d targets, want 1", len(got))
|
||||
}
|
||||
ts := got[0]
|
||||
if ts.Ready != tc.wantReady || ts.Reason != tc.wantReason {
|
||||
t.Errorf("verdict = ready %v reason %s (%s), want ready %v reason %s", ts.Ready, ts.Reason, ts.Message, tc.wantReady, tc.wantReason)
|
||||
}
|
||||
if !tc.wantReady && ts.Message == "" {
|
||||
t.Error("failed check has no message")
|
||||
}
|
||||
if tc.wantReady && ts.Message != "" {
|
||||
t.Errorf("successful check has a message: %q", ts.Message)
|
||||
}
|
||||
if tc.wantMessage != "" && !strings.Contains(ts.Message, tc.wantMessage) {
|
||||
t.Errorf("message = %q, want it to contain %q", ts.Message, tc.wantMessage)
|
||||
}
|
||||
if ts.Details["isoStorage"] != fakeISOStorage {
|
||||
t.Errorf("details = %v, want isoStorage whatever failed", ts.Details)
|
||||
}
|
||||
assertNoSecret(t, fx, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func setStorage(fx *fixture, fn func(s *storageStatus)) {
|
||||
fx.fake.mu.Lock()
|
||||
defer fx.fake.mu.Unlock()
|
||||
s := fx.fake.storages[fakeISOStorage]
|
||||
fn(&s)
|
||||
fx.fake.storages[fakeISOStorage] = s
|
||||
}
|
||||
|
||||
// TestCheck_reasonsOverHTTPClient takes the transport-level reasons from
|
||||
// the real client, so the errors checkReason sees are the ones net/http and
|
||||
// crypto/tls produce, and the Ready row replays the Phase 0 captures.
|
||||
func TestCheck_reasonsOverHTTPClient(t *testing.T) {
|
||||
t.Parallel()
|
||||
captures := func(t *testing.T) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
path := strings.TrimPrefix(r.URL.Path, "/api2/json/")
|
||||
name := map[string]string{
|
||||
"version": "version",
|
||||
"nodes/pv1-1/status": "node_status",
|
||||
"nodes/pv1-1/storage/local/status": "storage_status",
|
||||
"cluster/resources": "cluster_resources",
|
||||
}[path]
|
||||
if name == "" {
|
||||
t.Errorf("unexpected request %s", path)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
serveJSON(w, readCapture(t, name).Body)
|
||||
}
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
handler func(t *testing.T) http.HandlerFunc
|
||||
mutate func(t *testing.T, cfx *clientFixture)
|
||||
wantReason string
|
||||
}{
|
||||
{name: "captures", handler: captures, wantReason: provider.ReasonReachable},
|
||||
{name: "unknown authority", handler: captures, mutate: func(_ *testing.T, cfx *clientFixture) { cfx.cfg.CAFile = "" }, wantReason: reasonCertificateInvalid},
|
||||
{
|
||||
name: "host not in the certificate",
|
||||
handler: captures,
|
||||
mutate: func(_ *testing.T, cfx *clientFixture) {
|
||||
cfx.cfg.URL = strings.Replace(cfx.srv.URL, "127.0.0.1", "localhost", 1)
|
||||
},
|
||||
wantReason: reasonCertificateInvalid,
|
||||
},
|
||||
{
|
||||
name: "CA file missing",
|
||||
handler: captures,
|
||||
mutate: func(t *testing.T, cfx *clientFixture) { cfx.cfg.CAFile = filepath.Join(t.TempDir(), "ca.pem") },
|
||||
wantReason: reasonCertificateInvalid,
|
||||
},
|
||||
{
|
||||
name: "401 from pveproxy",
|
||||
handler: func(t *testing.T) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, _ *http.Request) { serveCapture(t, w, readCapture(t, "err_unauthorized")) }
|
||||
},
|
||||
wantReason: provider.ReasonUnauthorized,
|
||||
},
|
||||
{
|
||||
name: "596 from pveproxy",
|
||||
handler: func(t *testing.T) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, _ *http.Request) {
|
||||
serveCapture(t, w, capture{Status: 596, Reason: "Broken pipe"})
|
||||
}
|
||||
},
|
||||
wantReason: provider.ReasonUnreachable,
|
||||
},
|
||||
{
|
||||
name: "server gone",
|
||||
handler: captures,
|
||||
mutate: func(_ *testing.T, cfx *clientFixture) { cfx.srv.Close() },
|
||||
wantReason: provider.ReasonUnreachable,
|
||||
},
|
||||
{
|
||||
name: "connection cut mid response",
|
||||
handler: func(t *testing.T) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, _ *http.Request) {
|
||||
conn, _, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
t.Errorf("hijack: %v", err)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(conn, "HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{\"data\":")
|
||||
_ = conn.Close()
|
||||
}
|
||||
},
|
||||
wantReason: provider.ReasonUnreachable,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfx := newClientFixture(t, tc.handler(t))
|
||||
if tc.mutate != nil {
|
||||
tc.mutate(t, cfx)
|
||||
}
|
||||
cfg := testConfig()
|
||||
cfg.ISOStorage = "local"
|
||||
cfg.URL, cfg.TokenFile, cfg.CAFile = cfx.cfg.URL, cfx.cfg.TokenFile, cfx.cfg.CAFile
|
||||
p := newProvider(testProvider, cfg, cfx.api(t))
|
||||
got := p.Check(context.Background())
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("Check() returned %d targets, want 1", len(got))
|
||||
}
|
||||
if ts := got[0]; ts.Reason != tc.wantReason || ts.Ready != (tc.wantReason == provider.ReasonReachable) {
|
||||
t.Errorf("verdict = ready %v reason %s (%s), want %s", ts.Ready, ts.Reason, ts.Message, tc.wantReason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheck_detailsKeys(t *testing.T) {
|
||||
t.Parallel()
|
||||
full := map[string]string{
|
||||
"tokenID": "proxy-operator@pve!operator", "isoStorage": fakeISOStorage, "pveVersion": "9.2.20",
|
||||
"kernel": "7.0.14-17-pve", "isoStorageAvailGiB": "90.0", "templates": "1/1",
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
prep func(fx *fixture)
|
||||
want []string
|
||||
}{
|
||||
{name: "success", want: slices.Sorted(maps.Keys(full))},
|
||||
{name: "token unusable", prep: func(fx *fixture) { writeFile(fx.t, fx.p.cfg.TokenFile, "junk") }, want: []string{"isoStorage"}},
|
||||
{name: "node status fails", prep: func(fx *fixture) { fx.fake.failWith("NodeStatus", fakeErr(596, "Broken pipe")) }, want: []string{"isoStorage", "pveVersion", "tokenID"}},
|
||||
{name: "storage inactive", prep: func(fx *fixture) { setStorage(fx, func(s *storageStatus) { s.Active = 0 }) }, want: []string{"isoStorage", "isoStorageAvailGiB", "kernel", "pveVersion", "tokenID"}},
|
||||
{name: "template missing", prep: func(fx *fixture) { fx.fake.vms[testTemplateVMID].template = false }, want: slices.Sorted(maps.Keys(full))},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newCheckFixture(t)
|
||||
if tc.prep != nil {
|
||||
tc.prep(fx)
|
||||
}
|
||||
got := fx.p.Check(fx.ctx)[0]
|
||||
if keys := slices.Sorted(maps.Keys(got.Details)); !slices.Equal(keys, tc.want) {
|
||||
t.Errorf("details keys = %v, want %v", keys, tc.want)
|
||||
}
|
||||
for k, v := range got.Details {
|
||||
if want, ok := full[k]; ok && tc.name == "success" && v != want {
|
||||
t.Errorf("details[%s] = %q, want %q", k, v, want)
|
||||
}
|
||||
if v == "" {
|
||||
t.Errorf("details[%s] is empty", k)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("templates counts found over configured", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newCheckFixture(t)
|
||||
fx.p.cfg.Templates["ubuntu"] = provider.ProxmoxTemplate{VMID: 9002}
|
||||
fx.p.cfg.Templates["debian-again"] = provider.ProxmoxTemplate{VMID: testTemplateVMID}
|
||||
got := fx.p.Check(fx.ctx)[0]
|
||||
if got.Details["templates"] != "2/3" || got.Reason != reasonTemplateNotFound || !strings.Contains(got.Message, `template "ubuntu": VM 9002`) {
|
||||
t.Errorf("verdict = %s %q, details %v; want TemplateNotFound naming ubuntu, templates 2/3", got.Reason, got.Message, got.Details)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheck_logsVerdictPerNode(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newCheckFixture(t)
|
||||
fx.p.cfg.Nodes = []string{fakeNode, "pv1-2"}
|
||||
fx.p.Check(fx.ctx)
|
||||
checked := fx.logs.matching(`"msg"="proxmox node checked"`)
|
||||
if len(checked) != 2 {
|
||||
t.Fatalf("got %d verdict lines, want 2: %v", len(checked), checked)
|
||||
}
|
||||
for _, l := range checked {
|
||||
switch {
|
||||
case strings.Contains(l, `"node"="pv1-1"`):
|
||||
if !strings.Contains(l, `"ready"=true`) || !strings.Contains(l, `"reason"="Reachable"`) || !strings.Contains(l, `"latency"=`) {
|
||||
t.Errorf("pv1-1 verdict line = %s", l)
|
||||
}
|
||||
case strings.Contains(l, `"node"="pv1-2"`):
|
||||
if !strings.Contains(l, `"ready"=false`) || !strings.Contains(l, `"reason"="APIError"`) {
|
||||
t.Errorf("pv1-2 verdict line = %s", l)
|
||||
}
|
||||
default:
|
||||
t.Errorf("verdict line without a node: %s", l)
|
||||
}
|
||||
if !strings.Contains(l, `"provider"="`+testProvider+`"`) || !strings.Contains(l, `"level"=1`) {
|
||||
t.Errorf("verdict line lacks the provider or is not V(1): %s", l)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckReason_table(t *testing.T) {
|
||||
t.Parallel()
|
||||
dial := &url.Error{Op: "Get", URL: "https://192.0.2.8:8006/api2/json/version", Err: &net.OpError{Op: "dial", Net: "tcp", Err: context.DeadlineExceeded}}
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want string
|
||||
}{
|
||||
{name: "storage verdict", err: &checkFailure{reason: reasonStorageNotFound, msg: "x"}, want: reasonStorageNotFound},
|
||||
{name: "template verdict wrapped", err: fmt.Errorf("check: %w", &checkFailure{reason: reasonTemplateNotFound, msg: "x"}), want: reasonTemplateNotFound},
|
||||
{name: "token missing", err: fmt.Errorf("%w: %w: /t does not exist yet", provider.ErrTransient, errCredentials), want: reasonCredentialsUnavailable},
|
||||
{name: "token malformed", err: fmt.Errorf("%w: %w: /t is not a token", provider.ErrPermanent, errCredentials), want: reasonCredentialsUnavailable},
|
||||
{name: "CA file", err: fmt.Errorf("%w: %w: /ca holds no PEM certificate", provider.ErrPermanent, errCAFile), want: reasonCertificateInvalid},
|
||||
{name: "unknown authority", err: &url.Error{Err: x509.UnknownAuthorityError{}}, want: reasonCertificateInvalid},
|
||||
{name: "certificate invalid", err: &url.Error{Err: x509.CertificateInvalidError{Reason: x509.Expired}}, want: reasonCertificateInvalid},
|
||||
{name: "401", err: loadAPIError(t, "err_unauthorized"), want: provider.ReasonUnauthorized},
|
||||
{name: "403 wrapped", err: fmt.Errorf("reading: %w", loadAPIError(t, "err_forbidden_volume_delete")), want: provider.ReasonForbidden},
|
||||
{name: "595", err: fakeErr(595, "Connection refused"), want: provider.ReasonUnreachable},
|
||||
{name: "596", err: fakeErr(596, "Broken pipe"), want: provider.ReasonUnreachable},
|
||||
{name: "400", err: loadAPIError(t, "err_bad_param"), want: provider.ReasonAPIError},
|
||||
{name: "500", err: loadAPIError(t, "err_vm_missing"), want: provider.ReasonAPIError},
|
||||
{name: "dial ran out of its own budget", err: dial, want: provider.ReasonUnreachable},
|
||||
{name: "connection reset", err: &url.Error{Err: &net.OpError{Op: "read", Net: "tcp", Err: syscall.ECONNRESET}}, want: provider.ReasonUnreachable},
|
||||
{name: "request timeout", err: fmt.Errorf("reading: %w", &url.Error{Err: context.DeadlineExceeded}), want: provider.ReasonTimeout},
|
||||
{name: "cancelled", err: context.Canceled, want: provider.ReasonTimeout},
|
||||
{name: "server closed mid response", err: &url.Error{Err: io.ErrUnexpectedEOF}, want: provider.ReasonUnreachable},
|
||||
{name: "eof", err: io.EOF, want: provider.ReasonUnreachable},
|
||||
{name: "decode failure", err: errors.New("decoding GET version: invalid character '<'"), want: provider.ReasonAPIError},
|
||||
{name: "nil", err: nil, want: provider.ReasonAPIError},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := checkReason(tc.err); got != tc.want {
|
||||
t.Errorf("checkReason(%v) = %s, want %s", tc.err, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,10 @@ const (
|
||||
// Checker can report CredentialsUnavailable instead of an API failure.
|
||||
var errCredentials = errors.New("proxmox token file unusable")
|
||||
|
||||
// errCAFile marks a CA file that is missing or holds no certificate, so the
|
||||
// Checker can report CertificateInvalid: the admin fixes caFile.
|
||||
var errCAFile = errors.New("proxmox CA file unusable")
|
||||
|
||||
// tokenPattern is <user>@<realm>!<tokenid>=<secret> (spec entry 14).
|
||||
var tokenPattern = regexp.MustCompile(`^[^@\s]+@[^!\s]+![^=\s]+=\S+$`)
|
||||
|
||||
@@ -91,11 +95,11 @@ func (a *httpAPI) httpClient() (*http.Client, error) {
|
||||
if a.caFile != "" && !a.insecure {
|
||||
pem, err := os.ReadFile(a.caFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: reading CA file: %w", provider.ErrTransient, err)
|
||||
return nil, fmt.Errorf("%w: %w: reading CA file: %w", provider.ErrTransient, errCAFile, err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pem) {
|
||||
return nil, permanentf("CA file %s holds no PEM certificate", a.caFile)
|
||||
return nil, fmt.Errorf("%w: %w: %s holds no PEM certificate", provider.ErrPermanent, errCAFile, a.caFile)
|
||||
}
|
||||
tlsCfg.RootCAs = pool
|
||||
}
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package proxmox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
)
|
||||
|
||||
// Delete is spec entry 11: under the per-name lock, stop and destroy the VM
|
||||
// when it is ours, then delete its seed. Everything already gone is
|
||||
// success, and nothing of ours is left when nil is returned, so Get
|
||||
// afterwards is NotFound.
|
||||
func (p *Provider) Delete(ctx context.Context, id string) error {
|
||||
vmid, name, err := parseProviderID(id)
|
||||
if err != nil {
|
||||
return p.wrapErr("delete", id, err)
|
||||
}
|
||||
log := p.logger(ctx).WithValues("vmid", vmid, "name", name)
|
||||
unlock, err := p.lockName(ctx, name)
|
||||
if err != nil {
|
||||
return p.wrapErr("delete", id, err)
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
node, err := p.destroy(ctx, log, vmid, name)
|
||||
if err != nil {
|
||||
return p.wrapErr("delete", id, err)
|
||||
}
|
||||
if node == "" {
|
||||
err = p.deleteSeedEverywhere(ctx, log, name)
|
||||
} else {
|
||||
log = log.WithValues("node", node)
|
||||
err = p.deleteSeed(ctx, log, node, name)
|
||||
}
|
||||
if err != nil {
|
||||
return p.wrapErr("delete", id, err)
|
||||
}
|
||||
log.V(1).Info("proxmox vm deleted", "destroyed", node != "")
|
||||
return nil
|
||||
}
|
||||
|
||||
// destroy is steps 2 to 6. It returns the node of the VM it destroyed, or
|
||||
// "" when there was no VM of ours to destroy: then the seed's node is not
|
||||
// known, and the seed is looked for on every node.
|
||||
func (p *Provider) destroy(ctx context.Context, log logr.Logger, vmid int, name string) (string, error) {
|
||||
r, ok, err := p.resolve(ctx, vmid)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !ok {
|
||||
log.V(1).Info("proxmox vm already gone")
|
||||
return "", nil
|
||||
}
|
||||
log = log.WithValues("node", r.Node)
|
||||
if r.Lock != "" && r.Lock != lockSuspended {
|
||||
return "", transientf("VM %d is locked (%s)", vmid, r.Lock)
|
||||
}
|
||||
if r.Status == "unknown" {
|
||||
return "", transientf("VM %d has status unknown: its node %s is not answering", vmid, r.Node)
|
||||
}
|
||||
cfg, err := p.api.VMConfig(ctx, r.Node, vmid)
|
||||
if isNotExist(err) {
|
||||
log.V(1).Info("proxmox vm already gone")
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reading config of VM %d: %w", vmid, err)
|
||||
}
|
||||
if cfg["name"] != name {
|
||||
log.V(1).Info("proxmox vm at this vmid is not ours, left alone", "vmName", cfg["name"])
|
||||
return "", nil
|
||||
}
|
||||
if !parseDescription(cfg["description"]).Managed {
|
||||
log.V(1).Info("proxmox vm carries no proxy-operator mark, left alone")
|
||||
return "", nil
|
||||
}
|
||||
switch lock := cfg["lock"]; lock {
|
||||
case "":
|
||||
case lockSuspended:
|
||||
// Destroy refuses a locked VM and the lock never clears on its own,
|
||||
// so only an admin can move this on, as in Create's ownership check.
|
||||
return "", permanentf("VM %d named %s is hibernated (lock suspended); resume or destroy it in Proxmox", vmid, name)
|
||||
default:
|
||||
return "", transientf("VM %d is locked (%s)", vmid, lock)
|
||||
}
|
||||
|
||||
// The listing shows a paused VM as running, and Stop ends it too.
|
||||
if r.Status != "stopped" {
|
||||
gone, err := p.runTask(ctx, log, "stopping", vmid, p.budgets.stop, func() (string, error) { return p.api.Stop(ctx, r.Node, vmid) })
|
||||
if err != nil || gone {
|
||||
return r.Node, err
|
||||
}
|
||||
}
|
||||
if _, err := p.runTask(ctx, log, "destroying", vmid, p.budgets.destroy, func() (string, error) { return p.api.Destroy(ctx, r.Node, vmid) }); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return r.Node, nil
|
||||
}
|
||||
|
||||
// runTask submits a task on a VM and waits for it. A VM that is gone by the
|
||||
// time the task is submitted reports gone: it is what the task was for.
|
||||
func (p *Provider) runTask(ctx context.Context, log logr.Logger, what string, vmid int, budget time.Duration, submit func() (string, error)) (bool, error) {
|
||||
upid, err := submit()
|
||||
if isNotExist(err) {
|
||||
log.V(1).Info("proxmox vm already gone")
|
||||
return true, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s VM %d: %w", what, vmid, err)
|
||||
}
|
||||
if err := p.waitTask(ctx, log, upid, budget); err != nil {
|
||||
return false, fmt.Errorf("%s VM %d: %w", what, vmid, err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// deleteSeed is step 7 on one node. A missing volume still ends in a
|
||||
// successful task (F13), so nothing is matched.
|
||||
func (p *Provider) deleteSeed(ctx context.Context, log logr.Logger, node, name string) error {
|
||||
volid := seedVolID(p.cfg.ISOStorage, name)
|
||||
upid, err := p.api.DeleteVolume(ctx, node, p.cfg.ISOStorage, volid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting seed %s on %s: %w", volid, node, err)
|
||||
}
|
||||
if err := p.waitTask(ctx, log, upid, p.budgets.seedDelete); err != nil {
|
||||
return fmt.Errorf("deleting seed %s on %s: %w", volid, node, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteSeedEverywhere is step 7 when the VM's node is not known: every
|
||||
// configured node is tried, and one that fails is logged and skipped, so a
|
||||
// node that is down or lacks the storage cannot hold the Delete up. An
|
||||
// ended ctx fails the call instead.
|
||||
func (p *Provider) deleteSeedEverywhere(ctx context.Context, log logr.Logger, name string) error {
|
||||
for _, node := range p.cfg.Nodes {
|
||||
err := p.deleteSeed(ctx, log, node, name)
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return err
|
||||
}
|
||||
log.V(1).Info("proxmox seed delete failed on a node, skipped", "node", node, "reason", err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,717 @@
|
||||
package proxmox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// putSeed uploads testName's seed to the fake's ISO storage, as Create
|
||||
// leaves it.
|
||||
func (fx *fixture) putSeed() {
|
||||
fx.t.Helper()
|
||||
upid, err := fx.fake.UploadISO(context.Background(), fakeNode, fakeISOStorage, seedISOName(testName), strings.NewReader("seed"), 4)
|
||||
if err != nil {
|
||||
fx.t.Fatalf("UploadISO error = %v", err)
|
||||
}
|
||||
pollTask(fx.t, fx.fake, upid)
|
||||
}
|
||||
|
||||
func (fx *fixture) seedPresent() bool {
|
||||
_, ok := fx.fake.iso(fakeISOStorage, seedISOName(testName))
|
||||
return ok
|
||||
}
|
||||
|
||||
// deleteFails runs Delete on fx.id(), checks the error's class and returns
|
||||
// its text.
|
||||
func (fx *fixture) deleteFails(class error) string {
|
||||
fx.t.Helper()
|
||||
err := fx.p.Delete(fx.ctx, fx.id())
|
||||
assertProviderError(fx.t, err, "delete", class)
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
func (fx *fixture) delete() {
|
||||
fx.t.Helper()
|
||||
if err := fx.p.Delete(fx.ctx, fx.id()); err != nil {
|
||||
fx.t.Fatalf("Delete error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// seedDeleteNodes returns the node of every DeleteVolume call, read from
|
||||
// the decorator's lines; the fixture's api must be observed.
|
||||
func (fx *fixture) seedDeleteNodes() []string {
|
||||
lines := fx.logs.matching(`"call"="DeleteVolume"`)
|
||||
nodes := make([]string, 0, len(lines))
|
||||
for _, l := range lines {
|
||||
_, rest, _ := strings.Cut(l, `"node"="`)
|
||||
node, _, _ := strings.Cut(rest, `"`)
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
// twoNodeFixture is an observed fixture at V(1) with nodes pv1-1 and pv1-2.
|
||||
func twoNodeFixture(t *testing.T) *fixture {
|
||||
t.Helper()
|
||||
fx := newFixture(t, 1)
|
||||
fx.p.api = observe(fx.fake, testProvider)
|
||||
fx.p.cfg.Nodes = []string{fakeNode, "pv1-2"}
|
||||
return fx
|
||||
}
|
||||
|
||||
func TestDelete_malformedIDIsPermanent(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, id := range []string{"", "vms/150042", "vms/0150042/" + testName, "vm/150042/" + testName, "vms/abc/" + testName, "vms/150042/a/b", "vms/99/" + testName} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
assertProviderError(t, fx.p.Delete(fx.ctx, id), "delete", provider.ErrPermanent)
|
||||
if calls := fx.fake.callLog(); len(calls) != 0 {
|
||||
t.Errorf("calls = %v, want none", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_twiceSucceeds(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
fx.putSeed()
|
||||
fx.delete()
|
||||
fx.delete()
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); ok {
|
||||
t.Error("VM still exists")
|
||||
}
|
||||
if fx.seedPresent() {
|
||||
t.Error("seed still exists")
|
||||
}
|
||||
for method, want := range map[string]int{"Stop": 1, "Destroy": 1, "DeleteVolume": 2} {
|
||||
if got := fx.fake.callCount(method); got != want {
|
||||
t.Errorf("%s calls = %d, want %d", method, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_absentStillDeletesSeedOnEveryNode replaces the plan's
|
||||
// TestDelete_absentIsNoOp: spec entry 11 deletes the seed even when the VM
|
||||
// is gone, on every node, because the VM's node is no longer known.
|
||||
func TestDelete_absentStillDeletesSeedOnEveryNode(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.putSeed()
|
||||
fx.delete()
|
||||
if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{fakeNode, "pv1-2"}) {
|
||||
t.Errorf("seed deleted on %v, want every node in config order", got)
|
||||
}
|
||||
if fx.seedPresent() {
|
||||
t.Error("seed still exists")
|
||||
}
|
||||
for _, method := range []string{"VMConfig", "Stop", "Destroy"} {
|
||||
if n := fx.fake.callCount(method); n != 0 {
|
||||
t.Errorf("%s calls = %d, want 0 for an absent VM", method, n)
|
||||
}
|
||||
}
|
||||
if lines := fx.logs.matching(`"msg"="proxmox vm already gone"`); len(lines) != 1 {
|
||||
t.Errorf("already-gone lines = %v, want one", lines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_containerAtVMIDIsLeftAlone(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.addOurVM(testVMID, testName, "running").kind = "lxc"
|
||||
fx.delete()
|
||||
if vm, ok := fx.fake.vmSnapshot(testVMID); !ok || vm.status != "running" {
|
||||
t.Errorf("container = %+v, %v; want it untouched", vm, ok)
|
||||
}
|
||||
if got := fx.seedDeleteNodes(); len(got) != 2 {
|
||||
t.Errorf("seed deleted on %v, want every node", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_foreignNameLeavesVMAlone replaces the plan's
|
||||
// TestDelete_foreignNameIsNoOp: the VM is not touched, but our seed is
|
||||
// still deleted, on every node since ours is gone.
|
||||
func TestDelete_foreignNameLeavesVMAlone(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.addOurVM(testVMID, "proxy-someoneelse00000", "running")
|
||||
fx.putSeed()
|
||||
fx.delete()
|
||||
if vm, ok := fx.fake.vmSnapshot(testVMID); !ok || vm.status != "running" {
|
||||
t.Errorf("foreign VM = %+v, %v; want it running and untouched", vm, ok)
|
||||
}
|
||||
for _, method := range []string{"Stop", "Destroy", "SetVMConfig"} {
|
||||
if n := fx.fake.callCount(method); n != 0 {
|
||||
t.Errorf("%s calls = %d, want 0", method, n)
|
||||
}
|
||||
}
|
||||
if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{fakeNode, "pv1-2"}) {
|
||||
t.Errorf("seed deleted on %v, want every node", got)
|
||||
}
|
||||
lines := fx.logs.matching(`"msg"="proxmox vm at this vmid is not ours, left alone"`)
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], `"vmName"="proxy-someoneelse00000"`) {
|
||||
t.Errorf("no-op lines = %v, want one naming the other VM", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_unmarkedLeavesVMAlone replaces the plan's
|
||||
// TestDelete_unmarkedIsNoOp, for the same reason.
|
||||
func TestDelete_unmarkedLeavesVMAlone(t *testing.T) {
|
||||
t.Parallel()
|
||||
for name, desc := range map[string]string{
|
||||
"no description": "",
|
||||
"managed false": "proxy-operator-managed=false\nproxy-operator-uid=" + testUID,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
vm := fx.fake.addVM(testVMID, testName, "stopped")
|
||||
if desc != "" {
|
||||
vm.config["description"] = desc
|
||||
}
|
||||
fx.delete()
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); !ok {
|
||||
t.Error("unmarked VM was destroyed")
|
||||
}
|
||||
if n := fx.fake.callCount("Destroy"); n != 0 {
|
||||
t.Errorf("Destroy calls = %d, want 0", n)
|
||||
}
|
||||
if got := fx.seedDeleteNodes(); len(got) != 2 {
|
||||
t.Errorf("seed deleted on %v, want every node", got)
|
||||
}
|
||||
if lines := fx.logs.matching(`"msg"="proxmox vm carries no proxy-operator mark, left alone"`); len(lines) != 1 {
|
||||
t.Errorf("no-op lines = %v, want one", lines)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_runningVMStoppedFirst covers a paused VM too: the listing
|
||||
// shows it as running (F14), and Stop ends it. The fake refuses to destroy
|
||||
// a running VM, and each task takes two polls, so a Destroy that does not
|
||||
// wait for the stop fails.
|
||||
func TestDelete_runningVMStoppedFirst(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, qmp := range []string{"running", "paused", "prelaunch"} {
|
||||
t.Run(qmp, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.fake.taskPolls = 2
|
||||
fx.addOurVM(testVMID, testName, "running").qmpstatus = qmp
|
||||
fx.delete()
|
||||
log := fx.fake.callLog()
|
||||
stop, destroy := slices.Index(log, "Stop"), slices.Index(log, "Destroy")
|
||||
if stop < 0 || destroy < stop {
|
||||
t.Fatalf("calls = %v, want Stop before Destroy", log)
|
||||
}
|
||||
if n := callsBefore(log[stop:], "TaskStatus", "Destroy"); n != fx.fake.taskPolls+1 {
|
||||
t.Errorf("stop task polled %d times before Destroy, want %d", n, fx.fake.taskPolls+1)
|
||||
}
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); ok {
|
||||
t.Error("VM still exists")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_stoppedVMSkipsStop(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "stopped")
|
||||
fx.delete()
|
||||
want := []string{"ClusterResources", "VMConfig", "Destroy", "TaskStatus", "DeleteVolume", "TaskStatus"}
|
||||
if got := fx.fake.callLog(); !slices.Equal(got, want) {
|
||||
t.Errorf("calls = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_lockedIsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, lock := range []string{"backup", "migrate", "snapshot", "rollback", "create"} {
|
||||
t.Run("resources lock "+lock, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running").lock = lock
|
||||
fx.putSeed()
|
||||
if msg := fx.deleteFails(provider.ErrTransient); !strings.Contains(msg, "locked ("+lock+")") {
|
||||
t.Errorf("error = %s, want it to name the lock", msg)
|
||||
}
|
||||
if got := fx.fake.callLog(); !slices.Equal(got, []string{"UploadISO", "TaskStatus", "ClusterResources"}) {
|
||||
t.Errorf("calls = %v, want only the listing after the seed setup", got)
|
||||
}
|
||||
if !fx.seedPresent() {
|
||||
t.Error("seed deleted while the VM is locked")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("mid-clone nameless entry", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.fake.addTemplate(9001, "debian-13-template")
|
||||
if _, err := fx.fake.Clone(fx.ctx, fakeNode, 9001, cloneParams{NewID: testVMID, Name: testName, Description: formatDescription(testUID, testCreatedAt)}); err != nil {
|
||||
t.Fatalf("Clone error = %v", err)
|
||||
}
|
||||
fx.deleteFails(provider.ErrTransient)
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); !ok {
|
||||
t.Error("mid-clone VM is gone")
|
||||
}
|
||||
})
|
||||
t.Run("lock taken between the resources and config reads", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
vm := fx.addOurVM(testVMID, testName, "running")
|
||||
mutateOn(fx.fake, "VMConfig", func() { vm.lock = "backup" })
|
||||
fx.deleteFails(provider.ErrTransient)
|
||||
if n := fx.fake.callCount("Stop"); n != 0 {
|
||||
t.Errorf("Stop calls = %d, want 0", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDelete_unknownStatusIsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running").status = "unknown"
|
||||
if msg := fx.deleteFails(provider.ErrTransient); !strings.Contains(msg, fakeNode) {
|
||||
t.Errorf("error = %s, want it to name the node", msg)
|
||||
}
|
||||
if n := fx.fake.callCount("VMConfig"); n != 0 {
|
||||
t.Errorf("VMConfig calls = %d, want 0 while the node is unknown", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_hibernatedVMIsPermanentIfOurs matches Create's ownership
|
||||
// check: Destroy refuses the suspended lock, which never clears on its own,
|
||||
// so an admin must act. A hibernated VM that is not ours is left alone.
|
||||
func TestDelete_hibernatedVMIsPermanentIfOurs(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("ours", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "stopped").lock = lockSuspended
|
||||
fx.putSeed()
|
||||
if msg := fx.deleteFails(provider.ErrPermanent); !strings.Contains(msg, "resume or destroy it in Proxmox") {
|
||||
t.Errorf("error = %s, want the admin action", msg)
|
||||
}
|
||||
for _, method := range []string{"Stop", "Destroy", "DeleteVolume"} {
|
||||
if n := fx.fake.callCount(method); n != 0 {
|
||||
t.Errorf("%s calls = %d, want 0", method, n)
|
||||
}
|
||||
}
|
||||
if _, err := fx.fake.Destroy(fx.ctx, fakeNode, testVMID); !isLocked(err) {
|
||||
t.Errorf("fake Destroy on the hibernated VM error = %v, want the lock answer", err)
|
||||
}
|
||||
})
|
||||
t.Run("someone else's", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, "proxy-someoneelse00000", "stopped").lock = lockSuspended
|
||||
fx.delete()
|
||||
if vm, ok := fx.fake.vmSnapshot(testVMID); !ok || vm.lock != lockSuspended {
|
||||
t.Errorf("foreign hibernated VM = %+v, %v; want it untouched", vm, ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDelete_removesSeedISO(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.addOurVM(testVMID, testName, "stopped").node = "pv1-2"
|
||||
fx.putSeed()
|
||||
fx.delete()
|
||||
if fx.seedPresent() {
|
||||
t.Error("seed still exists")
|
||||
}
|
||||
if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{"pv1-2"}) {
|
||||
t.Errorf("seed deleted on %v, want only the VM's node pv1-2", got)
|
||||
}
|
||||
lines := fx.logs.matching(`"call"="DeleteVolume"`)
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], `"volid"="`+seedVolID(fakeISOStorage, testName)+`"`) || !strings.Contains(lines[0], `"storage"="`+fakeISOStorage+`"`) {
|
||||
t.Errorf("DeleteVolume lines = %v, want the seed's volid on the ISO storage", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_seedDeleteIsTaskAndIdempotent replaces the plan's
|
||||
// TestDelete_toleratesMissingISO: Proxmox ends a delete of a missing volume
|
||||
// with a successful imgdel task (F13), so nothing is matched, but the task
|
||||
// is still waited on.
|
||||
func TestDelete_seedDeleteIsTaskAndIdempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
missing := decodeData[taskStatus](t, "task_status_volume_missing")
|
||||
if missing.Type != "imgdel" || missing.ExitStatus != "OK" {
|
||||
t.Fatalf("capture task_status_volume_missing = %+v, want an imgdel task ending OK", missing)
|
||||
}
|
||||
for _, withSeed := range []bool{true, false} {
|
||||
t.Run("seed present "+strconv.FormatBool(withSeed), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "stopped")
|
||||
if withSeed {
|
||||
fx.putSeed()
|
||||
}
|
||||
fx.fake.taskPolls = 2
|
||||
fx.delete()
|
||||
log := fx.fake.callLog()
|
||||
last := len(log) - 1
|
||||
for log[last] != "DeleteVolume" {
|
||||
last--
|
||||
}
|
||||
if n := callsBefore(log[last:], "TaskStatus", ""); n != fx.fake.taskPolls+1 {
|
||||
t.Errorf("seed delete task polled %d times, want %d: %v", n, fx.fake.taskPolls+1, log)
|
||||
}
|
||||
if fx.seedPresent() {
|
||||
t.Error("seed still exists")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_seedDeleteFailureOnVMNodeFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(fx *fixture)
|
||||
class error
|
||||
}{
|
||||
{name: "token lacks Datastore.Allocate", setup: func(fx *fixture) {
|
||||
fx.fake.failWith("DeleteVolume", loadAPIError(t, "err_forbidden_volume_delete"))
|
||||
}, class: provider.ErrPermanent},
|
||||
{name: "pveproxy 596", setup: func(fx *fixture) { fx.fake.failWith("DeleteVolume", fakeErr(596, "Broken pipe")) }, class: provider.ErrTransient},
|
||||
{name: "task failed", setup: func(fx *fixture) {
|
||||
fx.fake.hookOn("TaskStatus", func() error {
|
||||
if fx.fake.callCount("DeleteVolume") > 0 {
|
||||
return fakeErr(500, "fake: task log unreadable")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}, class: provider.ErrTransient},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "stopped")
|
||||
tc.setup(fx)
|
||||
fx.deleteFails(tc.class)
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); ok {
|
||||
t.Error("VM still exists: the destroy should have run before the seed delete")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_seedDeleteNodeFailureSkippedWhenNodeUnknown(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("one node refuses", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.putSeed()
|
||||
var calls int
|
||||
fx.fake.hookOn("DeleteVolume", func() error {
|
||||
if calls++; calls == 1 {
|
||||
return fakeErr(500, "fake: storage 'proxy-seeds' is not available on node 'pv1-1'")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
fx.delete()
|
||||
if got := fx.seedDeleteNodes(); !slices.Equal(got, []string{fakeNode, "pv1-2"}) {
|
||||
t.Errorf("seed deleted on %v, want both nodes tried", got)
|
||||
}
|
||||
lines := fx.logs.matching(`"msg"="proxmox seed delete failed on a node, skipped"`)
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], `"node"="pv1-1"`) || !strings.Contains(lines[0], "not available") {
|
||||
t.Errorf("skip lines = %v, want one naming pv1-1 and the reason", lines)
|
||||
}
|
||||
if fx.seedPresent() {
|
||||
t.Error("seed still exists after pv1-2's delete")
|
||||
}
|
||||
})
|
||||
t.Run("ended ctx fails the call", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
ctx, cancel := context.WithCancel(fx.ctx)
|
||||
defer cancel()
|
||||
fx.fake.hookOn("DeleteVolume", func() error {
|
||||
cancel()
|
||||
return context.Canceled
|
||||
})
|
||||
err := fx.p.Delete(ctx, fx.id())
|
||||
assertProviderError(t, err, "delete", provider.ErrTransient)
|
||||
if n := fx.fake.callCount("DeleteVolume"); n != 1 {
|
||||
t.Errorf("DeleteVolume calls = %d, want 1: the ended ctx stops the loop", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDelete_destroyTaskFailureIsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "stopped")
|
||||
fx.putSeed()
|
||||
fx.fake.destroyFailsWith = "fake: unable to remove disk"
|
||||
if msg := fx.deleteFails(provider.ErrTransient); !strings.Contains(msg, "unable to remove disk") {
|
||||
t.Errorf("error = %s, want the task's exitstatus", msg)
|
||||
}
|
||||
if _, ok := fx.fake.vmSnapshot(testVMID); !ok {
|
||||
t.Error("VM gone after a failed destroy task")
|
||||
}
|
||||
if !fx.seedPresent() {
|
||||
t.Error("seed deleted although the VM is still there")
|
||||
}
|
||||
if lines := fx.logs.matching(`"msg"="proxmox task failed"`); len(lines) != 1 {
|
||||
t.Errorf("task failed lines = %v, want one", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_goneMidwayIsSuccess: the VM vanishing at any step counts as
|
||||
// done, and F2's does-not-exist answer is what each step then sees.
|
||||
func TestDelete_goneMidwayIsSuccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
status string
|
||||
method string
|
||||
wantDestroy int
|
||||
wantNodes []string
|
||||
}{
|
||||
{name: "before the config read", status: "stopped", method: "VMConfig", wantNodes: []string{fakeNode, "pv1-2"}},
|
||||
{name: "before the stop", status: "running", method: "Stop", wantNodes: []string{fakeNode}},
|
||||
{name: "before the destroy", status: "stopped", method: "Destroy", wantDestroy: 1, wantNodes: []string{fakeNode}},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := twoNodeFixture(t)
|
||||
fx.addOurVM(testVMID, testName, tc.status)
|
||||
mutateOn(fx.fake, tc.method, func() { delete(fx.fake.vms, testVMID) })
|
||||
fx.delete()
|
||||
if n := fx.fake.callCount("Destroy"); n != tc.wantDestroy {
|
||||
t.Errorf("Destroy calls = %d, want %d", n, tc.wantDestroy)
|
||||
}
|
||||
if got := fx.seedDeleteNodes(); !slices.Equal(got, tc.wantNodes) {
|
||||
t.Errorf("seed deleted on %v, want %v", got, tc.wantNodes)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_resourcesFailureIsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
fx.fake.failWith("ClusterResources", fakeErr(595, "Connection refused"))
|
||||
fx.deleteFails(provider.ErrTransient)
|
||||
if n := fx.fake.callCount("DeleteVolume"); n != 0 {
|
||||
t.Errorf("DeleteVolume calls = %d, want 0 when the VM's state is unknown", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_usesPurgeAndDestroyUnreferencedDisks runs Delete over the real
|
||||
// client against Phase 0's destroy captures: the DELETE carries both flags
|
||||
// and the seed delete sends the volid as one escaped segment.
|
||||
func TestDelete_usesPurgeAndDestroyUnreferencedDisks(t *testing.T) {
|
||||
t.Parallel()
|
||||
const (
|
||||
vmid = 199999
|
||||
name = "proxy-reconaaaaaaaaaaa"
|
||||
)
|
||||
var (
|
||||
mu sync.Mutex
|
||||
seen []string
|
||||
)
|
||||
cfx := newClientFixture(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
path := strings.TrimPrefix(r.URL.EscapedPath(), "/api2/json/")
|
||||
mu.Lock()
|
||||
seen = append(seen, r.Method+" "+path+"?"+r.URL.RawQuery)
|
||||
mu.Unlock()
|
||||
capture := ""
|
||||
switch {
|
||||
case r.Method == http.MethodGet && path == "cluster/resources":
|
||||
capture = "cluster_resources_during_destroy"
|
||||
case r.Method == http.MethodGet && path == "nodes/pv1-1/qemu/199999/config":
|
||||
capture = "config_during_destroy"
|
||||
case r.Method == http.MethodDelete && path == "nodes/pv1-1/qemu/199999":
|
||||
capture = "destroy_response"
|
||||
case r.Method == http.MethodDelete && path == "nodes/pv1-1/storage/proxy-seeds/content/proxy-seeds:iso%2F"+name+"-seed.iso":
|
||||
capture = "content_delete_response"
|
||||
case r.Method == http.MethodGet && strings.Contains(path, ":imgdel:"):
|
||||
capture = "task_status_volume_missing"
|
||||
case r.Method == http.MethodGet && strings.Contains(path, ":qmdestroy:"):
|
||||
capture = "task_status_stopped_ok"
|
||||
default:
|
||||
t.Errorf("unexpected request %s %s", r.Method, path)
|
||||
http.Error(w, "unexpected", http.StatusNotImplemented)
|
||||
return
|
||||
}
|
||||
serveJSON(w, readCapture(t, capture).Body)
|
||||
})
|
||||
cfg := testConfig()
|
||||
cfg.URL, cfg.TokenFile, cfg.CAFile = cfx.cfg.URL, cfx.cfg.TokenFile, cfx.cfg.CAFile
|
||||
p := newProvider(testProvider, cfg, cfx.api(t))
|
||||
p.budgets = testBudgets()
|
||||
if err := p.Delete(context.Background(), formatProviderID(vmid, name)); err != nil {
|
||||
t.Fatalf("Delete error = %v", err)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !slices.Contains(seen, "DELETE nodes/pv1-1/qemu/199999?destroy-unreferenced-disks=1&purge=1") {
|
||||
t.Errorf("requests = %v, want the destroy with purge=1 and destroy-unreferenced-disks=1", seen)
|
||||
}
|
||||
if !slices.ContainsFunc(seen, func(s string) bool { return strings.HasPrefix(s, "DELETE nodes/pv1-1/storage/proxy-seeds/content/") }) {
|
||||
t.Errorf("requests = %v, want the seed delete", seen)
|
||||
}
|
||||
if slices.ContainsFunc(seen, func(s string) bool { return strings.Contains(s, "status/stop") }) {
|
||||
t.Errorf("requests = %v, want no stop for the captured stopped VM", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_concurrentSameNameSafe(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
release := fx.fake.blockOn("Stop")
|
||||
errs := make([]error, 2)
|
||||
first := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
wg.Go(func() {
|
||||
defer close(first)
|
||||
errs[0] = fx.p.Delete(fx.ctx, fx.id())
|
||||
})
|
||||
for fx.fake.callCount("Stop") == 0 {
|
||||
select {
|
||||
case <-first:
|
||||
t.Fatalf("first Delete returned %v without stopping the VM", errs[0])
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
wg.Go(func() { errs[1] = fx.p.Delete(fx.ctx, fx.id()) })
|
||||
close(release)
|
||||
wg.Wait()
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
t.Errorf("Delete %d error = %v", i, err)
|
||||
}
|
||||
}
|
||||
for method, want := range map[string]int{"Stop": 1, "Destroy": 1} {
|
||||
if got := fx.fake.callCount(method); got != want {
|
||||
t.Errorf("%s calls = %d, want %d", method, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelete_waitsOnTheNameLock: Delete takes the per-name lock before any
|
||||
// call, so it cannot interleave with a Create or another Delete of that
|
||||
// name (spec entry 19).
|
||||
func TestDelete_waitsOnTheNameLock(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
release, err := fx.p.lockName(fx.ctx, testName)
|
||||
if err != nil {
|
||||
t.Fatalf("lockName error = %v", err)
|
||||
}
|
||||
defer release()
|
||||
ctx, cancel := context.WithTimeout(fx.ctx, 20*time.Millisecond)
|
||||
defer cancel()
|
||||
err = fx.p.Delete(ctx, fx.id())
|
||||
assertProviderError(t, err, "delete", provider.ErrTransient)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Errorf("error = %v, want it to wrap DeadlineExceeded", err)
|
||||
}
|
||||
if calls := fx.fake.callLog(); len(calls) != 0 {
|
||||
t.Errorf("calls = %v, want none while the name is locked", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_cancelledMidWaitReturnsPromptly(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.p.budgets = defaultBudgets()
|
||||
fx.p.budgets.poll = time.Millisecond
|
||||
fx.fake.taskPolls = 1 << 30
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
ctx, cancel := context.WithCancel(fx.ctx)
|
||||
defer cancel()
|
||||
fx.fake.hookOn("TaskStatus", func() error {
|
||||
cancel()
|
||||
return nil
|
||||
})
|
||||
start := time.Now()
|
||||
err := fx.p.Delete(ctx, fx.id())
|
||||
if elapsed := time.Since(start); elapsed > 5*time.Second {
|
||||
t.Errorf("Delete took %v after cancellation", elapsed)
|
||||
}
|
||||
assertProviderError(t, err, "delete", provider.ErrTransient)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("error = %v, want it to wrap context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_outcomeLine(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(fx *fixture)
|
||||
want []string
|
||||
never string
|
||||
}{
|
||||
{
|
||||
name: "destroyed",
|
||||
setup: func(fx *fixture) { fx.addOurVM(testVMID, testName, "stopped") },
|
||||
want: []string{`"node"="` + fakeNode + `"`, `"destroyed"=true`},
|
||||
},
|
||||
{name: "already gone", setup: func(*fixture) {}, want: []string{`"destroyed"=false`}, never: `"node"=`},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
tc.setup(fx)
|
||||
fx.delete()
|
||||
lines := fx.logs.matching(`"msg"="proxmox vm deleted"`)
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("outcome lines = %d, want 1:\n%s", len(lines), strings.Join(fx.logs.all(), "\n"))
|
||||
}
|
||||
for _, want := range append(tc.want, `"vmid"=150042`, `"name"="`+testName+`"`, `"provider"="`+testProvider+`"`) {
|
||||
if !strings.Contains(lines[0], want) {
|
||||
t.Errorf("outcome line lacks %s: %s", want, lines[0])
|
||||
}
|
||||
}
|
||||
if tc.never != "" && strings.Contains(lines[0], tc.never) {
|
||||
t.Errorf("outcome line has %s: %s", tc.never, lines[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_noOpLinesAreSilentAtV0(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := map[string]func(fx *fixture){
|
||||
"absent": func(*fixture) {},
|
||||
"foreign": func(fx *fixture) { fx.addOurVM(testVMID, "proxy-someoneelse00000", "running") },
|
||||
"unmarked": func(fx *fixture) { fx.fake.addVM(testVMID, testName, "running") },
|
||||
"ours": func(fx *fixture) { fx.addOurVM(testVMID, testName, "running") },
|
||||
}
|
||||
for name, setup := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 0)
|
||||
fx.p.api = observe(fx.fake, testProvider)
|
||||
setup(fx)
|
||||
fx.delete()
|
||||
if lines := fx.logs.all(); len(lines) != 0 {
|
||||
t.Errorf("V(0) lines = %v, want none", lines)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -39,9 +39,10 @@ type fakeAPI struct {
|
||||
// taskPolls is how many TaskStatus polls answer "running" before a
|
||||
// task stops.
|
||||
taskPolls int
|
||||
// cloneFailsWith and startFailsWith, when set, are the exitstatus of
|
||||
// every clone or start task. A failed clone removes its VM.
|
||||
cloneFailsWith, startFailsWith string
|
||||
// cloneFailsWith, startFailsWith and destroyFailsWith, when set, are
|
||||
// the exitstatus of every clone, start or destroy task. A failed clone
|
||||
// removes its VM; a failed destroy keeps it.
|
||||
cloneFailsWith, startFailsWith, destroyFailsWith string
|
||||
// agentNotRunning makes the agent of every running VM with agent=1
|
||||
// answer "QEMU guest agent is not running".
|
||||
agentNotRunning bool
|
||||
@@ -510,8 +511,12 @@ func (f *fakeAPI) Destroy(ctx context.Context, node string, vmid int) (string, e
|
||||
if vm.status == "running" {
|
||||
return "", fakeErr(500, fmt.Sprintf("fake: VM %d is running - destroy failed", vmid))
|
||||
}
|
||||
return f.newTask(node, "qmdestroy", strconv.Itoa(vmid), "OK", func() {
|
||||
if f.vms[vmid] == vm {
|
||||
exit := "OK"
|
||||
if f.destroyFailsWith != "" {
|
||||
exit = f.destroyFailsWith
|
||||
}
|
||||
return f.newTask(node, "qmdestroy", strconv.Itoa(vmid), exit, func() {
|
||||
if exit == "OK" && f.vms[vmid] == vm {
|
||||
delete(f.vms, vmid)
|
||||
}
|
||||
}), nil
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -474,3 +475,60 @@ func TestGet_resourcesFailureIsTransient(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_afterDeleteReturnsNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running").ifaces = capturedIfaces(t)
|
||||
if inst := fx.get(fx.id()); inst.State != provider.StateRunning {
|
||||
t.Fatalf("Get before Delete = %+v, want Running", inst)
|
||||
}
|
||||
if err := fx.p.Delete(fx.ctx, fx.id()); err != nil {
|
||||
t.Fatalf("Delete error = %v", err)
|
||||
}
|
||||
assertGetFails(t, fx, provider.ErrNotFound)
|
||||
}
|
||||
|
||||
// TestGet_duringDeleteStillReturnsInstance holds the destroy task running:
|
||||
// destroying sets no lock and keeps the VM listed until the task ends, as
|
||||
// Phase 0's mid-destroy captures show, so Get sees it Stopped, not gone.
|
||||
func TestGet_duringDeleteStillReturnsInstance(t *testing.T) {
|
||||
t.Parallel()
|
||||
if _, locked := capturedConfig(t, "config_during_destroy")["lock"]; locked {
|
||||
t.Fatal("capture config_during_destroy has a lock; the fake models none")
|
||||
}
|
||||
listed := decodeData[[]vmResource](t, "cluster_resources_during_destroy")
|
||||
if !slices.ContainsFunc(listed, func(r vmResource) bool { return r.VMID == 199999 && r.Status == "stopped" && r.Lock == "" }) {
|
||||
t.Fatal("capture cluster_resources_during_destroy does not list VM 199999 stopped and unlocked")
|
||||
}
|
||||
|
||||
fx := newFixture(t, 1)
|
||||
fx.addOurVM(testVMID, testName, "running")
|
||||
inTask, release := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
fx.fake.hookOn("TaskStatus", func() error {
|
||||
if fx.fake.callCount("Destroy") > 0 {
|
||||
once.Do(func() {
|
||||
close(inTask)
|
||||
<-release
|
||||
})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- fx.p.Delete(fx.ctx, fx.id()) }()
|
||||
select {
|
||||
case <-inTask:
|
||||
case err := <-done:
|
||||
t.Fatalf("Delete returned %v before its destroy task was polled", err)
|
||||
}
|
||||
inst := fx.get(fx.id())
|
||||
if inst.State != provider.StateStopped || inst.UID != testUID || inst.ID != fx.id() {
|
||||
t.Errorf("Get during the destroy = %+v, want the instance Stopped with its uid", inst)
|
||||
}
|
||||
close(release)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("Delete error = %v", err)
|
||||
}
|
||||
assertGetFails(t, fx, provider.ErrNotFound)
|
||||
}
|
||||
|
||||
@@ -172,9 +172,3 @@ func (p *Provider) lockName(ctx context.Context, name string) (func(), error) {
|
||||
}
|
||||
return release, nil
|
||||
}
|
||||
|
||||
// Delete is a stub until Phase 7. It answers Transient, so a Proxy on this
|
||||
// provider retries rather than latching.
|
||||
func (p *Provider) Delete(_ context.Context, id string) error {
|
||||
return p.wrapErr("delete", id, transientf("proxmox Delete is not implemented yet"))
|
||||
}
|
||||
|
||||
@@ -215,16 +215,6 @@ func TestWrapErr_carriesProviderNameOpAndID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestStubs_answerTransient keeps a Proxy on this provider retrying, not
|
||||
// latched, until Delete is implemented in Phase 7.
|
||||
func TestStubs_answerTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := newProvider("proxmox-home", testConfig(), nil)
|
||||
if err := p.Delete(context.Background(), "vms/150042/proxy-x"); !errors.Is(err, provider.ErrTransient) {
|
||||
t.Errorf("delete error = %v, want ErrTransient", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLockName_givesUpWithContextAsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := newProvider("proxmox-home", testConfig(), nil)
|
||||
|
||||
Reference in New Issue
Block a user