Add orphan GC sweeper and Prometheus metrics with explicit registration
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
125
internal/gc/gc.go
Normal file
125
internal/gc/gc.go
Normal file
@@ -0,0 +1,125 @@
|
||||
// Package gc implements orphan garbage collection: a periodic sweep that
|
||||
// deletes provider instances tagged by this operator whose owning Proxy CR
|
||||
// no longer exists — the safety net for crashes between a provider Create
|
||||
// and the status write that records it.
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// Sweeper is the manager Runnable running the sweep loop.
|
||||
type Sweeper struct {
|
||||
// Reader lists Proxies from the manager's cache to establish the live
|
||||
// UID set.
|
||||
Reader client.Reader
|
||||
// Providers are the configured backends; each is swept independently.
|
||||
Providers map[string]provider.Provider
|
||||
|
||||
// Interval between sweeps (default 10m). The first sweep runs one full
|
||||
// interval after start, not immediately — right after startup the
|
||||
// cache is coldest and an in-flight create is most likely.
|
||||
Interval time.Duration
|
||||
// MinAge exempts young instances (default 10m): an instance mid-create
|
||||
// may not have its status write landed yet; deleting it would race the
|
||||
// reconciler.
|
||||
MinAge time.Duration
|
||||
|
||||
// NamespaceRestricted must be set when the manager cache is limited to
|
||||
// one namespace. Then the live-UID set is incomplete, and a sweep
|
||||
// would delete VMs owned by Proxies the cache cannot see — so Start
|
||||
// refuses unless AllowNamespaced (--gc-allow-namespaced) is explicit.
|
||||
NamespaceRestricted bool
|
||||
AllowNamespaced bool
|
||||
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// NeedLeaderElection is true: the sweep is destructive and must have a
|
||||
// single writer.
|
||||
func (s *Sweeper) NeedLeaderElection() bool { return true }
|
||||
|
||||
// Start runs the sweep loop until ctx ends.
|
||||
func (s *Sweeper) Start(ctx context.Context) error {
|
||||
if s.NamespaceRestricted && !s.AllowNamespaced {
|
||||
return errors.New(
|
||||
"orphan GC refuses to run against a namespace-restricted cache: proxies outside the namespace " +
|
||||
"would count as orphans and their instances would be deleted; pass --gc-allow-namespaced to override")
|
||||
}
|
||||
if s.Interval == 0 {
|
||||
s.Interval = 10 * time.Minute
|
||||
}
|
||||
if s.MinAge == 0 {
|
||||
s.MinAge = 10 * time.Minute
|
||||
}
|
||||
if s.now == nil {
|
||||
s.now = time.Now
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(s.Interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
s.sweep(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sweep deletes tagged instances whose UID matches no existing Proxy CR.
|
||||
// A CR with a deletionTimestamp still counts as live: its finalizer owns
|
||||
// that deletion, and GC racing it would double-delete. A UID becomes
|
||||
// orphan-eligible only once the object is fully gone.
|
||||
func (s *Sweeper) sweep(ctx context.Context) {
|
||||
log := logf.FromContext(ctx).WithName("orphan-gc")
|
||||
|
||||
var list crawlv1alpha1.ProxyList
|
||||
if err := s.Reader.List(ctx, &list); err != nil {
|
||||
// Without the live set nothing can be proven orphaned; skip the
|
||||
// whole sweep rather than guess.
|
||||
log.Error(err, "listing proxies; skipping this sweep")
|
||||
return
|
||||
}
|
||||
live := make(map[string]bool, len(list.Items))
|
||||
for i := range list.Items {
|
||||
live[string(list.Items[i].UID)] = true
|
||||
}
|
||||
|
||||
for name, prov := range s.Providers {
|
||||
instances, err := prov.ListByTag(ctx)
|
||||
if err != nil {
|
||||
// One broken provider must not abort the sweep for the rest.
|
||||
log.Error(err, "listing instances; skipping this provider", "provider", name)
|
||||
continue
|
||||
}
|
||||
for _, inst := range instances {
|
||||
switch {
|
||||
case inst.UID == "":
|
||||
// Managed label without a UID label shouldn't exist for
|
||||
// anything this operator created; without ownership proof,
|
||||
// never delete.
|
||||
continue
|
||||
case live[inst.UID]:
|
||||
continue
|
||||
case s.now().Sub(inst.CreatedAt) < s.MinAge:
|
||||
continue
|
||||
}
|
||||
log.Info("WARNING: deleting orphaned instance",
|
||||
"provider", name, "providerID", inst.ID, "uid", inst.UID)
|
||||
if err := prov.Delete(ctx, inst.ID); err != nil {
|
||||
log.Error(err, "deleting orphaned instance",
|
||||
"provider", name, "providerID", inst.ID, "uid", inst.UID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
214
internal/gc/gc_test.go
Normal file
214
internal/gc/gc_test.go
Normal file
@@ -0,0 +1,214 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// listProvider serves a canned instance list and records deletions.
|
||||
type listProvider struct {
|
||||
mu sync.Mutex
|
||||
instances []provider.Instance
|
||||
listErr error
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (l *listProvider) Create(context.Context, provider.CreateRequest) (string, error) {
|
||||
return "", errors.New("not used")
|
||||
}
|
||||
|
||||
func (l *listProvider) Get(context.Context, string) (*provider.Instance, error) {
|
||||
return nil, provider.ErrNotFound
|
||||
}
|
||||
|
||||
func (l *listProvider) Delete(_ context.Context, id string) error {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.deleted = append(l.deleted, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *listProvider) ListByTag(context.Context) ([]provider.Instance, error) {
|
||||
return l.instances, l.listErr
|
||||
}
|
||||
|
||||
func (l *listProvider) deletedIDs() []string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return append([]string(nil), l.deleted...)
|
||||
}
|
||||
|
||||
func proxyWithUID(name, uid string, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy {
|
||||
p := &crawlv1alpha1.Proxy{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", UID: types.UID(uid)},
|
||||
Spec: crawlv1alpha1.ProxySpec{
|
||||
Mode: crawlv1alpha1.ModeManaged,
|
||||
Provider: "stub",
|
||||
},
|
||||
}
|
||||
for _, m := range mut {
|
||||
m(p)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func newReader(t *testing.T, objs ...client.Object) client.Reader {
|
||||
t.Helper()
|
||||
s := runtime.NewScheme()
|
||||
if err := crawlv1alpha1.AddToScheme(s); err != nil {
|
||||
t.Fatalf("scheme: %v", err)
|
||||
}
|
||||
return fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build()
|
||||
}
|
||||
|
||||
func oldInstance(id, uid string) provider.Instance {
|
||||
return provider.Instance{ID: id, UID: uid, State: provider.StateRunning,
|
||||
CreatedAt: time.Now().Add(-time.Hour)}
|
||||
}
|
||||
|
||||
func newSweeper(reader client.Reader, providers map[string]provider.Provider) *Sweeper {
|
||||
return &Sweeper{
|
||||
Reader: reader,
|
||||
Providers: providers,
|
||||
Interval: 10 * time.Minute,
|
||||
MinAge: 10 * time.Minute,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweep_deletesOnlyTrueOrphans(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
deletingCR := proxyWithUID("deleting", "uid-deleting", func(p *crawlv1alpha1.Proxy) {
|
||||
now := metav1.Now()
|
||||
p.DeletionTimestamp = &now
|
||||
p.Finalizers = []string{crawlv1alpha1.FinalizerName}
|
||||
})
|
||||
prov := &listProvider{instances: []provider.Instance{
|
||||
oldInstance("inst-live", "uid-live"),
|
||||
oldInstance("inst-orphan", "uid-orphan"),
|
||||
oldInstance("inst-deleting", "uid-deleting"),
|
||||
{ID: "inst-young", UID: "uid-young-orphan", State: provider.StateRunning,
|
||||
CreatedAt: time.Now().Add(-time.Minute)},
|
||||
oldInstance("inst-unlabelled", ""),
|
||||
}}
|
||||
s := newSweeper(
|
||||
newReader(t, proxyWithUID("live", "uid-live"), deletingCR),
|
||||
map[string]provider.Provider{"stub": prov},
|
||||
)
|
||||
|
||||
s.sweep(context.Background())
|
||||
|
||||
got := prov.deletedIDs()
|
||||
if len(got) != 1 || got[0] != "inst-orphan" {
|
||||
t.Errorf("deleted %v, want exactly [inst-orphan]:\n"+
|
||||
"live CR's instance must stay; a deleting CR still owns its instance (finalizer, not GC);\n"+
|
||||
"young instances may be mid-create; unlabelled instances have no ownership proof", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweep_providerErrorDoesNotAbortOthers(t *testing.T) {
|
||||
t.Parallel()
|
||||
broken := &listProvider{listErr: errors.New("cloud is down")}
|
||||
working := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
|
||||
s := newSweeper(newReader(t), map[string]provider.Provider{
|
||||
"broken": broken,
|
||||
"working": working,
|
||||
})
|
||||
|
||||
s.sweep(context.Background())
|
||||
|
||||
if got := working.deletedIDs(); len(got) != 1 {
|
||||
t.Errorf("working provider deletions = %v, want the orphan despite the broken provider", got)
|
||||
}
|
||||
}
|
||||
|
||||
// errReader fails every List: without the live set nothing can be proven
|
||||
// orphaned, so the sweep must delete nothing.
|
||||
type errReader struct{ client.Reader }
|
||||
|
||||
func (errReader) List(context.Context, client.ObjectList, ...client.ListOption) error {
|
||||
return errors.New("cache broken")
|
||||
}
|
||||
|
||||
func TestSweep_listFailureSkipsSweep(t *testing.T) {
|
||||
t.Parallel()
|
||||
prov := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
|
||||
s := newSweeper(errReader{}, map[string]provider.Provider{"stub": prov})
|
||||
|
||||
s.sweep(context.Background())
|
||||
|
||||
if got := prov.deletedIDs(); len(got) != 0 {
|
||||
t.Errorf("deleted %v with an unreadable live set, want nothing", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStart_namespaceGuard(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newSweeper(newReader(t), nil)
|
||||
s.NamespaceRestricted = true
|
||||
err := s.Start(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "--gc-allow-namespaced") {
|
||||
t.Errorf("Start with restricted cache = %v, want refusal naming the override flag", err)
|
||||
}
|
||||
|
||||
s2 := newSweeper(newReader(t), nil)
|
||||
s2.NamespaceRestricted = true
|
||||
s2.AllowNamespaced = true
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s2.Start(ctx) }()
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Errorf("Start with override = %v, want it to run until cancel", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Start did not stop on cancel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStart_sweepsOnIntervalAndStops(t *testing.T) {
|
||||
t.Parallel()
|
||||
prov := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
|
||||
s := newSweeper(newReader(t), map[string]provider.Provider{"stub": prov})
|
||||
s.Interval = 5 * time.Millisecond
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s.Start(ctx) }()
|
||||
|
||||
deadline := time.After(5 * time.Second)
|
||||
for len(prov.deletedIDs()) == 0 {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("no sweep ran")
|
||||
case <-time.After(2 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Errorf("Start = %v, want nil", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Start did not stop on cancel")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user