Add orphan GC sweeper and Prometheus metrics with explicit registration
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
125
internal/gc/gc.go
Normal file
125
internal/gc/gc.go
Normal file
@@ -0,0 +1,125 @@
|
||||
// Package gc implements orphan garbage collection: a periodic sweep that
|
||||
// deletes provider instances tagged by this operator whose owning Proxy CR
|
||||
// no longer exists — the safety net for crashes between a provider Create
|
||||
// and the status write that records it.
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// Sweeper is the manager Runnable running the sweep loop.
|
||||
type Sweeper struct {
|
||||
// Reader lists Proxies from the manager's cache to establish the live
|
||||
// UID set.
|
||||
Reader client.Reader
|
||||
// Providers are the configured backends; each is swept independently.
|
||||
Providers map[string]provider.Provider
|
||||
|
||||
// Interval between sweeps (default 10m). The first sweep runs one full
|
||||
// interval after start, not immediately — right after startup the
|
||||
// cache is coldest and an in-flight create is most likely.
|
||||
Interval time.Duration
|
||||
// MinAge exempts young instances (default 10m): an instance mid-create
|
||||
// may not have its status write landed yet; deleting it would race the
|
||||
// reconciler.
|
||||
MinAge time.Duration
|
||||
|
||||
// NamespaceRestricted must be set when the manager cache is limited to
|
||||
// one namespace. Then the live-UID set is incomplete, and a sweep
|
||||
// would delete VMs owned by Proxies the cache cannot see — so Start
|
||||
// refuses unless AllowNamespaced (--gc-allow-namespaced) is explicit.
|
||||
NamespaceRestricted bool
|
||||
AllowNamespaced bool
|
||||
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// NeedLeaderElection is true: the sweep is destructive and must have a
|
||||
// single writer.
|
||||
func (s *Sweeper) NeedLeaderElection() bool { return true }
|
||||
|
||||
// Start runs the sweep loop until ctx ends.
|
||||
func (s *Sweeper) Start(ctx context.Context) error {
|
||||
if s.NamespaceRestricted && !s.AllowNamespaced {
|
||||
return errors.New(
|
||||
"orphan GC refuses to run against a namespace-restricted cache: proxies outside the namespace " +
|
||||
"would count as orphans and their instances would be deleted; pass --gc-allow-namespaced to override")
|
||||
}
|
||||
if s.Interval == 0 {
|
||||
s.Interval = 10 * time.Minute
|
||||
}
|
||||
if s.MinAge == 0 {
|
||||
s.MinAge = 10 * time.Minute
|
||||
}
|
||||
if s.now == nil {
|
||||
s.now = time.Now
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(s.Interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
s.sweep(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sweep deletes tagged instances whose UID matches no existing Proxy CR.
|
||||
// A CR with a deletionTimestamp still counts as live: its finalizer owns
|
||||
// that deletion, and GC racing it would double-delete. A UID becomes
|
||||
// orphan-eligible only once the object is fully gone.
|
||||
func (s *Sweeper) sweep(ctx context.Context) {
|
||||
log := logf.FromContext(ctx).WithName("orphan-gc")
|
||||
|
||||
var list crawlv1alpha1.ProxyList
|
||||
if err := s.Reader.List(ctx, &list); err != nil {
|
||||
// Without the live set nothing can be proven orphaned; skip the
|
||||
// whole sweep rather than guess.
|
||||
log.Error(err, "listing proxies; skipping this sweep")
|
||||
return
|
||||
}
|
||||
live := make(map[string]bool, len(list.Items))
|
||||
for i := range list.Items {
|
||||
live[string(list.Items[i].UID)] = true
|
||||
}
|
||||
|
||||
for name, prov := range s.Providers {
|
||||
instances, err := prov.ListByTag(ctx)
|
||||
if err != nil {
|
||||
// One broken provider must not abort the sweep for the rest.
|
||||
log.Error(err, "listing instances; skipping this provider", "provider", name)
|
||||
continue
|
||||
}
|
||||
for _, inst := range instances {
|
||||
switch {
|
||||
case inst.UID == "":
|
||||
// Managed label without a UID label shouldn't exist for
|
||||
// anything this operator created; without ownership proof,
|
||||
// never delete.
|
||||
continue
|
||||
case live[inst.UID]:
|
||||
continue
|
||||
case s.now().Sub(inst.CreatedAt) < s.MinAge:
|
||||
continue
|
||||
}
|
||||
log.Info("WARNING: deleting orphaned instance",
|
||||
"provider", name, "providerID", inst.ID, "uid", inst.UID)
|
||||
if err := prov.Delete(ctx, inst.ID); err != nil {
|
||||
log.Error(err, "deleting orphaned instance",
|
||||
"provider", name, "providerID", inst.ID, "uid", inst.UID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user