kacerr/egress-proxies-operator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Let a Proxy finish deleting when its instance can no longer be reached

Browse Source 
reconcileDelete asked Get first and dropped the finalizer only on
not-found, so a permanent Get retried forever: a KVM host removed from
the config, or a providerID the provider cannot resolve, left the Proxy
in Terminating with no way out. It now tries Delete anyway, which is
idempotent and can still finish where Get cannot converge, and releases
the object when that succeeds.

crawl.example.com/skip-cleanup abandons the instance for the cases that
stay stuck, such as revoked credentials. It is checked before the
provider lookup, so it also frees a Proxy whose provider is gone from
the config.

In the KVM provider, Delete over an unconfigured host is now a no-op
success, mirroring removeVolumes over a missing pool. Get keeps failing
permanently there on purpose: the live path reads not-found as "rebuild
it", so a typo in hosts must not look like a missing VM.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
kacerr
2026-09-10 22:30:15 +00:00
co-authored by Claude Opus 5
parent ce365e2602
commit ba24d7e3d9
9 changed files with 213 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/controller/proxy_controller.go
+24 -3
View File
@@ -273,17 +273,38 @@ func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.
controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
return ctrl.Result{}, r.Update(ctx, p)
}
if p.Annotations[crawlv1alpha1.AnnotationSkipCleanup] == "true" {
// Checked before the provider lookup so it also releases a Proxy whose
// provider is gone from the config. The instance is left running.
logf.FromContext(ctx).Info("skip-cleanup annotation set; abandoning the instance",
"providerID", p.Status.ProviderID, "provider", p.Spec.Provider)
controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
return ctrl.Result{}, r.Update(ctx, p)
}
prov, ok := r.Providers[p.Spec.Provider]
if !ok {
return ctrl.Result{}, fmt.Errorf(
"provider %q is not configured; cannot clean up instance %s", p.Spec.Provider, p.Status.ProviderID)
}
_, err := prov.Get(ctx, p.Status.ProviderID)
if provider.Class(err) == provider.ErrNotFound {
switch {
case err == nil:
case provider.Class(err) == provider.ErrNotFound:
controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
return ctrl.Result{}, r.Update(ctx, p)
}
if err != nil {
case provider.Class(err) == provider.ErrPermanent:
// Get will never answer for this instance — a host retired from the
// config, a providerID the provider can no longer resolve. Delete is
// idempotent and may still finish, so try it instead of polling a Get
// that cannot converge, and release the object when it does.
if delErr := prov.Delete(ctx, p.Status.ProviderID); delErr != nil {
return r.deletionFailure(errors.Join(err, delErr))
}
logf.FromContext(ctx).Info("instance unreachable but cleanup succeeded; removing the finalizer",
"providerID", p.Status.ProviderID, "error", err.Error())
controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
return ctrl.Result{}, r.Update(ctx, p)
default:
return r.deletionFailure(err)
}
if err := prov.Delete(ctx, p.Status.ProviderID); err != nil {
internal/controller/reconcile_test.go
+67
View File
@@ -2,6 +2,7 @@ package controller
import (
"context"
"slices"
"testing"
"time"
@@ -78,6 +79,10 @@ func notFoundErr() error {
return provider.Wrap(provider.ErrNotFound, "get", "stub", "some-id", nil)
}
func permanentErr() error {
return provider.Wrap(provider.ErrPermanent, "get", "stub", "some-id", nil)
}
func testScheme(t *testing.T) *runtime.Scheme {
t.Helper()
s := runtime.NewScheme()
@@ -125,6 +130,15 @@ func withSpecHashAnnotation(hash string) func(*crawlv1alpha1.Proxy) {
}
}
func withSkipCleanup() func(*crawlv1alpha1.Proxy) {
return func(p *crawlv1alpha1.Proxy) {
if p.Annotations == nil {
p.Annotations = map[string]string{}
}
p.Annotations[crawlv1alpha1.AnnotationSkipCleanup] = "true"
}
}
func deleting() func(*crawlv1alpha1.Proxy) {
return func(p *crawlv1alpha1.Proxy) {
now := metav1.Now()
@@ -375,6 +389,59 @@ func TestReconcile_actionTable(t *testing.T) {
assertProxyGone(t, r)
},
},
{
// A permanent Get (a retired KVM host, a providerID the provider
// cannot resolve) used to retry forever with the finalizer on.
name: "deletion with an unreachable instance still deletes and releases",
proxy: managedProxy(deleting(), withProviderID("stub-id-1")),
stub: &stubProvider{getErr: permanentErr()},
wantResult: ctrl.Result{},
verify: func(t *testing.T, r *ProxyReconciler, stub *stubProvider) {
assertProxyGone(t, r)
if stub.deleteCalls != 1 {
t.Errorf("deleteCalls = %d, want 1 — Delete must be tried when Get cannot answer", stub.deleteCalls)
}
},
},
{
name: "deletion keeps the finalizer when cleanup itself fails",
proxy: managedProxy(deleting(), withProviderID("stub-id-1")),
stub: &stubProvider{
getErr: permanentErr(),
deleteErr: provider.Wrap(provider.ErrPermanent, "delete", "stub", "stub-id-1", nil),
},
wantResult: ctrl.Result{},
wantErr: true,
verify: func(t *testing.T, r *ProxyReconciler, _ *stubProvider) {
// getProxy fails the test if the object is gone, which is what
// finalizer removal would have caused.
if p := getProxy(t, r); !slices.Contains(p.Finalizers, crawlv1alpha1.FinalizerName) {
t.Error("finalizer removed although cleanup failed; the instance would leak silently")
}
},
},
{
name: "skip-cleanup annotation abandons the instance",
proxy: managedProxy(deleting(), withProviderID("stub-id-1"), withSkipCleanup()),
stub: &stubProvider{getInst: &provider.Instance{ID: "stub-id-1", State: provider.StateRunning}},
wantResult: ctrl.Result{},
verify: func(t *testing.T, r *ProxyReconciler, stub *stubProvider) {
assertProxyGone(t, r)
if stub.getCalls != 0 || stub.deleteCalls != 0 {
t.Errorf("getCalls/deleteCalls = %d/%d, want 0/0 — the provider must not be asked", stub.getCalls, stub.deleteCalls)
}
},
},
{
name: "skip-cleanup annotation releases a proxy whose provider is gone",
proxy: managedProxy(deleting(), withProviderID("stub-id-1"), withSkipCleanup(),
func(p *crawlv1alpha1.Proxy) { p.Spec.Provider = "retired" }),
stub: &stubProvider{},
wantResult: ctrl.Result{},
verify: func(t *testing.T, r *ProxyReconciler, _ *stubProvider) {
assertProxyGone(t, r)
},
},
{
name: "deletion deletes the instance and polls",
proxy: managedProxy(deleting(), withProviderID("stub-id-1")),