egress-proxies-operator/internal/controller/proxy_controller_test.go

/*
Copyright 2026.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"time"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	apimeta "k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	ctrl "sigs.k8s.io/controller-runtime"

	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

// These specs drive the reconciler against a real envtest API server, so
// CRD structural defaulting and CEL validation are live — the parts the
// fake-client action-table tests can't cover. The provider stays a stub:
// envtest has no kubelet or cloud, so instance state is simulated by
// mutating the stub between reconciles.
var _ = Describe("Proxy controller", func() {
	const ns = "default"

	newEnvtestReconciler := func(stub *stubProvider) *ProxyReconciler {
		return &ProxyReconciler{
			Client:           k8sClient,
			Scheme:           k8sClient.Scheme(),
			Providers:        map[string]provider.Provider{"stub": stub},
			ProvisioningPoll: 50 * time.Millisecond,
			DriftPoll:        100 * time.Millisecond,
			DeletionPoll:     50 * time.Millisecond,
			QuotaRetry:       200 * time.Millisecond,
			RequeueNow:       10 * time.Millisecond,
		}
	}

	envReconcile := func(r *ProxyReconciler, name string) (ctrl.Result, error) {
		return r.Reconcile(ctx, ctrl.Request{
			NamespacedName: types.NamespacedName{Namespace: ns, Name: name},
		})
	}

	fetch := func(name string) *crawlv1alpha1.Proxy {
		p := &crawlv1alpha1.Proxy{}
		Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, p)).To(Succeed())
		return p
	}

	// cleanup drives a Managed proxy's finalizer to completion so one spec's
	// leftovers can't leak into another. Registered via DeferCleanup so it
	// runs even when the spec body fails mid-way.
	cleanup := func(r *ProxyReconciler, stub *stubProvider, name string) {
		p := &crawlv1alpha1.Proxy{}
		err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, p)
		if apierrors.IsNotFound(err) {
			return
		}
		Expect(err).NotTo(HaveOccurred())
		Expect(k8sClient.Delete(ctx, p)).To(Succeed())
		stub.getErr = provider.Wrap(provider.ErrNotFound, "get", "stub", "", nil)
		for range 3 {
			if _, err := envReconcile(r, name); err != nil {
				break
			}
			if apierrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, p)) {
				return
			}
		}
		Fail("cleanup did not drive the proxy " + name + " to deletion")
	}

	managedSpec := func() crawlv1alpha1.ProxySpec {
		return crawlv1alpha1.ProxySpec{
			Mode:     crawlv1alpha1.ModeManaged,
			Provider: "stub",
		}
	}

	It("provisions a Managed proxy through to Running", func() {
		const name = "e2e-provision"
		stub := &stubProvider{createID: "inst-1"}
		r := newEnvtestReconciler(stub)
		DeferCleanup(func() { cleanup(r, stub, name) })

		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
			Spec:       managedSpec(),
		})).To(Succeed())

		By("adding the finalizer on the first pass")
		res, err := envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		Expect(res).To(Equal(ctrl.Result{}))
		Expect(fetch(name).Finalizers).To(ContainElement(crawlv1alpha1.FinalizerName))

		By("creating the instance on the second pass")
		res, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		Expect(res.RequeueAfter).To(Equal(r.ProvisioningPoll))
		p := fetch(name)
		Expect(p.Status.ProviderID).To(Equal("inst-1"))
		Expect(p.Annotations).To(HaveKey(crawlv1alpha1.AnnotationSpecHash))
		// The real API server defaulted spec.port; the create request must
		// have seen it.
		Expect(p.Spec.Port).To(Equal(crawlv1alpha1.DefaultPort))
		Expect(stub.lastCreate.Port).To(Equal(crawlv1alpha1.DefaultPort))
		Expect(stub.lastCreate.Name).To(Equal(provider.NameFromUID(p.UID)))

		By("polling while the instance provisions")
		stub.getInst = &provider.Instance{ID: "inst-1", State: provider.StateProvisioning}
		res, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		Expect(res.RequeueAfter).To(Equal(r.ProvisioningPoll))
		Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseProvisioning))

		By("publishing the IP once the instance runs")
		stub.getInst = &provider.Instance{ID: "inst-1", IP: "10.9.8.7", State: provider.StateRunning}
		res, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		Expect(res.RequeueAfter).To(Equal(r.DriftPoll))
		p = fetch(name)
		Expect(p.Status.IP).To(Equal("10.9.8.7"))
		cond := apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionProvisioned)
		Expect(cond).NotTo(BeNil())
		Expect(cond.Status).To(Equal(metav1.ConditionTrue))
		Expect(p.Status.ObservedGeneration).To(Equal(p.Generation))
	})

	It("replaces the instance when the spec changes", func() {
		const name = "e2e-replace"
		stub := &stubProvider{createID: "inst-old"}
		r := newEnvtestReconciler(stub)
		DeferCleanup(func() { cleanup(r, stub, name) })

		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
			Spec:       managedSpec(),
		})).To(Succeed())
		_, err := envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		stub.getInst = &provider.Instance{ID: "inst-old", IP: "10.0.0.1", State: provider.StateRunning}
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		oldHash := fetch(name).Annotations[crawlv1alpha1.AnnotationSpecHash]

		By("editing a replacement-triggering field")
		p := fetch(name)
		p.Spec.Port = 8080
		Expect(k8sClient.Update(ctx, p)).To(Succeed())

		By("deleting the old instance first")
		res, err := envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		Expect(res.RequeueAfter).To(Equal(r.DeletionPoll))
		Expect(stub.deleteCalls).To(Equal(1))
		p = fetch(name)
		Expect(p.Annotations[crawlv1alpha1.AnnotationSpecHash]).To(Equal(oldHash),
			"hash must not advance while the old instance still exists")
		cond := apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionProvisioned)
		Expect(cond.Reason).To(Equal(ReasonReplacing))

		By("advancing the hash once the old instance is gone")
		stub.getErr = provider.Wrap(provider.ErrNotFound, "get", "stub", "inst-old", nil)
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		p = fetch(name)
		Expect(p.Status.ProviderID).To(BeEmpty())
		Expect(p.Annotations[crawlv1alpha1.AnnotationSpecHash]).NotTo(Equal(oldHash))

		By("creating the replacement")
		stub.createID = "inst-new"
		stub.getErr = nil
		stub.getInst = nil
		res, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		Expect(res.RequeueAfter).To(Equal(r.ProvisioningPoll))
		Expect(stub.createCalls).To(Equal(2))
		Expect(fetch(name).Status.ProviderID).To(Equal("inst-new"))
		Expect(stub.lastCreate.Port).To(Equal(int32(8080)))
	})

	It("cleans up the instance on delete via the finalizer", func() {
		const name = "e2e-delete"
		stub := &stubProvider{createID: "inst-del"}
		r := newEnvtestReconciler(stub)
		DeferCleanup(func() { cleanup(r, stub, name) })

		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
			Spec:       managedSpec(),
		})).To(Succeed())
		_, err := envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		stub.getInst = &provider.Instance{ID: "inst-del", IP: "10.0.0.2", State: provider.StateRunning}
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())

		By("deleting the CR — the finalizer holds it")
		Expect(k8sClient.Delete(ctx, fetch(name))).To(Succeed())
		res, err := envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		Expect(res.RequeueAfter).To(Equal(r.DeletionPoll))
		Expect(stub.deleteCalls).To(Equal(1))
		Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseDeleting))

		By("removing the finalizer once the instance is gone")
		stub.getErr = provider.Wrap(provider.ErrNotFound, "get", "stub", "inst-del", nil)
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		err = k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &crawlv1alpha1.Proxy{})
		Expect(apierrors.IsNotFound(err)).To(BeTrue(), "proxy should be fully deleted")
	})

	It("reaches Ready once the health engine has a verdict", func() {
		const name = "e2e-ready"
		stub := &stubProvider{createID: "inst-rdy"}
		r := newEnvtestReconciler(stub)
		DeferCleanup(func() { cleanup(r, stub, name) })

		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
			Spec:       managedSpec(),
		})).To(Succeed())
		_, err := envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		stub.getInst = &provider.Instance{ID: "inst-rdy", IP: "10.3.3.3", State: provider.StateRunning}
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseProvisioning),
			"no health verdict yet — must not be Ready")

		By("supplying a healthy snapshot")
		r.Health = fakeSnapshotter{ok: true, snap: health.Snapshot{
			Healthy: true, Latency: 21 * time.Millisecond, LastProbe: time.Now(),
		}}
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		p := fetch(name)
		Expect(p.Status.Phase).To(Equal(crawlv1alpha1.PhaseReady))
		Expect(p.Status.LatencyMillis).To(Equal(int64(21)))
	})

	It("treats quota exhaustion as a wait and a permanent error as Failed", func() {
		const name = "e2e-errors"
		stub := &stubProvider{
			createErr: provider.Wrap(provider.ErrQuotaExceeded, "create", "stub", "", nil),
		}
		r := newEnvtestReconciler(stub)
		DeferCleanup(func() { cleanup(r, stub, name) })

		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
			Spec:       managedSpec(),
		})).To(Succeed())
		_, err := envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())

		By("quota: condition set, slow requeue, phase NOT Failed")
		res, err := envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred(), "quota must not count as an error (stays off the backoff curve)")
		Expect(res.RequeueAfter).To(Equal(r.QuotaRetry))
		p := fetch(name)
		cond := apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionProvisioned)
		Expect(cond.Reason).To(Equal(ReasonQuotaExceeded))
		Expect(p.Status.Phase).NotTo(Equal(crawlv1alpha1.PhaseFailed))

		By("permanent: phase Failed and no further provider calls")
		stub.createErr = provider.Wrap(provider.ErrPermanent, "create", "stub", "", nil)
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseFailed))
		callsAfterLatch := stub.createCalls
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		Expect(stub.createCalls).To(Equal(callsAfterLatch), "the latch must stop provider calls")
	})

	It("adopts an instance when the spec-hash annotation is stripped", func() {
		const name = "e2e-adopt"
		stub := &stubProvider{createID: "inst-adopt"}
		r := newEnvtestReconciler(stub)
		DeferCleanup(func() { cleanup(r, stub, name) })

		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
			Spec:       managedSpec(),
		})).To(Succeed())
		_, err := envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		stub.getInst = &provider.Instance{ID: "inst-adopt", IP: "10.4.4.4", State: provider.StateRunning}
		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		originalHash := fetch(name).Annotations[crawlv1alpha1.AnnotationSpecHash]
		Expect(originalHash).NotTo(BeEmpty())

		By("stripping the annotation, as an operator-version upgrade with a changed hash input would")
		p := fetch(name)
		delete(p.Annotations, crawlv1alpha1.AnnotationSpecHash)
		Expect(k8sClient.Update(ctx, p)).To(Succeed())

		_, err = envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		p = fetch(name)
		Expect(p.Annotations[crawlv1alpha1.AnnotationSpecHash]).To(Equal(originalHash), "hash must be restored")
		Expect(p.Status.ProviderID).To(Equal("inst-adopt"), "adoption must keep the instance")
		Expect(stub.deleteCalls).To(BeZero(), "adoption must never replace")
	})

	It("tracks an External proxy without touching providers", func() {
		const name = "e2e-external"
		stub := &stubProvider{}
		r := newEnvtestReconciler(stub)

		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
			Spec: crawlv1alpha1.ProxySpec{
				Mode:     crawlv1alpha1.ModeExternal,
				Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.7"},
			},
		})).To(Succeed())

		res, err := envReconcile(r, name)
		Expect(err).NotTo(HaveOccurred())
		Expect(res).To(Equal(ctrl.Result{}))

		p := fetch(name)
		Expect(p.Status.IP).To(Equal("203.0.113.7"))
		Expect(p.Finalizers).To(BeEmpty())
		cond := apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionProvisioned)
		Expect(cond).NotTo(BeNil())
		Expect(cond.Reason).To(Equal(ReasonExternalEndpoint))
		Expect(stub.createCalls + stub.getCalls + stub.deleteCalls).To(BeZero())

		By("deleting without any finalizer round-trip")
		Expect(k8sClient.Delete(ctx, p)).To(Succeed())
		err = k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &crawlv1alpha1.Proxy{})
		Expect(apierrors.IsNotFound(err)).To(BeTrue())
	})
})

// These specs assert the CRD's CEL rules and structural defaulting against
// the real envtest API server — the fake client runs neither, which is the
// documented caveat on the action-table unit tests.
var _ = Describe("Proxy CRD validation (CEL)", func() {
	const ns = "default"

	managed := func(name string) *crawlv1alpha1.Proxy {
		return &crawlv1alpha1.Proxy{
			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
			Spec: crawlv1alpha1.ProxySpec{
				Mode:     crawlv1alpha1.ModeManaged,
				Provider: "stub",
			},
		}
	}
	external := func(name string) *crawlv1alpha1.Proxy {
		return &crawlv1alpha1.Proxy{
			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
			Spec: crawlv1alpha1.ProxySpec{
				Mode:     crawlv1alpha1.ModeExternal,
				Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
			},
		}
	}
	mustCreate := func(p *crawlv1alpha1.Proxy) {
		GinkgoHelper()
		Expect(k8sClient.Create(ctx, p)).To(Succeed())
		DeferCleanup(func() { _ = k8sClient.Delete(ctx, p) })
	}

	It("rejects invalid creates", func() {
		invalid := []struct {
			about string
			spec  crawlv1alpha1.ProxySpec
			want  string
		}{
			{
				about: "Managed without provider",
				spec:  crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged},
				want:  "provider is required when mode is Managed",
			},
			{
				about: "External with provider",
				spec: crawlv1alpha1.ProxySpec{
					Mode: crawlv1alpha1.ModeExternal, Provider: "stub",
					Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
				},
				want: "provider must not be set when mode is External",
			},
			{
				about: "External without endpoint",
				spec:  crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeExternal},
				want:  "endpoint is required when mode is External",
			},
			{
				about: "Managed with endpoint",
				spec: crawlv1alpha1.ProxySpec{
					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
					Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
				},
				want: "endpoint must not be set when mode is Managed",
			},
			{
				about: "cloudInit with both inline and secretRef",
				spec: crawlv1alpha1.ProxySpec{
					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
					CloudInit: &crawlv1alpha1.CloudInitSpec{
						Inline:    "#cloud-config",
						SecretRef: &crawlv1alpha1.SecretKeySelector{Name: "s"},
					},
				},
				want: "exactly one of inline or secretRef",
			},
			{
				about: "cloudInit with neither inline nor secretRef",
				spec: crawlv1alpha1.ProxySpec{
					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
					CloudInit: &crawlv1alpha1.CloudInitSpec{},
				},
				want: "exactly one of inline or secretRef",
			},
		}
		for _, tc := range invalid {
			p := &crawlv1alpha1.Proxy{
				ObjectMeta: metav1.ObjectMeta{Name: "cel-invalid", Namespace: ns},
				Spec:       tc.spec,
			}
			err := k8sClient.Create(ctx, p)
			Expect(err).To(HaveOccurred(), tc.about)
			Expect(err.Error()).To(ContainSubstring(tc.want), tc.about)
		}
	})

	It("rejects mode mutation", func() {
		p := external("cel-mode-immutable")
		mustCreate(p)
		p.Spec = crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged, Provider: "stub"}
		err := k8sClient.Update(ctx, p)
		Expect(err).To(HaveOccurred())
		Expect(err.Error()).To(ContainSubstring("mode is immutable"))
	})

	It("rejects provider mutation and removal", func() {
		p := managed("cel-provider-immutable")
		mustCreate(p)

		p.Spec.Provider = "other"
		err := k8sClient.Update(ctx, p)
		Expect(err).To(HaveOccurred())
		Expect(err.Error()).To(ContainSubstring("provider is immutable"))

		// Removal must also be rejected — the has()==has() form exists
		// exactly because a field-level rule would not fire on absence.
		// (Dropping provider alone would also trip the required-iff rule,
		// so flip mode too and check the immutability rules win.)
		fresh := fetchProxy(ns, "cel-provider-immutable")
		fresh.Spec.Provider = ""
		fresh.Spec.Endpoint = &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"}
		fresh.Spec.Mode = crawlv1alpha1.ModeExternal
		err = k8sClient.Update(ctx, fresh)
		Expect(err).To(HaveOccurred())
		Expect(err.Error()).To(ContainSubstring("immutable"))
	})

	It("materializes every nested healthCheck default when healthCheck is omitted", func() {
		p := managed("cel-defaults")
		mustCreate(p)

		got := fetchProxy(ns, "cel-defaults")
		// The +kubebuilder:default={} assertion: structural defaulting only
		// descends into values that exist, so without it a nil healthCheck
		// would get none of these.
		hc := got.Spec.HealthCheck
		Expect(hc).NotTo(BeNil())
		Expect(hc.ProbeURL).To(Equal(crawlv1alpha1.DefaultProbeURL))
		Expect(hc.IntervalSeconds).To(Equal(crawlv1alpha1.DefaultHealthCheckIntervalSeconds))
		Expect(hc.TimeoutSeconds).To(Equal(crawlv1alpha1.DefaultHealthCheckTimeoutSeconds))
		Expect(hc.FailureThreshold).To(Equal(crawlv1alpha1.DefaultFailureThreshold))
		Expect(hc.SuccessThreshold).To(Equal(crawlv1alpha1.DefaultSuccessThreshold))
		Expect(hc.ExpectedStatusCodes).To(Equal(crawlv1alpha1.DefaultExpectedStatusCodes))

		Expect(got.Spec.Port).To(Equal(crawlv1alpha1.DefaultPort))
		Expect(got.Spec.MaxLeases).NotTo(BeNil())
		Expect(*got.Spec.MaxLeases).To(Equal(crawlv1alpha1.DefaultMaxLeases))
	})
})

func fetchProxy(ns, name string) *crawlv1alpha1.Proxy {
	GinkgoHelper()
	p := &crawlv1alpha1.Proxy{}
	Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, p)).To(Succeed())
	return p
}