egress-proxies-operator/internal/health/engine_test.go

package health

import (
	"context"
	"crypto/tls"
	"errors"
	"net/url"
	"strconv"
	"strings"
	"testing"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"
	"sigs.k8s.io/controller-runtime/pkg/event"

	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
)

var testKey = types.NamespacedName{Namespace: "default", Name: "p1"}

func testEngine() *Engine {
	e := &Engine{Events: make(chan event.GenericEvent, 8)}
	e.applyDefaults()
	return e
}

func testJob(failureThreshold, successThreshold int32) probeJob {
	return probeJob{
		key: testKey,
		uid: "uid-1",
		hc: crawlv1alpha1.HealthCheckSpec{
			FailureThreshold: failureThreshold,
			SuccessThreshold: successThreshold,
		},
		interval: 30 * time.Second,
	}
}

func drainOneEvent(t *testing.T, e *Engine) event.GenericEvent {
	t.Helper()
	select {
	case evt := <-e.Events:
		return evt
	default:
		t.Fatal("expected an event, channel is empty")
		return event.GenericEvent{}
	}
}

func assertNoEvent(t *testing.T, e *Engine) {
	t.Helper()
	select {
	case <-e.Events:
		t.Fatal("unexpected event emitted")
	default:
	}
}

func boolPtr(b bool) *bool { return &b }

func TestRecord_firstResultEmits(t *testing.T) {
	t.Parallel()
	e := testEngine()
	e.states[testKey] = &state{uid: "uid-1"}

	e.record(testJob(3, 1), probeResult{ok: true, latency: 30 * time.Millisecond}, time.Now())

	evt := drainOneEvent(t, e)
	if got := evt.Object.GetName(); got != "p1" {
		t.Errorf("event object name = %q, want p1", got)
	}
	snap, ok := e.Snapshot(testKey)
	if !ok || !snap.Healthy {
		t.Errorf("Snapshot = %+v, %v; want healthy verdict", snap, ok)
	}
	if snap.Latency != 30*time.Millisecond {
		t.Errorf("latency = %v, want 30ms", snap.Latency)
	}
}

func TestRecord_failureThresholdFlips(t *testing.T) {
	t.Parallel()
	e := testEngine()
	e.states[testKey] = &state{uid: "uid-1", healthy: boolPtr(true), reportedHealthy: boolPtr(true)}
	job := testJob(3, 1)
	probeErr := probeResult{err: errors.New("connect refused")}

	e.record(job, probeErr, time.Now())
	e.record(job, probeErr, time.Now())
	assertNoEvent(t, e)
	if snap, _ := e.Snapshot(testKey); !snap.Healthy {
		t.Fatal("flipped unhealthy before failureThreshold was reached")
	}

	e.record(job, probeErr, time.Now())
	drainOneEvent(t, e)
	snap, _ := e.Snapshot(testKey)
	if snap.Healthy {
		t.Error("still healthy after failureThreshold consecutive failures")
	}
	if snap.ConsecutiveFailures != 3 {
		t.Errorf("ConsecutiveFailures = %d, want 3", snap.ConsecutiveFailures)
	}
	if !strings.Contains(snap.LastError, "connect refused") {
		t.Errorf("LastError = %q, want the probe error", snap.LastError)
	}
}

func TestRecord_successThresholdFlips(t *testing.T) {
	t.Parallel()
	e := testEngine()
	e.states[testKey] = &state{uid: "uid-1", healthy: boolPtr(false), reportedHealthy: boolPtr(false)}
	job := testJob(3, 2)
	success := probeResult{ok: true, latency: 25 * time.Millisecond}

	e.record(job, success, time.Now())
	assertNoEvent(t, e)

	e.record(job, success, time.Now())
	drainOneEvent(t, e)
	if snap, _ := e.Snapshot(testKey); !snap.Healthy {
		t.Error("not healthy after successThreshold consecutive successes")
	}
}

func TestRecord_latencySuppression(t *testing.T) {
	t.Parallel()
	now := time.Now()

	tests := []struct {
		name            string
		reportedLatency time.Duration
		lastReport      time.Time
		newLatency      time.Duration
		wantEmit        bool
	}{
		{
			name:            "small change under the relative floor is suppressed",
			reportedLatency: 100 * time.Millisecond,
			lastReport:      now.Add(-2 * time.Minute),
			newLatency:      110 * time.Millisecond,
			wantEmit:        false,
		},
		{
			name:            "small absolute jitter at low latency is suppressed",
			reportedLatency: 5 * time.Millisecond,
			lastReport:      now.Add(-2 * time.Minute),
			newLatency:      20 * time.Millisecond, // >50% but under the 20ms floor
			wantEmit:        false,
		},
		{
			name:            "material change after the rate window emits",
			reportedLatency: 100 * time.Millisecond,
			lastReport:      now.Add(-2 * time.Minute),
			newLatency:      200 * time.Millisecond,
			wantEmit:        true,
		},
		{
			name:            "material change inside the rate window is suppressed",
			reportedLatency: 100 * time.Millisecond,
			lastReport:      now.Add(-10 * time.Second),
			newLatency:      400 * time.Millisecond,
			wantEmit:        false,
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			e := testEngine()
			e.states[testKey] = &state{
				uid:             "uid-1",
				healthy:         boolPtr(true),
				reportedHealthy: boolPtr(true),
				reportedLatency: tc.reportedLatency,
				lastReport:      tc.lastReport,
			}
			e.record(testJob(3, 1), probeResult{ok: true, latency: tc.newLatency}, now)
			if tc.wantEmit {
				drainOneEvent(t, e)
			} else {
				assertNoEvent(t, e)
			}
		})
	}
}

func TestRecord_droppedEventIsRetried(t *testing.T) {
	t.Parallel()
	e := testEngine()
	e.Events = make(chan event.GenericEvent) // unbuffered, nobody reading
	e.states[testKey] = &state{uid: "uid-1"}
	success := probeResult{ok: true, latency: 30 * time.Millisecond}

	e.record(testJob(3, 1), success, time.Now())
	e.mu.Lock()
	reported := e.states[testKey].reportedHealthy
	e.mu.Unlock()
	if reported != nil {
		t.Fatal("reported marker advanced although the event was dropped")
	}

	// Channel drains (reconciler recovers): the next probe re-emits.
	e.Events = make(chan event.GenericEvent, 1)
	e.record(testJob(3, 1), success, time.Now())
	drainOneEvent(t, e)
}

func TestRecord_staleJobIsIgnored(t *testing.T) {
	t.Parallel()
	e := testEngine()
	e.states[testKey] = &state{uid: "uid-NEW"}

	job := testJob(3, 1)
	job.uid = "uid-OLD"
	e.record(job, probeResult{ok: true, latency: time.Millisecond}, time.Now())

	assertNoEvent(t, e)
	if _, ok := e.Snapshot(testKey); ok {
		t.Error("stale probe produced a verdict for the new object")
	}
}

func externalProxy(name, host string, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy {
	p := &crawlv1alpha1.Proxy{
		ObjectMeta: metav1.ObjectMeta{
			Name: name, Namespace: "default", UID: types.UID("uid-" + name),
		},
		Spec: crawlv1alpha1.ProxySpec{
			Mode:     crawlv1alpha1.ModeExternal,
			Endpoint: &crawlv1alpha1.EndpointSpec{Host: host, Port: 3128},
		},
	}
	for _, m := range mut {
		m(p)
	}
	return p
}

func TestTick_schedulingAndPruning(t *testing.T) {
	t.Parallel()
	s := runtime.NewScheme()
	if err := crawlv1alpha1.AddToScheme(s); err != nil {
		t.Fatalf("scheme: %v", err)
	}

	probeable := externalProxy("probeable", "10.0.0.1")
	seeded := externalProxy("seeded", "10.0.0.2", func(p *crawlv1alpha1.Proxy) {
		p.Status.Conditions = []metav1.Condition{{
			Type: crawlv1alpha1.ConditionHealthy, Status: metav1.ConditionTrue,
			Reason: "ProbeSucceeded", LastTransitionTime: metav1.Now(),
		}}
		p.Status.LatencyMillis = 42
	})
	noIP := &crawlv1alpha1.Proxy{
		ObjectMeta: metav1.ObjectMeta{Name: "no-ip", Namespace: "default", UID: "uid-no-ip"},
		Spec:       crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged, Provider: "stub"},
	}

	e := testEngine()
	e.Reader = fake.NewClientBuilder().WithScheme(s).
		WithObjects(probeable, seeded, noIP).Build()
	// Stale entries: one for a proxy that no longer exists, one under a key
	// that now belongs to a different UID (delete + recreate).
	e.states[types.NamespacedName{Namespace: "default", Name: "gone"}] = &state{uid: "uid-gone"}
	e.states[types.NamespacedName{Namespace: "default", Name: "probeable"}] = &state{
		uid: "uid-previous-incarnation", healthy: boolPtr(false),
	}

	jobs := make(chan probeJob, 8)
	e.tick(context.Background(), time.Now(), jobs)

	var dispatched []probeJob
	for {
		select {
		case j := <-jobs:
			dispatched = append(dispatched, j)
			continue
		default:
		}
		break
	}

	if len(dispatched) != 1 {
		t.Fatalf("dispatched %d jobs, want exactly 1 (only the fresh probeable proxy)", len(dispatched))
	}
	j := dispatched[0]
	if j.key.Name != "probeable" || j.uid != "uid-probeable" {
		t.Errorf("dispatched job = %+v, want the recreated probeable proxy", j)
	}
	if want := "http://" + "10.0.0.1:" + strconv.Itoa(3128); j.proxyURL.String() != want {
		t.Errorf("proxyURL = %s, want %s", j.proxyURL, want)
	}

	e.mu.Lock()
	defer e.mu.Unlock()
	if _, ok := e.states[types.NamespacedName{Namespace: "default", Name: "gone"}]; ok {
		t.Error("state for a deleted proxy was not pruned")
	}
	if _, ok := e.states[types.NamespacedName{Namespace: "default", Name: "no-ip"}]; ok {
		t.Error("state was created for a proxy with no IP")
	}
	st := e.states[types.NamespacedName{Namespace: "default", Name: "probeable"}]
	if st == nil || st.uid != "uid-probeable" {
		t.Fatalf("state for recreated proxy = %+v, want fresh state with the new UID", st)
	}
	if st.healthy != nil && !*st.healthy {
		t.Error("recreated proxy inherited the previous incarnation's unhealthy verdict")
	}
	seededSt := e.states[types.NamespacedName{Namespace: "default", Name: "seeded"}]
	if seededSt == nil {
		t.Fatal("no state created for the seeded proxy")
	}
	if seededSt.healthy == nil || !*seededSt.healthy {
		t.Error("seeded proxy did not inherit its Healthy condition")
	}
	if seededSt.reportedHealthy == nil || !*seededSt.reportedHealthy {
		t.Error("seeded verdict must count as already reported, or restart would re-emit for the whole fleet")
	}
	if seededSt.reportedLatency != 42*time.Millisecond {
		t.Errorf("seeded reportedLatency = %v, want 42ms", seededSt.reportedLatency)
	}
	if seededSt.consecOK != 0 || seededSt.consecFail != 0 {
		t.Error("seeded counters must start at zero")
	}
}

func TestEngine_StartEndToEnd(t *testing.T) {
	t.Parallel()
	s := runtime.NewScheme()
	if err := crawlv1alpha1.AddToScheme(s); err != nil {
		t.Fatalf("scheme: %v", err)
	}

	e := testEngine()
	e.Tick = 5 * time.Millisecond
	e.Reader = fake.NewClientBuilder().WithScheme(s).
		WithObjects(externalProxy("p1", "192.0.2.1")).Build()
	e.probeFn = func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult {
		return probeResult{ok: true, latency: 12 * time.Millisecond}
	}

	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- e.Start(ctx) }()

	select {
	case evt := <-e.Events:
		if evt.Object.GetName() != "p1" {
			t.Errorf("event for %q, want p1", evt.Object.GetName())
		}
	case <-time.After(5 * time.Second):
		t.Fatal("no health event within 5s")
	}
	snap, ok := e.Snapshot(types.NamespacedName{Namespace: "default", Name: "p1"})
	if !ok || !snap.Healthy || snap.Latency != 12*time.Millisecond {
		t.Errorf("Snapshot = %+v, %v; want healthy at 12ms", snap, ok)
	}

	cancel()
	select {
	case err := <-done:
		if err != nil {
			t.Errorf("Start returned %v, want nil on context cancel", err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("Start did not stop within 5s of cancel")
	}
}