Add the health engine: through-proxy probes, thresholds, channel-fed transitions

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-09 15:01:10 +02:00
parent 71c00c40d1
commit 801a9fbe5f
10 changed files with 1350 additions and 34 deletions
--- a/internal/health/engine_test.go
+++ b/internal/health/engine_test.go
@@ -0,0 +1,372 @@
+package health
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"net/url"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+)
+
+var testKey = types.NamespacedName{Namespace: "default", Name: "p1"}
+
+func testEngine() *Engine {
+	e := &Engine{Events: make(chan event.GenericEvent, 8)}
+	e.applyDefaults()
+	return e
+}
+
+func testJob(failureThreshold, successThreshold int32) probeJob {
+	return probeJob{
+		key: testKey,
+		uid: "uid-1",
+		hc: crawlv1alpha1.HealthCheckSpec{
+			FailureThreshold: failureThreshold,
+			SuccessThreshold: successThreshold,
+		},
+		interval: 30 * time.Second,
+	}
+}
+
+func drainOneEvent(t *testing.T, e *Engine) event.GenericEvent {
+	t.Helper()
+	select {
+	case evt := <-e.Events:
+		return evt
+	default:
+		t.Fatal("expected an event, channel is empty")
+		return event.GenericEvent{}
+	}
+}
+
+func assertNoEvent(t *testing.T, e *Engine) {
+	t.Helper()
+	select {
+	case <-e.Events:
+		t.Fatal("unexpected event emitted")
+	default:
+	}
+}
+
+func boolPtr(b bool) *bool { return &b }
+
+func TestRecord_firstResultEmits(t *testing.T) {
+	t.Parallel()
+	e := testEngine()
+	e.states[testKey] = &state{uid: "uid-1"}
+
+	e.record(testJob(3, 1), probeResult{ok: true, latency: 30 * time.Millisecond}, time.Now())
+
+	evt := drainOneEvent(t, e)
+	if got := evt.Object.GetName(); got != "p1" {
+		t.Errorf("event object name = %q, want p1", got)
+	}
+	snap, ok := e.Snapshot(testKey)
+	if !ok || !snap.Healthy {
+		t.Errorf("Snapshot = %+v, %v; want healthy verdict", snap, ok)
+	}
+	if snap.Latency != 30*time.Millisecond {
+		t.Errorf("latency = %v, want 30ms", snap.Latency)
+	}
+}
+
+func TestRecord_failureThresholdFlips(t *testing.T) {
+	t.Parallel()
+	e := testEngine()
+	e.states[testKey] = &state{uid: "uid-1", healthy: boolPtr(true), reportedHealthy: boolPtr(true)}
+	job := testJob(3, 1)
+	probeErr := probeResult{err: errors.New("connect refused")}
+
+	e.record(job, probeErr, time.Now())
+	e.record(job, probeErr, time.Now())
+	assertNoEvent(t, e)
+	if snap, _ := e.Snapshot(testKey); !snap.Healthy {
+		t.Fatal("flipped unhealthy before failureThreshold was reached")
+	}
+
+	e.record(job, probeErr, time.Now())
+	drainOneEvent(t, e)
+	snap, _ := e.Snapshot(testKey)
+	if snap.Healthy {
+		t.Error("still healthy after failureThreshold consecutive failures")
+	}
+	if snap.ConsecutiveFailures != 3 {
+		t.Errorf("ConsecutiveFailures = %d, want 3", snap.ConsecutiveFailures)
+	}
+	if !strings.Contains(snap.LastError, "connect refused") {
+		t.Errorf("LastError = %q, want the probe error", snap.LastError)
+	}
+}
+
+func TestRecord_successThresholdFlips(t *testing.T) {
+	t.Parallel()
+	e := testEngine()
+	e.states[testKey] = &state{uid: "uid-1", healthy: boolPtr(false), reportedHealthy: boolPtr(false)}
+	job := testJob(3, 2)
+	success := probeResult{ok: true, latency: 25 * time.Millisecond}
+
+	e.record(job, success, time.Now())
+	assertNoEvent(t, e)
+
+	e.record(job, success, time.Now())
+	drainOneEvent(t, e)
+	if snap, _ := e.Snapshot(testKey); !snap.Healthy {
+		t.Error("not healthy after successThreshold consecutive successes")
+	}
+}
+
+func TestRecord_latencySuppression(t *testing.T) {
+	t.Parallel()
+	now := time.Now()
+
+	tests := []struct {
+		name            string
+		reportedLatency time.Duration
+		lastReport      time.Time
+		newLatency      time.Duration
+		wantEmit        bool
+	}{
+		{
+			name:            "small change under the relative floor is suppressed",
+			reportedLatency: 100 * time.Millisecond,
+			lastReport:      now.Add(-2 * time.Minute),
+			newLatency:      110 * time.Millisecond,
+			wantEmit:        false,
+		},
+		{
+			name:            "small absolute jitter at low latency is suppressed",
+			reportedLatency: 5 * time.Millisecond,
+			lastReport:      now.Add(-2 * time.Minute),
+			newLatency:      20 * time.Millisecond, // >50% but under the 20ms floor
+			wantEmit:        false,
+		},
+		{
+			name:            "material change after the rate window emits",
+			reportedLatency: 100 * time.Millisecond,
+			lastReport:      now.Add(-2 * time.Minute),
+			newLatency:      200 * time.Millisecond,
+			wantEmit:        true,
+		},
+		{
+			name:            "material change inside the rate window is suppressed",
+			reportedLatency: 100 * time.Millisecond,
+			lastReport:      now.Add(-10 * time.Second),
+			newLatency:      400 * time.Millisecond,
+			wantEmit:        false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			e := testEngine()
+			e.states[testKey] = &state{
+				uid:             "uid-1",
+				healthy:         boolPtr(true),
+				reportedHealthy: boolPtr(true),
+				reportedLatency: tc.reportedLatency,
+				lastReport:      tc.lastReport,
+			}
+			e.record(testJob(3, 1), probeResult{ok: true, latency: tc.newLatency}, now)
+			if tc.wantEmit {
+				drainOneEvent(t, e)
+			} else {
+				assertNoEvent(t, e)
+			}
+		})
+	}
+}
+
+func TestRecord_droppedEventIsRetried(t *testing.T) {
+	t.Parallel()
+	e := testEngine()
+	e.Events = make(chan event.GenericEvent) // unbuffered, nobody reading
+	e.states[testKey] = &state{uid: "uid-1"}
+	success := probeResult{ok: true, latency: 30 * time.Millisecond}
+
+	e.record(testJob(3, 1), success, time.Now())
+	e.mu.Lock()
+	reported := e.states[testKey].reportedHealthy
+	e.mu.Unlock()
+	if reported != nil {
+		t.Fatal("reported marker advanced although the event was dropped")
+	}
+
+	// Channel drains (reconciler recovers): the next probe re-emits.
+	e.Events = make(chan event.GenericEvent, 1)
+	e.record(testJob(3, 1), success, time.Now())
+	drainOneEvent(t, e)
+}
+
+func TestRecord_staleJobIsIgnored(t *testing.T) {
+	t.Parallel()
+	e := testEngine()
+	e.states[testKey] = &state{uid: "uid-NEW"}
+
+	job := testJob(3, 1)
+	job.uid = "uid-OLD"
+	e.record(job, probeResult{ok: true, latency: time.Millisecond}, time.Now())
+
+	assertNoEvent(t, e)
+	if _, ok := e.Snapshot(testKey); ok {
+		t.Error("stale probe produced a verdict for the new object")
+	}
+}
+
+func externalProxy(name, host string, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy {
+	p := &crawlv1alpha1.Proxy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name, Namespace: "default", UID: types.UID("uid-" + name),
+		},
+		Spec: crawlv1alpha1.ProxySpec{
+			Mode:     crawlv1alpha1.ModeExternal,
+			Endpoint: &crawlv1alpha1.EndpointSpec{Host: host, Port: 3128},
+		},
+	}
+	for _, m := range mut {
+		m(p)
+	}
+	return p
+}
+
+func TestTick_schedulingAndPruning(t *testing.T) {
+	t.Parallel()
+	s := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("scheme: %v", err)
+	}
+
+	probeable := externalProxy("probeable", "10.0.0.1")
+	seeded := externalProxy("seeded", "10.0.0.2", func(p *crawlv1alpha1.Proxy) {
+		p.Status.Conditions = []metav1.Condition{{
+			Type: crawlv1alpha1.ConditionHealthy, Status: metav1.ConditionTrue,
+			Reason: "ProbeSucceeded", LastTransitionTime: metav1.Now(),
+		}}
+		p.Status.LatencyMillis = 42
+	})
+	noIP := &crawlv1alpha1.Proxy{
+		ObjectMeta: metav1.ObjectMeta{Name: "no-ip", Namespace: "default", UID: "uid-no-ip"},
+		Spec:       crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged, Provider: "stub"},
+	}
+
+	e := testEngine()
+	e.Reader = fake.NewClientBuilder().WithScheme(s).
+		WithObjects(probeable, seeded, noIP).Build()
+	// Stale entries: one for a proxy that no longer exists, one under a key
+	// that now belongs to a different UID (delete + recreate).
+	e.states[types.NamespacedName{Namespace: "default", Name: "gone"}] = &state{uid: "uid-gone"}
+	e.states[types.NamespacedName{Namespace: "default", Name: "probeable"}] = &state{
+		uid: "uid-previous-incarnation", healthy: boolPtr(false),
+	}
+
+	jobs := make(chan probeJob, 8)
+	e.tick(context.Background(), time.Now(), jobs)
+
+	var dispatched []probeJob
+	for {
+		select {
+		case j := <-jobs:
+			dispatched = append(dispatched, j)
+			continue
+		default:
+		}
+		break
+	}
+
+	if len(dispatched) != 1 {
+		t.Fatalf("dispatched %d jobs, want exactly 1 (only the fresh probeable proxy)", len(dispatched))
+	}
+	j := dispatched[0]
+	if j.key.Name != "probeable" || j.uid != "uid-probeable" {
+		t.Errorf("dispatched job = %+v, want the recreated probeable proxy", j)
+	}
+	if want := "http://" + "10.0.0.1:" + strconv.Itoa(3128); j.proxyURL.String() != want {
+		t.Errorf("proxyURL = %s, want %s", j.proxyURL, want)
+	}
+
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if _, ok := e.states[types.NamespacedName{Namespace: "default", Name: "gone"}]; ok {
+		t.Error("state for a deleted proxy was not pruned")
+	}
+	if _, ok := e.states[types.NamespacedName{Namespace: "default", Name: "no-ip"}]; ok {
+		t.Error("state was created for a proxy with no IP")
+	}
+	st := e.states[types.NamespacedName{Namespace: "default", Name: "probeable"}]
+	if st == nil || st.uid != "uid-probeable" {
+		t.Fatalf("state for recreated proxy = %+v, want fresh state with the new UID", st)
+	}
+	if st.healthy != nil && !*st.healthy {
+		t.Error("recreated proxy inherited the previous incarnation's unhealthy verdict")
+	}
+	seededSt := e.states[types.NamespacedName{Namespace: "default", Name: "seeded"}]
+	if seededSt == nil {
+		t.Fatal("no state created for the seeded proxy")
+	}
+	if seededSt.healthy == nil || !*seededSt.healthy {
+		t.Error("seeded proxy did not inherit its Healthy condition")
+	}
+	if seededSt.reportedHealthy == nil || !*seededSt.reportedHealthy {
+		t.Error("seeded verdict must count as already reported, or restart would re-emit for the whole fleet")
+	}
+	if seededSt.reportedLatency != 42*time.Millisecond {
+		t.Errorf("seeded reportedLatency = %v, want 42ms", seededSt.reportedLatency)
+	}
+	if seededSt.consecOK != 0 || seededSt.consecFail != 0 {
+		t.Error("seeded counters must start at zero")
+	}
+}
+
+func TestEngine_StartEndToEnd(t *testing.T) {
+	t.Parallel()
+	s := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("scheme: %v", err)
+	}
+
+	e := testEngine()
+	e.Tick = 5 * time.Millisecond
+	e.Reader = fake.NewClientBuilder().WithScheme(s).
+		WithObjects(externalProxy("p1", "192.0.2.1")).Build()
+	e.probeFn = func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult {
+		return probeResult{ok: true, latency: 12 * time.Millisecond}
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- e.Start(ctx) }()
+
+	select {
+	case evt := <-e.Events:
+		if evt.Object.GetName() != "p1" {
+			t.Errorf("event for %q, want p1", evt.Object.GetName())
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("no health event within 5s")
+	}
+	snap, ok := e.Snapshot(types.NamespacedName{Namespace: "default", Name: "p1"})
+	if !ok || !snap.Healthy || snap.Latency != 12*time.Millisecond {
+		t.Errorf("Snapshot = %+v, %v; want healthy at 12ms", snap, ok)
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start returned %v, want nil on context cancel", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop within 5s of cancel")
+	}
+}