package health import ( "context" "crypto/tls" "errors" "net/url" "strconv" "strings" "testing" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/event" crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1" ) var testKey = types.NamespacedName{Namespace: "default", Name: "p1"} func testEngine() *Engine { e := &Engine{Events: make(chan event.GenericEvent, 8)} e.applyDefaults() return e } func testJob(failureThreshold, successThreshold int32) probeJob { return probeJob{ key: testKey, uid: "uid-1", hc: crawlv1alpha1.HealthCheckSpec{ FailureThreshold: failureThreshold, SuccessThreshold: successThreshold, }, interval: 30 * time.Second, } } func drainOneEvent(t *testing.T, e *Engine) event.GenericEvent { t.Helper() select { case evt := <-e.Events: return evt default: t.Fatal("expected an event, channel is empty") return event.GenericEvent{} } } func assertNoEvent(t *testing.T, e *Engine) { t.Helper() select { case <-e.Events: t.Fatal("unexpected event emitted") default: } } func boolPtr(b bool) *bool { return &b } func TestRecord_firstResultEmits(t *testing.T) { t.Parallel() e := testEngine() e.states[testKey] = &state{uid: "uid-1"} e.record(testJob(3, 1), probeResult{ok: true, latency: 30 * time.Millisecond}, time.Now()) evt := drainOneEvent(t, e) if got := evt.Object.GetName(); got != "p1" { t.Errorf("event object name = %q, want p1", got) } snap, ok := e.Snapshot(testKey) if !ok || !snap.Healthy { t.Errorf("Snapshot = %+v, %v; want healthy verdict", snap, ok) } if snap.Latency != 30*time.Millisecond { t.Errorf("latency = %v, want 30ms", snap.Latency) } } func TestRecord_failureThresholdFlips(t *testing.T) { t.Parallel() e := testEngine() e.states[testKey] = &state{uid: "uid-1", healthy: boolPtr(true), reportedHealthy: boolPtr(true)} job := testJob(3, 1) probeErr := probeResult{err: errors.New("connect refused")} e.record(job, probeErr, time.Now()) e.record(job, probeErr, time.Now()) assertNoEvent(t, e) if snap, _ := e.Snapshot(testKey); !snap.Healthy { t.Fatal("flipped unhealthy before failureThreshold was reached") } e.record(job, probeErr, time.Now()) drainOneEvent(t, e) snap, _ := e.Snapshot(testKey) if snap.Healthy { t.Error("still healthy after failureThreshold consecutive failures") } if snap.ConsecutiveFailures != 3 { t.Errorf("ConsecutiveFailures = %d, want 3", snap.ConsecutiveFailures) } if !strings.Contains(snap.LastError, "connect refused") { t.Errorf("LastError = %q, want the probe error", snap.LastError) } } func TestRecord_successThresholdFlips(t *testing.T) { t.Parallel() e := testEngine() e.states[testKey] = &state{uid: "uid-1", healthy: boolPtr(false), reportedHealthy: boolPtr(false)} job := testJob(3, 2) success := probeResult{ok: true, latency: 25 * time.Millisecond} e.record(job, success, time.Now()) assertNoEvent(t, e) e.record(job, success, time.Now()) drainOneEvent(t, e) if snap, _ := e.Snapshot(testKey); !snap.Healthy { t.Error("not healthy after successThreshold consecutive successes") } } func TestRecord_latencySuppression(t *testing.T) { t.Parallel() now := time.Now() tests := []struct { name string reportedLatency time.Duration lastReport time.Time newLatency time.Duration wantEmit bool }{ { name: "small change under the relative floor is suppressed", reportedLatency: 100 * time.Millisecond, lastReport: now.Add(-2 * time.Minute), newLatency: 110 * time.Millisecond, wantEmit: false, }, { name: "small absolute jitter at low latency is suppressed", reportedLatency: 5 * time.Millisecond, lastReport: now.Add(-2 * time.Minute), newLatency: 20 * time.Millisecond, // >50% but under the 20ms floor wantEmit: false, }, { name: "material change after the rate window emits", reportedLatency: 100 * time.Millisecond, lastReport: now.Add(-2 * time.Minute), newLatency: 200 * time.Millisecond, wantEmit: true, }, { name: "material change inside the rate window is suppressed", reportedLatency: 100 * time.Millisecond, lastReport: now.Add(-10 * time.Second), newLatency: 400 * time.Millisecond, wantEmit: false, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() e := testEngine() e.states[testKey] = &state{ uid: "uid-1", healthy: boolPtr(true), reportedHealthy: boolPtr(true), reportedLatency: tc.reportedLatency, lastReport: tc.lastReport, } e.record(testJob(3, 1), probeResult{ok: true, latency: tc.newLatency}, now) if tc.wantEmit { drainOneEvent(t, e) } else { assertNoEvent(t, e) } }) } } func TestRecord_droppedEventIsRetried(t *testing.T) { t.Parallel() e := testEngine() e.Events = make(chan event.GenericEvent) // unbuffered, nobody reading e.states[testKey] = &state{uid: "uid-1"} success := probeResult{ok: true, latency: 30 * time.Millisecond} e.record(testJob(3, 1), success, time.Now()) e.mu.Lock() reported := e.states[testKey].reportedHealthy e.mu.Unlock() if reported != nil { t.Fatal("reported marker advanced although the event was dropped") } // Channel drains (reconciler recovers): the next probe re-emits. e.Events = make(chan event.GenericEvent, 1) e.record(testJob(3, 1), success, time.Now()) drainOneEvent(t, e) } func TestRecord_staleJobIsIgnored(t *testing.T) { t.Parallel() e := testEngine() e.states[testKey] = &state{uid: "uid-NEW"} job := testJob(3, 1) job.uid = "uid-OLD" e.record(job, probeResult{ok: true, latency: time.Millisecond}, time.Now()) assertNoEvent(t, e) if _, ok := e.Snapshot(testKey); ok { t.Error("stale probe produced a verdict for the new object") } } func externalProxy(name, host string, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy { p := &crawlv1alpha1.Proxy{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "default", UID: types.UID("uid-" + name), }, Spec: crawlv1alpha1.ProxySpec{ Mode: crawlv1alpha1.ModeExternal, Endpoint: &crawlv1alpha1.EndpointSpec{Host: host, Port: 3128}, }, } for _, m := range mut { m(p) } return p } func TestTick_schedulingAndPruning(t *testing.T) { t.Parallel() s := runtime.NewScheme() if err := crawlv1alpha1.AddToScheme(s); err != nil { t.Fatalf("scheme: %v", err) } probeable := externalProxy("probeable", "10.0.0.1") seeded := externalProxy("seeded", "10.0.0.2", func(p *crawlv1alpha1.Proxy) { p.Status.Conditions = []metav1.Condition{{ Type: crawlv1alpha1.ConditionHealthy, Status: metav1.ConditionTrue, Reason: "ProbeSucceeded", LastTransitionTime: metav1.Now(), }} p.Status.LatencyMillis = 42 }) noIP := &crawlv1alpha1.Proxy{ ObjectMeta: metav1.ObjectMeta{Name: "no-ip", Namespace: "default", UID: "uid-no-ip"}, Spec: crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged, Provider: "stub"}, } e := testEngine() e.Reader = fake.NewClientBuilder().WithScheme(s). WithObjects(probeable, seeded, noIP).Build() // Stale entries: one for a proxy that no longer exists, one under a key // that now belongs to a different UID (delete + recreate). e.states[types.NamespacedName{Namespace: "default", Name: "gone"}] = &state{uid: "uid-gone"} e.states[types.NamespacedName{Namespace: "default", Name: "probeable"}] = &state{ uid: "uid-previous-incarnation", healthy: boolPtr(false), } jobs := make(chan probeJob, 8) e.tick(context.Background(), time.Now(), jobs) var dispatched []probeJob for { select { case j := <-jobs: dispatched = append(dispatched, j) continue default: } break } if len(dispatched) != 1 { t.Fatalf("dispatched %d jobs, want exactly 1 (only the fresh probeable proxy)", len(dispatched)) } j := dispatched[0] if j.key.Name != "probeable" || j.uid != "uid-probeable" { t.Errorf("dispatched job = %+v, want the recreated probeable proxy", j) } if want := "http://" + "10.0.0.1:" + strconv.Itoa(3128); j.proxyURL.String() != want { t.Errorf("proxyURL = %s, want %s", j.proxyURL, want) } e.mu.Lock() defer e.mu.Unlock() if _, ok := e.states[types.NamespacedName{Namespace: "default", Name: "gone"}]; ok { t.Error("state for a deleted proxy was not pruned") } if _, ok := e.states[types.NamespacedName{Namespace: "default", Name: "no-ip"}]; ok { t.Error("state was created for a proxy with no IP") } st := e.states[types.NamespacedName{Namespace: "default", Name: "probeable"}] if st == nil || st.uid != "uid-probeable" { t.Fatalf("state for recreated proxy = %+v, want fresh state with the new UID", st) } if st.healthy != nil && !*st.healthy { t.Error("recreated proxy inherited the previous incarnation's unhealthy verdict") } seededSt := e.states[types.NamespacedName{Namespace: "default", Name: "seeded"}] if seededSt == nil { t.Fatal("no state created for the seeded proxy") } if seededSt.healthy == nil || !*seededSt.healthy { t.Error("seeded proxy did not inherit its Healthy condition") } if seededSt.reportedHealthy == nil || !*seededSt.reportedHealthy { t.Error("seeded verdict must count as already reported, or restart would re-emit for the whole fleet") } if seededSt.reportedLatency != 42*time.Millisecond { t.Errorf("seeded reportedLatency = %v, want 42ms", seededSt.reportedLatency) } if seededSt.consecOK != 0 || seededSt.consecFail != 0 { t.Error("seeded counters must start at zero") } } func TestEngine_StartEndToEnd(t *testing.T) { t.Parallel() s := runtime.NewScheme() if err := crawlv1alpha1.AddToScheme(s); err != nil { t.Fatalf("scheme: %v", err) } e := testEngine() e.Tick = 5 * time.Millisecond e.Reader = fake.NewClientBuilder().WithScheme(s). WithObjects(externalProxy("p1", "192.0.2.1")).Build() e.probeFn = func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult { return probeResult{ok: true, latency: 12 * time.Millisecond} } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- e.Start(ctx) }() select { case evt := <-e.Events: if evt.Object.GetName() != "p1" { t.Errorf("event for %q, want p1", evt.Object.GetName()) } case <-time.After(5 * time.Second): t.Fatal("no health event within 5s") } snap, ok := e.Snapshot(types.NamespacedName{Namespace: "default", Name: "p1"}) if !ok || !snap.Healthy || snap.Latency != 12*time.Millisecond { t.Errorf("Snapshot = %+v, %v; want healthy at 12ms", snap, ok) } cancel() select { case err := <-done: if err != nil { t.Errorf("Start returned %v, want nil on context cancel", err) } case <-time.After(5 * time.Second): t.Fatal("Start did not stop within 5s of cancel") } }