egress-proxies-operator/internal/providerhealth/monitor_test.go

package providerhealth

import (
	"context"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/go-logr/logr"
	"github.com/go-logr/logr/funcr"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	apimeta "k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"

	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)

var (
	t0 = time.Date(2026, 9, 11, 15, 0, 0, 0, time.UTC)
	t1 = t0.Add(time.Minute)
)

type checkFunc func(ctx context.Context) []provider.TargetStatus

func (f checkFunc) Check(ctx context.Context) []provider.TargetStatus { return f(ctx) }

func verdicts(ts ...provider.TargetStatus) checkFunc {
	return func(context.Context) []provider.TargetStatus { return ts }
}

func up(name string) provider.TargetStatus {
	return provider.TargetStatus{Name: name, Address: name + ":22", Ready: true, Reason: provider.ReasonReachable}
}

func down(name, reason string) provider.TargetStatus {
	return provider.TargetStatus{Name: name, Address: name + ":22", Reason: reason, Message: name + " is down"}
}

func managed(name string) *crawlv1alpha1.ProviderHealth {
	return &crawlv1alpha1.ProviderHealth{ObjectMeta: metav1.ObjectMeta{
		Name:   name,
		Labels: map[string]string{labelManagedBy: managedByValue},
	}}
}

// newFakeClient registers ProviderHealth as cluster-scoped; the fake's
// default mapper would treat it as namespaced.
func newFakeClient(t *testing.T, objs ...client.Object) client.WithWatch {
	t.Helper()
	scheme := runtime.NewScheme()
	if err := crawlv1alpha1.AddToScheme(scheme); err != nil {
		t.Fatal(err)
	}
	mapper := apimeta.NewDefaultRESTMapper([]schema.GroupVersion{crawlv1alpha1.GroupVersion})
	mapper.Add(crawlv1alpha1.GroupVersion.WithKind("ProviderHealth"), apimeta.RESTScopeRoot)
	return fake.NewClientBuilder().
		WithScheme(scheme).
		WithRESTMapper(mapper).
		WithStatusSubresource(&crawlv1alpha1.ProviderHealth{}).
		WithObjects(objs...).
		Build()
}

func newTestMonitor(c client.Client, targets ...Target) *Monitor {
	return &Monitor{
		Client:   c,
		Reader:   c,
		Targets:  targets,
		Interval: time.Minute,
		Timeout:  time.Second,
		now:      func() time.Time { return t1 },
	}
}

func getHealth(t *testing.T, c client.Reader, name string) *crawlv1alpha1.ProviderHealth {
	t.Helper()
	var ph crawlv1alpha1.ProviderHealth
	if err := c.Get(context.Background(), client.ObjectKey{Name: name}, &ph); err != nil {
		t.Fatalf("Get ProviderHealth %s: %v", name, err)
	}
	return &ph
}

func readyCondition(ph *crawlv1alpha1.ProviderHealth) *metav1.Condition {
	return apimeta.FindStatusCondition(ph.Status.Conditions, crawlv1alpha1.ConditionReady)
}

func TestMonitor_checkOnce_createsLabelledObjects(t *testing.T) {
	t.Parallel()
	c := newFakeClient(t)
	m := newTestMonitor(c,
		Target{Name: "kvm-home", Type: "kvm", Checker: verdicts(up("nas1"), down("nas2", provider.ReasonUnreachable))},
		Target{Name: "local", Type: "kubernetes", Checker: verdicts(up("apiserver"))},
	)

	m.checkOnce(context.Background())

	kvm := getHealth(t, c, "kvm-home")
	if kvm.Labels[labelManagedBy] != managedByValue {
		t.Errorf("labels = %v, want %s=%s", kvm.Labels, labelManagedBy, managedByValue)
	}
	if cond := readyCondition(kvm); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != reasonTargetsNotReady {
		t.Errorf("kvm-home Ready = %+v, want False/%s", cond, reasonTargetsNotReady)
	}
	if kvm.Status.Type != "kvm" || kvm.Status.TargetsReady != "1/2" || kvm.Status.LastCheckTime == nil || !kvm.Status.LastCheckTime.Time.Equal(t1) {
		t.Errorf("kvm-home status = %+v, want type kvm, 1/2 ready, checked at %s", kvm.Status, t1)
	}
	local := getHealth(t, c, "local")
	if cond := readyCondition(local); cond == nil || cond.Status != metav1.ConditionTrue {
		t.Errorf("local Ready = %+v, want True", cond)
	}
}

func TestMonitor_checkOnce_patchesExisting(t *testing.T) {
	t.Parallel()
	existing := managed("local")
	existing.Status = crawlv1alpha1.ProviderHealthStatus{
		Type:    "kubernetes",
		Targets: []crawlv1alpha1.ProviderTarget{readyAt("apiserver", t0)},
	}
	c := newFakeClient(t, existing)
	m := newTestMonitor(c, Target{Name: "local", Type: "kubernetes", Checker: verdicts(up("apiserver"))})

	m.checkOnce(context.Background())

	got := getHealth(t, c, "local")
	if got.Status.LastCheckTime == nil || !got.Status.LastCheckTime.Time.Equal(t1) {
		t.Errorf("lastCheckTime = %v, want %s", got.Status.LastCheckTime, t1)
	}
	if lt := got.Status.Targets[0].LastTransitionTime.Time; !lt.Equal(t0) {
		t.Errorf("apiserver lastTransitionTime = %s, want %s kept: it was ready before too", lt, t0)
	}
}

func TestMonitor_checkOnce_timeoutBoundsEachRound(t *testing.T) {
	t.Parallel()
	c := newFakeClient(t)
	var hadDeadline atomic.Bool
	hanging := checkFunc(func(ctx context.Context) []provider.TargetStatus {
		_, ok := ctx.Deadline()
		hadDeadline.Store(ok)
		<-ctx.Done()
		return []provider.TargetStatus{down("nas1", provider.ReasonTimeout)}
	})
	m := newTestMonitor(c,
		Target{Name: "kvm-home", Type: "kvm", Checker: hanging},
		Target{Name: "local", Type: "kubernetes", Checker: verdicts(up("apiserver"))},
	)
	m.Timeout = 50 * time.Millisecond

	done := make(chan struct{})
	go func() {
		m.checkOnce(context.Background())
		close(done)
	}()
	select {
	case <-done:
	case <-time.After(5 * time.Second):
		t.Fatal("checkOnce still running 5s after a 50ms timeout")
	}

	if !hadDeadline.Load() {
		t.Error("the checker got a context without a deadline")
	}
	if cond := readyCondition(getHealth(t, c, "kvm-home")); cond == nil || cond.Status != metav1.ConditionFalse {
		t.Errorf("kvm-home Ready = %+v, want False", cond)
	}
	if cond := readyCondition(getHealth(t, c, "local")); cond == nil || cond.Status != metav1.ConditionTrue {
		t.Errorf("local Ready = %+v, want True despite the slow neighbour", cond)
	}
}

func TestMonitor_checkOnce_noCheckerIsUnknown(t *testing.T) {
	t.Parallel()
	c := newFakeClient(t)
	m := newTestMonitor(c, Target{Name: "gcp-eu", Type: "gcp"})

	m.checkOnce(context.Background())

	got := getHealth(t, c, "gcp-eu")
	if cond := readyCondition(got); cond == nil || cond.Status != metav1.ConditionUnknown || cond.Reason != reasonChecksNotImplemented {
		t.Errorf("Ready = %+v, want Unknown/%s", cond, reasonChecksNotImplemented)
	}
}

func TestMonitor_checkOnce_invalidNameSkipsTheObject(t *testing.T) {
	t.Parallel()
	c := newFakeClient(t)
	m := newTestMonitor(c, Target{Name: "Home_KVM", Type: "kvm", Checker: verdicts(up("nas1"))})

	m.checkOnce(context.Background())

	var list crawlv1alpha1.ProviderHealthList
	if err := c.List(context.Background(), &list); err != nil {
		t.Fatal(err)
	}
	if len(list.Items) != 0 {
		t.Errorf("wrote %d objects, want none for an invalid name", len(list.Items))
	}
	snap := m.Snapshot()
	if len(snap) != 1 || snap[0].Provider != "Home_KVM" || !snap[0].Targets[0].Ready {
		t.Errorf("Snapshot() = %+v, want the result kept for metrics", snap)
	}
}

func TestMonitor_logsOnlyTransitions(t *testing.T) {
	t.Parallel()
	var mu sync.Mutex
	var lines []string
	log := funcr.New(func(_, args string) {
		mu.Lock()
		defer mu.Unlock()
		lines = append(lines, args)
	}, funcr.Options{})
	ctx := logr.NewContext(context.Background(), log)
	rounds := [][]provider.TargetStatus{
		{up("nas1")},
		{up("nas1")},
		{down("nas1", provider.ReasonUnreachable)},
		{down("nas1", provider.ReasonUnreachable)},
		{down("nas1", provider.ReasonTimeout)},
		{up("nas1")},
	}
	round := 0
	m := newTestMonitor(newFakeClient(t), Target{Name: "kvm-home", Type: "kvm", Checker: checkFunc(
		func(context.Context) []provider.TargetStatus { return rounds[round] })})

	for round = range rounds {
		m.checkOnce(ctx)
	}

	mu.Lock()
	defer mu.Unlock()
	want := [][]string{
		{`"msg"="WARNING: provider target not ready"`, `"reason"="Unreachable"`},
		{`"msg"="WARNING: provider target not ready"`, `"reason"="Timeout"`},
		{`"msg"="Provider target ready"`},
	}
	if len(lines) != len(want) {
		t.Fatalf("logged %d lines, want %d:\n%s", len(lines), len(want), strings.Join(lines, "\n"))
	}
	for i, subs := range want {
		for _, sub := range subs {
			if !strings.Contains(lines[i], sub) {
				t.Errorf("line %d missing %s:\n%s", i, sub, lines[i])
			}
		}
	}
}

func TestMonitor_cleanup_deletesOnlyLabelledUnconfigured(t *testing.T) {
	t.Parallel()
	foreign := &crawlv1alpha1.ProviderHealth{ObjectMeta: metav1.ObjectMeta{Name: "foreign"}}
	c := newFakeClient(t, managed("kvm-home"), managed("retired"), foreign)
	m := newTestMonitor(c)

	m.cleanup(context.Background(), map[string]bool{"kvm-home": true})

	getHealth(t, c, "kvm-home")
	getHealth(t, c, "foreign")
	var gone crawlv1alpha1.ProviderHealth
	if err := c.Get(context.Background(), client.ObjectKey{Name: "retired"}, &gone); !apierrors.IsNotFound(err) {
		t.Errorf("Get retired = %v, want NotFound", err)
	}
}

func TestMonitor_Start_offCleansUpAndReturns(t *testing.T) {
	t.Parallel()
	c := newFakeClient(t, managed("kvm-home"))
	var checked atomic.Bool
	m := newTestMonitor(c, Target{Name: "kvm-home", Type: "kvm", Checker: checkFunc(
		func(context.Context) []provider.TargetStatus { checked.Store(true); return nil })})
	m.Interval = 0

	errc := make(chan error, 1)
	go func() { errc <- m.Start(context.Background()) }()
	select {
	case err := <-errc:
		if err != nil {
			t.Fatalf("Start() = %v, want nil", err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("Start did not return with checking off")
	}

	if checked.Load() {
		t.Error("a check ran with checking off")
	}
	var gone crawlv1alpha1.ProviderHealth
	if err := c.Get(context.Background(), client.ObjectKey{Name: "kvm-home"}, &gone); !apierrors.IsNotFound(err) {
		t.Errorf("Get kvm-home = %v, want NotFound so no stale Ready is left behind", err)
	}
}

func TestMonitor_Start_checksAtOnceAndStops(t *testing.T) {
	t.Parallel()
	c := newFakeClient(t)
	m := newTestMonitor(c, Target{Name: "local", Type: "kubernetes", Checker: verdicts(up("apiserver"))})
	m.Interval = time.Hour
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	errc := make(chan error, 1)
	go func() { errc <- m.Start(ctx) }()
	deadline := time.Now().Add(5 * time.Second)
	for len(m.Snapshot()) == 0 {
		if time.Now().After(deadline) {
			t.Fatal("no check within 5s of Start, want one right away")
		}
		time.Sleep(10 * time.Millisecond)
	}
	cancel()

	select {
	case err := <-errc:
		if err != nil {
			t.Errorf("Start() = %v, want nil", err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("Start did not return after its context ended")
	}
}

func TestValidateSchedule(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name              string
		interval, timeout time.Duration
		wantErr           bool
	}{
		{"off", 0, 0, false},
		{"defaults", time.Minute, 15 * time.Second, false},
		{"shortest interval", 10 * time.Second, 10 * time.Second, false},
		{"too frequent", 5 * time.Second, time.Second, true},
		{"negative interval", -time.Minute, time.Second, true},
		{"timeout left to derive, short interval", 12 * time.Second, 0, false},
		{"negative timeout", time.Minute, -time.Second, true},
		{"timeout longer than interval", time.Minute, 2 * time.Minute, true},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			if err := ValidateSchedule(tc.interval, tc.timeout); (err != nil) != tc.wantErr {
				t.Errorf("ValidateSchedule(%s, %s) = %v, want error %v", tc.interval, tc.timeout, err, tc.wantErr)
			}
		})
	}
}

func TestMonitor_setDefaults_timeout(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name                    string
		interval, timeout, want time.Duration
	}{
		{"default", time.Minute, 0, 15 * time.Second},
		{"interval shorter than the default", 12 * time.Second, 0, 12 * time.Second},
		{"explicit", time.Minute, 20 * time.Second, 20 * time.Second},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			m := &Monitor{Interval: tc.interval, Timeout: tc.timeout}
			m.setDefaults()
			if m.Timeout != tc.want {
				t.Errorf("Timeout = %s, want %s", m.Timeout, tc.want)
			}
		})
	}
}