Add the provider health Monitor

A leader-only runnable checks every configured provider concurrently
under one timeout per round (first round at start, then every interval),
keeps the latest results for the metrics collector, logs targets going
down, changing reason or coming back, and publishes one ProviderHealth
object per provider: created with a managed-by label when missing, then
a status merge patch. Per-target lastTransitionTime survives rounds in
which ready does not flip. Objects of providers that are no longer
configured are deleted once at start; with checks turned off, all are.
Reads go through the API reader, so no cluster-wide informer is needed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

internal/providerhealth/envtest_test.go

@@ -0,0 +1,100 @@
+package providerhealth
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+	"unicode/utf8"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// TestMonitor_envtestRoundTrip writes through a real API server, which is
+// the only thing that validates the generated schema: cluster scope, the
+// status subresource, required fields, list-map keys and the message limit.
+func TestMonitor_envtestRoundTrip(t *testing.T) {
+	if testing.Short() {
+		t.Skip("envtest skipped under -short")
+	}
+	env := &envtest.Environment{
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "config", "crd", "bases")},
+		ErrorIfCRDPathMissing: true,
+		BinaryAssetsDirectory: envtestBinaryDir(),
+	}
+	cfg, err := env.Start()
+	if err != nil {
+		t.Fatalf("starting envtest (run `make setup-envtest` first): %v", err)
+	}
+	t.Cleanup(func() { _ = env.Stop() })
+	scheme := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(scheme); err != nil {
+		t.Fatal(err)
+	}
+	c, err := client.New(cfg, client.Options{Scheme: scheme})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	m := &Monitor{Client: c, Reader: c}
+
+	first := Result{Provider: "kvm-home", Type: "kvm", Supported: true, CheckedAt: t0, Targets: []provider.TargetStatus{
+		{Name: "nas1", Address: "nas1:22", Ready: true, Reason: provider.ReasonReachable,
+			Latency: 42 * time.Millisecond, Details: map[string]string{"libvirtVersion": "10.2.0"}},
+		{Name: "nas2", Address: "nas2:22", Reason: provider.ReasonUnreachable, Message: strings.Repeat("x", 2000)},
+	}}
+	if err := m.publish(ctx, first); err != nil {
+		t.Fatalf("first publish: %v", err)
+	}
+	got := getHealth(t, c, "kvm-home")
+	if got.Status.TargetsReady != "1/2" || got.Status.Targets[0].Details["libvirtVersion"] != "10.2.0" ||
+		got.Status.Targets[0].LatencyMillis != 42 || utf8.RuneCountInString(got.Status.Targets[1].Message) != maxMessageLen {
+		t.Errorf("status after the first round = %+v", got.Status)
+	}
+
+	second := Result{Provider: "kvm-home", Type: "kvm", Supported: true, CheckedAt: t1,
+		Targets: []provider.TargetStatus{up("nas1"), up("nas2")}}
+	if err := m.publish(ctx, second); err != nil {
+		t.Fatalf("second publish: %v", err)
+	}
+	got = getHealth(t, c, "kvm-home")
+	if cond := readyCondition(got); cond == nil || cond.Reason != reasonAllTargetsReady {
+		t.Errorf("Ready after recovery = %+v, want %s", cond, reasonAllTargetsReady)
+	}
+	if lt := got.Status.Targets[0].LastTransitionTime.Time; !lt.Equal(t0) {
+		t.Errorf("nas1 lastTransitionTime = %s, want %s", lt, t0)
+	}
+	if lt := got.Status.Targets[1].LastTransitionTime.Time; !lt.Equal(t1) {
+		t.Errorf("nas2 lastTransitionTime = %s, want %s", lt, t1)
+	}
+
+	m.cleanup(ctx, nil)
+	var gone crawlv1alpha1.ProviderHealth
+	if err := c.Get(ctx, client.ObjectKey{Name: "kvm-home"}, &gone); !apierrors.IsNotFound(err) {
+		t.Errorf("Get after cleanup = %v, want NotFound", err)
+	}
+}
+
+// envtestBinaryDir finds the binaries `make setup-envtest` downloads, so
+// the test also runs outside make (which sets KUBEBUILDER_ASSETS instead).
+func envtestBinaryDir() string {
+	base := filepath.Join("..", "..", "bin", "k8s")
+	entries, err := os.ReadDir(base)
+	if err != nil {
+		return ""
+	}
+	for _, e := range entries {
+		if e.IsDir() {
+			return filepath.Join(base, e.Name())
+		}
+	}
+	return ""
+}

internal/providerhealth/monitor.go

@@ -0,0 +1,290 @@
+// Package providerhealth checks every configured provider on an interval
+// and publishes the results: one ProviderHealth object per provider, a log
+// line whenever a target goes down or comes back, and a snapshot the
+// metrics collector reads at scrape time.
+package providerhealth
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"sync"
+	"time"
+
+	"github.com/go-logr/logr"
+	"go.opentelemetry.io/otel/attribute"
+	"k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+const (
+	// labelManagedBy marks the ProviderHealth objects this operator owns;
+	// cleanup never deletes an object without it.
+	labelManagedBy = "app.kubernetes.io/managed-by"
+	managedByValue = "egress-proxies-operator"
+
+	minInterval    = 10 * time.Second
+	defaultTimeout = 15 * time.Second
+)
+
+// Target is one configured provider. Checker is nil for a provider that
+// cannot check itself.
+type Target struct {
+	Name    string
+	Type    string
+	Checker provider.Checker
+}
+
+// Result is the outcome of one provider's latest check.
+type Result struct {
+	Provider string
+	Type     string
+	// Supported is false when the provider has no Checker.
+	Supported bool
+	Targets   []provider.TargetStatus
+	CheckedAt time.Time
+}
+
+// controller-gen reads RBAC markers only at package level, so they stay
+// out of Monitor's doc comment.
+// +kubebuilder:rbac:groups=crawl.example.com,resources=providerhealths,verbs=get;list;create;delete
+// +kubebuilder:rbac:groups=crawl.example.com,resources=providerhealths/status,verbs=get;patch
+
+// Monitor is the manager Runnable that checks the providers.
+type Monitor struct {
+	// Client writes the ProviderHealth objects.
+	Client client.Client
+	// Reader reads them; pass the manager's API reader. The objects are
+	// cluster-scoped, and reading them through the cache would start a
+	// cluster-wide informer needing watch rights, whatever --proxy-namespace
+	// says, for a handful of reads a minute.
+	Reader client.Reader
+	// Targets are the configured providers, in config order.
+	Targets []Target
+	// Interval between rounds of checks; 0 turns checking off. The first
+	// round runs at start: nothing races it, and early visibility is the
+	// point.
+	Interval time.Duration
+	// Timeout bounds each round (default 15s).
+	Timeout time.Duration
+
+	now func() time.Time
+
+	mu     sync.Mutex
+	latest []Result
+
+	// prev holds each target's previous verdict for transition logging.
+	// Only the loop goroutine touches it.
+	prev map[string]provider.TargetStatus
+}
+
+// NeedLeaderElection is true: only the leader dials the hypervisors and
+// writes the objects, and only its gauges describe the checks.
+func (m *Monitor) NeedLeaderElection() bool { return true }
+
+// ValidateSchedule checks --provider-check-interval and
+// --provider-check-timeout. The interval is 0 (off) or at least 10s, which
+// is what keeps one status write per provider per round cheap; the timeout
+// is positive and no longer than the interval.
+func ValidateSchedule(interval, timeout time.Duration) error {
+	switch {
+	case interval == 0:
+		return nil
+	case interval < minInterval:
+		return fmt.Errorf("--provider-check-interval must be 0 (off) or at least %s, got %s", minInterval, interval)
+	case timeout <= 0 || timeout > interval:
+		return fmt.Errorf("--provider-check-timeout must be positive and at most the interval (%s), got %s", interval, timeout)
+	}
+	return nil
+}
+
+// Start deletes the objects of providers that are no longer configured,
+// then checks every Interval until ctx ends. The providers config is read
+// once per process, so cleaning up once is enough. With checking off,
+// every object goes, so none is left claiming a stale Ready.
+func (m *Monitor) Start(ctx context.Context) error {
+	if m.Timeout == 0 {
+		m.Timeout = defaultTimeout
+	}
+	if m.now == nil {
+		m.now = time.Now
+	}
+	log := logf.FromContext(ctx).WithName("provider-health")
+	ctx = logf.IntoContext(ctx, log)
+
+	keep := map[string]bool{}
+	if m.Interval > 0 {
+		for _, t := range m.Targets {
+			keep[t.Name] = true
+		}
+	}
+	m.cleanup(ctx, keep)
+	if m.Interval == 0 {
+		log.Info("Provider checks are off (--provider-check-interval=0)")
+		return nil
+	}
+	for _, t := range m.Targets {
+		if errs := validation.IsDNS1123Subdomain(t.Name); len(errs) > 0 {
+			log.Info("WARNING: provider name is not a valid object name; its ProviderHealth object is not written, its metrics are",
+				"provider", t.Name, "problems", errs)
+		}
+	}
+
+	m.checkOnce(ctx)
+	ticker := time.NewTicker(m.Interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			m.checkOnce(ctx)
+		}
+	}
+}
+
+// Snapshot returns the latest results in config order.
+func (m *Monitor) Snapshot() []Result {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	out := make([]Result, len(m.latest))
+	for i, r := range m.latest {
+		r.Targets = slices.Clone(r.Targets)
+		out[i] = r
+	}
+	return out
+}
+
+// checkOnce runs every provider's check concurrently under one timeout,
+// then records, logs and publishes the results provider by provider. A
+// failed write is logged and retried by the next round.
+func (m *Monitor) checkOnce(ctx context.Context) {
+	ctx, span := tracing.Start(ctx, "providerhealth.check")
+	defer span.End()
+	log := logf.FromContext(ctx)
+
+	checkCtx, cancel := context.WithTimeout(ctx, m.Timeout)
+	defer cancel()
+	results := make([]Result, len(m.Targets))
+	var wg sync.WaitGroup
+	for i, t := range m.Targets {
+		wg.Go(func() { results[i] = m.run(checkCtx, t) })
+	}
+	wg.Wait()
+
+	m.mu.Lock()
+	m.latest = results
+	m.mu.Unlock()
+
+	notReady := 0
+	for _, r := range results {
+		m.logTransitions(log, r)
+		for _, ts := range r.Targets {
+			if !ts.Ready {
+				notReady++
+			}
+		}
+		if err := m.publish(ctx, r); err != nil {
+			log.Error(err, "Writing provider health", "provider", r.Provider)
+		}
+	}
+	span.SetAttributes(
+		attribute.Int("providerhealth.providers", len(results)),
+		attribute.Int("providerhealth.targets_not_ready", notReady),
+	)
+}
+
+func (m *Monitor) run(ctx context.Context, t Target) Result {
+	r := Result{Provider: t.Name, Type: t.Type, Supported: t.Checker != nil}
+	if t.Checker != nil {
+		r.Targets = t.Checker.Check(ctx)
+	}
+	r.CheckedAt = m.now()
+	return r
+}
+
+// logTransitions logs a target going down, staying down for a different
+// reason, or coming back. Steady states stay quiet; a target that is ready
+// the first time it is seen only logs at V(1).
+func (m *Monitor) logTransitions(log logr.Logger, r Result) {
+	if m.prev == nil {
+		m.prev = map[string]provider.TargetStatus{}
+	}
+	for _, ts := range r.Targets {
+		key := r.Provider + "/" + ts.Name
+		prev, seen := m.prev[key]
+		m.prev[key] = ts
+		kv := []any{"provider", r.Provider, "target", ts.Name, "address", ts.Address, "reason", ts.Reason}
+		switch {
+		case !ts.Ready && (!seen || prev.Ready || prev.Reason != ts.Reason):
+			log.Info("WARNING: provider target not ready", append(kv, "message", ts.Message)...)
+		case ts.Ready && seen && !prev.Ready:
+			log.Info("Provider target ready", append(kv, "latency", ts.Latency.Round(time.Millisecond))...)
+		case ts.Ready && !seen:
+			log.V(1).Info("Provider target ready", append(kv, "latency", ts.Latency.Round(time.Millisecond))...)
+		}
+	}
+}
+
+// publish creates the provider's ProviderHealth object when missing and
+// patches its status. The status changes on every round, since
+// lastCheckTime moves; the comparison only skips the patch when a clock
+// did not advance.
+func (m *Monitor) publish(ctx context.Context, r Result) error {
+	if len(validation.IsDNS1123Subdomain(r.Provider)) > 0 {
+		return nil
+	}
+	var ph crawlv1alpha1.ProviderHealth
+	err := m.Reader.Get(ctx, client.ObjectKey{Name: r.Provider}, &ph)
+	switch {
+	case apierrors.IsNotFound(err):
+		ph = crawlv1alpha1.ProviderHealth{ObjectMeta: metav1.ObjectMeta{
+			Name:   r.Provider,
+			Labels: map[string]string{labelManagedBy: managedByValue},
+		}}
+		if err := m.Client.Create(ctx, &ph); err != nil {
+			return fmt.Errorf("creating ProviderHealth %s: %w", r.Provider, err)
+		}
+	case err != nil:
+		return fmt.Errorf("reading ProviderHealth %s: %w", r.Provider, err)
+	}
+	base := ph.DeepCopy()
+	ph.Status = buildStatus(ph.Status, r)
+	if equality.Semantic.DeepEqual(base.Status, ph.Status) {
+		return nil
+	}
+	if err := m.Client.Status().Patch(ctx, &ph, client.MergeFrom(base)); err != nil {
+		return fmt.Errorf("patching ProviderHealth %s status: %w", r.Provider, err)
+	}
+	return nil
+}
+
+// cleanup deletes this operator's ProviderHealth objects whose name is not
+// in keep.
+func (m *Monitor) cleanup(ctx context.Context, keep map[string]bool) {
+	log := logf.FromContext(ctx)
+	var list crawlv1alpha1.ProviderHealthList
+	if err := m.Reader.List(ctx, &list, client.MatchingLabels{labelManagedBy: managedByValue}); err != nil {
+		log.Error(err, "Listing provider health objects; stale ones stay until the next start")
+		return
+	}
+	for i := range list.Items {
+		ph := &list.Items[i]
+		if keep[ph.Name] {
+			continue
+		}
+		if err := m.Client.Delete(ctx, ph); client.IgnoreNotFound(err) != nil {
+			log.Error(err, "Deleting provider health of an unconfigured provider", "name", ph.Name)
+			continue
+		}
+		log.Info("Deleted provider health of an unconfigured provider", "name", ph.Name)
+	}
+}

internal/providerhealth/monitor_test.go

@@ -0,0 +1,354 @@
+package providerhealth
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/go-logr/logr"
+	"github.com/go-logr/logr/funcr"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+var (
+	t0 = time.Date(2026, 9, 11, 15, 0, 0, 0, time.UTC)
+	t1 = t0.Add(time.Minute)
+)
+
+type checkFunc func(ctx context.Context) []provider.TargetStatus
+
+func (f checkFunc) Check(ctx context.Context) []provider.TargetStatus { return f(ctx) }
+
+func verdicts(ts ...provider.TargetStatus) checkFunc {
+	return func(context.Context) []provider.TargetStatus { return ts }
+}
+
+func up(name string) provider.TargetStatus {
+	return provider.TargetStatus{Name: name, Address: name + ":22", Ready: true, Reason: provider.ReasonReachable}
+}
+
+func down(name, reason string) provider.TargetStatus {
+	return provider.TargetStatus{Name: name, Address: name + ":22", Reason: reason, Message: name + " is down"}
+}
+
+func managed(name string) *crawlv1alpha1.ProviderHealth {
+	return &crawlv1alpha1.ProviderHealth{ObjectMeta: metav1.ObjectMeta{
+		Name:   name,
+		Labels: map[string]string{labelManagedBy: managedByValue},
+	}}
+}
+
+// newFakeClient registers ProviderHealth as cluster-scoped; the fake's
+// default mapper would treat it as namespaced.
+func newFakeClient(t *testing.T, objs ...client.Object) client.WithWatch {
+	t.Helper()
+	scheme := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(scheme); err != nil {
+		t.Fatal(err)
+	}
+	mapper := apimeta.NewDefaultRESTMapper([]schema.GroupVersion{crawlv1alpha1.GroupVersion})
+	mapper.Add(crawlv1alpha1.GroupVersion.WithKind("ProviderHealth"), apimeta.RESTScopeRoot)
+	return fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithRESTMapper(mapper).
+		WithStatusSubresource(&crawlv1alpha1.ProviderHealth{}).
+		WithObjects(objs...).
+		Build()
+}
+
+func newTestMonitor(c client.Client, targets ...Target) *Monitor {
+	return &Monitor{
+		Client:   c,
+		Reader:   c,
+		Targets:  targets,
+		Interval: time.Minute,
+		Timeout:  time.Second,
+		now:      func() time.Time { return t1 },
+	}
+}
+
+func getHealth(t *testing.T, c client.Reader, name string) *crawlv1alpha1.ProviderHealth {
+	t.Helper()
+	var ph crawlv1alpha1.ProviderHealth
+	if err := c.Get(context.Background(), client.ObjectKey{Name: name}, &ph); err != nil {
+		t.Fatalf("Get ProviderHealth %s: %v", name, err)
+	}
+	return &ph
+}
+
+func readyCondition(ph *crawlv1alpha1.ProviderHealth) *metav1.Condition {
+	return apimeta.FindStatusCondition(ph.Status.Conditions, crawlv1alpha1.ConditionReady)
+}
+
+func TestMonitor_checkOnce_createsLabelledObjects(t *testing.T) {
+	t.Parallel()
+	c := newFakeClient(t)
+	m := newTestMonitor(c,
+		Target{Name: "kvm-home", Type: "kvm", Checker: verdicts(up("nas1"), down("nas2", provider.ReasonUnreachable))},
+		Target{Name: "local", Type: "kubernetes", Checker: verdicts(up("apiserver"))},
+	)
+
+	m.checkOnce(context.Background())
+
+	kvm := getHealth(t, c, "kvm-home")
+	if kvm.Labels[labelManagedBy] != managedByValue {
+		t.Errorf("labels = %v, want %s=%s", kvm.Labels, labelManagedBy, managedByValue)
+	}
+	if cond := readyCondition(kvm); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != reasonTargetsNotReady {
+		t.Errorf("kvm-home Ready = %+v, want False/%s", cond, reasonTargetsNotReady)
+	}
+	if kvm.Status.Type != "kvm" || kvm.Status.TargetsReady != "1/2" || kvm.Status.LastCheckTime == nil || !kvm.Status.LastCheckTime.Time.Equal(t1) {
+		t.Errorf("kvm-home status = %+v, want type kvm, 1/2 ready, checked at %s", kvm.Status, t1)
+	}
+	local := getHealth(t, c, "local")
+	if cond := readyCondition(local); cond == nil || cond.Status != metav1.ConditionTrue {
+		t.Errorf("local Ready = %+v, want True", cond)
+	}
+}
+
+func TestMonitor_checkOnce_patchesExisting(t *testing.T) {
+	t.Parallel()
+	existing := managed("local")
+	existing.Status = crawlv1alpha1.ProviderHealthStatus{
+		Type:    "kubernetes",
+		Targets: []crawlv1alpha1.ProviderTarget{readyAt("apiserver", t0)},
+	}
+	c := newFakeClient(t, existing)
+	m := newTestMonitor(c, Target{Name: "local", Type: "kubernetes", Checker: verdicts(up("apiserver"))})
+
+	m.checkOnce(context.Background())
+
+	got := getHealth(t, c, "local")
+	if got.Status.LastCheckTime == nil || !got.Status.LastCheckTime.Time.Equal(t1) {
+		t.Errorf("lastCheckTime = %v, want %s", got.Status.LastCheckTime, t1)
+	}
+	if lt := got.Status.Targets[0].LastTransitionTime.Time; !lt.Equal(t0) {
+		t.Errorf("apiserver lastTransitionTime = %s, want %s kept: it was ready before too", lt, t0)
+	}
+}
+
+func TestMonitor_checkOnce_timeoutBoundsEachRound(t *testing.T) {
+	t.Parallel()
+	c := newFakeClient(t)
+	var hadDeadline atomic.Bool
+	hanging := checkFunc(func(ctx context.Context) []provider.TargetStatus {
+		_, ok := ctx.Deadline()
+		hadDeadline.Store(ok)
+		<-ctx.Done()
+		return []provider.TargetStatus{down("nas1", provider.ReasonTimeout)}
+	})
+	m := newTestMonitor(c,
+		Target{Name: "kvm-home", Type: "kvm", Checker: hanging},
+		Target{Name: "local", Type: "kubernetes", Checker: verdicts(up("apiserver"))},
+	)
+	m.Timeout = 50 * time.Millisecond
+
+	done := make(chan struct{})
+	go func() {
+		m.checkOnce(context.Background())
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("checkOnce still running 5s after a 50ms timeout")
+	}
+
+	if !hadDeadline.Load() {
+		t.Error("the checker got a context without a deadline")
+	}
+	if cond := readyCondition(getHealth(t, c, "kvm-home")); cond == nil || cond.Status != metav1.ConditionFalse {
+		t.Errorf("kvm-home Ready = %+v, want False", cond)
+	}
+	if cond := readyCondition(getHealth(t, c, "local")); cond == nil || cond.Status != metav1.ConditionTrue {
+		t.Errorf("local Ready = %+v, want True despite the slow neighbour", cond)
+	}
+}
+
+func TestMonitor_checkOnce_noCheckerIsUnknown(t *testing.T) {
+	t.Parallel()
+	c := newFakeClient(t)
+	m := newTestMonitor(c, Target{Name: "gcp-eu", Type: "gcp"})
+
+	m.checkOnce(context.Background())
+
+	got := getHealth(t, c, "gcp-eu")
+	if cond := readyCondition(got); cond == nil || cond.Status != metav1.ConditionUnknown || cond.Reason != reasonChecksNotImplemented {
+		t.Errorf("Ready = %+v, want Unknown/%s", cond, reasonChecksNotImplemented)
+	}
+}
+
+func TestMonitor_checkOnce_invalidNameSkipsTheObject(t *testing.T) {
+	t.Parallel()
+	c := newFakeClient(t)
+	m := newTestMonitor(c, Target{Name: "Home_KVM", Type: "kvm", Checker: verdicts(up("nas1"))})
+
+	m.checkOnce(context.Background())
+
+	var list crawlv1alpha1.ProviderHealthList
+	if err := c.List(context.Background(), &list); err != nil {
+		t.Fatal(err)
+	}
+	if len(list.Items) != 0 {
+		t.Errorf("wrote %d objects, want none for an invalid name", len(list.Items))
+	}
+	snap := m.Snapshot()
+	if len(snap) != 1 || snap[0].Provider != "Home_KVM" || !snap[0].Targets[0].Ready {
+		t.Errorf("Snapshot() = %+v, want the result kept for metrics", snap)
+	}
+}
+
+func TestMonitor_logsOnlyTransitions(t *testing.T) {
+	t.Parallel()
+	var mu sync.Mutex
+	var lines []string
+	log := funcr.New(func(_, args string) {
+		mu.Lock()
+		defer mu.Unlock()
+		lines = append(lines, args)
+	}, funcr.Options{})
+	ctx := logr.NewContext(context.Background(), log)
+	rounds := [][]provider.TargetStatus{
+		{up("nas1")},
+		{up("nas1")},
+		{down("nas1", provider.ReasonUnreachable)},
+		{down("nas1", provider.ReasonUnreachable)},
+		{down("nas1", provider.ReasonTimeout)},
+		{up("nas1")},
+	}
+	round := 0
+	m := newTestMonitor(newFakeClient(t), Target{Name: "kvm-home", Type: "kvm", Checker: checkFunc(
+		func(context.Context) []provider.TargetStatus { return rounds[round] })})
+
+	for round = range rounds {
+		m.checkOnce(ctx)
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	want := [][]string{
+		{`"msg"="WARNING: provider target not ready"`, `"reason"="Unreachable"`},
+		{`"msg"="WARNING: provider target not ready"`, `"reason"="Timeout"`},
+		{`"msg"="Provider target ready"`},
+	}
+	if len(lines) != len(want) {
+		t.Fatalf("logged %d lines, want %d:\n%s", len(lines), len(want), strings.Join(lines, "\n"))
+	}
+	for i, subs := range want {
+		for _, sub := range subs {
+			if !strings.Contains(lines[i], sub) {
+				t.Errorf("line %d missing %s:\n%s", i, sub, lines[i])
+			}
+		}
+	}
+}
+
+func TestMonitor_cleanup_deletesOnlyLabelledUnconfigured(t *testing.T) {
+	t.Parallel()
+	foreign := &crawlv1alpha1.ProviderHealth{ObjectMeta: metav1.ObjectMeta{Name: "foreign"}}
+	c := newFakeClient(t, managed("kvm-home"), managed("retired"), foreign)
+	m := newTestMonitor(c)
+
+	m.cleanup(context.Background(), map[string]bool{"kvm-home": true})
+
+	getHealth(t, c, "kvm-home")
+	getHealth(t, c, "foreign")
+	var gone crawlv1alpha1.ProviderHealth
+	if err := c.Get(context.Background(), client.ObjectKey{Name: "retired"}, &gone); !apierrors.IsNotFound(err) {
+		t.Errorf("Get retired = %v, want NotFound", err)
+	}
+}
+
+func TestMonitor_Start_offCleansUpAndReturns(t *testing.T) {
+	t.Parallel()
+	c := newFakeClient(t, managed("kvm-home"))
+	var checked atomic.Bool
+	m := newTestMonitor(c, Target{Name: "kvm-home", Type: "kvm", Checker: checkFunc(
+		func(context.Context) []provider.TargetStatus { checked.Store(true); return nil })})
+	m.Interval = 0
+
+	errc := make(chan error, 1)
+	go func() { errc <- m.Start(context.Background()) }()
+	select {
+	case err := <-errc:
+		if err != nil {
+			t.Fatalf("Start() = %v, want nil", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not return with checking off")
+	}
+
+	if checked.Load() {
+		t.Error("a check ran with checking off")
+	}
+	var gone crawlv1alpha1.ProviderHealth
+	if err := c.Get(context.Background(), client.ObjectKey{Name: "kvm-home"}, &gone); !apierrors.IsNotFound(err) {
+		t.Errorf("Get kvm-home = %v, want NotFound so no stale Ready is left behind", err)
+	}
+}
+
+func TestMonitor_Start_checksAtOnceAndStops(t *testing.T) {
+	t.Parallel()
+	c := newFakeClient(t)
+	m := newTestMonitor(c, Target{Name: "local", Type: "kubernetes", Checker: verdicts(up("apiserver"))})
+	m.Interval = time.Hour
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	errc := make(chan error, 1)
+	go func() { errc <- m.Start(ctx) }()
+	deadline := time.Now().Add(5 * time.Second)
+	for len(m.Snapshot()) == 0 {
+		if time.Now().After(deadline) {
+			t.Fatal("no check within 5s of Start, want one right away")
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	cancel()
+
+	select {
+	case err := <-errc:
+		if err != nil {
+			t.Errorf("Start() = %v, want nil", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not return after its context ended")
+	}
+}
+
+func TestValidateSchedule(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name              string
+		interval, timeout time.Duration
+		wantErr           bool
+	}{
+		{"off", 0, 0, false},
+		{"defaults", time.Minute, 15 * time.Second, false},
+		{"shortest interval", 10 * time.Second, 10 * time.Second, false},
+		{"too frequent", 5 * time.Second, time.Second, true},
+		{"negative interval", -time.Minute, time.Second, true},
+		{"no timeout", time.Minute, 0, true},
+		{"timeout longer than interval", time.Minute, 2 * time.Minute, true},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if err := ValidateSchedule(tc.interval, tc.timeout); (err != nil) != tc.wantErr {
+				t.Errorf("ValidateSchedule(%s, %s) = %v, want error %v", tc.interval, tc.timeout, err, tc.wantErr)
+			}
+		})
+	}
+}

internal/providerhealth/status.go

@@ -0,0 +1,98 @@
+package providerhealth
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+	"unicode/utf8"
+
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+)
+
+// Reasons on the Ready condition.
+const (
+	reasonAllTargetsReady      = "AllTargetsReady"
+	reasonTargetsNotReady      = "TargetsNotReady"
+	reasonChecksNotImplemented = "ChecksNotImplemented"
+	reasonNoTargets            = "NoTargets"
+)
+
+// maxMessageLen matches the CRD's limit on a target's message.
+const maxMessageLen = 1024
+
+// buildStatus derives a ProviderHealth status from the previous one and a
+// fresh result. Every timestamp comes from r.CheckedAt, and prev is left
+// untouched. A target keeps its lastTransitionTime while ready does not
+// flip, which is how several hosts each show when they went down.
+func buildStatus(prev crawlv1alpha1.ProviderHealthStatus, r Result) crawlv1alpha1.ProviderHealthStatus {
+	now := metav1.NewTime(r.CheckedAt)
+	out := crawlv1alpha1.ProviderHealthStatus{
+		Type:          r.Type,
+		Conditions:    slices.Clone(prev.Conditions),
+		LastCheckTime: &now,
+	}
+	before := make(map[string]crawlv1alpha1.ProviderTarget, len(prev.Targets))
+	for _, t := range prev.Targets {
+		before[t.Name] = t
+	}
+	ready := 0
+	var failing []string
+	for _, ts := range r.Targets {
+		t := crawlv1alpha1.ProviderTarget{
+			Name:               ts.Name,
+			Address:            ts.Address,
+			Ready:              ts.Ready,
+			Reason:             ts.Reason,
+			Message:            truncate(ts.Message, maxMessageLen),
+			LatencyMillis:      ts.Latency.Milliseconds(),
+			LastTransitionTime: now,
+			Details:            ts.Details,
+		}
+		if b, ok := before[ts.Name]; ok && b.Ready == ts.Ready {
+			t.LastTransitionTime = b.LastTransitionTime
+		}
+		if ts.Ready {
+			ready++
+		} else {
+			failing = append(failing, ts.Name+": "+ts.Reason)
+		}
+		out.Targets = append(out.Targets, t)
+	}
+	if r.Supported {
+		out.TargetsReady = fmt.Sprintf("%d/%d", ready, len(r.Targets))
+	}
+
+	cond := metav1.Condition{Type: crawlv1alpha1.ConditionReady, LastTransitionTime: now}
+	switch {
+	case !r.Supported:
+		cond.Status = metav1.ConditionUnknown
+		cond.Reason = reasonChecksNotImplemented
+		cond.Message = fmt.Sprintf("%s providers have no health check", r.Type)
+	case len(r.Targets) == 0:
+		cond.Status = metav1.ConditionUnknown
+		cond.Reason = reasonNoTargets
+		cond.Message = "the provider has nothing to check"
+	case len(failing) == 0:
+		cond.Status = metav1.ConditionTrue
+		cond.Reason = reasonAllTargetsReady
+		cond.Message = fmt.Sprintf("%d of %d targets ready", ready, len(r.Targets))
+	default:
+		cond.Status = metav1.ConditionFalse
+		cond.Reason = reasonTargetsNotReady
+		cond.Message = truncate(strings.Join(failing, "; "), maxMessageLen)
+	}
+	apimeta.SetStatusCondition(&out.Conditions, cond)
+	return out
+}
+
+// truncate cuts s to at most n characters, which is how the API server
+// counts maxLength.
+func truncate(s string, n int) string {
+	if utf8.RuneCountInString(s) <= n {
+		return s
+	}
+	return string([]rune(s)[:n-1]) + "…"
+}

internal/providerhealth/status_test.go

@@ -0,0 +1,135 @@
+package providerhealth
+
+import (
+	"strings"
+	"testing"
+	"time"
+	"unicode/utf8"
+
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func readyAt(name string, at time.Time) crawlv1alpha1.ProviderTarget {
+	return crawlv1alpha1.ProviderTarget{Name: name, Ready: true, Reason: provider.ReasonReachable, LastTransitionTime: metav1.NewTime(at)}
+}
+
+func TestBuildStatus(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name             string
+		prev             crawlv1alpha1.ProviderHealthStatus
+		result           Result
+		wantStatus       metav1.ConditionStatus
+		wantReason       string
+		wantMessage      string
+		wantTargetsReady string
+		// wantTransitions is each target's lastTransitionTime, in order.
+		wantTransitions []time.Time
+	}{
+		{
+			name:             "all ready",
+			result:           Result{Type: "kvm", Supported: true, Targets: []provider.TargetStatus{up("nas1"), up("nas2")}, CheckedAt: t1},
+			wantStatus:       metav1.ConditionTrue,
+			wantReason:       reasonAllTargetsReady,
+			wantMessage:      "2 of 2 targets ready",
+			wantTargetsReady: "2/2",
+			wantTransitions:  []time.Time{t1, t1},
+		},
+		{
+			name:             "one target down",
+			result:           Result{Type: "kvm", Supported: true, Targets: []provider.TargetStatus{up("nas1"), down("nas2", provider.ReasonUnreachable)}, CheckedAt: t1},
+			wantStatus:       metav1.ConditionFalse,
+			wantReason:       reasonTargetsNotReady,
+			wantMessage:      "nas2: Unreachable",
+			wantTargetsReady: "1/2",
+			wantTransitions:  []time.Time{t1, t1},
+		},
+		{
+			name: "only a flip moves a target's transition time",
+			prev: crawlv1alpha1.ProviderHealthStatus{Targets: []crawlv1alpha1.ProviderTarget{readyAt("nas1", t0), readyAt("nas2", t0)}},
+			result: Result{Type: "kvm", Supported: true, CheckedAt: t1,
+				Targets: []provider.TargetStatus{up("nas1"), down("nas2", provider.ReasonTimeout)}},
+			wantStatus:       metav1.ConditionFalse,
+			wantReason:       reasonTargetsNotReady,
+			wantMessage:      "nas2: Timeout",
+			wantTargetsReady: "1/2",
+			wantTransitions:  []time.Time{t0, t1},
+		},
+		{
+			name:       "no checker",
+			result:     Result{Type: "gcp", CheckedAt: t1},
+			wantStatus: metav1.ConditionUnknown,
+			wantReason: reasonChecksNotImplemented,
+		},
+		{
+			name:             "nothing to check",
+			result:           Result{Type: "kvm", Supported: true, CheckedAt: t1},
+			wantStatus:       metav1.ConditionUnknown,
+			wantReason:       reasonNoTargets,
+			wantTargetsReady: "0/0",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got := buildStatus(tc.prev, tc.result)
+
+			cond := apimeta.FindStatusCondition(got.Conditions, crawlv1alpha1.ConditionReady)
+			if cond == nil || cond.Status != tc.wantStatus || cond.Reason != tc.wantReason {
+				t.Fatalf("Ready = %+v, want %s/%s", cond, tc.wantStatus, tc.wantReason)
+			}
+			if tc.wantMessage != "" && cond.Message != tc.wantMessage {
+				t.Errorf("Ready message = %q, want %q", cond.Message, tc.wantMessage)
+			}
+			if got.TargetsReady != tc.wantTargetsReady {
+				t.Errorf("targetsReady = %q, want %q", got.TargetsReady, tc.wantTargetsReady)
+			}
+			if got.Type != tc.result.Type || got.LastCheckTime == nil || !got.LastCheckTime.Time.Equal(t1) {
+				t.Errorf("type/lastCheckTime = %q/%v, want %q/%s", got.Type, got.LastCheckTime, tc.result.Type, t1)
+			}
+			if len(got.Targets) != len(tc.wantTransitions) {
+				t.Fatalf("got %d targets, want %d", len(got.Targets), len(tc.wantTransitions))
+			}
+			for i, want := range tc.wantTransitions {
+				if lt := got.Targets[i].LastTransitionTime.Time; !lt.Equal(want) {
+					t.Errorf("target %s lastTransitionTime = %s, want %s", got.Targets[i].Name, lt, want)
+				}
+			}
+		})
+	}
+}
+
+func TestBuildStatus_conditionKeepsTransitionWhileUnchanged(t *testing.T) {
+	t.Parallel()
+	first := buildStatus(crawlv1alpha1.ProviderHealthStatus{},
+		Result{Type: "kvm", Supported: true, Targets: []provider.TargetStatus{up("nas1")}, CheckedAt: t0})
+	firstCond := apimeta.FindStatusCondition(first.Conditions, crawlv1alpha1.ConditionReady)
+
+	second := buildStatus(first, Result{Type: "kvm", Supported: true, Targets: []provider.TargetStatus{up("nas1")}, CheckedAt: t1})
+
+	cond := apimeta.FindStatusCondition(second.Conditions, crawlv1alpha1.ConditionReady)
+	if !cond.LastTransitionTime.Time.Equal(t0) {
+		t.Errorf("Ready lastTransitionTime = %s, want %s from the first round", cond.LastTransitionTime, t0)
+	}
+	if !firstCond.LastTransitionTime.Time.Equal(t0) || len(first.Conditions) != 1 {
+		t.Errorf("buildStatus modified the previous status: %+v", first.Conditions)
+	}
+}
+
+func TestBuildStatus_truncatesLongMessages(t *testing.T) {
+	t.Parallel()
+	long := strings.Repeat("é", 3000)
+	got := buildStatus(crawlv1alpha1.ProviderHealthStatus{}, Result{Type: "kvm", Supported: true, CheckedAt: t1,
+		Targets: []provider.TargetStatus{{Name: "nas1", Reason: provider.ReasonUnreachable, Message: long}}})
+
+	if n := utf8.RuneCountInString(got.Targets[0].Message); n != maxMessageLen {
+		t.Errorf("message has %d characters, want %d", n, maxMessageLen)
+	}
+	if !strings.HasSuffix(got.Targets[0].Message, "…") {
+		t.Error("truncated message does not end with an ellipsis")
+	}
+}