Wire the provider health monitor into the manager, with gauges

--provider-check-interval (1m; 0 = off, else at least 10s) and
--provider-check-timeout (15s) are validated before the providers config
loads. Checkers are collected from the unwrapped providers in config
order, so checks never count as provider requests. Three scrape-time
gauges per {provider,target} read the monitor's snapshot: up, check
duration and check timestamp. The e2e suite waits for the kubernetes
ProviderHealth to turn Ready and asserts its gauge in the scrape.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

cmd/main.go

@@ -16,8 +16,8 @@ limitations under the License.
 
 // Package main is the composition root: it loads the provider config (fail
 // fast), assembles the provider registry, and wires the reconciler, health
-// engine, lease store, discovery API, orphan GC, and metrics onto one
-// controller-runtime manager.
+// engine, lease store, discovery API, orphan GC, provider health checks and
+// metrics onto one controller-runtime manager.
 package main
 
 import (
@@ -60,6 +60,7 @@ import (
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kvm"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/providerhealth"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version"
 	// +kubebuilder:scaffold:imports
@@ -97,6 +98,7 @@ func main() {
 	var showVersion bool
 	var gcpWireFullPayloads bool
 	var traceHealthProbes bool
+	var providerCheckInterval, providerCheckTimeout time.Duration
 
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -140,6 +142,12 @@ func main() {
 	flag.BoolVar(&traceHealthProbes, "trace-health-probes", false,
 		"Emit one trace span per health probe. Off by default: probes run about "+
 			"once per second per proxy and would dominate trace volume.")
+	flag.DurationVar(&providerCheckInterval, "provider-check-interval", time.Minute,
+		"Interval between provider health checks (can the cluster reach each hypervisor, API "+
+			"and account), published as ProviderHealth objects and metrics. 0 turns them off; "+
+			"otherwise at least 10s.")
+	flag.DurationVar(&providerCheckTimeout, "provider-check-timeout", 15*time.Second,
+		"Time limit for one round of provider health checks; at most the interval.")
 
 	opts := zap.Options{
 		Development: true,
@@ -174,6 +182,10 @@ func main() {
 		setupLog.Error(nil, "--providers-config is required")
 		os.Exit(1)
 	}
+	if err := providerhealth.ValidateSchedule(providerCheckInterval, providerCheckTimeout); err != nil {
+		setupLog.Error(err, "Invalid provider check flags")
+		os.Exit(1)
+	}
 	cfg, err := provider.LoadConfigFile(providersConfig)
 	if err != nil {
 		setupLog.Error(err, "Failed to load providers config", "path", providersConfig)
@@ -195,6 +207,14 @@ func main() {
 		os.Exit(1)
 	}
 	m := metrics.New()
+	// Checkers come from the unwrapped providers, in config order: the
+	// metrics and tracing wrappers implement only Provider, and a check is
+	// not a provider request to be counted anyway.
+	checkTargets := make([]providerhealth.Target, 0, len(cfg.Providers))
+	for _, pc := range cfg.Providers {
+		checker, _ := providers[pc.Name].(provider.Checker)
+		checkTargets = append(checkTargets, providerhealth.Target{Name: pc.Name, Type: pc.Type, Checker: checker})
+	}
 	for name, p := range providers {
 		// Tracing outermost: the span covers the metrics recording too.
 		providers[name] = provider.WithTracing(name, provider.WithMetrics(name, p, m))
@@ -314,6 +334,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	monitor := &providerhealth.Monitor{
+		Client:   mgr.GetClient(),
+		Reader:   mgr.GetAPIReader(),
+		Targets:  checkTargets,
+		Interval: providerCheckInterval,
+		Timeout:  providerCheckTimeout,
+	}
+	if err := mgr.Add(monitor); err != nil {
+		setupLog.Error(err, "Failed to add provider health monitor")
+		os.Exit(1)
+	}
+
 	if err := m.Register(ctrlmetrics.Registry,
 		proxyPhaseCounts(mgr.GetClient()),
 		func() int {
@@ -327,6 +359,10 @@ func main() {
 		setupLog.Error(err, "Failed to register metrics")
 		os.Exit(1)
 	}
+	if err := metrics.RegisterProviderTargets(ctrlmetrics.Registry, providerTargetSamples(monitor)); err != nil {
+		setupLog.Error(err, "Failed to register provider check metrics")
+		os.Exit(1)
+	}
 
 	if err := (&controller.ProxyReconciler{
 		Client:       mgr.GetClient(),
@@ -389,3 +425,23 @@ func proxyPhaseCounts(c client.Reader) func() map[string]int {
 		return counts
 	}
 }
+
+// providerTargetSamples flattens the monitor's latest results for the
+// scrape-time provider-check gauges.
+func providerTargetSamples(mon *providerhealth.Monitor) func() []metrics.ProviderTarget {
+	return func() []metrics.ProviderTarget {
+		var out []metrics.ProviderTarget
+		for _, r := range mon.Snapshot() {
+			for _, ts := range r.Targets {
+				out = append(out, metrics.ProviderTarget{
+					Provider:  r.Provider,
+					Target:    ts.Name,
+					Up:        ts.Ready,
+					Duration:  ts.Latency,
+					CheckedAt: r.CheckedAt,
+				})
+			}
+		}
+		return out
+	}
+}