Add orphan GC sweeper and Prometheus metrics with explicit registration

Co-Authored-By: Claude <noreply@anthropic.com>

internal/metrics/metrics.go

@@ -0,0 +1,119 @@
+// Package metrics defines the operator's Prometheus metrics. Nothing here
+// registers itself — no init(), per house rules — the composition root
+// calls Register explicitly, which also lets every test use a fresh
+// registry.
+package metrics
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metrics holds the vector metrics the operator's components feed. The
+// consuming packages (health, discovery, provider) each define their own
+// small recorder interface; *Metrics satisfies all of them structurally,
+// so none of them import this package's prometheus surface.
+type Metrics struct {
+	healthcheckDuration *prometheus.HistogramVec
+	healthcheckFailures *prometheus.CounterVec
+	leaseRequests       *prometheus.CounterVec
+	providerRequests    *prometheus.CounterVec
+}
+
+// New builds the metric set, unregistered.
+func New() *Metrics {
+	return &Metrics{
+		healthcheckDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "proxy_operator_healthcheck_duration_seconds",
+			Help:    "Duration of through-the-proxy health probes.",
+			Buckets: prometheus.DefBuckets,
+		}, []string{"proxy"}),
+		healthcheckFailures: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_healthcheck_failures_total",
+			Help: "Failed health probes.",
+		}, []string{"proxy"}),
+		leaseRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_lease_requests_total",
+			Help: "Lease acquisition requests by outcome.",
+		}, []string{"outcome"}),
+		providerRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_provider_requests_total",
+			Help: "Provider API calls by operation and classified result.",
+		}, []string{"provider", "op", "result"}),
+	}
+}
+
+// Register registers the vectors plus the two scrape-time collectors.
+// proxyPhases and activeLeases are read at every scrape: gauges derived
+// from reconcile-time increments inevitably drift and leak series on
+// delete; reading the source of truth cannot.
+func (m *Metrics) Register(reg prometheus.Registerer, proxyPhases func() map[string]int, activeLeases func() int) error {
+	collectors := []prometheus.Collector{
+		m.healthcheckDuration,
+		m.healthcheckFailures,
+		m.leaseRequests,
+		m.providerRequests,
+		&constCollector{
+			desc: prometheus.NewDesc("proxy_operator_proxies",
+				"Proxy objects by phase.", []string{"phase"}, nil),
+			read: proxyPhases,
+		},
+		&constCollector{
+			desc: prometheus.NewDesc("proxy_operator_leases_active",
+				"Currently active leases.", nil, nil),
+			read: func() map[string]int { return map[string]int{"": activeLeases()} },
+		},
+	}
+	for _, c := range collectors {
+		if err := reg.Register(c); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ObserveProbe records one health probe. Called on every probe — metrics
+// are the home for high-frequency signal that must never touch status.
+func (m *Metrics) ObserveProbe(proxy string, latency time.Duration, success bool) {
+	m.healthcheckDuration.WithLabelValues(proxy).Observe(latency.Seconds())
+	if !success {
+		m.healthcheckFailures.WithLabelValues(proxy).Inc()
+	}
+}
+
+// ForgetProxy drops the per-proxy series when the health engine prunes its
+// state — without this, series for deleted proxies leak forever.
+func (m *Metrics) ForgetProxy(proxy string) {
+	m.healthcheckDuration.DeleteLabelValues(proxy)
+	m.healthcheckFailures.DeleteLabelValues(proxy)
+}
+
+// LeaseRequest records a lease acquisition outcome ("granted"|"no_match").
+func (m *Metrics) LeaseRequest(outcome string) {
+	m.leaseRequests.WithLabelValues(outcome).Inc()
+}
+
+// ProviderRequest records one provider API call with its classified result.
+func (m *Metrics) ProviderRequest(provider, op, result string) {
+	m.providerRequests.WithLabelValues(provider, op, result).Inc()
+}
+
+// constCollector reads a label→value map at scrape time and emits one
+// gauge sample per entry. An empty-string label key means "no labels".
+type constCollector struct {
+	desc *prometheus.Desc
+	read func() map[string]int
+}
+
+func (c *constCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.desc }
+
+func (c *constCollector) Collect(ch chan<- prometheus.Metric) {
+	for label, value := range c.read() {
+		if label == "" {
+			ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(value))
+			continue
+		}
+		ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(value), label)
+	}
+}

internal/metrics/metrics_test.go

@@ -0,0 +1,113 @@
+package metrics
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+// register wires a fresh registry — the reason Register exists instead of
+// init()-time self-registration.
+func register(t *testing.T, m *Metrics, phases map[string]int, active int) *prometheus.Registry {
+	t.Helper()
+	reg := prometheus.NewRegistry()
+	err := m.Register(reg,
+		func() map[string]int { return phases },
+		func() int { return active },
+	)
+	if err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+	return reg
+}
+
+func TestRegister_scrapeTimeCollectors(t *testing.T) {
+	t.Parallel()
+	m := New()
+	reg := register(t, m, map[string]int{"Ready": 3, "Provisioning": 1}, 7)
+
+	expected := `
+# HELP proxy_operator_leases_active Currently active leases.
+# TYPE proxy_operator_leases_active gauge
+proxy_operator_leases_active 7
+# HELP proxy_operator_proxies Proxy objects by phase.
+# TYPE proxy_operator_proxies gauge
+proxy_operator_proxies{phase="Provisioning"} 1
+proxy_operator_proxies{phase="Ready"} 3
+`
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(expected),
+		"proxy_operator_proxies", "proxy_operator_leases_active"); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestObserveProbe_andForget(t *testing.T) {
+	t.Parallel()
+	m := New()
+	reg := register(t, m, nil, 0)
+
+	m.ObserveProbe("default/p1", 30*time.Millisecond, true)
+	m.ObserveProbe("default/p1", 40*time.Millisecond, false)
+	m.ObserveProbe("default/p2", 10*time.Millisecond, true)
+
+	if got := testutil.CollectAndCount(m.healthcheckDuration); got != 2 {
+		t.Errorf("duration series = %d, want 2 (one per proxy)", got)
+	}
+	if got := testutil.ToFloat64(m.healthcheckFailures.WithLabelValues("default/p1")); got != 1 {
+		t.Errorf("p1 failures = %v, want 1 (only the failed probe)", got)
+	}
+
+	m.ForgetProxy("default/p1")
+	if got := testutil.CollectAndCount(m.healthcheckDuration); got != 1 {
+		t.Errorf("duration series after ForgetProxy = %d, want 1 — series must not leak", got)
+	}
+	if got := testutil.CollectAndCount(m.healthcheckFailures); got != 0 {
+		t.Errorf("failure series after ForgetProxy = %d, want 0", got)
+	}
+	_ = reg
+}
+
+func TestLeaseRequest(t *testing.T) {
+	t.Parallel()
+	m := New()
+	register(t, m, nil, 0)
+
+	m.LeaseRequest("granted")
+	m.LeaseRequest("granted")
+	m.LeaseRequest("no_match")
+
+	if got := testutil.ToFloat64(m.leaseRequests.WithLabelValues("granted")); got != 2 {
+		t.Errorf("granted = %v, want 2", got)
+	}
+	if got := testutil.ToFloat64(m.leaseRequests.WithLabelValues("no_match")); got != 1 {
+		t.Errorf("no_match = %v, want 1", got)
+	}
+}
+
+func TestProviderRequest(t *testing.T) {
+	t.Parallel()
+	m := New()
+	register(t, m, nil, 0)
+
+	m.ProviderRequest("gcp-eu", "create", "ok")
+	m.ProviderRequest("gcp-eu", "create", "quota_exceeded")
+
+	if got := testutil.ToFloat64(m.providerRequests.WithLabelValues("gcp-eu", "create", "ok")); got != 1 {
+		t.Errorf("ok = %v, want 1", got)
+	}
+	if got := testutil.ToFloat64(m.providerRequests.WithLabelValues("gcp-eu", "create", "quota_exceeded")); got != 1 {
+		t.Errorf("quota_exceeded = %v, want 1", got)
+	}
+}
+
+func TestRegister_freshRegistryPerTest(t *testing.T) {
+	t.Parallel()
+	// Registering the same metric set on two registries must both succeed —
+	// the property init()-style global registration would break.
+	m1, m2 := New(), New()
+	register(t, m1, nil, 0)
+	register(t, m2, nil, 0)
+}