kacerr/egress-proxies-operator
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add the health engine: through-proxy probes, thresholds, channel-fed transitions

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Jan Novak
2026-08-09 15:01:10 +02:00
parent 71c00c40d1
commit 801a9fbe5f
10 changed files with 1350 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
internal/controller/health_test.go Normal file
View File

@@ -0,0 +1,175 @@
package controller
import (
"strings"
"testing"
"time"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
type fakeSnapshotter struct {
snap health.Snapshot
ok bool
}
func (f fakeSnapshotter) Snapshot(types.NamespacedName) (health.Snapshot, bool) {
return f.snap, f.ok
}
// TestReconcile_healthRepresentation covers the reconciler's half of the
// health split: turning the engine's Snapshot into the Healthy condition,
// the latency fields, and ultimately the Ready/Unhealthy phases.
func TestReconcile_healthRepresentation(t *testing.T) {
t.Parallel()
probeTime := time.Now()
freshHash := specHash(managedProxy(), "")
runningStub := func() *stubProvider {
return &stubProvider{
getInst: &provider.Instance{ID: "stub-id-1", IP: "10.1.2.3", State: provider.StateRunning},
}
}
tests := []struct {
name string
proxy *crawlv1alpha1.Proxy
stub *stubProvider
health HealthSnapshotter
wantPhase crawlv1alpha1.ProxyPhase
verify func(t *testing.T, r *ProxyReconciler)
}{
{
name: "running and healthy becomes Ready",
proxy: managedProxy(withProviderID("stub-id-1"), withSpecHashAnnotation(freshHash)),
stub: runningStub(),
health: fakeSnapshotter{ok: true, snap: health.Snapshot{
Healthy: true, Latency: 37 * time.Millisecond, LastProbe: probeTime,
}},
wantPhase: crawlv1alpha1.PhaseReady,
verify: func(t *testing.T, r *ProxyReconciler) {
p := getProxy(t, r)
assertCondition(t, p, crawlv1alpha1.ConditionHealthy, metav1.ConditionTrue, ReasonProbeSucceeded)
if p.Status.LatencyMillis != 37 {
t.Errorf("latencyMillis = %d, want 37", p.Status.LatencyMillis)
}
if p.Status.LastHealthCheckTime == nil {
t.Error("lastHealthCheckTime not set")
}
},
},
{
name: "running but unhealthy becomes Unhealthy",
proxy: managedProxy(withProviderID("stub-id-1"), withSpecHashAnnotation(freshHash)),
stub: runningStub(),
health: fakeSnapshotter{ok: true, snap: health.Snapshot{
Healthy: false, LastProbe: probeTime,
LastError: "CONNECT refused", ConsecutiveFailures: 3,
}},
wantPhase: crawlv1alpha1.PhaseUnhealthy,
verify: func(t *testing.T, r *ProxyReconciler) {
p := getProxy(t, r)
assertCondition(t, p, crawlv1alpha1.ConditionHealthy, metav1.ConditionFalse, ReasonProbeFailed)
cond := apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionHealthy)
if !strings.Contains(cond.Message, "CONNECT refused") {
t.Errorf("condition message %q does not carry the probe error", cond.Message)
}
},
},
{
name: "no verdict yet stays Provisioning without a Healthy condition",
proxy: managedProxy(withProviderID("stub-id-1"), withSpecHashAnnotation(freshHash)),
stub: runningStub(),
health: fakeSnapshotter{ok: false},
wantPhase: crawlv1alpha1.PhaseProvisioning,
verify: func(t *testing.T, r *ProxyReconciler) {
p := getProxy(t, r)
if apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionHealthy) != nil {
t.Error("Healthy condition present without an engine verdict")
}
},
},
{
name: "external proxy with a healthy verdict becomes Ready",
proxy: managedProxy(func(p *crawlv1alpha1.Proxy) {
p.Finalizers = nil
p.Spec = crawlv1alpha1.ProxySpec{
Mode: crawlv1alpha1.ModeExternal,
Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.7"},
}
}),
stub: &stubProvider{},
health: fakeSnapshotter{ok: true, snap: health.Snapshot{
Healthy: true, Latency: 5 * time.Millisecond, LastProbe: probeTime,
}},
wantPhase: crawlv1alpha1.PhaseReady,
verify: func(t *testing.T, r *ProxyReconciler) {
assertCondition(t, getProxy(t, r), crawlv1alpha1.ConditionHealthy, metav1.ConditionTrue, ReasonProbeSucceeded)
},
},
{
name: "creating a replacement clears the stale Healthy verdict",
proxy: managedProxy(func(p *crawlv1alpha1.Proxy) {
p.Status.Conditions = []metav1.Condition{{
Type: crawlv1alpha1.ConditionHealthy, Status: metav1.ConditionTrue,
Reason: ReasonProbeSucceeded, LastTransitionTime: metav1.Now(),
}}
p.Status.LatencyMillis = 42
p.Status.LastHealthCheckTime = &metav1.Time{Time: probeTime}
}),
stub: &stubProvider{createID: "stub-id-2"},
health: fakeSnapshotter{ok: false},
wantPhase: crawlv1alpha1.PhaseProvisioning,
verify: func(t *testing.T, r *ProxyReconciler) {
p := getProxy(t, r)
if apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionHealthy) != nil {
t.Error("stale Healthy condition survived instance creation")
}
if p.Status.LatencyMillis != 0 || p.Status.LastHealthCheckTime != nil {
t.Error("stale latency fields survived instance creation")
}
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
r := newTestReconciler(t, tc.stub, tc.proxy)
r.Health = tc.health
if _, err := doReconcile(t, r); err != nil {
t.Fatalf("Reconcile: %v", err)
}
if got := getProxy(t, r).Status.Phase; got != tc.wantPhase {
t.Errorf("phase = %s, want %s", got, tc.wantPhase)
}
tc.verify(t, r)
})
}
}
// A nil Health snapshotter must disable representation entirely.
func TestReconcile_nilHealthSnapshotter(t *testing.T) {
t.Parallel()
freshHash := specHash(managedProxy(), "")
r := newTestReconciler(t,
&stubProvider{getInst: &provider.Instance{ID: "stub-id-1", IP: "10.1.2.3", State: provider.StateRunning}},
managedProxy(withProviderID("stub-id-1"), withSpecHashAnnotation(freshHash)))
if _, err := doReconcile(t, r); err != nil {
t.Fatalf("Reconcile: %v", err)
}
p := getProxy(t, r)
if apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionHealthy) != nil {
t.Error("Healthy condition written with no snapshotter configured")
}
if p.Status.Phase != crawlv1alpha1.PhaseProvisioning {
t.Errorf("phase = %s, want Provisioning", p.Status.Phase)
}
}

36
internal/controller/proxy_controller.go
View File

@@ -27,18 +27,30 @@ import (
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
)
// HealthSnapshotter provides the current probe verdict for a proxy. The
// health engine implements it; the reconciler is its only consumer, turning
// snapshots into the Healthy condition — the engine owns health state, the
// reconciler owns its representation.
type HealthSnapshotter interface {
Snapshot(key types.NamespacedName) (health.Snapshot, bool)
}
// ProxyReconciler reconciles Proxy objects as a state machine: every
// reconcile derives exactly one action from (spec, status, provider Get),
// performs it, and requeues. Status is written at most once per reconcile,
@@ -50,6 +62,13 @@ type ProxyReconciler struct {
// Providers maps spec.provider values to configured backends.
Providers map[string]provider.Provider
// Health supplies probe verdicts; nil disables health representation
// (the Healthy condition simply never appears).
Health HealthSnapshotter
// HealthEvents, when non-nil, is watched as a raw source so the health
// engine can enqueue proxies on status-affecting transitions.
HealthEvents <-chan event.GenericEvent
// Poll intervals are struct fields, never consts, so tests can shrink
// them to milliseconds.
ProvisioningPoll time.Duration // while waiting for an instance to reach Running
@@ -142,6 +161,12 @@ func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1
p.Status.ProviderID = id
p.Status.IP = ""
setProvisioned(p, metav1.ConditionFalse, ReasonProvisioning, "instance created; waiting for it to run")
// Any Healthy verdict belonged to the previous instance; the health
// engine starts fresh for the new one (its state was pruned while
// the proxy had no IP), and so must the status.
apimeta.RemoveStatusCondition(&p.Status.Conditions, crawlv1alpha1.ConditionHealthy)
p.Status.LatencyMillis = 0
p.Status.LastHealthCheckTime = nil
return ctrl.Result{RequeueAfter: r.ProvisioningPoll}, nil
}
@@ -176,6 +201,7 @@ func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1
case provider.StateRunning:
p.Status.IP = inst.IP
setProvisioned(p, metav1.ConditionTrue, ReasonCreated, "instance is running")
r.applyHealth(p)
return ctrl.Result{RequeueAfter: r.DriftPoll}, nil
default: // Stopped, Terminated: cattle, not pets — delete and recreate.
if err := prov.Delete(ctx, p.Status.ProviderID); err != nil {
@@ -262,6 +288,7 @@ func (r *ProxyReconciler) reconcileExternal(_ context.Context, p *crawlv1alpha1.
}
p.Status.IP = p.Spec.Endpoint.Host
setProvisioned(p, metav1.ConditionTrue, ReasonExternalEndpoint, "tracking an external endpoint")
r.applyHealth(p)
return ctrl.Result{}, nil
}
@@ -371,12 +398,15 @@ func (r *ProxyReconciler) proxiesForSecret(ctx context.Context, obj client.Objec
// (cmd/main.go) restricts that cache to labelled cloud-init Secrets.
func (r *ProxyReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.applyDefaults()
return ctrl.NewControllerManagedBy(mgr).
b := ctrl.NewControllerManagedBy(mgr).
For(&crawlv1alpha1.Proxy{}).
Named("proxy").
Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.proxiesForSecret)).
WithOptions(controller.Options{MaxConcurrentReconciles: 3}).
Complete(r)
WithOptions(controller.Options{MaxConcurrentReconciles: 3})
if r.HealthEvents != nil {
b = b.WatchesRawSource(source.Channel(r.HealthEvents, &handler.EnqueueRequestForObject{}))
}
return b.Complete(r)
}
func (r *ProxyReconciler) applyDefaults() {

42
internal/controller/status.go
View File

@@ -2,6 +2,7 @@ package controller
import (
"context"
"fmt"
"k8s.io/apimachinery/pkg/api/equality"
apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -11,8 +12,9 @@ import (
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
)
// Reasons used on the Provisioned condition. The Healthy condition is owned
// by the health engine (internal/health) and only represented here.
// Reasons used on the Provisioned condition, plus the two the reconciler
// writes on the Healthy condition when representing the health engine's
// verdict (the engine owns the state; only the reconciler writes status).
const (
ReasonProvisioning = "Provisioning"
ReasonCreated = "Created"
@@ -23,6 +25,9 @@ const (
ReasonCloudInitError = "CloudInitError"
ReasonExternalEndpoint = "ExternalEndpoint"
ReasonDeleting = "Deleting"
ReasonProbeSucceeded = "ProbeSucceeded"
ReasonProbeFailed = "ProbeFailed"
)
// setProvisioned stages the Provisioned condition on p. Nothing is written
@@ -39,6 +44,39 @@ func setProvisioned(p *crawlv1alpha1.Proxy, status metav1.ConditionStatus, reaso
})
}
// applyHealth stages the Healthy condition and the latency fields from the
// health engine's current snapshot. Called only from states where the proxy
// is reachable (Running, External); everywhere else the condition is either
// left as-is or removed by the create branch.
func (r *ProxyReconciler) applyHealth(p *crawlv1alpha1.Proxy) {
if r.Health == nil {
return
}
snap, ok := r.Health.Snapshot(client.ObjectKeyFromObject(p))
if !ok {
return
}
cond := metav1.Condition{
Type: crawlv1alpha1.ConditionHealthy,
ObservedGeneration: p.Generation,
}
if snap.Healthy {
cond.Status = metav1.ConditionTrue
cond.Reason = ReasonProbeSucceeded
cond.Message = "probe succeeded through the proxy"
} else {
cond.Status = metav1.ConditionFalse
cond.Reason = ReasonProbeFailed
cond.Message = fmt.Sprintf("%d consecutive probe failures; last: %s",
snap.ConsecutiveFailures, snap.LastError)
}
apimeta.SetStatusCondition(&p.Status.Conditions, cond)
p.Status.LatencyMillis = snap.Latency.Milliseconds()
if !snap.LastProbe.IsZero() {
p.Status.LastHealthCheckTime = &metav1.Time{Time: snap.LastProbe}
}
}
// computePhase derives status.phase from deletionTimestamp and the
// Provisioned/Healthy conditions. Pure, so the truth table is unit-testable.
func computePhase(p *crawlv1alpha1.Proxy) crawlv1alpha1.ProxyPhase {