Add the health engine: through-proxy probes, thresholds, channel-fed transitions

Co-Authored-By: Claude <noreply@anthropic.com>

internal/health/probe_test.go

@@ -0,0 +1,169 @@
+package health
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+)
+
+// startConnectProxy runs a minimal but real HTTP proxy: CONNECT tunneling
+// for https targets, absolute-URI forwarding for plain http ones. With
+// refuseConnect it answers CONNECT with 502 — the "accepts TCP but cannot
+// egress" failure mode the probe must classify as unhealthy.
+func startConnectProxy(t *testing.T, refuseConnect bool) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodConnect {
+			if refuseConnect {
+				http.Error(w, "no egress", http.StatusBadGateway)
+				return
+			}
+			dst, err := net.DialTimeout("tcp", r.Host, time.Second)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadGateway)
+				return
+			}
+			conn, bufrw, err := http.NewResponseController(w).Hijack()
+			if err != nil {
+				_ = dst.Close()
+				t.Errorf("hijack: %v", err)
+				return
+			}
+			_, _ = bufrw.WriteString("HTTP/1.1 200 Connection established\r\n\r\n")
+			_ = bufrw.Flush()
+			done := make(chan struct{}, 2)
+			go func() { _, _ = io.Copy(dst, bufrw); done <- struct{}{} }()
+			go func() { _, _ = io.Copy(conn, dst); done <- struct{}{} }()
+			<-done
+			_ = conn.Close()
+			_ = dst.Close()
+			return
+		}
+		out := r.Clone(r.Context())
+		out.RequestURI = ""
+		resp, err := http.DefaultTransport.RoundTrip(out)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadGateway)
+			return
+		}
+		defer func() { _ = resp.Body.Close() }()
+		for k, vv := range resp.Header {
+			for _, v := range vv {
+				w.Header().Add(k, v)
+			}
+		}
+		w.WriteHeader(resp.StatusCode)
+		_, _ = io.Copy(w, resp.Body)
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func startTLSTarget(t *testing.T, status int) (*httptest.Server, *tls.Config) {
+	t.Helper()
+	target := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(status)
+	}))
+	t.Cleanup(target.Close)
+	pool := x509.NewCertPool()
+	pool.AddCert(target.Certificate())
+	return target, &tls.Config{RootCAs: pool}
+}
+
+func proxyURL(t *testing.T, srv *httptest.Server) *url.URL {
+	t.Helper()
+	u, err := url.Parse(srv.URL)
+	if err != nil {
+		t.Fatalf("parsing proxy URL: %v", err)
+	}
+	return u
+}
+
+func testHC(probeTarget string) crawlv1alpha1.HealthCheckSpec {
+	return crawlv1alpha1.HealthCheckSpec{
+		ProbeURL:            probeTarget,
+		IntervalSeconds:     30,
+		TimeoutSeconds:      5,
+		FailureThreshold:    3,
+		SuccessThreshold:    1,
+		ExpectedStatusCodes: []int32{200, 204},
+	}
+}
+
+func TestProbe_connectTunnelSucceeds(t *testing.T) {
+	t.Parallel()
+	target, tlsCfg := startTLSTarget(t, http.StatusNoContent)
+	proxy := startConnectProxy(t, false)
+
+	res := probe(context.Background(), proxyURL(t, proxy), testHC(target.URL), tlsCfg)
+	if !res.ok {
+		t.Fatalf("probe failed through working proxy: %v", res.err)
+	}
+	if res.latency <= 0 {
+		t.Errorf("latency = %v, want > 0", res.latency)
+	}
+}
+
+func TestProbe_refusedConnectFails(t *testing.T) {
+	t.Parallel()
+	target, tlsCfg := startTLSTarget(t, http.StatusNoContent)
+	proxy := startConnectProxy(t, true)
+
+	res := probe(context.Background(), proxyURL(t, proxy), testHC(target.URL), tlsCfg)
+	if res.ok {
+		t.Fatal("probe succeeded through a proxy that refuses CONNECT")
+	}
+	if res.err == nil {
+		t.Error("expected an error from the refused CONNECT")
+	}
+}
+
+func TestProbe_unexpectedStatusFails(t *testing.T) {
+	t.Parallel()
+	target, tlsCfg := startTLSTarget(t, http.StatusInternalServerError)
+	proxy := startConnectProxy(t, false)
+
+	res := probe(context.Background(), proxyURL(t, proxy), testHC(target.URL), tlsCfg)
+	if res.ok {
+		t.Fatal("probe succeeded on a 500 response")
+	}
+}
+
+func TestProbe_unreachableProxyFails(t *testing.T) {
+	t.Parallel()
+	// A listener that is immediately closed: guaranteed-refused port.
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("reserving port: %v", err)
+	}
+	dead := &url.URL{Scheme: "http", Host: l.Addr().String()}
+	_ = l.Close()
+
+	res := probe(context.Background(), dead, testHC("https://example.invalid/"), nil)
+	if res.ok {
+		t.Fatal("probe succeeded against a dead proxy")
+	}
+}
+
+func TestProbe_plainHTTPForwardSucceeds(t *testing.T) {
+	t.Parallel()
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	t.Cleanup(target.Close)
+	proxy := startConnectProxy(t, false)
+
+	res := probe(context.Background(), proxyURL(t, proxy), testHC(target.URL), nil)
+	if !res.ok {
+		t.Fatalf("plain-http probe failed: %v", res.err)
+	}
+}