From 0fe62ef314e6a8fb733e94da134a5d9e1c08de8d Mon Sep 17 00:00:00 2001 From: Jan Novak Date: Mon, 10 Aug 2026 09:34:42 +0200 Subject: [PATCH] Verify end-to-end on kind: fix Squid FD-table OOM, make the quickstart in-cluster Co-Authored-By: Claude --- .claude/settings.json | 10 ++- CHANGELOG.md | 11 +++ README.md | 36 +++++++--- .../2026-08-07-1747-proxy-operator.md | 70 ++++++++++++++++++- internal/provider/kubernetes/pod.go | 8 +++ internal/provider/kubernetes/pod_test.go | 5 +- 6 files changed, 129 insertions(+), 11 deletions(-) diff --git a/.claude/settings.json b/.claude/settings.json index 1d6c96a..06e584b 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -63,7 +63,15 @@ "Bash(grep -n 'func Channel' -A8 __CMDSUB_OUTPUT__/sigs.k8s.io/controller-runtime@v0.24.1/pkg/source/source.go)", "Bash(grep -n 'type GenericEvent' __CMDSUB_OUTPUT__/sigs.k8s.io/controller-runtime@v0.24.1/pkg/event/event.go)", "Bash(KUBEBUILDER_ASSETS=__TRACKED_VAR__/bin/k8s/1.36.2-darwin-arm64 go test -race ./...)", - "Bash(cat >> *)" + "Bash(cat >> *)", + "Bash(make run-dev *)", + "Bash(kubectl get *)", + "Bash(kubectl delete *)", + "Bash(make docker-build *)", + "Bash(kind load *)", + "Bash(make deploy *)", + "Bash(kubectl -n egress-proxies-operator-system rollout status deploy/egress-proxies-operator-controller-manager --timeout=120s)", + "Bash(kubectl -n egress-proxies-operator-system rollout restart deploy/egress-proxies-operator-controller-manager)" ], "additionalDirectories": [ "/Users/jan.novak/srv/go/egress-proxies-operator/.claude", diff --git a/CHANGELOG.md b/CHANGELOG.md index aa02966..4fe773f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 2026-08-10 09:34 CEST — kind e2e verified; fix Squid OOM in containers; quickstart goes in-cluster + +- Full end-to-end pass on a throwaway kind cluster: Squid pod Ready with a real CONNECT + probe (89 ms), lease grant/report/cooldown-409/release through the discovery API, + finalizer cleanup on delete. +- Fixed the kubernetes provider's generated squid.conf: `max_filedescriptors 1024` + (squid sizes FD tables from the container's effectively-unlimited RLIMIT_NOFILE and + was OOM-killed at startup under kind/containerd) + `cache_mem 16 MB`. +- README quickstart now deploys the operator in-cluster: `make run-dev` on a laptop + cannot reach kind pod IPs, so health probes fail by construction there (documented). + ## 2026-08-09 17:27 CEST — Operator wired end to end: reconciler, health, leases, discovery, GC, two providers - `cmd/main.go` is now the full composition root: `--providers-config` (required, fail-fast), diff --git a/README.md b/README.md index 1311928..106552f 100644 --- a/README.md +++ b/README.md @@ -31,17 +31,26 @@ Details, diagrams, and recorded design decisions: [docs/architecture.md](docs/ar ## Quickstart on kind (~5 minutes) -Requires: kind, kubectl, Go 1.26, jq (optional). The kubernetes-pod -provider needs no cloud account — proxies are real `ubuntu/squid` pods in -the kind cluster itself. +Requires: kind, kubectl, docker, Go 1.26, jq (optional). The +kubernetes-pod provider needs no cloud account — proxies are real +`ubuntu/squid` pods in the kind cluster itself. + +The operator runs **in-cluster** for this quickstart. (Running it on your +laptop with `make run-dev` provisions pods fine, but the health probe then +originates on your machine, which cannot reach kind's pod IPs — the proxy +would sit at `Unhealthy` forever. In-cluster, probes run where the pod +network is routable.) ```sh kind create cluster --name proxy-operator-demo -make install # install the CRD -make run-dev # run the operator locally (foreground) +make install # install the CRD +make docker-build IMG=egress-proxies-operator:dev +kind load docker-image egress-proxies-operator:dev --name proxy-operator-demo +make deploy IMG=egress-proxies-operator:dev +kubectl -n egress-proxies-operator-system rollout status deploy/egress-proxies-operator-controller-manager ``` -In a second terminal: +Create a proxy and watch it come up: ```sh kubectl apply -f config/samples/proxy_kubernetes.yaml @@ -50,7 +59,12 @@ kubectl get px -w # proxy-kubernetes-sample Managed kubernetes Ready 10.244.x.x True ``` -Once it's `Ready`, use the discovery API: +Once it's `Ready`, port-forward the discovery API and use it: + +```sh +kubectl -n egress-proxies-operator-system port-forward \ + svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090 & +``` ```sh # List healthy proxies @@ -156,11 +170,17 @@ Makefile's derived version to 1.36.3, which has none). ## Development ```sh -make test # unit + envtest suites (sets up envtest binaries itself) +make test # unit + envtest suites, with -race (sets up envtest binaries itself) go test -short ./... # skip the envtest suite make run-dev # run against the current kubeconfig context ``` +`make run-dev` is for iterating on the operator itself: provisioning, +replacement, the discovery API, and External proxies all work from your +laptop. Health checks against in-cluster pods do **not** (see the +quickstart note) — use the in-cluster deploy to see a kubernetes-provider +proxy go `Ready`. + Project layout, reconcile-loop diagrams, and the decision log are in [docs/architecture.md](docs/architecture.md); the build history is in [docs/plans-executions/](docs/plans-executions/). diff --git a/docs/plans-executions/2026-08-07-1747-proxy-operator.md b/docs/plans-executions/2026-08-07-1747-proxy-operator.md index 1291560..1c73c49 100644 --- a/docs/plans-executions/2026-08-07-1747-proxy-operator.md +++ b/docs/plans-executions/2026-08-07-1747-proxy-operator.md @@ -16,7 +16,7 @@ Pairs with [docs/plans/2026-08-07-1747-proxy-operator.md](../plans/2026-08-07-17 - [x] Step 9 — Orphan GC + metrics - [x] Step 10 — Wiring, config, docs - [x] Step 11 — Tests -- [ ] Verification (vet/test/kind e2e) + commit, push, open MR +- [x] Verification (vet/test/kind e2e) + commit, push, open MR ## Step 0 — Branch and scaffold @@ -1061,3 +1061,71 @@ test flips mode and adds an endpoint in the same update to isolate the immutability rules as the thing that rejects. The plan's remaining checklist item is Verification: the throwaway-kind-cluster run of the README quickstart, then push + MR. + +## Verification — kind end-to-end + +Static checks first (`go vet ./...`, `make build`, full `make test` with +`-race`): all green. Then the real thing, per the spec's §13 "run the kind +quickstart yourself and fix what breaks" — and two things broke, both now +fixed. + +**Finding 1 — `make run-dev` cannot produce a Ready proxy on kind.** The +operator on the host provisions the pod fine (Provisioned=True, IP +published), but the health probe originates on the host, and kind pod IPs +(10.244.x.x) are not host-routable — every probe fails by construction +and the proxy latches `Unhealthy`: + +```text +Healthy=False: Get "https://www.gstatic.com/generate_204": + proxyconnect tcp: dial tcp 10.244.0.5:3128: connect: connection refused +``` + +Everything around the failure worked exactly as designed (thresholds, +condition, phase, and the finalizer delete ran clean from the host). Fix: +the README quickstart now deploys the operator **in-cluster** +(docker-build → kind load → deploy → port-forward 8090), with the +run-dev limitation documented in both the quickstart and the Development +section. + +**Finding 2 — Squid was OOM-killed at startup in-cluster.** With the +operator deployed in-cluster the pod crash-looped (`OOMKilled`, empty +logs). Root cause: squid sizes its file-descriptor tables from +`RLIMIT_NOFILE`, and containerd under kind sets that effectively +unlimited (~10^9) — squid allocates gigabytes before it ever listens. +Fix in the generated config (`internal/provider/kubernetes/pod.go`): +`max_filedescriptors 1024` (the load-bearing line) plus `cache_mem 16 MB` +(a crawling forward proxy gains nothing from squid's 256 MB default), +with a regression assertion added to `pod_test.go`. + +**With both fixes, the full pass:** + +```bash +kind create cluster --name proxy-operator-demo +make install +make docker-build IMG=egress-proxies-operator:dev +kind load docker-image egress-proxies-operator:dev --name proxy-operator-demo +make deploy IMG=egress-proxies-operator:dev +kubectl apply -f config/samples/proxy_kubernetes.yaml +# → Ready 10.244.0.9 lat=89ms Provisioned=True Healthy=True (~30 s) +kubectl -n egress-proxies-operator-system port-forward svc/...-discovery-service 8090:8090 & +curl -s 'localhost:8090/v1/proxies?healthy=true' # count:1, latencyMillis:89 +curl -s -XPOST localhost:8090/v1/leases -d '{"selector":{"geo":"local"},"ttlSeconds":300}' +# → 201 {leaseID, proxy(activeLeases:1), expiresAt, ttlSeconds:300} +curl -XPOST .../report -d '{"result":"rate_limited","target":"example.com"}' # 204 +curl -XPOST /v1/leases -d '{...,"target":"example.com"}' # 409 {inCooldown:1} ✓ +curl -XDELETE /v1/leases/ # 204, and 204 again ✓ +kubectl delete -f config/samples/proxy_kubernetes.yaml # finalizer: pod Terminating, CR gone +kind delete cluster --name proxy-operator-demo +``` + +A real Squid pod went Ready through a real CONNECT probe, a lease was +held on it, the cooldown machinery answered a 409 with correct +arithmetic, and the finalizer cleaned up — the plan's success bar, met +with the actual product. + +Worth noting: `make deploy` runs `kustomize edit set image` and mutates +`config/manager/kustomization.yaml` in the working tree — reverted before +committing (the repo keeps the pinned stanza). The health probe's ~89 ms +latency is gstatic-through-squid from a kind pod on this machine; +metrics-side observations were not separately checked in-cluster (covered +by unit tests). diff --git a/internal/provider/kubernetes/pod.go b/internal/provider/kubernetes/pod.go index 4e2d467..deb1367 100644 --- a/internal/provider/kubernetes/pod.go +++ b/internal/provider/kubernetes/pod.go @@ -58,10 +58,18 @@ func buildPod(image string, req provider.CreateRequest) *corev1.Pod { // there, never interpreted. via/forwarded_for are turned off so the proxy // doesn't leak the Pod's identity to the origin. func squidConf(port int32) string { + // max_filedescriptors is load-bearing in containers: squid sizes its FD + // tables from RLIMIT_NOFILE at startup, and containerd commonly sets + // that to effectively unlimited (kind: ~10^9) — squid then allocates + // gigabytes and is OOM-killed before it ever listens. cache_mem is + // trimmed because a forwarding proxy for crawling gains nothing from + // squid's 256 MB default cache. return fmt.Sprintf(`http_port %d acl all src 0.0.0.0/0 http_access allow all via off forwarded_for off +max_filedescriptors 1024 +cache_mem 16 MB `, port) } diff --git a/internal/provider/kubernetes/pod_test.go b/internal/provider/kubernetes/pod_test.go index 1062885..de80a94 100644 --- a/internal/provider/kubernetes/pod_test.go +++ b/internal/provider/kubernetes/pod_test.go @@ -72,7 +72,10 @@ func TestBuildPod_usesRequestPort(t *testing.T) { func TestSquidConf_permissive(t *testing.T) { t.Parallel() conf := squidConf(3128) - for _, want := range []string{"http_port 3128", "http_access allow all", "via off", "forwarded_for off"} { + // max_filedescriptors guards against squid sizing its FD tables from a + // container's effectively-unlimited RLIMIT_NOFILE and getting OOM-killed + // at startup — found by the kind verification run, must not regress. + for _, want := range []string{"http_port 3128", "http_access allow all", "via off", "forwarded_for off", "max_filedescriptors 1024"} { if !strings.Contains(conf, want) { t.Errorf("squidConf() = %q, want it to contain %q", conf, want) }