Record the provider health validation and add the changelog entry

The user ran the real-hypervisor checks against beelink and confirmed they
passed, which completes Step 7 of the provider health plan.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.claude/agents/operator-reviewer.md

@@ -0,0 +1,15 @@
+---
+name: operator-reviewer
+description: Reviews Kubernetes operator PRs for controller-runtime correctness, reconcile semantics, and API design
+tools: Read, Grep, Glob, Bash
+---
+You are a senior reviewer specializing in Kubernetes operators.
+Review with focus on:
+- Reconcile idempotency and requeue behavior; no state assumptions between reconciles
+- Informer cache reads vs direct API reads; stale-cache races
+- Finalizer handling, deletion flow, orphaned resources
+- CRD schema evolution, conversion webhooks, status subresource / conditions conventions
+- RBAC minimality vs what the controller actually touches
+- Leader election, watch predicates, event filtering for churn reduction
+- Go: context propagation, error wrapping, client.Object handling
+Output: findings ranked by severity, with file:line refs. No praise padding.

.claude/settings.json

@@ -63,7 +63,43 @@
       "Bash(grep -n 'func Channel' -A8 __CMDSUB_OUTPUT__/sigs.k8s.io/controller-runtime@v0.24.1/pkg/source/source.go)",
       "Bash(grep -n 'type GenericEvent' __CMDSUB_OUTPUT__/sigs.k8s.io/controller-runtime@v0.24.1/pkg/event/event.go)",
       "Bash(KUBEBUILDER_ASSETS=__TRACKED_VAR__/bin/k8s/1.36.2-darwin-arm64 go test -race ./...)",
-      "Bash(cat >> *)"
+      "Bash(cat >> *)",
+      "Bash(make run-dev *)",
+      "Bash(kubectl get *)",
+      "Bash(kubectl delete *)",
+      "Bash(make docker-build *)",
+      "Bash(kind load *)",
+      "Bash(make deploy *)",
+      "Bash(kubectl -n egress-proxies-operator-system rollout status deploy/egress-proxies-operator-controller-manager --timeout=120s)",
+      "Bash(kubectl -n egress-proxies-operator-system rollout restart deploy/egress-proxies-operator-controller-manager)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator add docs/plans/2026-08-11-1742-gcp-provider-verbose-logging.md)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator commit -m 'Add plan: verbose V-level logging in the GCP provider *)",
+      "Bash(echo \"exit: $?\")",
+      "Bash(echo \"tests exit: $?\")",
+      "Bash(./bin/manager --version)",
+      "Bash(./bin/manager-stamped --version)",
+      "Bash(./bin/manager-pkg --version)",
+      "Bash(docker run *)",
+      "Bash(kubectl -n egress-proxies-operator-system get pods -o wide)",
+      "Bash(kubectl -n egress-proxies-operator-system get deploy egress-proxies-operator-controller-manager -o jsonpath='{.spec.template.spec.containers[0].args}')",
+      "Bash(kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager)",
+      "Bash(python3 -c \"import json; d=json.load\\(open\\('docs/deploy/sa_key.json'\\)\\); print\\(d.get\\('type'\\), d.get\\('client_email'\\)\\)\")",
+      "Bash(tea pr *)",
+      "Bash(git worktree *)",
+      "Bash(python3 -c \"import yaml; yaml.safe_load\\(open\\('.gitea/workflows/build.yaml'\\)\\); print\\('YAML OK'\\)\")",
+      "Bash(ruby -ryaml -e \"YAML.load_file\\('.gitea/workflows/build.yaml'\\); puts 'YAML OK'\")",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator status --short --branch)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator log --oneline -1)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator tag 0.01)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator push origin 0.01)",
+      "Bash(chmod +x docs/demo/run-demo.sh)",
+      "Bash(bash -n docs/demo/run-demo.sh)",
+      "Bash(git add *)",
+      "Bash(git commit -q -m 'Add plan: KVM/libvirt provider *)",
+      "Bash(go get *)",
+      "Bash(go build *)",
+      "Bash(go test *)",
+      "Bash(git commit *)"
     ],
     "additionalDirectories": [
       "/Users/jan.novak/srv/go/egress-proxies-operator/.claude",

.gitea/workflows/build.yaml

@@ -0,0 +1,76 @@
+name: Build and Push
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Image tag'
+        required: true
+        default: 'latest'
+  push:
+    tags:
+      - '*'
+
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Vet
+        run: go vet ./...
+      - name: Build
+        run: go build ./...
+      - name: Test (short)
+        run: go test -short ./...
+
+  build:
+    needs: check
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Compute image tags
+        id: meta
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            TAG="${{ inputs.tag }}"
+          else
+            TAG="${{ github.ref_name }}"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "sha=sha-$(echo '${{ github.sha }}' | cut -c1-12)" >> "$GITHUB_OUTPUT"
+
+      - name: Login to Gitea registry
+        run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login -u ${{ github.actor }} --password-stdin gitea.home.hrajfrisbee.cz
+
+      - name: Build and push
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker build \
+            --build-arg GIT_COMMIT=$(echo '${{ github.sha }}' | cut -c1-12) \
+            --label org.opencontainers.image.source=https://gitea.home.hrajfrisbee.cz/${{ github.repository }} \
+            --label org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
+            -t "$IMAGE:${{ steps.meta.outputs.tag }}" \
+            -t "$IMAGE:${{ steps.meta.outputs.sha }}" \
+            .
+          docker push "$IMAGE:${{ steps.meta.outputs.tag }}"
+          docker push "$IMAGE:${{ steps.meta.outputs.sha }}"
+
+      # Only real tag pushes move :latest — an ad-hoc dispatch of an old ref must not clobber it.
+      - name: Push latest (tag builds only)
+        if: github.event_name == 'push'
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker tag "$IMAGE:${{ steps.meta.outputs.tag }}" "$IMAGE:latest"
+          docker push "$IMAGE:latest"

.gitignore

@@ -28,3 +28,6 @@ go.work
 
 # Kubeconfig might contain secrets
 *.kubeconfig
+
+# GCP service-account keys (created per docs/gcp-in-specific-project.md)
+sa_key.json

CHANGELOG.md

@@ -1 +1,132 @@
 # Changelog
+
+## 2026-09-16 08:45 UTC — Provider health checks: ProviderHealth CRD, per-provider checks, gauges
+
+- **Each configured provider is now checked on a schedule** and reported in a
+  cluster-scoped `ProviderHealth` object (`kubectl get ph`), one per provider, with a
+  `Ready` condition, a reason, and per-target verdicts.
+- **Checks:** kubernetes lists one object from the API server; GCP fetches a single
+  instance page; KVM checks SSH/libvirt reachability and the storage pool on every
+  host, in parallel. Failures map to fixed reasons (`Unreachable`,
+  `SSHHandshakeFailed`, `PoolNotFound`, `Timeout`, …).
+- **The KVM check can't hurt reconciles:** it uses `hostPool.probe`, which gives up on
+  a slow RPC without dropping the shared connection, and waiting for a host slot can
+  be interrupted.
+- **Metrics:** `proxy_operator_provider_target_up`, `_check_duration_seconds` and
+  `_check_timestamp_seconds`. The flags are `--provider-check-interval` and
+  `--provider-check-timeout` (defaults to `min(15s, interval)`).
+- Validated with `make test`, `make test-e2e` and the real-hypervisor KVM run against
+  beelink. `make lint` is blocked by a Go 1.27 toolchain/staticcheck panic that has
+  nothing to do with this change.
+- Key files: `api/v1alpha1/providerhealth_types.go`, `internal/provider/check.go`,
+  `internal/provider/kvm/`, `internal/providerhealth/`, `docs/architecture.md`.
+
+## 2026-09-10 22:58 UTC — Code-review fixes on the KVM branch: deletion, validation, lazy SSH, guest agent
+
+Five findings from a review of `feat/kvm-provider`, all fixed on the branch.
+
+- **A Proxy can always finish deleting.** The delete path used to drop the finalizer
+  only when `Get` answered not-found, so a permanent error there retried forever —
+  a KVM host removed from the config was enough. It now calls `Delete` anyway, since
+  that is idempotent and can finish where `Get` cannot, and
+  `crawl.example.com/skip-cleanup: "true"` abandons the instance for whatever stays
+  stuck. In the KVM provider, deleting over an unconfigured host is a no-op success;
+  `Get` still fails permanently there so a typo cannot look like a missing VM.
+- **The guest agent ships as a vendor-data script**, superseding the cloud-config
+  merge in the entry below. Merging could never be made safe: cloud-init drops
+  vendor-data keys the user-data defines, and the strategy cannot be overridden from
+  either document. The Proxy's cloud-init now reaches the seed byte for byte in any
+  format — Jinja, MIME, gzip, scripts — where the round-trip previously re-typed
+  unquoted scalars (`version: 1.10` arrived as `1.1`), dropped comments and reordered
+  keys. New `installGuestAgent` setting, default on.
+- **A missing `kvm-ssh` Secret no longer crash-loops the manager.** The SSH material
+  is read on first use and cached, so only KVM Proxies fail and they recover when the
+  Secret appears. A missing file is transient, an unparseable key permanent, and
+  startup still logs a warning.
+- **Host names and image keys are validated.** Either can end up in a providerID or a
+  volume file name, where a slash made the VM unaddressable for good.
+- Verified end to end: unit suite, the integration test against a real hypervisor
+  (22 s, agent reporting the IP), both in-cluster Proxies Ready in 2 min with 204
+  through squid, the guest showing `qemu-guest-agent` active with its cloud-config
+  comments intact, and the manager staying up with the Secret pointed away.
+- Key files: `internal/provider/kvm/{seed.go,conn.go,kvm.go}`,
+  `internal/controller/proxy_controller.go`, `internal/provider/config_kvm.go`,
+  `api/v1alpha1/proxy_types.go`.
+
+## 2026-09-10 21:04 UTC — KVM provider passes in-cluster; guest agent merged into cloud-config
+
+- Two KVM Proxies with real cloud-configs reach `Ready` from inside the kind cluster
+  (2 min from apply, IP via the guest agent, 204 through squid from the LAN).
+- The `qemu-guest-agent` bootstrap is now merged into each Proxy's cloud-config
+  (`packages` + `runcmd`) instead of shipped as vendor-data: cloud-init's default
+  merger drops vendor-data keys the user-data also defines, so any Proxy with its own
+  `packages:` never got the agent and stayed `Provisioning` forever. `vendorData` is
+  now plain extra vendor-data with no default; non-cloud-config user-data (scripts,
+  `#include`) still gets the bootstrap as vendor-data. A cloud-config that is not a
+  YAML mapping fails the Proxy permanently instead of booting without its config.
+- The manager pod mounts a 64 MiB `emptyDir` on `/tmp`: the seed ISO writer stages
+  files on disk and the root filesystem is read-only.
+- Errors already classified with a provider sentinel no longer drop the cached
+  libvirt SSH connection; only libvirt and transport errors do.
+- Key files: `internal/provider/kvm/{seed.go,conn.go,errors.go}`,
+  `config/manager/manager.yaml`, `docs/kvm-provider.md`.
+
+## 2026-09-10 10:07 UTC — KVM provider validated on a real hypervisor; UEFI default; libvirt call tracing
+
+- `make test-integration-kvm` passes end to end against a libvirt 11.6 / QEMU 10.1
+  host (Debian 13 image, UEFI, guest agent reporting the IP after ~20 s). Four
+  defects found only on the real host, all fixed on the branch:
+  - domain disks are now `type="file"` with resolved paths: pool-volume references
+    get no AppArmor rule from `virt-aa-helper`, so QEMU was denied the disks;
+  - domain metadata is parsed by local name: libvirt returns the element without
+    its namespace declaration;
+  - guests boot **UEFI by default** (`images.<key>.firmware: uefi|bios`, OVMF now a
+    host prerequisite): Debian 13's image hangs in GRUB under SeaBIOS on a headless
+    VM with a serial console (Debian bug #1111240, open); bookworm is unaffected;
+  - the image cache's raw-upload path is confirmed to produce a byte-identical file.
+- Every libvirt RPC is logged at `--zap-log-level=debug` as one `libvirt call` line
+  (`rpc=virDomainCreate`, identifiers, duration, `libvirtError` on failure); level 2
+  adds XML, metadata and addresses; cloud-init is never logged. The integration test
+  shows the trace in `-v` output (`KVM_TEST_LOG_LEVEL`, default 1).
+- Key files: `internal/provider/kvm/{domainxml.go,metadata.go,trace.go,conn.go}`,
+  `internal/provider/config_kvm.go`, `docs/kvm-provider.md`.
+
+## 2026-09-08 11:00 UTC — KVM/libvirt provider
+
+- New `kvm` provider type: Managed proxies as VMs on your own libvirt/KVM hosts. Each
+  host is reached over SSH with the libvirt unix socket forwarded through the
+  connection (pure Go, nothing to expose on the hypervisors beyond sshd); one provider
+  instance can span several hosts, selected by `placement.zone`.
+- Base images (`images:` map or a direct URL in `placement.image`) are downloaded by the
+  operator into each host's storage pool on first use, verified by sha256 when configured,
+  and cached there. Per-VM disks are qcow2 overlays; cloud-init arrives on a NoCloud seed
+  ISO built in-process. GC identity lives in the domain's `<metadata>` element.
+- Wiring: `"kvm": kvm.New` in `cmd/main.go`; optional `kvm-ssh` Secret mounted at
+  `/etc/proxy-operator/kvm`; samples `config/samples/proxy_kvm.yaml` and a `kvm-home`
+  block in `providers-config.yaml`; `docs/kvm-provider.md`; opt-in integration test
+  (`make test-integration-kvm`) against a real hypervisor.
+- Not validated on a real host at the time of this entry; see 2026-09-10 above.
+
+## 2026-08-10 09:34 CEST — kind e2e verified; fix Squid OOM in containers; quickstart goes in-cluster
+
+- Full end-to-end pass on a throwaway kind cluster: Squid pod Ready with a real CONNECT
+  probe (89 ms), lease grant/report/cooldown-409/release through the discovery API,
+  finalizer cleanup on delete.
+- Fixed the kubernetes provider's generated squid.conf: `max_filedescriptors 1024`
+  (squid sizes FD tables from the container's effectively-unlimited RLIMIT_NOFILE and
+  was OOM-killed at startup under kind/containerd) + `cache_mem 16 MB`.
+- README quickstart now deploys the operator in-cluster: `make run-dev` on a laptop
+  cannot reach kind pod IPs, so health probes fail by construction there (documented).
+
+## 2026-08-09 17:27 CEST — Operator wired end to end: reconciler, health, leases, discovery, GC, two providers
+
+- `cmd/main.go` is now the full composition root: `--providers-config` (required, fail-fast),
+  `--discovery-addr`, `--proxy-namespace`, `--health-workers`, `--gc-interval`, `--gc-min-age`,
+  `--gc-allow-namespaced`, `--lease-cooldown`, `--max-lease-ttl`; wires the kubernetes + gcp
+  providers (metrics-instrumented), health engine, lease store, discovery API, orphan GC, and
+  Prometheus metrics onto one manager.
+- Deploy manifests: providers ConfigMap mount, optional `DISCOVERY_TOKEN` Secret env,
+  discovery port 8090 + Service; pods RBAC for the kubernetes provider.
+- Samples for all three proxy flavors + providers-config; `make run-dev` for local development.
+- README rewritten (kind quickstart, GCP setup, the two load-bearing caveats); architecture doc
+  completed with components table and the full decision log.

CLAUDE.md

@@ -25,10 +25,11 @@ entrypoint yet, so there's nothing to run locally.
 
 ## Architecture
 
-Not yet built. TODO: once the controller-runtime manager and reconcile loop exist,
-document the data-flow diagram and package/role table here, following the standard
-reconcile-loop shape: fetch → resolve → list related → compute desired state → merge
-→ patch only on diff.
+Documented in [docs/architecture.md](docs/architecture.md). It holds the component
+table, the reconcile flow, and one section per runnable: health engine, discovery and
+lease API, orphan GC, metrics, tracing, and provider health checks. Provider-specific
+detail is in `docs/kvm-provider.md` and the GCP docs; the test layout is in
+`docs/testing.md`.
 
 ## Conventions
 
@@ -199,12 +200,24 @@ Always append a `Co-Authored-By` trailer to indicate AI assistance:
 
     Co-Authored-By: Claude <noreply@anthropic.com>
 
-TODO: no `.gitea/workflows/` CI pipeline exists yet — add a CI/CD subsection here once
-one is set up.
+### CI/CD
+
+`.gitea/workflows/build.yaml` builds the manager image and pushes it to the Gitea
+registry. Triggers: any tag push, or manual `workflow_dispatch` with a `tag` input.
+A lightweight `check` job (`go vet` / `go build` / `go test -short`) gates the build.
+
+- Images: `gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator:<tag>` plus an
+  immutable `sha-<12-char-commit>` tag on every build; `:latest` moves only on real
+  tag pushes, never on manual dispatch.
+- Requires the `REGISTRY_TOKEN` repo secret (Gitea PAT with `write:package`),
+  same convention as the other projects on this Gitea instance.
+- The commit is baked into the binary via the `GIT_COMMIT` build arg (see Dockerfile).
 
 ## Gotchas
 
-- This repo currently contains only `go.mod` (module
-  `gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator`) — no controller code,
-  `main.go`, Dockerfile, k8s manifests, Makefile, or CI config exist yet. Nothing in
-  this file describes working software; it's the convention scaffold to build against.
+- The "What this is" and "Commands" sections above predate the code and are stale.
+  The Makefile targets (`make test`, `make manifests generate`, `make test-e2e`,
+  `make test-integration-kvm`) and `docs/testing.md` are authoritative.
+- `+kubebuilder:rbac` markers are package-level. Keep them in their own comment block,
+  never inside a type's or function's doc comment: controller-gen silently ignores
+  them there. After `make manifests`, check `git diff config/rbac/role.yaml`.

Dockerfile

@@ -2,6 +2,7 @@
 FROM golang:1.26 AS builder
 ARG TARGETOS
 ARG TARGETARCH
+ARG GIT_COMMIT=unknown
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -19,11 +20,15 @@ COPY . .
 # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a \
+    -ldflags "-X gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version.Commit=${GIT_COMMIT}" \
+    -o manager cmd/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
+ARG GIT_COMMIT=unknown
+LABEL org.opencontainers.image.revision="${GIT_COMMIT}"
 WORKDIR /
 COPY --from=builder /workspace/manager .
 USER 65532:65532

Makefile

@@ -2,6 +2,9 @@
 IMG ?= controller:latest
 # YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header.
 YEAR ?= $(shell date +%Y)
+# GIT_COMMIT is baked into the image (-ldflags in the Dockerfile); -dirty
+# covers staged and untracked changes too, which `git diff --quiet` misses.
+GIT_COMMIT ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)$(shell test -z "$$(git status --porcelain 2>/dev/null)" || echo -dirty)
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -61,7 +64,11 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet setup-envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test -race $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+
+.PHONY: test-integration-kvm
+test-integration-kvm: ## Run the KVM provider test against a real hypervisor. Needs KVM_TEST_HOST/USER/SSH_KEY/BRIDGE/IMAGE_URL; see docs/testing.md.
+	go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/
 
 # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
 # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
@@ -87,7 +94,8 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist
 
 .PHONY: test-e2e
 test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
-	KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v
+	KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) TEMPO_URL=$(TEMPO_URL) OTLP_ENDPOINT=$(OTLP_ENDPOINT) \
+		go test -tags=e2e ./test/e2e/ -v -ginkgo.v -timeout 30m
 	$(MAKE) cleanup-test-e2e
 
 .PHONY: cleanup-test-e2e
@@ -110,18 +118,22 @@ lint-config: golangci-lint ## Verify golangci-lint linter configuration
 
 .PHONY: build
 build: manifests generate fmt vet ## Build manager binary.
-	go build -o bin/manager cmd/main.go
+	go build -o bin/manager ./cmd
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./cmd/main.go
 
+.PHONY: run-dev
+run-dev: manifests generate fmt vet ## Run locally against the current kubeconfig with the kubernetes-pod provider.
+	go run ./cmd/main.go --providers-config hack/providers-dev.yaml --metrics-bind-address :8080 --metrics-secure=false
+
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
 docker-build: ## Build docker image with the manager.
-	$(CONTAINER_TOOL) build -t ${IMG} .
+	$(CONTAINER_TOOL) build --build-arg GIT_COMMIT=$(GIT_COMMIT) -t ${IMG} .
 
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
@@ -140,7 +152,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform
 	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
 	- $(CONTAINER_TOOL) buildx create --name egress-proxies-operator-builder
 	$(CONTAINER_TOOL) buildx use egress-proxies-operator-builder
-	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
+	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --build-arg GIT_COMMIT=$(GIT_COMMIT) --tag ${IMG} -f Dockerfile.cross .
 	- $(CONTAINER_TOOL) buildx rm egress-proxies-operator-builder
 	rm Dockerfile.cross
 
@@ -166,6 +178,20 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 	@out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \
 	if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi
 
+KVM_SSH_KEY ?= $(HOME)/.ssh/proxy-operator_ed25519
+KVM_KNOWN_HOSTS ?= $(HOME)/.ssh/kvm_known_hosts
+KVM_SSH_NAMESPACE ?= egress-proxies-operator-system
+
+.PHONY: kvm-ssh-secret
+kvm-ssh-secret: ## Create or update the kvm-ssh Secret from KVM_SSH_KEY (private key) and KVM_KNOWN_HOSTS.
+	@test -f "$(KVM_SSH_KEY)" || { echo "KVM_SSH_KEY=$(KVM_SSH_KEY) not found"; exit 1; }
+	@test -f "$(KVM_KNOWN_HOSTS)" || { echo "KVM_KNOWN_HOSTS=$(KVM_KNOWN_HOSTS) not found"; exit 1; }
+	@grep -q 'PRIVATE KEY' "$(KVM_SSH_KEY)" || { echo "KVM_SSH_KEY=$(KVM_SSH_KEY) is not a private key (did you pass the .pub?)"; exit 1; }
+	"$(KUBECTL)" -n $(KVM_SSH_NAMESPACE) create secret generic kvm-ssh \
+		--from-file=id_ed25519="$(KVM_SSH_KEY)" \
+		--from-file=known_hosts="$(KVM_KNOWN_HOSTS)" \
+		--dry-run=client -o yaml | "$(KUBECTL)" apply -f -
+
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG}

PROJECT

@@ -18,4 +18,11 @@ resources:
   kind: Proxy
   path: gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+  domain: example.com
+  group: crawl
+  kind: ProviderHealth
+  path: gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1
+  version: v1alpha1
 version: "3"

README.md

@@ -1,135 +1,334 @@
 # egress-proxies-operator
-// TODO(user): Add simple overview of use/purpose
 
-## Description
-// TODO(user): An in-depth paragraph about your project and overview of use
+A Kubernetes operator that manages a fleet of HTTP egress proxies for
+crawling: each proxy is a `Proxy` custom resource that the operator
+provisions (or merely tracks), actively health-checks **through the proxy
+itself**, and hands out to crawler clients via an HTTP list/lease API.
 
-## Getting Started
+## Architecture in 60 seconds
 
-### Prerequisites
-- go version v1.24.6+
-- docker version 17.03+.
-- kubectl version v1.11.3+.
-- Access to a Kubernetes v1.11.3+ cluster.
+- **`Proxy` CRD** (`crawl.example.com/v1alpha1`, namespaced, `kubectl get px`):
+  `Managed` proxies are provisioned by a configured provider; `External`
+  proxies exist elsewhere and are only tracked and health-checked.
+- **Reconciler** — a crash-safe state machine: every reconcile derives one
+  action from (spec, status, provider Get). Proxies are **immutable
+  cattle**: any meaningful spec change (placement, cloud-init, port)
+  deletes and recreates the VM — never in-place mutation.
+- **Providers** behind one minimal interface: `kubernetes` (a real Squid
+  pod in this cluster — local dev/CI), `gcp` (Compute Engine VMs with
+  ephemeral external IPs — the real egress fleet) and `kvm` (VMs on your
+  own libvirt hypervisors, reached over SSH). Config is a YAML file
+  (`--providers-config`) with named instances (`gcp-eu`, `kvm-home`, ...).
+- **Health engine** probes every proxy by fetching a URL *through* it (a
+  real CONNECT tunnel — a proxy that accepts TCP but can't egress goes
+  Unhealthy), with threshold logic and transition-only status writes.
+- **Discovery API** (`:8090`): list healthy proxies filtered by
+  attributes, lease one (least-loaded, TTL-based), release, and report
+  rate-limiting — reports put the proxy in a per-target cooldown.
+- **Orphan GC** sweeps each provider for tagged instances whose owning CR
+  is gone — the safety net for crashes mid-create.
 
-### To Deploy on the cluster
-**Build and push your image to the location specified by `IMG`:**
+Details, diagrams, and recorded design decisions: [docs/architecture.md](docs/architecture.md).
+
+## Quickstart on kind (~5 minutes)
+
+Requires: kind, kubectl, docker, Go 1.26, jq (optional). The
+kubernetes-pod provider needs no cloud account — proxies are real
+`ubuntu/squid` pods in the kind cluster itself.
+
+The operator runs **in-cluster** for this quickstart. (Running it on your
+laptop with `make run-dev` provisions pods fine, but the health probe then
+originates on your machine, which cannot reach kind's pod IPs — the proxy
+would sit at `Unhealthy` forever. In-cluster, probes run where the pod
+network is routable.)
 
 ```sh
-make docker-build docker-push IMG=<some-registry>/egress-proxies-operator:tag
+kind create cluster --name proxy-operator-demo
+make install                                          # install the CRD
+make docker-build IMG=egress-proxies-operator:dev
+kind load docker-image egress-proxies-operator:dev --name proxy-operator-demo
+make deploy IMG=egress-proxies-operator:dev
+kubectl -n egress-proxies-operator-system rollout status deploy/egress-proxies-operator-controller-manager
 ```
 
-**NOTE:** This image ought to be published in the personal registry you specified.
-And it is required to have access to pull the image from the working environment.
-Make sure you have the proper permission to the registry if the above commands don’t work.
-
-**Install the CRDs into the cluster:**
+Create a proxy and watch it come up:
 
 ```sh
-make install
+kubectl apply -f config/samples/proxy_kubernetes.yaml
+kubectl get px -w
+# NAME                      MODE      PROVIDER     PHASE   IP           HEALTHY
+# proxy-kubernetes-sample   Managed   kubernetes   Ready   10.244.x.x   True
 ```
 
-**Deploy the Manager to the cluster with the image specified by `IMG`:**
+Once it's `Ready`, port-forward the discovery API and use it (full
+reference with schemas and error codes: [docs/api.md](docs/api.md)):
 
 ```sh
-make deploy IMG=<some-registry>/egress-proxies-operator:tag
+kubectl -n egress-proxies-operator-system port-forward \
+  svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090 &
 ```
 
-> **NOTE**: If you encounter RBAC errors, you may need to grant yourself cluster-admin
-privileges or be logged in as admin.
-
-**Create instances of your solution**
-You can apply the samples (examples) from the config/sample:
-
 ```sh
-kubectl apply -k config/samples/
+# List healthy proxies
+curl -s 'localhost:8090/v1/proxies?healthy=true' | jq
+
+# Lease one (5-minute TTL)
+curl -s -XPOST localhost:8090/v1/leases \
+  -d '{"selector":{"geo":"local"},"ttlSeconds":300}' | jq
+# → {"leaseID":"...", "proxy":{"id":"default/proxy-kubernetes-sample", "ip":..., ...}}
+
+# Actually crawl through it (from inside the cluster, or port-forward the pod)
+# curl -x http://<proxy-ip>:3128 https://example.com
+
+# Report the proxy got rate-limited by a site → 15-minute cooldown for that target
+curl -s -XPOST localhost:8090/v1/leases/<leaseID>/report \
+  -d '{"result":"rate_limited","target":"example.com"}'
+
+# Release early (idempotent — 204 both times)
+curl -si -XDELETE localhost:8090/v1/leases/<leaseID>
 ```
 
->**NOTE**: Ensure that the samples has default values to test it out.
-
-### To Uninstall
-**Delete the instances (CRs) from the cluster:**
+Tear down:
 
 ```sh
-kubectl delete -k config/samples/
+kubectl delete -f config/samples/proxy_kubernetes.yaml   # finalizer deletes the pod
+kind delete cluster --name proxy-operator-demo
 ```
 
-**Delete the APIs(CRDs) from the cluster:**
+## Deploying in-cluster
 
 ```sh
-make uninstall
+make docker-build IMG=<registry>/egress-proxies-operator:dev
+make deploy IMG=<registry>/egress-proxies-operator:dev
 ```
 
-**UnDeploy the controller from the cluster:**
+- Provider config comes from the `providers-config` ConfigMap
+  ([config/manager/providers_config.yaml](config/manager/providers_config.yaml));
+  the default ships only the kubernetes provider.
+- The discovery API is exposed by the
+  `controller-manager-discovery-service` Service on port 8090.
+- Auth: create the token Secret, or the API serves **unauthenticated**
+  (it warns loudly at startup):
+
+  ```sh
+  kubectl -n egress-proxies-operator-system create secret generic discovery-token \
+    --from-literal=token="$(openssl rand -hex 24)"
+  ```
+
+## GCP setup
+
+1. Add a `gcp` entry to the providers config (see
+   [config/samples/providers-config.yaml](config/samples/providers-config.yaml)) —
+   only `project` is required.
+2. Credentials are **Application Default Credentials**: workload identity
+   in-cluster, `gcloud auth application-default login` locally. No
+   key-file plumbing exists.
+3. The identity needs `roles/compute.instanceAdmin.v1` on the project —
+   plus `roles/iam.serviceAccountUser` if instances attach a service
+   account.
+4. Managed GCP proxies must set all of `placement.zone`,
+   `placement.machineType`, and `placement.image`
+   (see [config/samples/proxy_gcp.yaml](config/samples/proxy_gcp.yaml),
+   which also installs Squid via cloud-init). A missing field fails the
+   Proxy with a message naming it.
+
+## KVM setup
+
+Managed proxies as VMs on libvirt/KVM hosts you own. Full guide:
+[docs/kvm-provider.md](docs/kvm-provider.md).
+
+1. Each host needs `libvirtd`, an SSH user in the `libvirt` group, a
+   bridge to a network with DHCP, and a storage pool.
+2. Create the `kvm-ssh` Secret from the operator's **private** key and a
+   `known_hosts` with each host's key; the manager mounts it at
+   `/etc/proxy-operator/kvm`:
+
+   ```sh
+   ssh-keyscan -H <host> >> ~/.ssh/kvm_known_hosts
+   kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
+     --from-file=id_ed25519=$HOME/.ssh/proxy-operator_ed25519 \
+     --from-file=known_hosts=$HOME/.ssh/kvm_known_hosts
+   # or: make kvm-ssh-secret   (same paths by default; KVM_SSH_KEY / KVM_KNOWN_HOSTS to override)
+   ```
+
+3. Add a `kvm` entry to the providers config with `images`,
+   `machineTypes` and `hosts` (see
+   [config/samples/providers-config.yaml](config/samples/providers-config.yaml)).
+   Base images are downloaded into each host's pool on first use.
+4. A Proxy names a host (`placement.zone`, optional with one host), a
+   size (`placement.machineType`) and an image (`placement.image`) — see
+   [config/samples/proxy_kvm.yaml](config/samples/proxy_kvm.yaml).
+
+Cloud-init from a Secret: the Secret **must** carry the label
+`crawl.example.com/cloud-init: "true"` — the operator's cache only holds
+labelled Secrets, so an unlabelled one is invisible (the Proxy reports
+`CloudInitError`). Rotating the Secret's content triggers VM replacement.
+
+Stuck deletion: a Proxy stays `Terminating` while the operator keeps failing
+to clean its instance up — credentials revoked, a provider or a KVM host
+dropped from the config. Annotate it with
+`crawl.example.com/skip-cleanup: "true"` to abandon the instance and release
+the object. Whatever is still running then has to be removed by hand.
+
+## Providers
+
+`Managed` proxies are provisioned by a provider — a small compute backend
+behind one minimal interface. Providers are configured in the
+`--providers-config` YAML file as **named instances**: `spec.provider` on a
+Proxy refers to an entry's `name`, not its `type`, so `gcp-eu` and `gcp-us`
+can be two differently-configured instances of the same `gcp` type (see
+[config/samples/providers-config.yaml](config/samples/providers-config.yaml)).
+
+### Implemented providers
+
+| Type | Creates | Per-instance config | Notes |
+| --- | --- | --- | --- |
+| `kubernetes` | A real Squid pod (`ubuntu/squid:6.6-24.04_edge` by default) in the same cluster the operator runs in | `image` (optional) | Needs no cloud account — local dev, CI, and the kind quickstart. Pods share the cluster's egress IP, so it exercises the full lifecycle but not distinct egress paths. |
+| `gcp` | A Compute Engine VM with an ephemeral external IP | `project` (required), `network`, `networkTag`, `diskSizeGb` | The real egress fleet. Deliberately uses only four API calls (Insert / Get / Delete / AggregatedList), all fire-and-forget: `Create` returns as soon as the operation is submitted and the reconciler discovers progress by polling `Get`. Auth is Application Default Credentials — workload identity in-cluster, `gcloud` ADC locally; no key-file plumbing. |
+
+### Adding a provider
+
+A new backend (Hetzner, AWS, ...) is four pieces; the contract lives in
+[internal/provider/provider.go](internal/provider/provider.go):
+
+1. **Implement the 4-method `Provider` interface** in a new
+   `internal/provider/<type>/` package:
+   - `Create` submits and returns — it never blocks until the VM runs, and
+     must be idempotent keyed on `req.Name` (a deterministic name derived
+     from the Proxy's UID), so a repeat call after a crash finds the
+     existing instance instead of duplicating it.
+   - `Get` returns `provider.ErrNotFound` as a *normal* outcome — the
+     reconciler branches on it for replacement and adoption, so don't
+     treat it as exceptional.
+   - `Delete` is idempotent: deleting an already-gone instance is not an
+     error.
+   - `ListByTag` returns every instance the operator ever tagged, for
+     orphan GC.
+2. **Tag every created resource** with `LabelManaged=true` and
+   `LabelUID=<Proxy UID>`, and report `CreatedAt` — orphan GC relies on
+   all three to find owned resources and skip in-flight creates.
+3. **Classify every returned error** with `provider.Wrap` into the
+   four-sentinel taxonomy in
+   [internal/provider/errors.go](internal/provider/errors.go)
+   (`ErrNotFound` / `ErrQuotaExceeded` / `ErrTransient` / `ErrPermanent`)
+   — the reconciler decides retry, slow backoff, or latching `Failed`
+   purely from that classification, never from provider-specific types.
+4. **Wire it up**: add a type-specific config block in
+   [internal/provider/config.go](internal/provider/config.go), and
+   register the constructor in the builtins map in
+   [cmd/main.go](cmd/main.go) (`"<type>": <pkg>.New`). The registry
+   ([internal/provider/registry](internal/provider/registry/registry.go))
+   handles named instances, and the metrics wrapper is applied
+   automatically.
+
+Test against a fake API seam rather than the real cloud — see the
+`instancesAPI` seam in
+[internal/provider/gcp/gcp.go](internal/provider/gcp/gcp.go) for the
+pattern.
+
+## Tracing
+
+OpenTelemetry tracing is built in but **off by default** — the operator
+only exports spans when pointed at an OTLP receiver:
+
+```bash
+# In config/manager/manager.yaml (commented-out block is already there):
+#   OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger-collector.observability:4318
+# Local dev: print spans to stdout instead
+OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml
+```
+
+Every reconcile, discovery API request, and GC sweep becomes a trace, with
+child spans for provider and Kubernetes API calls; log lines inside a traced
+operation carry matching `traceID`/`spanID` fields (point Grafana/Loki
+derived fields at `traceID`). Sampling and endpoints follow the standard
+`OTEL_*` env vars; `--trace-health-probes` additionally traces each health
+probe (high volume). Details in `docs/architecture.md` §10.
+
+## Caveats — read these two
+
+**Changing a proxy changes its IP.** Proxies are immutable cattle: editing
+`placement`, `cloudInit` (or rotating its Secret), or `port` deletes the
+VM and creates a replacement with the **same name but a new IP**. Clients
+discover the new address via the discovery API; anything that pinned the
+old IP breaks by design.
+
+**Operator restart drops all leases and cooldowns.** Lease state is
+in-memory (`replicas: 1` accordingly). Clients must tolerate a lease
+vanishing — requests through the proxy keep working; they just re-lease.
+The lease store sits behind an interface so a persistent backend can
+replace it without touching the API handlers.
+
+Smaller notes:
+
+- `status.lastHealthCheckTime` is the time of the last *status-affecting*
+  probe, not the most recent probe — status writes are transition-only by
+  design. True probe recency lives in the metrics
+  (`proxy_operator_healthcheck_*`).
+- The discovery API is served by every replica but is not leader-elected;
+  the operator ships with `replicas: 1` (see the lease caveat above).
+
+## Version pins
+
+Built and verified against the spec's pins with **no substitutions
+needed**: Go 1.26, kubebuilder v4.15.0, controller-runtime v0.24.1,
+k8s.io/* v0.36.3 (Kubernetes 1.36 API level), controller-tools v0.21.0,
+cloud.google.com/go/compute v1.65.0. envtest uses the 1.36.2 binary
+bundle (the latest 1.36 patch with published binaries — do not "fix" the
+Makefile's derived version to 1.36.3, which has none).
+
+## Gitea CI
+
+[.gitea/workflows/build.yaml](.gitea/workflows/build.yaml) builds the
+manager image and pushes it to this Gitea instance's container registry.
+It runs on **any tag push** or manually via **Run workflow** (with a `tag`
+input) — never on branch pushes. A lightweight `check` job (`go vet`,
+`go build`, `go test -short`) gates the build.
+
+Every build pushes two tags to
+`gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator`:
+
+- the human tag (the git tag, or the dispatch input), and
+- an immutable `sha-<12-char-commit>` tag — pin deployments to this one.
+
+`:latest` is additionally updated on real tag pushes only, so a manual
+dispatch of an old ref can never clobber it. The commit is baked into the
+binary (`internal/version.Commit`) via the `GIT_COMMIT` build arg.
+
+### Mandatory Gitea secrets
+
+Set under **Settings → Actions → Secrets** in this repo:
+
+| Secret           | Required by                   | What it is                               |
+| ---------------- | ----------------------------- | ---------------------------------------- |
+| `REGISTRY_TOKEN` | `build.yaml` (registry login) | Gitea PAT with the `write:package` scope |
+
+The token is paired with `${{ github.actor }}` as the username, so it
+must belong to the user triggering the workflow — same convention as the
+other projects on this instance.
+
+Without `REGISTRY_TOKEN` the `check` job still passes but the build job
+fails at the `docker login` step. No other secrets are needed — the
+workflow does not deploy anywhere.
+
+## Development
 
 ```sh
-make undeploy
+make test        # unit + envtest suites, with -race (sets up envtest binaries itself)
+go test -short ./...   # skip the envtest suite
+make run-dev     # run against the current kubeconfig context
 ```
 
-## Project Distribution
+`make run-dev` is for iterating on the operator itself: provisioning,
+replacement, the discovery API, and External proxies all work from your
+laptop. Health checks against in-cluster pods do **not** (see the
+quickstart note) — use the in-cluster deploy to see a kubernetes-provider
+proxy go `Ready`.
 
-Following the options to release and provide this solution to the users.
-
-### By providing a bundle with all YAML files
-
-1. Build the installer for the image built and published in the registry:
-
-```sh
-make build-installer IMG=<some-registry>/egress-proxies-operator:tag
-```
-
-**NOTE:** The makefile target mentioned above generates an 'install.yaml'
-file in the dist directory. This file contains all the resources built
-with Kustomize, which are necessary to install this project without its
-dependencies.
-
-2. Using the installer
-
-Users can just run 'kubectl apply -f <URL for YAML BUNDLE>' to install
-the project, i.e.:
-
-```sh
-kubectl apply -f https://raw.githubusercontent.com/<org>/egress-proxies-operator/<tag or branch>/dist/install.yaml
-```
-
-### By providing a Helm Chart
-
-1. Build the chart using the optional helm plugin
-
-```sh
-kubebuilder edit --plugins=helm/v2-alpha
-```
-
-2. See that a chart was generated under 'dist/chart', and users
-can obtain this solution from there.
-
-**NOTE:** If you change the project, you need to update the Helm Chart
-using the same command above to sync the latest changes. Furthermore,
-if you create webhooks, you need to use the above command with
-the '--force' flag and manually ensure that any custom configuration
-previously added to 'dist/chart/values.yaml' or 'dist/chart/manager/manager.yaml'
-is manually re-applied afterwards.
-
-## Contributing
-// TODO(user): Add detailed information on how you would like others to contribute to this project
-
-**NOTE:** Run `make help` for more information on all potential `make` targets
-
-More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html)
-
-## License
-
-Copyright 2026.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+The full test inventory — what each suite covers, the deliberate gaps,
+and the manual kind verification procedure — is in
+[docs/testing.md](docs/testing.md).
 
+Project layout, reconcile-loop diagrams, and the decision log are in
+[docs/architecture.md](docs/architecture.md); the build history is in
+[docs/plans-executions/](docs/plans-executions/).

api/v1alpha1/providerhealth_types.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ConditionReady summarizes a ProviderHealth's targets: True when every
+// target passed its last check.
+const ConditionReady = "Ready"
+
+// ProviderTarget is the latest check verdict for one thing a provider
+// depends on: a KVM hypervisor, the Kubernetes API server, a GCP project.
+type ProviderTarget struct {
+	// name identifies the target within its provider: the KVM host name,
+	// "apiserver", or the GCP project.
+	// +kubebuilder:validation:MinLength=1
+	// +required
+	Name string `json:"name"`
+
+	// address is where the check connected to.
+	// +optional
+	Address string `json:"address,omitempty"`
+
+	// ready is true when the last check succeeded.
+	// +required
+	Ready bool `json:"ready"`
+
+	// reason is a CamelCase verdict such as Reachable, Unreachable,
+	// Forbidden, SSHHandshakeFailed or PoolNotActive.
+	// +optional
+	Reason string `json:"reason,omitempty"`
+
+	// message is human-readable detail, usually the error of a failed check.
+	// +kubebuilder:validation:MaxLength=1024
+	// +optional
+	Message string `json:"message,omitempty"`
+
+	// latencyMillis is how long the last check took.
+	// +optional
+	LatencyMillis int64 `json:"latencyMillis,omitempty"`
+
+	// lastTransitionTime is when ready last changed.
+	// +required
+	LastTransitionTime metav1.Time `json:"lastTransitionTime"`
+
+	// details are provider-specific facts from the last check, e.g. the
+	// libvirt version and free storage-pool space of a KVM host.
+	// +optional
+	Details map[string]string `json:"details,omitempty"`
+}
+
+// ProviderHealthStatus is the result of the latest check of one provider.
+type ProviderHealthStatus struct {
+	// type is the provider's type from the providers config: kubernetes,
+	// gcp or kvm.
+	// +optional
+	Type string `json:"type,omitempty"`
+
+	// conditions hold Ready, which summarizes the targets.
+	// +listType=map
+	// +listMapKey=type
+	// +optional
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+	// targets hold one verdict per checked target.
+	// +listType=map
+	// +listMapKey=name
+	// +optional
+	Targets []ProviderTarget `json:"targets,omitempty"`
+
+	// targetsReady counts the ready targets as "ready/total".
+	// +optional
+	TargetsReady string `json:"targetsReady,omitempty"`
+
+	// lastCheckTime is when the latest check finished.
+	// +optional
+	LastCheckTime *metav1.Time `json:"lastCheckTime,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster,shortName=ph
+// +kubebuilder:printcolumn:name="Type",type=string,JSONPath=`.status.type`
+// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`
+// +kubebuilder:printcolumn:name="Targets",type=string,JSONPath=`.status.targetsReady`
+// +kubebuilder:printcolumn:name="Last Check",type=date,JSONPath=`.status.lastCheckTime`
+// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
+
+// ProviderHealth reports the periodic health check of one configured
+// provider. It is named after the provider, and the operator creates,
+// updates and deletes it itself; there is no spec to edit.
+type ProviderHealth struct {
+	metav1.TypeMeta `json:",inline"`
+
+	// metadata is a standard object metadata
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitzero"`
+
+	// status is the result of the latest check
+	// +optional
+	Status ProviderHealthStatus `json:"status,omitzero"`
+}
+
+// +kubebuilder:object:root=true
+
+// ProviderHealthList contains a list of ProviderHealth
+type ProviderHealthList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitzero"`
+	Items           []ProviderHealth `json:"items"`
+}

api/v1alpha1/proxy_types.go

@@ -61,6 +61,21 @@ const (
 	// provision. A mismatch against the freshly computed hash means the VM
 	// must be replaced.
 	AnnotationSpecHash = "crawl.example.com/spec-hash"
+
+	// AnnotationSkipCleanup abandons the provider-side instance: set it to
+	// "true" and deletion removes the finalizer without asking the provider
+	// to clean anything up. It is the escape hatch for a cleanup that cannot
+	// succeed — a provider dropped from the config, credentials revoked, a
+	// backend that answers permanently — where the object would otherwise
+	// stay in Terminating forever. The instance is left running and becomes
+	// the administrator's to remove.
+	AnnotationSkipCleanup = "crawl.example.com/skip-cleanup"
+
+	// LabelCloudInit must be set (to "true") on every Secret referenced by
+	// spec.cloudInit.secretRef: the manager's cache only holds Secrets
+	// carrying this label, so an unlabelled Secret is invisible to the
+	// operator — both to the resolve step and to the rotation watch.
+	LabelCloudInit = "crawl.example.com/cloud-init"
 )
 
 // Defaults, applied both by CRD structural defaulting (kubebuilder:default
@@ -346,7 +361,7 @@ type ProxyList struct {
 
 func init() {
 	SchemeBuilder.Register(func(s *runtime.Scheme) error {
-		s.AddKnownTypes(SchemeGroupVersion, &Proxy{}, &ProxyList{})
+		s.AddKnownTypes(SchemeGroupVersion, &Proxy{}, &ProxyList{}, &ProviderHealth{}, &ProviderHealthList{})
 		return nil
 	})
 }

api/v1alpha1/zz_generated.deepcopy.go

@@ -95,6 +95,120 @@ func (in *PlacementSpec) DeepCopy() *PlacementSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderHealth) DeepCopyInto(out *ProviderHealth) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderHealth.
+func (in *ProviderHealth) DeepCopy() *ProviderHealth {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderHealth)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProviderHealth) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderHealthList) DeepCopyInto(out *ProviderHealthList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]ProviderHealth, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderHealthList.
+func (in *ProviderHealthList) DeepCopy() *ProviderHealthList {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderHealthList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProviderHealthList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderHealthStatus) DeepCopyInto(out *ProviderHealthStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]v1.Condition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Targets != nil {
+		in, out := &in.Targets, &out.Targets
+		*out = make([]ProviderTarget, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.LastCheckTime != nil {
+		in, out := &in.LastCheckTime, &out.LastCheckTime
+		*out = (*in).DeepCopy()
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderHealthStatus.
+func (in *ProviderHealthStatus) DeepCopy() *ProviderHealthStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderHealthStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderTarget) DeepCopyInto(out *ProviderTarget) {
+	*out = *in
+	in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
+	if in.Details != nil {
+		in, out := &in.Details, &out.Details
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderTarget.
+func (in *ProviderTarget) DeepCopy() *ProviderTarget {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderTarget)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Proxy) DeepCopyInto(out *Proxy) {
 	*out = *in

cmd/main.go

@@ -14,28 +14,55 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package main is the composition root: it loads the provider config (fail
+// fast), assembles the provider registry, and wires the reconciler, health
+// engine, lease store, discovery API, orphan GC, provider health checks and
+// metrics onto one controller-runtime manager.
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
+	"fmt"
 	"os"
+	goruntime "runtime"
+	"time"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/controller"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/discovery"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/gc"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/metrics"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kvm"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/providerhealth"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -60,6 +87,19 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var tlsOpts []func(*tls.Config)
+
+	var providersConfig string
+	var discoveryAddr string
+	var proxyNamespace string
+	var healthWorkers int
+	var gcInterval, gcMinAge time.Duration
+	var gcAllowNamespaced bool
+	var leaseCooldown, maxLeaseTTL time.Duration
+	var showVersion bool
+	var gcpWireFullPayloads bool
+	var traceHealthProbes bool
+	var providerCheckInterval, providerCheckTimeout time.Duration
+
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -74,13 +114,112 @@ func main() {
 	flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics server")
+
+	flag.StringVar(&providersConfig, "providers-config", "",
+		"Path to the providers config YAML. Required.")
+	flag.StringVar(&discoveryAddr, "discovery-addr", ":8090",
+		"Listen address of the discovery/lease HTTP API.")
+	flag.StringVar(&proxyNamespace, "proxy-namespace", "",
+		"Restrict the manager's cache to one namespace. Empty watches all namespaces. "+
+			"Restricting also disables orphan GC unless --gc-allow-namespaced is set.")
+	flag.IntVar(&healthWorkers, "health-workers", 8,
+		"Number of concurrent health-probe workers.")
+	flag.DurationVar(&gcInterval, "gc-interval", 10*time.Minute,
+		"Interval between orphan GC sweeps.")
+	flag.DurationVar(&gcMinAge, "gc-min-age", 10*time.Minute,
+		"Minimum instance age before orphan GC may delete it.")
+	flag.BoolVar(&gcAllowNamespaced, "gc-allow-namespaced", false,
+		"Allow orphan GC to run although the cache is namespace-restricted. Dangerous: proxies "+
+			"outside the namespace count as orphans and their instances get deleted.")
+	flag.DurationVar(&leaseCooldown, "lease-cooldown", 15*time.Minute,
+		"How long a reported proxy/target pair is excluded from lease selection.")
+	flag.DurationVar(&maxLeaseTTL, "max-lease-ttl", time.Hour,
+		"Maximum lease TTL a client may request.")
+	flag.BoolVar(&showVersion, "version", false,
+		"Print the commit the binary was built from and exit.")
+	flag.BoolVar(&gcpWireFullPayloads, "gcp-wire-log-full-payloads", false,
+		"Log GCP V(5) wire payloads verbatim instead of eliding fields larger than 1KiB.")
+	flag.BoolVar(&traceHealthProbes, "trace-health-probes", false,
+		"Emit one trace span per health probe. Off by default: probes run about "+
+			"once per second per proxy and would dominate trace volume.")
+	flag.DurationVar(&providerCheckInterval, "provider-check-interval", time.Minute,
+		"Interval between provider health checks (can the cluster reach each hypervisor, API "+
+			"and account), published as ProviderHealth objects and metrics. 0 turns them off; "+
+			"otherwise at least 10s.")
+	flag.DurationVar(&providerCheckTimeout, "provider-check-timeout", 0,
+		"Time limit for one round of provider health checks: 15s, or the interval when that is "+
+			"shorter. If set, at most the interval.")
+
 	opts := zap.Options{
 		Development: true,
 	}
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
+	if showVersion {
+		fmt.Println(version.Resolve())
+		os.Exit(0)
+	}
+
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+	setupLog.Info("Starting egress-proxies-operator",
+		"commit", version.Resolve(), "goVersion", goruntime.Version())
+
+	// Off (no-op spans, unchanged logs) unless OTEL_* env opts in; see
+	// internal/tracing. Shutdown is called explicitly after mgr.Start
+	// returns — the os.Exit paths below skip defers.
+	tracingShutdown, err := tracing.Setup(logf.IntoContext(context.Background(), setupLog),
+		"egress-proxies-operator", version.Resolve())
+	if err != nil {
+		setupLog.Error(err, "Failed to set up tracing")
+		os.Exit(1)
+	}
+
+	ctx := ctrl.SetupSignalHandler()
+
+	// Providers load first and fail fast: a manager that comes up without
+	// its backends would just convert every Proxy into an error loop.
+	if providersConfig == "" {
+		setupLog.Error(nil, "--providers-config is required")
+		os.Exit(1)
+	}
+	if err := providerhealth.ValidateSchedule(providerCheckInterval, providerCheckTimeout); err != nil {
+		setupLog.Error(err, "Invalid provider check flags")
+		os.Exit(1)
+	}
+	cfg, err := provider.LoadConfigFile(providersConfig)
+	if err != nil {
+		setupLog.Error(err, "Failed to load providers config", "path", providersConfig)
+		os.Exit(1)
+	}
+	providers, err := registry.Build(ctx, cfg, map[string]registry.Constructor{
+		"kubernetes": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+			// The kubernetes provider builds its own uncached client;
+			// wrap its transport so its API calls join the caller's trace.
+			return kubernetes.NewWithTransportWrapper(ctx, pc, tracing.RestConfigWrapper())
+		},
+		"gcp": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+			return gcp.NewWithWireOptions(ctx, pc, gcp.WireLogOptions{FullPayloads: gcpWireFullPayloads})
+		},
+		"kvm": kvm.New,
+	})
+	if err != nil {
+		setupLog.Error(err, "Failed to build providers")
+		os.Exit(1)
+	}
+	m := metrics.New()
+	// Checkers come from the unwrapped providers, in config order: the
+	// metrics and tracing wrappers implement only Provider, and a check is
+	// not a provider request to be counted anyway.
+	checkTargets := make([]providerhealth.Target, 0, len(cfg.Providers))
+	for _, pc := range cfg.Providers {
+		checker, _ := providers[pc.Name].(provider.Checker)
+		checkTargets = append(checkTargets, providerhealth.Target{Name: pc.Name, Type: pc.Type, Checker: checker})
+	}
+	for name, p := range providers {
+		// Tracing outermost: the span covers the metrics recording too.
+		providers[name] = provider.WithTracing(name, provider.WithMetrics(name, p, m))
+	}
 
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
@@ -128,32 +267,110 @@ func main() {
 		metricsServerOptions.KeyName = metricsCertKey
 	}
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	// The Secret cache is restricted to labelled cloud-init Secrets: the
+	// operator has cluster-wide Secret read RBAC, and without the label
+	// selector it would cache every Secret in scope.
+	cacheOpts := cache.Options{
+		ByObject: map[client.Object]cache.ByObject{
+			&corev1.Secret{}: {
+				Label: labels.SelectorFromSet(labels.Set{crawlv1alpha1.LabelCloudInit: "true"}),
+			},
+		},
+	}
+	if proxyNamespace != "" {
+		cacheOpts.DefaultNamespaces = map[string]cache.Config{proxyNamespace: {}}
+	}
+
+	restCfg := ctrl.GetConfigOrDie()
+	restCfg.Wrap(tracing.RestConfigWrapper())
+	mgr, err := ctrl.NewManager(restCfg, ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
+		Cache:                  cacheOpts,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "b47711d1.example.com",
-		// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
-		// when the Manager ends. This requires the binary to immediately end when the
-		// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
-		// speeds up voluntary leader transitions as the new leader don't have to wait
-		// LeaseDuration time first.
-		//
-		// In the default scaffold provided, the program ends immediately after
-		// the manager stops, so would be fine to enable this option. However,
-		// if you are doing or is intended to do any operation such as perform cleanups
-		// after the manager stops then its usage might be unsafe.
-		// LeaderElectionReleaseOnCancel: true,
 	})
 	if err != nil {
 		setupLog.Error(err, "Failed to start manager")
 		os.Exit(1)
 	}
 
+	store := lease.NewStore(leaseCooldown)
+	if err := mgr.Add(store); err != nil {
+		setupLog.Error(err, "Failed to add lease store")
+		os.Exit(1)
+	}
+
+	engine := health.NewEngine(mgr.GetClient())
+	engine.Workers = healthWorkers
+	engine.Metrics = m
+	engine.TraceProbes = traceHealthProbes
+	if err := mgr.Add(engine); err != nil {
+		setupLog.Error(err, "Failed to add health engine")
+		os.Exit(1)
+	}
+
+	if err := mgr.Add(&discovery.Server{
+		Reader:      mgr.GetClient(),
+		Store:       store,
+		Addr:        discoveryAddr,
+		Token:       os.Getenv("DISCOVERY_TOKEN"),
+		MaxLeaseTTL: maxLeaseTTL,
+		Metrics:     m,
+	}); err != nil {
+		setupLog.Error(err, "Failed to add discovery server")
+		os.Exit(1)
+	}
+
+	if err := mgr.Add(&gc.Sweeper{
+		Reader:              mgr.GetClient(),
+		Providers:           providers,
+		Interval:            gcInterval,
+		MinAge:              gcMinAge,
+		NamespaceRestricted: proxyNamespace != "",
+		AllowNamespaced:     gcAllowNamespaced,
+	}); err != nil {
+		setupLog.Error(err, "Failed to add orphan GC")
+		os.Exit(1)
+	}
+
+	monitor := &providerhealth.Monitor{
+		Client:   mgr.GetClient(),
+		Reader:   mgr.GetAPIReader(),
+		Targets:  checkTargets,
+		Interval: providerCheckInterval,
+		Timeout:  providerCheckTimeout,
+	}
+	if err := mgr.Add(monitor); err != nil {
+		setupLog.Error(err, "Failed to add provider health monitor")
+		os.Exit(1)
+	}
+
+	if err := m.Register(ctrlmetrics.Registry,
+		proxyPhaseCounts(mgr.GetClient()),
+		func() int {
+			total := 0
+			for _, n := range store.Counts() {
+				total += n
+			}
+			return total
+		},
+	); err != nil {
+		setupLog.Error(err, "Failed to register metrics")
+		os.Exit(1)
+	}
+	if err := metrics.RegisterProviderTargets(ctrlmetrics.Registry, providerTargetSamples(monitor)); err != nil {
+		setupLog.Error(err, "Failed to register provider check metrics")
+		os.Exit(1)
+	}
+
 	if err := (&controller.ProxyReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:       mgr.GetClient(),
+		Scheme:       mgr.GetScheme(),
+		Providers:    providers,
+		Health:       engine,
+		HealthEvents: engine.Events,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "Failed to create controller", "controller", "proxy")
 		os.Exit(1)
@@ -170,8 +387,62 @@ func main() {
 	}
 
 	setupLog.Info("Starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
-		setupLog.Error(err, "Failed to run manager")
+	startErr := mgr.Start(ctx)
+
+	// Flush pending spans on the way out, error path included. Fresh
+	// context: the signal ctx is already cancelled by the time Start
+	// returns.
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	if err := tracingShutdown(shutdownCtx); err != nil {
+		setupLog.Error(err, "Failed to flush traces on shutdown")
+	}
+	cancel()
+
+	if startErr != nil {
+		setupLog.Error(startErr, "Failed to run manager")
 		os.Exit(1)
 	}
 }
+
+// proxyPhaseCounts reads phase counts from the cache at scrape time. Before
+// the cache has synced (or on any list error) it reports nothing rather
+// than something wrong.
+func proxyPhaseCounts(c client.Reader) func() map[string]int {
+	return func() map[string]int {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		var list crawlv1alpha1.ProxyList
+		if err := c.List(ctx, &list); err != nil {
+			return nil
+		}
+		counts := map[string]int{}
+		for i := range list.Items {
+			phase := string(list.Items[i].Status.Phase)
+			if phase == "" {
+				phase = string(crawlv1alpha1.PhasePending)
+			}
+			counts[phase]++
+		}
+		return counts
+	}
+}
+
+// providerTargetSamples flattens the monitor's latest results for the
+// scrape-time provider-check gauges.
+func providerTargetSamples(mon *providerhealth.Monitor) func() []metrics.ProviderTarget {
+	return func() []metrics.ProviderTarget {
+		var out []metrics.ProviderTarget
+		for _, r := range mon.Snapshot() {
+			for _, ts := range r.Targets {
+				out = append(out, metrics.ProviderTarget{
+					Provider:  r.Provider,
+					Target:    ts.Name,
+					Up:        ts.Ready,
+					Duration:  ts.Latency,
+					CheckedAt: r.CheckedAt,
+				})
+			}
+		}
+		return out
+	}
+}

config/crd/bases/crawl.example.com_providerhealths.yaml

@@ -0,0 +1,193 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.21.0
+  name: providerhealths.crawl.example.com
+spec:
+  group: crawl.example.com
+  names:
+    kind: ProviderHealth
+    listKind: ProviderHealthList
+    plural: providerhealths
+    shortNames:
+    - ph
+    singular: providerhealth
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .status.type
+      name: Type
+      type: string
+    - jsonPath: .status.conditions[?(@.type=="Ready")].status
+      name: Ready
+      type: string
+    - jsonPath: .status.targetsReady
+      name: Targets
+      type: string
+    - jsonPath: .status.lastCheckTime
+      name: Last Check
+      type: date
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: |-
+          ProviderHealth reports the periodic health check of one configured
+          provider. It is named after the provider, and the operator creates,
+          updates and deletes it itself; there is no spec to edit.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          status:
+            description: status is the result of the latest check
+            properties:
+              conditions:
+                description: conditions hold Ready, which summarizes the targets.
+                items:
+                  description: Condition contains details for one aspect of the current
+                    state of this API Resource.
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - type
+                x-kubernetes-list-type: map
+              lastCheckTime:
+                description: lastCheckTime is when the latest check finished.
+                format: date-time
+                type: string
+              targets:
+                description: targets hold one verdict per checked target.
+                items:
+                  description: |-
+                    ProviderTarget is the latest check verdict for one thing a provider
+                    depends on: a KVM hypervisor, the Kubernetes API server, a GCP project.
+                  properties:
+                    address:
+                      description: address is where the check connected to.
+                      type: string
+                    details:
+                      additionalProperties:
+                        type: string
+                      description: |-
+                        details are provider-specific facts from the last check, e.g. the
+                        libvirt version and free storage-pool space of a KVM host.
+                      type: object
+                    lastTransitionTime:
+                      description: lastTransitionTime is when ready last changed.
+                      format: date-time
+                      type: string
+                    latencyMillis:
+                      description: latencyMillis is how long the last check took.
+                      format: int64
+                      type: integer
+                    message:
+                      description: message is human-readable detail, usually the error
+                        of a failed check.
+                      maxLength: 1024
+                      type: string
+                    name:
+                      description: |-
+                        name identifies the target within its provider: the KVM host name,
+                        "apiserver", or the GCP project.
+                      minLength: 1
+                      type: string
+                    ready:
+                      description: ready is true when the last check succeeded.
+                      type: boolean
+                    reason:
+                      description: |-
+                        reason is a CamelCase verdict such as Reachable, Unreachable,
+                        Forbidden, SSHHandshakeFailed or PoolNotActive.
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - name
+                  - ready
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
+              targetsReady:
+                description: targetsReady counts the ready targets as "ready/total".
+                type: string
+              type:
+                description: |-
+                  type is the provider's type from the providers config: kubernetes,
+                  gcp or kvm.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

config/crd/kustomization.yaml

@@ -3,4 +3,5 @@
 # It should be run by config/default
 resources:
 - bases/crawl.example.com_proxies.yaml
+- bases/crawl.example.com_providerhealths.yaml
 # +kubebuilder:scaffold:crdkustomizeresource

config/default/discovery_service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: controller-manager
+    app.kubernetes.io/name: egress-proxies-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: controller-manager-discovery-service
+  namespace: system
+spec:
+  ports:
+  - name: discovery
+    port: 8090
+    protocol: TCP
+    targetPort: discovery
+  selector:
+    control-plane: controller-manager
+    app.kubernetes.io/name: egress-proxies-operator

config/default/kustomization.yaml

@@ -22,6 +22,8 @@ resources:
 #- ../prometheus
 # [METRICS] Expose the controller manager metrics service.
 - metrics_service.yaml
+# Expose the discovery/lease HTTP API inside the cluster.
+- discovery_service.yaml
 
 # Uncomment the patches line if you enable Metrics
 patches:

config/manager/kustomization.yaml

@@ -1,8 +1,9 @@
 resources:
 - manager.yaml
+- providers_config.yaml
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: example.com/egress-proxies-operator
-  newTag: v0.0.1
+  newName: egress-proxies-operator
+  newTag: kvm-dev

config/manager/manager.yaml

@@ -63,12 +63,61 @@ spec:
         args:
           - --leader-elect
           - --health-probe-bind-address=:8081
+          - --providers-config=/etc/proxy-operator/providers.yaml
+          # Add --trace-health-probes to emit one span per health probe
+          # (high volume: ~1/s per proxy).
+          # Provider health checks (kubectl get providerhealth) run every
+          # minute; tune with --provider-check-interval (0 turns them off,
+          # otherwise at least 10s) and --provider-check-timeout (15s, or
+          # the interval if that is shorter).
+        env:
+          # Bearer token for the discovery API. Optional: without the
+          # Secret the API serves unauthenticated (with a loud warning).
+          # Create it with:
+          #   kubectl -n egress-proxies-operator-system create secret \
+          #     generic discovery-token --from-literal=token=<your-token>
+          - name: DISCOVERY_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: discovery-token
+                key: token
+                optional: true
+          # --- OpenTelemetry tracing ---
+          # Tracing is OFF until OTEL_EXPORTER_OTLP_ENDPOINT (or
+          # OTEL_TRACES_EXPORTER) is set; without it the operator runs
+          # exactly as before. The downward-API vars must stay listed
+          # before OTEL_RESOURCE_ATTRIBUTES: $(VAR) expansion only sees
+          # earlier-listed vars.
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: OTEL_SERVICE_NAME
+            value: egress-proxies-operator
+          - name: OTEL_RESOURCE_ATTRIBUTES
+            value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)
+          # Point at an OTLP receiver (Tempo, Jaeger, otel-collector) to
+          # enable tracing; http/protobuf (4318) is the default protocol,
+          # set OTEL_EXPORTER_OTLP_PROTOCOL=grpc for 4317.
+          # - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          #   value: http://jaeger-collector.observability:4318
+          # - name: OTEL_TRACES_SAMPLER
+          #   value: parentbased_traceidratio
+          # - name: OTEL_TRACES_SAMPLER_ARG
+          #   value: "0.1"
         image: controller:latest
         name: manager
         ports:
         - containerPort: 8081
           name: health
           protocol: TCP
+        - containerPort: 8090
+          name: discovery
+          protocol: TCP
         securityContext:
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
@@ -96,7 +145,34 @@ spec:
           requests:
             cpu: 10m
             memory: 64Mi
-        volumeMounts: []
-      volumes: []
+        volumeMounts:
+        - name: providers-config
+          mountPath: /etc/proxy-operator
+          readOnly: true
+        # SSH material for kvm providers (sshKeyFile / knownHostsFile in the
+        # providers config point here). Optional: the mount is empty when the
+        # Secret does not exist, and only kvm entries need it. Create it with:
+        #   kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
+        #     --from-file=id_ed25519=<private key> \
+        #     --from-file=known_hosts=<known_hosts with every host's key>
+        - name: kvm-ssh
+          mountPath: /etc/proxy-operator/kvm
+          readOnly: true
+        # Scratch space: the root filesystem is read-only, and the kvm
+        # provider builds each VM's cloud-init seed ISO through a temp
+        # directory (a few hundred KB, deleted right after upload).
+        - name: tmp
+          mountPath: /tmp
+      volumes:
+      - name: providers-config
+        configMap:
+          name: providers-config
+      - name: kvm-ssh
+        secret:
+          secretName: kvm-ssh
+          optional: true
+      - name: tmp
+        emptyDir:
+          sizeLimit: 64Mi
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10

config/manager/providers_config.yaml

@@ -0,0 +1,54 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: providers-config
+  namespace: system
+  labels:
+    app.kubernetes.io/name: egress-proxies-operator
+    app.kubernetes.io/managed-by: kustomize
+data:
+  # Mounted at /etc/proxy-operator/providers.yaml (--providers-config).
+  # The default ships only the kubernetes-pod provider so the operator runs
+  # out of the box; add gcp entries (type: gcp, gcp.project: ...) or kvm
+  # entries (type: kvm, hosts reached over SSH) for real egress fleets —
+  # see config/samples/providers-config.yaml.
+  providers.yaml: |
+    providers:
+      - name: kubernetes
+        type: kubernetes
+      - name: kvm-home
+        type: kvm
+        kvm:
+          # SSH to each host with the libvirt unix socket forwarded through the
+          # connection; nothing to open on the hypervisors beyond sshd. Both
+          # files come from the optional kvm-ssh Secret mounted by
+          # config/manager/manager.yaml — see docs/kvm-provider.md.
+          sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+          knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+          # insecureIgnoreHostKey: true   # skip host-key checks (not recommended)
+          # diskSizeGb: 10                # per-VM overlay; cloud-init grows the root fs into it
+          # vendorData: |                 # extra vendor-data; the qemu-guest-agent bootstrap is merged
+          #   #cloud-config               # into each Proxy's cloud-config, not configured here
+          #   ...
+          images:
+            # placement.image names a key here (or is an https:// URL directly).
+            # Fetched into each host's pool on first use and cached there as
+            # base-<key>.qcow2; do not change an entry while Proxies use it.
+            debian-13:
+              url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+              # sha256: "<64 hex>"        # optional; verified after download
+              # firmware: uefi            # default; "bios" only for images with no EFI partition
+              #                           # (Debian 13 hangs in GRUB under BIOS headless: Debian bug #1111240)
+            ubuntu-24.04:
+              url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+          machineTypes:                   # placement.machineType names a key here
+            small:  {vcpus: 1, memoryMiB: 1024}
+            medium: {vcpus: 2, memoryMiB: 2048}
+          hosts:                          # placement.zone names a host; optional with one host
+            - name: beelink
+              address: beelink.tail8c69c.ts.net:22
+              user: novakj                  # must be in the libvirt group
+              bridge: br0                 # to a LAN with DHCP; the VM's IP comes from there
+              # socket: /var/run/libvirt/libvirt-sock
+              # pool: default
+

config/rbac/kustomization.yaml

@@ -25,4 +25,7 @@ resources:
 - proxy_admin_role.yaml
 - proxy_editor_role.yaml
 - proxy_viewer_role.yaml
+# ProviderHealth objects are written only by the operator, so they get a
+# viewer role and nothing else.
+- providerhealth_viewer_role.yaml
 

config/rbac/providerhealth_viewer_role.yaml

@@ -0,0 +1,29 @@
+# This rule is not used by the project egress-proxies-operator itself.
+# It is provided to allow the cluster admin to help manage permissions for users.
+#
+# Grants read-only access to ProviderHealth objects, the results of the
+# operator's periodic provider checks. The operator creates, updates and
+# deletes them itself, so no admin or editor role is provided.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: egress-proxies-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: providerhealth-viewer-role
+rules:
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths/status
+  verbs:
+  - get

config/rbac/role.yaml

@@ -4,6 +4,16 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -12,6 +22,22 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths/status
+  verbs:
+  - get
+  - patch
 - apiGroups:
   - crawl.example.com
   resources:

config/samples/crawl_v1alpha1_proxy.yaml

@@ -1,9 +0,0 @@
-apiVersion: crawl.example.com/v1alpha1
-kind: Proxy
-metadata:
-  labels:
-    app.kubernetes.io/name: egress-proxies-operator
-    app.kubernetes.io/managed-by: kustomize
-  name: proxy-sample
-spec:
-  # TODO(user): Add fields here

config/samples/kustomization.yaml

@@ -1,4 +1,9 @@
 ## Append samples of your project ##
+# providers-config.yaml is deliberately absent: it is a sample
+# --providers-config file, not a Kubernetes manifest.
 resources:
-- crawl_v1alpha1_proxy.yaml
+- proxy_kubernetes.yaml
+- proxy_gcp.yaml
+- proxy_kvm.yaml
+- proxy_external.yaml
 # +kubebuilder:scaffold:manifestskustomizesamples

config/samples/providers-config.yaml

@@ -0,0 +1,57 @@
+# Sample --providers-config file (not a Kubernetes manifest). In-cluster
+# this content lives in the providers-config ConfigMap
+# (config/manager/providers_config.yaml); for `make run-dev` a
+# kubernetes-only variant is at hack/providers-dev.yaml.
+#
+# Named provider instances: "gcp-eu" and "gcp-us" are two configs of the
+# same type. spec.provider on a Proxy refers to the name, not the type.
+providers:
+  - name: kubernetes
+    type: kubernetes
+    # kubernetes:
+    #   image: ubuntu/squid:6.6-24.04_edge   # the default
+  - name: gcp-eu
+    type: gcp
+    gcp:
+      project: my-project
+      # network: default            # VPC network name
+      # networkTag: proxy-operator  # firewall tag on created instances
+      # diskSizeGb: 10
+      # auth: Application Default Credentials (workload identity
+      # in-cluster, gcloud ADC locally). No key-file plumbing.
+  - name: kvm-home
+    type: kvm
+    kvm:
+      # SSH to each host with the libvirt unix socket forwarded through the
+      # connection; nothing to open on the hypervisors beyond sshd. Both
+      # files come from the optional kvm-ssh Secret mounted by
+      # config/manager/manager.yaml — see docs/kvm-provider.md.
+      sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+      knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+      # insecureIgnoreHostKey: true   # skip host-key checks (not recommended)
+      # diskSizeGb: 10                # per-VM overlay; cloud-init grows the root fs into it
+      # installGuestAgent: false      # skip the vendor-data script that installs qemu-guest-agent
+      # vendorData: |                 # extra vendor-data, carried beside the agent script as a
+      #   #cloud-config               # MIME multipart. The Proxy's own cloud-init is never rewritten.
+      #   ...
+      images:
+        # placement.image names a key here (or is an https:// URL directly).
+        # Fetched into each host's pool on first use and cached there as
+        # base-<key>.qcow2; do not change an entry while Proxies use it.
+        debian-13:
+          url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+          # sha256: "<64 hex>"        # optional; verified after download
+          # firmware: uefi            # default; "bios" only for images with no EFI partition
+          #                           # (Debian 13 hangs in GRUB under BIOS headless: Debian bug #1111240)
+        ubuntu-24.04:
+          url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+      machineTypes:                   # placement.machineType names a key here
+        small:  {vcpus: 1, memoryMiB: 1024}
+        medium: {vcpus: 2, memoryMiB: 2048}
+      hosts:                          # placement.zone names a host; optional with one host
+        - name: beelink
+          address: beelink:22
+          user: novakj                  # must be in the libvirt group
+          bridge: br0                 # to a LAN with DHCP; the VM's IP comes from there
+          # socket: /var/run/libvirt/libvirt-sock
+          # pool: default

config/samples/proxy_external.yaml

@@ -0,0 +1,15 @@
+# An External proxy: the VM exists outside the operator's control; the
+# operator only tracks and health-checks it through the endpoint. No
+# finalizer, no provider calls, and deleting the CR touches nothing.
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-external-sample
+spec:
+  mode: External
+  endpoint:
+    host: 203.0.113.7
+    port: 3128
+  attributes:
+    geo: eu
+    purpose: crawl

config/samples/proxy_gcp.yaml

@@ -0,0 +1,36 @@
+# A Managed proxy backed by GCP: the operator creates a VM with an
+# ephemeral external IP and installs Squid via cloud-init. Requires a
+# providers-config entry named "gcp-eu" (see providers-config.yaml) and
+# Application Default Credentials with compute.instanceAdmin.v1.
+#
+# All three placement fields are required for GCP; the operator sets the
+# Proxy to Failed with a message naming any missing one.
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-gcp-sample
+spec:
+  mode: Managed
+  provider: gcp-eu
+  placement:
+    zone: europe-west1-b
+    machineType: e2-micro
+    image: projects/debian-cloud/global/images/family/debian-12
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_port 3128
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+  attributes:
+    geo: eu
+    purpose: crawl

config/samples/proxy_kubernetes.yaml

@@ -0,0 +1,14 @@
+# A Managed proxy backed by the kubernetes-pod provider: the operator runs
+# a real Squid pod in this cluster. This is the local-dev/CI sample — pods
+# share the cluster's egress IP, so it exercises the full lifecycle but
+# does not provide a distinct egress path (use the gcp provider for that).
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-kubernetes-sample
+spec:
+  mode: Managed
+  provider: kubernetes
+  attributes:
+    geo: local
+    purpose: crawl

config/samples/proxy_kvm.yaml

@@ -0,0 +1,43 @@
+# A Managed proxy backed by a libvirt/KVM hypervisor on the home network:
+# the operator boots a stock Debian cloud image as a VM on the named host,
+# with a NIC on the host's bridge (IP from the LAN's DHCP), and installs
+# Squid via cloud-init. Requires a providers-config entry named "kvm-home"
+# (see providers-config.yaml) and the kvm-ssh Secret (docs/kvm-provider.md).
+#
+# placement.zone is the host name from kvm.hosts (optional when only one
+# host is configured); machineType and image are keys in the provider's
+# machineTypes and images maps. The first Proxy for an image on a host
+# waits for the image download; later ones reuse the cached copy.
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-kvm-sample
+spec:
+  mode: Managed
+  provider: kvm-home
+  placement:
+    zone: nas1
+    machineType: small
+    image: debian-13
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_port 3128
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+      # Optional: log in with `ssh debian@<ip>` (the Debian image's default
+      # user) to look around; `virsh console <name>` on the host works too.
+      # ssh_authorized_keys:
+      #   - ssh-ed25519 AAAA... you@laptop
+  attributes:
+    geo: home
+    purpose: crawl

docs/api.md

@@ -0,0 +1,323 @@
+# Discovery API reference
+
+The operator serves an HTTP API (the *discovery API*) that crawler clients
+use to find and lease egress proxies: list healthy proxies filtered by
+attributes, acquire a TTL-based lease on one, release it early, and report
+how a target site treated the proxy. It is implemented in
+[`internal/discovery`](../internal/discovery/) with lease state in
+[`internal/lease`](../internal/lease/); the only Kubernetes interaction is
+reading `Proxy` resources from the manager's cache.
+
+## Base URL
+
+The API listens on `:8090` (`--discovery-addr`) inside the manager pod and
+is exposed by a Service
+([config/default/discovery_service.yaml](../config/default/discovery_service.yaml)).
+
+In-cluster:
+
+```text
+http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090
+```
+
+From a workstation, port-forward:
+
+```sh
+kubectl -n egress-proxies-operator-system port-forward \
+  svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090 &
+```
+
+Set `BASE_URL` to wherever you reach the API; all examples below use it:
+
+```sh
+export BASE_URL=localhost:8090   # via the port-forward above
+# or, from inside the cluster:
+# export BASE_URL=http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090
+```
+
+## Authentication
+
+A single static bearer token, read from the `DISCOVERY_TOKEN` environment
+variable at startup. The shipped Deployment populates it from the
+`discovery-token` Secret (key `token`), which is **optional** — if the
+Secret is absent or the token is empty, the API serves **unauthenticated**
+(the manager logs a loud warning at startup). Create the Secret:
+
+```sh
+kubectl -n egress-proxies-operator-system create secret generic discovery-token \
+  --from-literal=token="$(openssl rand -hex 24)"
+```
+
+Send the token on every request:
+
+```sh
+export TOKEN=<the token>
+curl -s -H "Authorization: Bearer $TOKEN" "$BASE_URL/v1/proxies" | jq
+```
+
+A missing or wrong token gets `401 {"error":"unauthorized",...}`.
+`GET /healthz` is always exempt.
+
+The curl examples below omit the `-H "Authorization: Bearer $TOKEN"` flag
+for brevity — add it to every call when auth is enabled.
+
+## Conventions
+
+- Requests and responses are JSON. Errors share one envelope:
+
+  ```json
+  {"error": "<machine_code>", "message": "<human-readable text>"}
+  ```
+
+- Request bodies are capped at **64 KiB** (larger bodies fail the JSON
+  decode with `400 invalid_body`).
+- Proxies with a deletion timestamp (being finalized) are excluded from
+  every response and never offered for lease.
+
+## Configuration
+
+| Setting | Default | Meaning |
+|---|---|---|
+| `--discovery-addr` | `:8090` | Listen address of the API |
+| `--max-lease-ttl` | `1h` | Maximum `ttlSeconds` a client may request |
+| `--lease-cooldown` | `15m` | Cooldown window applied on `rate_limited`/`banned` reports |
+| `DISCOVERY_TOKEN` (env) | empty | Bearer token; empty disables auth |
+
+The shipped Deployment passes none of these flags, so the defaults apply.
+
+## Endpoints
+
+### `GET /healthz`
+
+Liveness check. Unauthenticated, always `200` with body `ok`.
+
+```sh
+curl -s "$BASE_URL/healthz"
+```
+
+### `GET /v1/proxies` — list proxies
+
+Query parameters (all optional):
+
+| Parameter | Values | Effect |
+|---|---|---|
+| `healthy` | `true` \| `false` | Keep only proxies whose `Healthy` condition matches. Any other value → `400 invalid_query`. |
+| `attr.<key>` | any string | Exact match on `spec.attributes[<key>]`. Repeatable; **all** given pairs must match. |
+
+List everything:
+
+```sh
+curl -s "$BASE_URL/v1/proxies" | jq
+```
+
+List healthy proxies in a given geo:
+
+```sh
+curl -s "$BASE_URL/v1/proxies?healthy=true&attr.geo=eu" | jq
+```
+
+Response — `200`, proxies sorted by `id`, an empty match is `200` with
+`"count": 0` (never `404`):
+
+```json
+{
+  "proxies": [
+    {
+      "id": "default/proxy-kubernetes-sample",
+      "ip": "10.244.1.7",
+      "port": 3128,
+      "attributes": {"geo": "local"},
+      "phase": "Ready",
+      "healthy": true,
+      "latencyMillis": 42,
+      "activeLeases": 1,
+      "maxLeases": 5
+    }
+  ],
+  "count": 1
+}
+```
+
+Proxy object fields (the same shape appears inside lease responses):
+
+| Field | Meaning |
+|---|---|
+| `id` | `namespace/name` of the `Proxy` resource; used as the stable key everywhere |
+| `ip` | Effective host — `spec.endpoint.host` for `External` proxies, `status.ip` for `Managed` (empty until the backing VM/pod is up) |
+| `port` | Effective port (default `3128`) |
+| `attributes` | `spec.attributes` — free-form selection labels (`geo`, `asn`, `purpose`, …); omitted when empty |
+| `phase` | `Pending` \| `Provisioning` \| `Ready` \| `Unhealthy` \| `Deleting` \| `Failed` |
+| `healthy` | `true` iff the `Healthy` condition is `True` (the through-the-proxy health probe passes) |
+| `latencyMillis` | Latency of the last status-affecting health probe |
+| `activeLeases` | Currently active leases on this proxy |
+| `maxLeases` | Lease capacity (default `5`; an explicit `0` means unleasable) |
+
+### `POST /v1/leases` — acquire a lease
+
+Picks a healthy proxy with free capacity matching the selector and grants
+an exclusive-slot, TTL-based lease on it.
+
+Request body (every field optional; `{}` is valid):
+
+```json
+{
+  "selector": {"geo": "eu"},
+  "ttlSeconds": 300,
+  "target": "example.com"
+}
+```
+
+| Field | Default | Meaning |
+|---|---|---|
+| `selector` | none | Attribute equality filter, same semantics as `attr.<key>` above |
+| `ttlSeconds` | `300` (5 min) | Lease lifetime; must be ≤ `--max-lease-ttl` (default 1 h), else `400 invalid_ttl` |
+| `target` | none | The site you intend to crawl; enables per-target cooldowns (see below) |
+
+```sh
+curl -s -XPOST "$BASE_URL/v1/leases" \
+  -d '{"selector":{"geo":"eu"},"ttlSeconds":300,"target":"example.com"}' | jq
+```
+
+Success — `201`:
+
+```json
+{
+  "leaseID": "P3X6HHQTPCM5UTGVGE3B5UPS3A",
+  "proxy": {
+    "id": "default/proxy-eu-1",
+    "ip": "34.88.10.20",
+    "port": 3128,
+    "attributes": {"geo": "eu"},
+    "phase": "Ready",
+    "healthy": true,
+    "latencyMillis": 42,
+    "activeLeases": 1,
+    "maxLeases": 5
+  },
+  "expiresAt": "2026-08-11T22:05:00Z",
+  "ttlSeconds": 300
+}
+```
+
+Use `proxy.ip` and `proxy.port` as an HTTP proxy for the lease's lifetime:
+
+```sh
+curl -x http://34.88.10.20:3128 https://example.com
+```
+
+No match — `409` with diagnostic counts explaining why nothing qualified:
+
+```json
+{
+  "error": "no_match",
+  "message": "no healthy proxy with free capacity matched the selector",
+  "considered": 3,
+  "atCapacity": 1,
+  "inCooldown": 1,
+  "unhealthy": 1
+}
+```
+
+| Count | Meaning |
+|---|---|
+| `considered` | Proxies that matched the selector (before health/capacity checks) |
+| `atCapacity` | Skipped because `activeLeases >= maxLeases` |
+| `inCooldown` | Skipped because of an active cooldown for this target (or a global one) |
+| `unhealthy` | Skipped because the `Healthy` condition is not `True` |
+
+Leases expire on their own — releasing is only needed to free the slot
+early. There is no renew/extend endpoint; acquire a new lease instead.
+
+### `DELETE /v1/leases/{id}` — release early
+
+Frees the lease's capacity slot immediately. Idempotent: always `204`,
+including for unknown or already-expired lease IDs.
+
+```sh
+curl -si -XDELETE "$BASE_URL/v1/leases/P3X6HHQTPCM5UTGVGE3B5UPS3A"
+```
+
+### `POST /v1/leases/{id}/report` — report an outcome
+
+Tell the operator how the target site treated the proxy. This is the
+feedback signal that drives cooldowns.
+
+Request body:
+
+```json
+{"result": "rate_limited", "target": "example.com"}
+```
+
+| Field | Values | Meaning |
+|---|---|---|
+| `result` | `ok` \| `rate_limited` \| `banned` | Anything else → `400 invalid_result` |
+| `target` | optional | Which site produced the result; falls back to the lease's `target`, then to global |
+
+```sh
+curl -si -XPOST "$BASE_URL/v1/leases/P3X6HHQTPCM5UTGVGE3B5UPS3A/report" \
+  -d '{"result":"rate_limited","target":"example.com"}'
+```
+
+Responses: `204` on success, `404 unknown_lease` if the lease ID was never
+issued or has aged out.
+
+Semantics:
+
+- `ok` is a pure acknowledgement — nothing is recorded.
+- `rate_limited` and `banned` currently behave **identically**: both put
+  the proxy in one cooldown window (default 15 min, `--lease-cooldown`)
+  for the resolved target.
+- An expired lease remains reportable for one cooldown window past its
+  TTL, so a late "we got rate-limited" still lands.
+
+## Proxy selection and cooldowns
+
+How `POST /v1/leases` picks among eligible proxies (healthy, matching the
+selector, not being deleted, not at capacity, not in cooldown), in order:
+
+1. fewest `activeLeases` (least-loaded),
+2. lowest `latencyMillis`,
+3. lexicographic `id` (deterministic tie-break).
+
+Cooldowns are keyed by **(proxy, target)**:
+
+- A report **with a target** blocks that proxy only for lease requests
+  naming the **same target**. Other targets — and requests with no
+  target — still get the proxy.
+- A report **without a target**, on a lease that also had no target,
+  creates a **global** cooldown: the proxy is blocked for *all* lease
+  requests until the window passes. Always pass `target` on leases and
+  reports unless you really mean "this proxy is bad for everyone".
+
+## End-to-end example
+
+```sh
+# 1. Acquire a lease for crawling example.com through an EU proxy
+LEASE=$(curl -s -XPOST "$BASE_URL/v1/leases" \
+  -H "Authorization: Bearer $TOKEN" \
+  -d '{"selector":{"geo":"eu"},"ttlSeconds":600,"target":"example.com"}')
+LEASE_ID=$(echo "$LEASE" | jq -r .leaseID)
+PROXY=$(echo "$LEASE" | jq -r '"\(.proxy.ip):\(.proxy.port)"')
+
+# 2. Crawl through the leased proxy
+curl -x "http://$PROXY" https://example.com/some/page
+
+# 3. Got a 429? Report it — example.com-bound leases will avoid this
+#    proxy for the next 15 minutes
+curl -s -XPOST "$BASE_URL/v1/leases/$LEASE_ID/report" \
+  -H "Authorization: Bearer $TOKEN" \
+  -d '{"result":"rate_limited","target":"example.com"}'
+
+# 4. Done early? Release the slot (otherwise the TTL frees it)
+curl -s -XDELETE "$BASE_URL/v1/leases/$LEASE_ID" \
+  -H "Authorization: Bearer $TOKEN"
+```
+
+## Caveats
+
+- **Lease and cooldown state is in-memory and per-process.** An operator
+  restart drops all active leases and cooldowns. Clients must tolerate a
+  granted lease disappearing (a subsequent report returns `404`).
+- **Run a single replica.** The API is served by every manager replica but
+  is not leader-elected, and lease state is not shared between replicas;
+  the shipped Deployment pins `replicas: 1`.

docs/architecture.md

@@ -1,10 +1,22 @@
 # Architecture
 
-> **Status:** the operator is built through Step 5 (health engine) of
-> [docs/plans/2026-08-07-1747-proxy-operator.md](plans/2026-08-07-1747-proxy-operator.md).
-> This document currently covers the event/reconcile flow; the components
-> table and the Decisions section arrive with Step 10, and the diagrams
-> below grow as the lease store, discovery API, and orphan GC land.
+## Components
+
+| Component | Package | Runs as | Leader-elected | Role |
+|---|---|---|---|---|
+| Proxy + ProviderHealth CRDs | `api/v1alpha1` | types | — | `Proxy` spec/status, CEL validation, defaulting, pure helpers; the status-only, cluster-scoped `ProviderHealth` |
+| Reconciler | `internal/controller` | controller | yes (with the manager) | the state machine: provision, replace, delete, represent health |
+| Provider contract | `internal/provider` | library | — | `Provider` interface, optional `Checker`, error taxonomy, deterministic naming, config, metrics decorator |
+| kubernetes provider | `internal/provider/kubernetes` | library | — | real Squid pods in this cluster (local dev/CI) |
+| gcp provider | `internal/provider/gcp` | library | — | Compute Engine VMs, four API calls, fire-and-forget ops |
+| kvm provider | `internal/provider/kvm` | library | — | VMs on libvirt hosts over SSH-forwarded sockets; image cache, overlay disks, NoCloud seeds |
+| Health engine | `internal/health` | Runnable | yes | through-the-proxy probes, thresholds, transition events |
+| Lease store | `internal/lease` | Runnable (expiry sweep) | no | in-memory leases + cooldowns, single mutex |
+| Discovery API | `internal/discovery` | Runnable | no | HTTP list/lease/release/report on `:8090` |
+| Orphan GC | `internal/gc` | Runnable | yes | deletes tagged instances whose CR is gone |
+| Provider health monitor | `internal/providerhealth` | Runnable | yes | periodic provider checks → one `ProviderHealth` per provider, target gauges, transition logs |
+| Metrics | `internal/metrics` | library | — | explicit registration, scrape-time collectors |
+| Composition root | `cmd/main.go` | binary | — | flags, provider registry, wires everything onto one manager |
 
 ## Event flow: cluster events → reconciler functions
 
@@ -117,20 +129,51 @@ reconcileDelete(ctx, p)                          reconcileExternal(ctx, p)
         ──► RequeueAfter: DeletionPoll (poll until gone)
 ```
 
-### 5. What provider calls do back in the cluster (kubernetes pod provider)
+### 5. What provider calls do in the outside world
 
 ```text
+kubernetes pod provider (internal/provider/kubernetes/)
 prov.Create ──► buildPod (pure) ──► client.Create(corev1.Pod)   ─┐ these cause Pod events,
 prov.Get    ──► client.Get(Pod) → phase/IP → InstanceState       │ but the operator does NOT
 prov.Delete ──► client.Delete(Pod, tolerate NotFound)            │ watch Pods — it observes
 prov.ListByTag ─► client.List(Pods by labels, all namespaces)   ─┘ them by polling prov.Get
                                                                     on each RequeueAfter tick
+
+gcp provider (internal/provider/gcp/) — instances.{Insert,Get,Delete,AggregatedList}, nothing else
+prov.Create ──► buildInsertRequest (pure) ──► instances.Insert   ─┐ fire-and-forget:
+                  409 alreadyExists = success (idempotent retry)  │ Operation.Wait is never
+prov.Get    ──► instances.Get → status/NatIP → InstanceState      │ called; readiness is
+                  RUNNING without NatIP = still Provisioning      │ discovered by Get polls,
+prov.Delete ──► instances.Delete (404 = success)                  │ exactly like the pod
+prov.ListByTag ─► AggregatedList(label filter,                   ─┘ provider
+                    ReturnPartialSuccess: true)
+providerID = zones/<zone>/instances/<name> — zone-qualified, so Get/Delete
+stay correct even mid-replacement after a zone edit
+
+kvm provider (internal/provider/kvm/) — libvirt RPC over an SSH-forwarded unix socket, per host
+prov.Create ──► DomainLookupByName: exists → start if needed, done      ─┐ every step tolerates
+                else: ensure base image in pool (download on first use) │ the previous attempt
+                      overlay volume (exists = reuse)                    │ having got that far;
+                      seed ISO (always rebuilt — a cut-off upload must   │ the seed is the one
+                        never be trusted)                                │ thing redone
+                      DomainDefineXML + DomainCreate                    ─┘
+prov.Get    ──► DomainGetState + DomainGetMetadata + DomainInterfaceAddresses
+                  (ARP, then guest agent; first routable IPv4)
+                  running without IP = Provisioning; nostate = Provisioning
+prov.Delete ──► destroy (not running = fine) → undefine → delete overlay + seed by name
+prov.ListByTag ─► ConnectListAllDomains on every host, keep domains whose
+                    <metadata> carries managed=true; one host down fails the sweep,
+                    one unreadable domain is skipped
+providerID = hosts/<host>/domains/<name>; GC identity lives in the domain's
+<metadata> element (libvirt has no labels). All RPCs run under a context-
+bounded call: go-libvirt never times out on its own, so a cancelled reconcile
+closes the SSH connection to unblock it.
 ```
 
-The reconciler never watches provider-side resources (Pods now, GCP VMs
-later). All instance-state observation is poll-based through the
-`Provider` interface, so the same flow works identically for a cloud API
-that has no watch mechanism at all.
+The reconciler never watches provider-side resources (Pods or GCP VMs).
+All instance-state observation is poll-based through the `Provider`
+interface, so the same flow works identically for a cloud API that has no
+watch mechanism at all.
 
 ### 6. Health engine (`internal/health/`) — probes and transitions
 
@@ -183,3 +226,390 @@ Consequence worth knowing: `status.lastHealthCheckTime` is the time of the
 last *status-affecting* probe, not the most recent probe — suppressed
 probes deliberately never write status. True probe recency will live in
 metrics (Step 9).
+
+### 7. Discovery + lease API (`internal/discovery/`, `internal/lease/`)
+
+HTTP-driven, not cluster-event-driven: crawler clients call in; the only
+Kubernetes interaction is reading Proxies from the manager's cache. The
+server is a non-leader-elected Runnable (all replicas would serve, but the
+deployment ships `replicas: 1` because lease state is per-process — an
+operator restart drops all leases and cooldowns, a documented caveat).
+Client-facing reference with request/response schemas and curl examples:
+[api.md](api.md).
+
+```text
+crawler client
+  │  Authorization: Bearer $DISCOVERY_TOKEN   (empty token = auth disabled, loud startup warning)
+  ▼
+Server.handler()      middleware, outermost first        (server.go)
+  recover → request-log → MaxBytesReader(64KiB) → bearer auth (constant-time; /healthz exempt)
+  │
+  ├─ GET  /healthz            ──► 200 ok (unauthenticated)
+  │
+  ├─ GET  /v1/proxies?attr.k=v&healthy=true            (handlers.go)
+  │     Reader.List(Proxies) ── manager cache
+  │     filter: attributes equality + Healthy condition
+  │     + Store.Counts() for activeLeases
+  │     ──► 200 {"proxies":[...], "count":N}   (empty list is 200, not 404)
+  │
+  ├─ POST /v1/leases  {"selector":{...},"ttlSeconds":300,"target":"..."}
+  │     Reader.List → filter selector; unhealthy matches counted, not offered
+  │     Store.Acquire(healthy candidates, target, ttl)   ── one lock: select+insert
+  │     │    selection: fewest active leases, then latency, then name
+  │     ├─ granted ──► 201 {leaseID, proxy:{...}, expiresAt, ttlSeconds}
+  │     └─ ErrNoMatch ──► 409 {"error":"no_match", considered, atCapacity,
+  │                            inCooldown, unhealthy}
+  │
+  ├─ DELETE /v1/leases/{id} ──► Store.Release ──► always 204 (idempotent)
+  │
+  └─ POST /v1/leases/{id}/report {"result":"ok|rate_limited|banned","target":"..."}
+        Store.Report ── rate_limited/banned ⇒ cooldown[{proxy,target}] for
+        │               CooldownWindow (target falls back: report → lease → global)
+        ├─ 204 │ 400 invalid_result │ 404 unknown_lease
+        └─ an expired lease still resolves for CooldownWindow past its TTL —
+           a late report lands exactly when the proxy is being rate-limited
+
+Store.Start(ctx)  ── manager Runnable, NOT leader-elected: sweeps expired
+                     leases + cooldowns; correctness never depends on the
+                     sweep (every read checks ExpiresAt against the clock)
+```
+
+### 8. Orphan GC (`internal/gc/`) — the crash-safety net
+
+Timer-driven, leader-elected (destructive ⇒ single writer). Exists for the
+one gap the reconciler cannot close alone: a crash after a provider Create
+but before the status write that records the instance.
+
+```text
+Sweeper.Start(ctx)   ── refuses to run when the cache is namespace-
+  │                     restricted unless --gc-allow-namespaced is explicit
+  │                     (an incomplete live set would "orphan" live VMs)
+  └─ every Interval (10m; first sweep a full interval after start):
+       sweep(ctx)
+         │ Reader.List(Proxies) → live UID set
+         │   List fails → skip the whole sweep (never guess)
+         │   a CR with deletionTimestamp still counts as LIVE — its
+         │   finalizer owns that deletion; GC racing it double-deletes
+         └ per provider: ListByTag
+             │ error → log, continue with the next provider
+             └ delete only when ALL hold:
+                 has the proxy-operator-uid label   (ownership proof)
+                 older than MinAge (10m)            (not mid-create)
+                 UID matches no existing CR         (truly orphaned)
+               each kill logged loudly with provider, providerID, UID
+```
+
+### 9. Metrics (`internal/metrics/`)
+
+Registered explicitly from `cmd/main.go` (no `init()`; tests use fresh
+registries). Two kinds:
+
+- **Scrape-time collectors** — `proxy_operator_proxies{phase}` and
+  `proxy_operator_leases_active` read the cache / lease store at every
+  scrape; reconcile-incremented gauges inevitably drift and leak series.
+  The provider-check gauges (§11) read the health monitor's snapshot the
+  same way.
+- **Fed vectors** — `healthcheck_duration_seconds{proxy}` and
+  `healthcheck_failures_total{proxy}` observe EVERY probe (status writes
+  are transition-only; metrics carry the high-frequency signal), and the
+  health engine deletes a proxy's series when it prunes its state;
+  `lease_requests_total{outcome}` from the discovery handlers;
+  `provider_requests_total{provider,op,result}` from the
+  `provider.WithMetrics` decorator — the one place `Class()` is called
+  purely for observability.
+
+Each consuming package defines its own small recorder interface
+(`health.ProbeMetrics`, `discovery.LeaseMetrics`, `provider.RequestRecorder`);
+`metrics.Metrics` satisfies all of them structurally, so no package other
+than `cmd/main.go` imports the metrics package.
+
+### 10. Tracing (`internal/tracing/`)
+
+OpenTelemetry tracing, integrated with — not replacing — the logr/zap
+logging. Everything hangs off standard `OTEL_*` env vars: with no
+`OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` set, no SDK is
+installed, spans are no-ops, and logs are byte-for-byte what they were.
+`OTEL_SDK_DISABLED=true` and `OTEL_TRACES_EXPORTER=none` force it off;
+`console` prints spans to stdout for local dev; sampling follows
+`OTEL_TRACES_SAMPLER(_ARG)`.
+
+Span topology — each unit of work is a new trace (watch events carry no
+incoming trace context):
+
+- `Reconcile Proxy` (root, via a `reconcile.Reconciler` decorator) →
+  `reconcile.managed` / `reconcile.replaceInstance` / `reconcile.delete`,
+  plus `status.patch` from the deferred flush. `reconcileExternal` does no
+  I/O and is unspanned.
+- `provider.create|get|delete|list` (client spans, `provider.WithTracing`
+  decorator wired outermost around `WithMetrics`); the GCP SDK's own
+  otelhttp transport contributes HTTP child spans automatically.
+- Kubernetes API calls become client child spans via a wrapped
+  `rest.Config` transport — gated on an existing parent span, so informer
+  list/watch long-polls and leader-election renewals never create root
+  spans. The kube-apiserver ignores incoming `traceparent` by design;
+  these spans are leaves.
+- Discovery API: one server span per request (named from the route
+  pattern), incoming W3C `traceparent` honored so clients' traces continue
+  into the operator; `/healthz` excluded.
+- `gc.sweep` per GC pass; `providerhealth.check` per round of provider
+  checks; `health.probe` per probe **only** with
+  `--trace-health-probes` (default off — probes run ~1/s per proxy).
+
+Log correlation: logr sinks never see a context, so trace IDs ride on the
+logger — `tracing.Start` re-derives the ctx logger from a captured base
+with `traceID`/`spanID` values (lowerCamel, matching `reconcileID`;
+Grafana/Loki derived-field regexes must match `traceID`, not `trace_id`).
+Re-deriving from the base rather than layering keeps zap from emitting
+duplicate keys on nested spans. GCP V(5) wire logs get the same keys from
+the slog handler's ctx.
+
+### 11. Provider health checks (`internal/providerhealth/`)
+
+Answers "can this cluster reach each provider's backend right now?"
+without waiting for a Proxy to fail. A provider opts in by implementing
+`provider.Checker`: `Check(ctx) []TargetStatus` returns one verdict per
+target and never fails as a whole. All three providers do:
+
+| Provider | Target | What one check does |
+|---|---|---|
+| kvm | each `kvm.hosts` entry | over the pooled SSH+libvirt connection: libvirt version and hostname, then the configured pool must exist and be running; reports pool capacity and free space ([details](kvm-provider.md#health-check)) |
+| kubernetes | `apiserver` | lists at most one managed Pod cluster-wide: the verb and scope `ListByTag` and GC rely on |
+| gcp | the project | one page of an aggregated instance list with `maxResults=1`: credentials, Compute API enabled, `compute.instances.list` |
+
+```text
+Monitor.Start(ctx)   ── leader-only; --provider-check-interval (1m; 0 = off,
+  │                     otherwise ≥ 10s), --provider-check-timeout (15s, or
+  │                     the interval if that is shorter)
+  ├─ once: delete managed-by-labelled ProviderHealth objects whose provider
+  │   is not configured (all of them when checks are off) — the providers
+  │   config is read once per process
+  └─ at start, then every Interval: checkOnce
+       │ every provider's Check concurrently, under one context.WithTimeout
+       │ snapshot → the scrape-time gauges
+       │ log transitions: down or reason changed → "WARNING: provider target
+       │   not ready"; back up → "Provider target ready"; steady → silent
+       └ per provider: APIReader.Get → Create if missing → status merge patch
+```
+
+- **One status write per provider per round**, since `lastCheckTime`
+  moves every round. At the 10s minimum interval that is still far below
+  leader-election renewals. Proxy status avoids writes like this only
+  because its probes run once a second per proxy.
+- **Reads go through the API reader**, not the cache. The objects are
+  cluster-scoped; a cached Get would start a cluster-wide informer (and
+  need `watch`) even with `--proxy-namespace`.
+- **Checks bypass the provider decorators.** `cmd/main.go` collects the
+  Checkers before wrapping the providers in `WithTracing(WithMetrics(...))`,
+  so a check never counts as a provider request. KVM RPCs still appear in
+  the V(1) `libvirt call` trace, and each round is one
+  `providerhealth.check` span.
+- **Not wired to `/readyz`.** A dead hypervisor must not make the manager
+  unready while the other providers keep working.
+- **Gauges** `proxy_operator_provider_target_up`,
+  `proxy_operator_provider_check_duration_seconds` and
+  `proxy_operator_provider_check_timestamp_seconds`, each labelled
+  `{provider,target}`. Only the leader exports them.
+
+Illustrative output:
+
+```sh
+$ kubectl get providerhealth          # short name: ph
+NAME         TYPE         READY   TARGETS   LAST CHECK   AGE
+kubernetes   kubernetes   True    1/1       12s          3d
+kvm-home     kvm          False   0/1       12s          3d
+$ kubectl get ph kvm-home -o yaml     # per target: reason, message, details
+```
+
+## Decisions
+
+Judgment calls the spec left open, and deliberate deviations — recorded so
+they read as choices, not accidents. Chronological by build step.
+
+- **Registry takes its constructor map as a parameter** instead of holding
+  a package-level map: avoids the provider⇄registry import cycle and puts
+  the wiring at the composition root, where it is visible.
+- **Mock provider replaced by the kubernetes-pod provider** (user
+  decision, mid-build): a simulated in-memory provider was too far from
+  the real system to build confidence in. Local dev/CI now runs real
+  `ubuntu/squid` pods (Canonical's actively maintained image, verified
+  50M+ pulls, pinned tag) in the operator's own cluster. Trade-off
+  accepted: envtest has no kubelet, so end-to-end proof lives in the kind
+  quickstart, and cluster pods share one egress IP — distinct egress
+  paths remain the GCP provider's job.
+- **`RequeueAfter: RequeueNow` instead of the plan's `Requeue: true`:**
+  `ctrl.Result.Requeue` is deprecated in controller-runtime v0.24; a fifth
+  configurable interval (default 1s) keeps identical semantics and stays
+  shrinkable in tests.
+- **Quota exhaustion is a wait, not a failure:** `ErrQuotaExceeded` sets a
+  condition and requeues slowly (5m) with a nil error — off the backoff
+  curve, out of the error log, and never `phase: Failed`. Only
+  `ErrPermanent` latches Failed, keyed to the generation so a spec edit
+  auto-recovers.
+- **The finalizer path never latches permanent failures:** a permanent
+  error during deletion keeps retrying visibly instead — latching there
+  would wedge the object forever with no path out but manual finalizer
+  surgery.
+- **Health transitions travel reconciler-ward over a channel**
+  (`source.Channel`), not direct status patches: `phase` derives from both
+  provisioning and health, so two status writers would race and flap. One
+  writer of status; the engine owns health *state*, the reconciler its
+  *representation*; write-only-on-transition falls out for free.
+- **Health state seeds from the existing Healthy condition on leader
+  handover** (verdict kept, counters zeroed, first probe jittered), so a
+  healthy fleet doesn't flap to Unknown on restart — but a real
+  transition still needs a full threshold run. A never-probed proxy skips
+  the jitter and probes on the next tick: startup spread matters for
+  restarts, not for a single new proxy.
+- **Latency suppression is `max(20ms, 50%)` + a 60s rate limit, and only
+  while the verdict is healthy.** The spec's bare ">50% change" is
+  undefined at 0 and lets a proxy jittering 40↔61ms write status forever;
+  the healthy-only guard (found by test) stops a below-threshold success
+  streak from emitting latency updates for a proxy still reported
+  unhealthy. Consequence: `status.lastHealthCheckTime` means "last
+  status-affecting probe" — true probe recency is in the metrics.
+- **Deterministic instance names** are `proxy-` + 16 chars of
+  base32(SHA-256(CR UID)): legal for both GCP (`[a-z2-7]` ⊂ `[-a-z0-9]`,
+  22 ≤ 63 chars) and Pod names, 80 bits against birthday collisions at a
+  fleet of tens. The replacement VM therefore has the *same name* as the
+  one being deleted — which is why replacement polls to NotFound before
+  recreating instead of racing a 409.
+- **`banned` and `rate_limited` share one cooldown window:** a second
+  duration knob the spec doesn't ask for; the report's semantic
+  difference is preserved in the API but not the store.
+- **Report targets fall back report → lease → global**, so a client that
+  leased with a target can't accidentally poison the proxy's global pool
+  by omitting the target in its report.
+- **The 409 body's `considered` counts unhealthy matches too** (the store
+  only ever sees healthy candidates): `considered = atCapacity +
+  inCooldown + unhealthy + eligible-but-outranked`, keeping the numbers
+  additive for a human debugging "why no proxy?".
+- **TTLs above `--max-lease-ttl` are a 400, not a silent clamp** — a
+  client asking for a week should find out.
+- **Discovery is not leader-elected and ships `replicas: 1`:** caches
+  start before non-leader-election runnables (verified in
+  controller-runtime's ordering), and a leader-elected server would leave
+  non-leader replicas as broken Service endpoints. One replica because
+  lease state is per-process.
+- **GCP `Create` requires zone, machineType, and image** and fails
+  `ErrPermanent` naming the missing field — inventing machine-type
+  defaults would silently create billable VMs of arbitrary shape.
+- **Unknown GCP instance statuses map to `Stopped`:** the reconciler's
+  answer to Stopped is delete-and-recreate, the always-safe move for
+  cattle when the API grows a new state.
+- **KVM hosts are reached over SSH with the libvirt socket forwarded,**
+  not libvirtd's TCP/TLS listener: nothing to configure or expose on the
+  hypervisors beyond sshd, pure Go on the operator side (no cgo, no
+  libvirt libraries in the image), and the hosts list can grow by a
+  config edit because the providerID carries the host name.
+- **KVM base images are fetched by the operator into the host's pool**
+  through libvirt's upload stream, rather than expected to be pre-placed:
+  the operator then needs no filesystem access on the host and a new
+  host is usable with zero manual staging. The cost is a synchronous
+  first-time download inside Create.
+- **The KVM seed ISO is rebuilt on every Create attempt** before the
+  domain exists, instead of "exists → skip": a reconcile cancelled mid
+  upload leaves a truncated seed, and the compensating delete cannot run
+  because the connection was already closed to unblock the RPC. Skipping
+  it would boot a VM with no cloud-init and no retry would ever repair it.
+- **KVM `nostate` maps to `Provisioning`, not `Stopped`:** unlike GCP's
+  unknown statuses, `nostate` means libvirt could not determine the
+  state at all; answering that with delete-and-recreate is too drastic,
+  so the next poll decides.
+- **KVM guests boot UEFI by default, secure boot off.** Found on the
+  first real host: Debian 13's generic image hangs in GRUB under SeaBIOS
+  on a headless VM (Debian bug #1111240: `terminal_output gfxterm
+  serial` with a serial console and no video device) regardless of CPU
+  model or seed, and boots under OVMF with the serial console working.
+  The alternatives were a virtio console (loses the kernel's serial
+  output) or a video device the VMs have no use for. Every
+  current distro cloud image supports UEFI, so it is the default and
+  `firmware: bios` is the per-image escape; libvirt's firmware
+  autoselection (`<os firmware="efi">`) keeps the OVMF path out of the
+  operator's config. Secure boot is off so unsigned images boot too.
+- **A KVM domain GC cannot read is skipped, a host it cannot reach fails
+  the sweep.** GC never deletes what it does not see, so leaving one
+  domain out is safe; one corrupt domain blocking GC for every other one
+  is not. A whole host missing from the view would hide many, so that
+  still aborts.
+- **Kubernetes 403s classify as `ErrPermanent`** even though quota
+  exhaustion also surfaces as 403 (indistinguishable from RBAC denial in
+  `apierrors`): not hammering an API server that may never allow the
+  request is the safer default; a real ResourceQuota 403 forgoes the
+  gentler quota backoff. Documented at the classification site.
+- **GC kills log at Info with a `WARNING:` prefix** — logr has no Warn
+  level; the plan's "log at Warn" is met in spirit with provider,
+  providerID, and UID always attached. Same convention as the
+  discovery server's empty-token warning.
+- **GC trusts only provable orphans:** instances without the UID label
+  are never deleted, a CR with a deletionTimestamp still counts as live
+  (its finalizer owns that deletion), and an unreadable Proxy list skips
+  the whole sweep. The namespace guard refuses to sweep a
+  namespace-restricted cache without `--gc-allow-namespaced`.
+- **Cloud-init Secrets must carry `crawl.example.com/cloud-init: "true"`:**
+  the manager caches only labelled Secrets (the operator holds
+  cluster-wide Secret read RBAC — an unrestricted cache would hold every
+  Secret in scope). Unlabelled referenced Secrets are invisible by
+  construction, surfacing as `CloudInitError`.
+- **Deletion never gives up on its own, but can be released by hand:** a
+  permanent cleanup failure keeps retrying with the finalizer on, because
+  dropping it automatically would leak a running instance on any provider
+  that answers "permission denied". Where `Get` fails permanently the
+  reconciler still calls `Delete`, since it is idempotent and may finish
+  where `Get` cannot converge. `crawl.example.com/skip-cleanup: "true"`
+  abandons the instance and releases the object; it is checked before the
+  provider lookup, so it also frees a Proxy whose provider is gone from the
+  config.
+- **Events RBAC from the plan is omitted:** nothing wires an
+  EventRecorder in the prototype, and granting verbs nothing uses would
+  be RBAC lint noise. Add the marker together with the recorder if events
+  land later.
+- **logr, not slog, inside controller paths:** the repo convention says
+  `slog`, but `log.FromContext(ctx)` hands controller-runtime's logr
+  logger to everything running under the manager — fighting that would
+  mean two logging systems in one process. Noted as a deviation rather
+  than silently ignored.
+- **Tracing is env-gated, not flag-gated:** it activates only when
+  `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` is set (user
+  decision). No collector configured means no SDK installed, no-op spans,
+  unchanged logs — the safe default for every existing deployment. The
+  one flag is `--trace-health-probes`, off by default, because probes at
+  ~1/s per proxy would dominate trace volume.
+- **Trace keys are `traceID`/`spanID`**, lowerCamel like `reconcileID` and
+  `providerID`, deliberately not the `trace_id` many Grafana derived-field
+  examples assume — configure the derived-field regex accordingly. logr
+  sinks can't read ctx, so the IDs ride on the ctx logger, re-derived from
+  a captured base per span so zap never emits duplicate keys.
+- **Probe→reconcile trace links are not attempted:** the health engine's
+  `GenericEvent` carries only namespace/name (no ctx), and the workqueue
+  coalesces events, so any link would be a guess. A health-triggered
+  reconcile starts a fresh trace; the probe that caused it is findable via
+  its own (opt-in) span and shared proxy attributes.
+- **No `traceparent` toward probe targets:** probe transports stay
+  uninstrumented so trace headers can never leak through a proxy to
+  external sites. The kube-apiserver ignores incoming `traceparent` by
+  design (public endpoint), so k8s client spans are leaves — in-process
+  traces, not cross-process ones.
+- **Provider health is a status-only, cluster-scoped CRD** (user
+  decision). `kubectl get providerhealth` works without Prometheus, which
+  the deployment does not ship (its ServiceMonitor is commented out). There
+  is no spec: the providers config file stays the one place providers are
+  configured, and the object only reports on it. The gauges carry the same
+  verdicts for alerting.
+- **KVM checks reuse the pooled connection instead of dialing fresh.** A
+  fresh dial would not pick up a rotated Secret anyway, since the SSH
+  material is cached. It would also open an sshd session per host per
+  minute. And a working cached connection is exactly what reconciles use.
+  Sharing must not cost the reconciles anything, so the check calls
+  through `hostPool.probe`. A check that runs out of time abandons its
+  RPCs instead of closing the connection, which would kill an upload or a
+  Create running on it. At most one abandoned check per host is left
+  waiting, and until it returns, the next check reports `Timeout` at once.
+  Waiting for a host's connection slot also ends with the caller's
+  deadline, so a reconcile redialing a dead host cannot stall the round.
+- **Provider transitions are logs, not Events**, in line with the
+  EventRecorder decision above. They log at Info with a `WARNING:` prefix,
+  and each target's `lastTransitionTime` in the object records when it
+  flipped.
+- **A provider name that is not a valid object name** gets metrics but no
+  `ProviderHealth` object, with a warning at start. Tightening config
+  validation instead would crash-loop existing deployments on upgrade.

docs/demo/create-gcp-proxies.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+# Demo: create N Managed proxies backed by the gcp provider, named
+# proxy-gcp-demo-1 .. proxy-gcp-demo-N, each in a randomly picked EU zone
+# so the fleet gets egress IPs from different locations.
+#
+# Usage:
+#   ./create-gcp-proxies.sh <count>
+#
+# Optional environment:
+#   NAMESPACE     namespace to create the proxies in (default: current context)
+#   GCP_PROVIDER  provider NAME from providers.yaml (default: gcp-eu)
+#   ZONES         space-separated zone list to pick from (default: EU zones below)
+set -euo pipefail
+
+if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
+    echo "usage: $(basename "$0") <count>  (positive integer)" >&2
+    exit 1
+fi
+count="$1"
+provider="${GCP_PROVIDER:-gcp-eu}"
+
+# GCP zones in the EU where e2-micro is generally available. Override with
+# ZONES="zone1 zone2 ..." if your project has quota only in some of them.
+default_zones=(
+    europe-west1-b europe-west1-c europe-west1-d       # Belgium
+    europe-west2-a europe-west2-b europe-west2-c       # London
+    europe-west3-a europe-west3-b europe-west3-c       # Frankfurt
+    europe-west4-a europe-west4-b europe-west4-c       # Netherlands
+    europe-west6-a europe-west6-b europe-west6-c       # Zurich
+    europe-west8-a europe-west8-b europe-west8-c       # Milan
+    europe-west9-a europe-west9-b europe-west9-c       # Paris
+    europe-central2-a europe-central2-b europe-central2-c   # Warsaw
+    europe-north1-a europe-north1-b europe-north1-c    # Finland
+    europe-southwest1-a europe-southwest1-b europe-southwest1-c   # Madrid
+)
+if [ -n "${ZONES:-}" ]; then
+    read -r -a zones <<< "${ZONES}"
+else
+    zones=("${default_zones[@]}")
+fi
+
+if ! command -v kubectl &> /dev/null; then
+    echo "ERROR: kubectl is required but not installed" >&2
+    exit 1
+fi
+
+ns_args=()
+if [ -n "${NAMESPACE:-}" ]; then
+    ns_args=(-n "${NAMESPACE}")
+fi
+
+for i in $(seq 1 "${count}"); do
+    zone="${zones[RANDOM % ${#zones[@]}]}"
+    echo "Creating proxy-gcp-demo-${i} in ${zone} ..."
+    kubectl apply "${ns_args[@]}" -f - << EOF
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-gcp-demo-${i}
+spec:
+  mode: Managed
+  provider: ${provider}              # must match a provider NAME in providers.yaml
+  placement:
+    zone: ${zone}
+    machineType: e2-micro
+    # debian-cloud images have no cloud-init, so spec.cloudInit (passed as
+    # user-data metadata) would be silently ignored there. Ubuntu images do.
+    image: projects/ubuntu-os-cloud/global/images/family/ubuntu-2404-lts-amd64
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      package_update: true
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+  attributes:
+    geo: eu
+    zone: ${zone}
+    purpose: crawl
+EOF
+done
+
+echo ""
+echo "Created ${count} proxies. VMs take a few minutes to provision and pass"
+echo "the health check. Watch them come up with:"
+echo "  kubectl get px ${ns_args[*]:-} -w"

docs/demo/create-kubernetes-proxies.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Demo: create N Managed proxies backed by the kubernetes-pod provider,
+# named proxy-kubernetes-demo-1 .. proxy-kubernetes-demo-N. Pods share the
+# cluster's egress IP — this exercises the full lifecycle, not distinct
+# egress paths (use create-gcp-proxies.sh for that).
+#
+# Usage:
+#   ./create-kubernetes-proxies.sh <count>
+#
+# Optional environment:
+#   NAMESPACE  namespace to create the proxies in (default: current context)
+set -euo pipefail
+
+if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
+    echo "usage: $(basename "$0") <count>  (positive integer)" >&2
+    exit 1
+fi
+count="$1"
+
+if ! command -v kubectl &> /dev/null; then
+    echo "ERROR: kubectl is required but not installed" >&2
+    exit 1
+fi
+
+ns_args=()
+if [ -n "${NAMESPACE:-}" ]; then
+    ns_args=(-n "${NAMESPACE}")
+fi
+
+for i in $(seq 1 "${count}"); do
+    echo "Creating proxy-kubernetes-demo-${i} ..."
+    kubectl apply "${ns_args[@]}" -f - << EOF
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-kubernetes-demo-${i}
+spec:
+  mode: Managed
+  provider: kubernetes
+  attributes:
+    geo: local
+    purpose: crawl
+EOF
+done
+
+echo ""
+echo "Created ${count} proxies. Watch them come up with:"
+echo "  kubectl get px ${ns_args[*]:-} -w"

docs/demo/run-demo.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Demo driver: opens a tmux session with a 2x2 pane grid:
+#
+#   top-left:     show-egress-ips-table.sh in a 10s loop, run inside the
+#                 netshoot pod against the in-cluster discovery Service
+#   top-right:    watch -n3 kubectl get px
+#   bottom-left:  create-kubernetes-proxies.sh <count>
+#   bottom-right: create-gcp-proxies.sh <count>
+#
+# Usage:
+#   ./run-demo.sh
+#
+# Optional environment:
+#   COUNT         proxies each create script makes (default: 4)
+#   SESSION       tmux session name (default: proxy-demo; an existing
+#                 session with this name is killed and recreated)
+#   NETSHOOT_POD  pod to exec into for the egress-IP loop (default: netshoot)
+#   DEMO_DIR      where the demo scripts live (default: this script's dir)
+set -euo pipefail
+
+COUNT="${COUNT:-4}"
+SESSION="${SESSION:-proxy-demo}"
+NETSHOOT_POD="${NETSHOOT_POD:-netshoot}"
+DEMO_DIR="${DEMO_DIR:-$(cd "$(dirname "$0")" && pwd)}"
+BASE_URL="http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090"
+
+for tool in tmux kubectl; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+if ! kubectl get pod "${NETSHOOT_POD}" &> /dev/null; then
+    echo "ERROR: pod ${NETSHOOT_POD} not found — start one with:" >&2
+    echo "  kubectl run netshoot --image=nicolaka/netshoot -- sleep infinity" >&2
+    exit 1
+fi
+
+echo "Copying show-egress-ips-table.sh into pod ${NETSHOOT_POD} ..."
+kubectl cp "${DEMO_DIR}/show-egress-ips-table.sh" "${NETSHOOT_POD}:/tmp/show-egress-ips-table.sh"
+
+if tmux has-session -t "${SESSION}" 2> /dev/null; then
+    echo "Killing existing tmux session ${SESSION}"
+    tmux kill-session -t "${SESSION}"
+fi
+
+# 2x2 grid: after these splits pane indexes are 0 top-left, 1 top-right,
+# 2 bottom-left, 3 bottom-right; tiled layout evens them into quarters.
+tmux new-session -d -s "${SESSION}"
+tmux split-window -h -t "${SESSION}:0"
+tmux split-window -v -t "${SESSION}:0.0"
+tmux split-window -v -t "${SESSION}:0.1"
+tmux select-layout -t "${SESSION}:0" tiled
+
+loop_cmd="BASE_URL=${BASE_URL}; while true; do bash /tmp/show-egress-ips-table.sh \"\$BASE_URL\"; echo; sleep 10; done"
+tmux send-keys -t "${SESSION}:0.0" "kubectl exec -it ${NETSHOOT_POD} -- bash -c '${loop_cmd}'" C-m
+tmux send-keys -t "${SESSION}:0.1" "watch -n3 kubectl get px" C-m
+tmux send-keys -t "${SESSION}:0.2" "bash ${DEMO_DIR}/create-kubernetes-proxies.sh ${COUNT}" C-m
+tmux send-keys -t "${SESSION}:0.3" "bash ${DEMO_DIR}/create-gcp-proxies.sh ${COUNT}" C-m
+
+tmux select-pane -t "${SESSION}:0.2"
+if [ -n "${TMUX:-}" ]; then
+    tmux switch-client -t "${SESSION}"
+else
+    tmux attach-session -t "${SESSION}"
+fi

docs/demo/show-egress-ips-table.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+# Demo: condensed-table variant of show-egress-ips.sh, made to fit a small
+# tmux pane. One line per proxy: which proxy the request goes through, its
+# endpoint, location (zone/geo attribute), and the egress IP the IP-echo
+# site saw — or unhealthy/FAILED.
+#
+# Usage:
+#   ./show-egress-ips-table.sh <BASE_URL>    e.g. ./show-egress-ips-table.sh localhost:8090
+#
+# Optional environment:
+#   TOKEN        bearer token for the discovery API (see docs/api.md)
+#   IP_ECHO_URL  site that returns the caller's IP as JSON with an "ip" field
+#                (default: https://api.ipify.org?format=json)
+set -euo pipefail
+
+if [ $# -ne 1 ]; then
+    echo "usage: $(basename "$0") <BASE_URL>  (e.g. localhost:8090)" >&2
+    exit 1
+fi
+BASE_URL="$1"
+IP_ECHO_URL="${IP_ECHO_URL:-https://api.ipify.org?format=json}"
+
+for tool in curl jq; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+auth_args=()
+if [ -n "${TOKEN:-}" ]; then
+    auth_args=(-H "Authorization: Bearer ${TOKEN}")
+fi
+
+if ! proxies_json=$(curl -sS --fail "${auth_args[@]}" "${BASE_URL}/v1/proxies") \
+    || ! echo "${proxies_json}" | jq -e . > /dev/null 2>&1; then
+    echo "ERROR: could not fetch proxy list from ${BASE_URL}" >&2
+    exit 1
+fi
+
+total=$(echo "${proxies_json}" | jq -r '.count')
+fmt="%-31s %-21s %-21s %s\n"
+
+echo "${total} proxies @ $(date +%H:%M:%S)"
+# shellcheck disable=SC2059
+printf "${fmt}" "PROXY" "ENDPOINT" "LOCATION" "EGRESS-IP"
+
+probed=0
+skipped=0
+failed=0
+while IFS= read -r proxy; do
+    id=$(echo "${proxy}" | jq -r '.id')
+    endpoint=$(echo "${proxy}" | jq -r '"\(.ip):\(.port)"')
+    location=$(echo "${proxy}" | jq -r '.attributes.zone // .attributes.geo // "-"')
+    healthy=$(echo "${proxy}" | jq -r '.healthy')
+
+    if [ "${healthy}" != "true" ]; then
+        # shellcheck disable=SC2059
+        printf "${fmt}" "${id}" "${endpoint}" "${location}" "(unhealthy)"
+        skipped=$((skipped + 1))
+        continue
+    fi
+
+    if response=$(curl -sS --max-time 10 -x "http://${endpoint}" "${IP_ECHO_URL}" 2> /dev/null); then
+        egress=$(echo "${response}" | jq -r '.ip // "?"' 2> /dev/null || echo "?")
+        probed=$((probed + 1))
+    else
+        egress="FAILED"
+        failed=$((failed + 1))
+    fi
+    # shellcheck disable=SC2059
+    printf "${fmt}" "${id}" "${endpoint}" "${location}" "${egress}"
+done < <(echo "${proxies_json}" | jq -c '.proxies[]')
+
+echo "-- ${probed} probed, ${skipped} unhealthy, ${failed} failed --"

docs/demo/show-egress-ips.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Demo: list proxies from the discovery API and show the egress IP each
+# healthy one provides, by calling an IP-echo site through it.
+#
+# Usage:
+#   ./show-egress-ips.sh <BASE_URL>          e.g. ./show-egress-ips.sh localhost:8090
+#
+# Optional environment:
+#   TOKEN        bearer token for the discovery API (see docs/api.md)
+#   IP_ECHO_URL  site that returns the caller's IP as JSON
+#                (default: https://api.ipify.org?format=json)
+set -euo pipefail
+
+if [ $# -ne 1 ]; then
+    echo "usage: $(basename "$0") <BASE_URL>  (e.g. localhost:8090)" >&2
+    exit 1
+fi
+BASE_URL="$1"
+IP_ECHO_URL="${IP_ECHO_URL:-https://api.ipify.org?format=json}"
+
+for tool in curl jq; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+auth_args=()
+if [ -n "${TOKEN:-}" ]; then
+    auth_args=(-H "Authorization: Bearer ${TOKEN}")
+fi
+
+echo "Fetching proxies from ${BASE_URL}/v1/proxies ..."
+if ! proxies_json=$(curl -sS --fail "${auth_args[@]}" "${BASE_URL}/v1/proxies"); then
+    echo "ERROR: could not fetch proxy list from ${BASE_URL}" >&2
+    exit 1
+fi
+if ! echo "${proxies_json}" | jq -e . > /dev/null; then
+    echo "ERROR: response from ${BASE_URL}/v1/proxies is not valid JSON" >&2
+    exit 1
+fi
+
+total=$(echo "${proxies_json}" | jq -r '.count')
+echo "Found ${total} proxies"
+echo ""
+
+probed=0
+skipped=0
+failed=0
+while IFS= read -r proxy; do
+    id=$(echo "${proxy}" | jq -r '.id')
+    ip=$(echo "${proxy}" | jq -r '.ip')
+    port=$(echo "${proxy}" | jq -r '.port')
+    healthy=$(echo "${proxy}" | jq -r '.healthy')
+
+    if [ "${healthy}" != "true" ]; then
+        echo "--- skipping ${id} (unhealthy) ---"
+        echo ""
+        skipped=$((skipped + 1))
+        continue
+    fi
+
+    echo "=== via ${id} — http://${ip}:${port} ==="
+    if response=$(curl -sS --max-time 10 -x "http://${ip}:${port}" "${IP_ECHO_URL}"); then
+        echo "${response}" | jq . 2> /dev/null || echo "${response}"
+        probed=$((probed + 1))
+    else
+        echo "WARNING: request through ${id} failed" >&2
+        failed=$((failed + 1))
+    fi
+    echo ""
+done < <(echo "${proxies_json}" | jq -c '.proxies[]')
+
+echo "Done: ${total} proxies — ${probed} probed, ${skipped} skipped (unhealthy), ${failed} failed"

docs/gcp-in-specific-project.md

@@ -0,0 +1,255 @@
+## Project egress-proxy
+
+```bash
+PROJECT_ID=egress-proxy
+
+# 1. Create the service account
+gcloud iam service-accounts create proxy-operator \
+  --project ${PROJECT_ID} \
+  --display-name "egress-proxies-operator"
+
+# Output: 
+# Created service account [proxy-operator].
+# Service account email: proxy-operator@egress-proxy.iam.gserviceaccount.com
+
+# 2. Grant compute.instanceAdmin.v1 on the project
+gcloud projects add-iam-policy-binding ${PROJECT_ID} \
+  --member "serviceAccount:proxy-operator@${PROJECT_ID}.iam.gserviceaccount.com" \
+  --role roles/compute.instanceAdmin.v1
+
+# Output:
+# ---------
+# Updated IAM policy for project [egress-proxy].
+# bindings:
+# - members:
+#   - serviceAccount:proxy-operator@egress-proxy.iam.gserviceaccount.com
+#   role: roles/compute.instanceAdmin.v1
+# - members:
+#   - serviceAccount:541231138892@cloudservices.gserviceaccount.com
+#   role: roles/compute.instanceGroupManagerServiceAgent
+# - members:
+#   - serviceAccount:service-541231138892@compute-system.iam.gserviceaccount.com
+#   role: roles/compute.serviceAgent
+# - members:
+#   - user:admin@fujultimate.cz
+#   role: roles/owner
+# etag: BwZYuFXko24=
+# version: 1
+
+# 3. Create the JSON key (this is what goes into the Secret)
+SA_KEY_PATH=sa_key.json
+gcloud iam service-accounts keys create $SA_KEY_PATH \
+  --iam-account proxy-operator@${PROJECT_ID}.iam.gserviceaccount.com
+
+# output:
+# created key [fdff85174a8e80bbd684e76c4d9fe28e2f4b2ddf] of type [json] as [sa_key.json] for [proxy-operator@egress-proxy.iam.gserviceaccount.com]
+
+# 4. A **firewall rule**: created VMs get network tag `proxy-operator` (the
+#    default; configurable as `gcp.networkTag`), an ephemeral external IP,
+#    and Squid listening on 3128.
+
+gcloud compute firewall-rules create allow-proxy-operator \
+    --project $PROJECT_ID \
+    --network default \
+    --allow tcp:3128 \
+    --target-tags proxy-operator \
+    --source-ranges 94.230.145.216/32
+```
+
+## Phase 2 - resources in kube
+
+```bash
+SA_KEY_PATH=sa_key.json
+kubectl -n egress-proxies-operator-system create secret generic gcp-credentials \
+  --from-file=key.json=$SA_KEY_PATH
+```
+
+
+## Appendix - full manifests
+
+```bash
+# crawl CR
+kubectl apply -f - <<'EOF'
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-gcp-sample
+spec:
+  mode: Managed
+  provider: gcp-eu                   # must match a provider NAME in providers.yaml
+  placement:
+    zone: europe-west1-b
+    machineType: e2-micro
+    # debian-cloud images have no cloud-init, so spec.cloudInit (passed as
+    # user-data metadata) would be silently ignored there. Ubuntu images do.
+    image: projects/ubuntu-os-cloud/global/images/family/ubuntu-2404-lts-amd64
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      package_update: true
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+  attributes:
+    geo: eu
+    purpose: crawl
+EOF
+
+
+
+# configmap
+kubectl apply -f - <<'EOF'
+apiVersion: v1
+data:
+  providers.yaml: |
+    providers:
+      - name: kubernetes
+        type: kubernetes
+      - name: gcp-eu          # spec.provider on a Proxy refers to this NAME, not the type
+        type: gcp
+        gcp:
+            project: egress-proxy
+            # network: default            # these three default as shown
+            # networkTag: proxy-operator
+            # diskSizeGb: 10
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: egress-proxies-operator
+  name: egress-proxies-operator-providers-config
+  namespace: egress-proxies-operator-system
+EOF
+
+# operator deployment
+kubectl apply -f - <<'EOF'
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    deployment.kubernetes.io/revision: "2"
+  labels:
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: egress-proxies-operator
+    control-plane: controller-manager
+  name: egress-proxies-operator-controller-manager
+  namespace: egress-proxies-operator-system
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: egress-proxies-operator
+      control-plane: controller-manager
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/default-container: manager
+      labels:
+        app.kubernetes.io/name: egress-proxies-operator
+        control-plane: controller-manager
+    spec:
+      containers:
+        - args:
+            - --metrics-bind-address=:8443
+            - --leader-elect
+            - --health-probe-bind-address=:8081
+            - --providers-config=/etc/proxy-operator/providers.yaml
+          command:
+            - /manager
+          env:
+            - name: DISCOVERY_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  key: token
+                  name: discovery-token
+                  optional: true
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: /var/secrets/gcp/key.json
+          image: egress-proxies-operator:dev
+          imagePullPolicy: IfNotPresent
+          livenessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthz
+              port: 8081
+              scheme: HTTP
+            initialDelaySeconds: 15
+            periodSeconds: 20
+            successThreshold: 1
+            timeoutSeconds: 1
+          name: manager
+          ports:
+            - containerPort: 8081
+              name: health
+              protocol: TCP
+            - containerPort: 8090
+              name: discovery
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /readyz
+              port: 8081
+              scheme: HTTP
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          resources:
+            limits:
+              cpu: 500m
+              memory: 128Mi
+            requests:
+              cpu: 10m
+              memory: 64Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: true
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /etc/proxy-operator
+              name: providers-config
+              readOnly: true
+            - mountPath: /var/secrets/gcp
+              name: gcp-credentials
+              readOnly: true
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      serviceAccount: egress-proxies-operator-controller-manager
+      serviceAccountName: egress-proxies-operator-controller-manager
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - configMap:
+            defaultMode: 420
+            name: egress-proxies-operator-providers-config
+          name: providers-config
+        - name: gcp-credentials
+          secret:
+            defaultMode: 420
+            secretName: gcp-credentials
+EOF
+```

docs/gcp-vm-validation.md

@@ -0,0 +1,158 @@
+# Validating real VM creation on GCP
+
+Recipe for wiring the GCP provider into a live cluster and watching a
+`Proxy` CR create a real Compute Engine VM. Angle brackets mark values you
+supply: `<PROJECT_ID>`, `<SA_KEY_PATH>`, `<ZONE>`, `<CLUSTER_EGRESS_IP>`,
+`<REGISTRY_IMAGE>`.
+
+The one important fact up front: **the operator takes no GCP credentials
+through its own config.** The client is built with Application Default
+Credentials (`internal/provider/gcp/gcp.go`, `New()`); there is no
+key-file field in the providers config. The only secret to prepare is a
+service-account JSON key, injected via the standard
+`GOOGLE_APPLICATION_CREDENTIALS` mechanism. On GKE you would use workload
+identity instead and skip the key entirely.
+
+## 1. GCP-side prerequisites (prepared outside the cluster)
+
+1. A project — `<PROJECT_ID>` — with the **Compute Engine API enabled**.
+2. A **service account** with `roles/compute.instanceAdmin.v1` on the
+   project. The operator only calls instances
+   `Insert`/`Get`/`Delete`/`AggregatedList` and does not attach a service
+   account to the VMs it creates, so no `iam.serviceAccountUser` is
+   needed.
+3. A **JSON key** for that service account, saved at `<SA_KEY_PATH>`.
+4. A **firewall rule**: created VMs get network tag `proxy-operator` (the
+   default; configurable as `gcp.networkTag`), an ephemeral external IP,
+   and Squid listening on 3128.
+
+   ```sh
+   gcloud compute firewall-rules create allow-proxy-operator \
+     --project <PROJECT_ID> \
+     --network default \
+     --allow tcp:3128 \
+     --target-tags proxy-operator \
+     --source-ranges <CLUSTER_EGRESS_IP>/32
+   ```
+
+   The source range must cover the cluster's egress IP — the operator's
+   CONNECT health probes originate there, and without the rule the Proxy
+   hangs at `Running`/unhealthy instead of reaching `Ready`. ⚠️ The
+   sample cloud-init configures `http_access allow all`, so on a public
+   IP this is an open proxy — keep the source ranges tight.
+
+## 2. Create the credentials Secret
+
+Namespace is `egress-proxies-operator-system` after kustomize prefixing:
+
+```sh
+kubectl -n egress-proxies-operator-system create secret generic gcp-credentials \
+  --from-file=key.json=<SA_KEY_PATH>
+```
+
+## 3. Add a GCP entry to the providers ConfigMap
+
+Edit `config/manager/providers_config.yaml` (mounted at
+`/etc/proxy-operator/providers.yaml`):
+
+```yaml
+providers:
+  - name: kubernetes
+    type: kubernetes
+  - name: gcp-eu          # spec.provider on a Proxy refers to this NAME, not the type
+    type: gcp
+    gcp:
+      project: <PROJECT_ID>
+      # network: default            # these three default as shown
+      # networkTag: proxy-operator
+      # diskSizeGb: 10
+```
+
+The config is validated fail-fast at startup — a typo shows up
+immediately in the manager log, not on first use.
+
+## 4. Mount the Secret and point ADC at it
+
+In `config/manager/manager.yaml`, add to the manager container:
+
+```yaml
+env:
+  - name: GOOGLE_APPLICATION_CREDENTIALS
+    value: /var/secrets/gcp/key.json
+volumeMounts:
+  - name: gcp-credentials
+    mountPath: /var/secrets/gcp
+    readOnly: true
+volumes:
+  - name: gcp-credentials
+    secret:
+      secretName: gcp-credentials
+```
+
+(`volumeMounts` merges into the existing container list; `volumes` into
+the existing pod-level list.)
+
+## 5. Deploy and create the Proxy
+
+```sh
+make deploy IMG=<REGISTRY_IMAGE>
+```
+
+`config/samples/proxy_gcp.yaml` is usable as-is once `spec.provider`
+matches the name from step 3. All three placement fields are mandatory
+for GCP — a missing one sets the Proxy to `Failed` with a message naming
+it:
+
+```yaml
+spec:
+  mode: Managed
+  provider: gcp-eu
+  placement:
+    zone: <ZONE>                     # e.g. europe-west1-b
+    machineType: e2-micro
+    image: projects/debian-cloud/global/images/family/debian-12
+```
+
+```sh
+kubectl apply -f config/samples/proxy_gcp.yaml
+```
+
+## 6. What you should see
+
+```sh
+kubectl get proxy -w
+```
+
+`Provisioning` → `Running` (VM's external IP published in status) →
+`Ready` (CONNECT health probe succeeded through the public IP). Then:
+
+```sh
+# the VM exists and carries the GC labels
+gcloud compute instances list --project <PROJECT_ID> \
+  --filter 'labels.proxy-operator-managed=yes'
+
+# the proxy actually tunnels — should print the VM's external IP
+curl -x http://<EXTERNAL_IP>:3128 https://ifconfig.me
+```
+
+Cleanup — the finalizer deletes the VM:
+
+```sh
+kubectl delete proxy proxy-gcp-sample
+gcloud compute instances list --project <PROJECT_ID>   # should be empty again
+```
+
+## Gotchas
+
+- **The orphan GC sweeps the whole project**: any VM labeled
+  `proxy-operator-managed=yes` whose UID does not match a live Proxy CR
+  in *this* cluster is deleted once past the age threshold. Do not point
+  two operator installs at the same project, and do not hand-create VMs
+  with that label.
+- **VM creation is fire-and-forget** — the provider never waits on the
+  insert operation; progress is discovered by polling `Get`. A quota
+  error or bad image name surfaces on the Proxy's status/conditions a
+  reconcile later, not synchronously. `kubectl describe proxy` is the
+  place to look when something stalls.
+- **e2-micro costs pennies but is not free everywhere** — remember to
+  delete the CR (or check `gcloud compute instances list`) when done.

docs/kvm-provider.md

@@ -0,0 +1,373 @@
+# The KVM provider
+
+Runs Managed proxies as VMs on your own libvirt/KVM hypervisors. The
+operator reaches each host over SSH, forwards the libvirt unix socket
+through that connection, and talks libvirt's own RPC over it — so the
+hosts need nothing beyond `sshd` and `libvirtd`, and the operator image
+carries no libvirt libraries. VMs boot a stock distro cloud image that the
+operator downloads into the host's storage pool on first use, with the
+Proxy's cloud-init delivered on a NoCloud seed ISO built in-process.
+
+## Hypervisor prerequisites
+
+Per host, once:
+
+1. **libvirt + KVM** (`libvirtd` running, `/dev/kvm` present).
+2. **An SSH user in the `libvirt` group**, so it may open
+   `/var/run/libvirt/libvirt-sock`. The operator authenticates with a
+   private key; nothing else on the host is touched.
+3. **A bridge to a network with DHCP** (e.g. `br0` bridged onto the LAN).
+   Each VM gets one virtio NIC on it and takes its IP from that network's
+   DHCP; the cluster must be able to reach those IPs for health probes
+   and for crawlers to use the proxy.
+4. **A storage pool** (default `default`, usually `/var/lib/libvirt/images`)
+   with room for one base image per distro (a few hundred MB) plus the
+   per-VM overlays (`diskSizeGb` each, thin-provisioned).
+5. **UEFI firmware for guests**: the `ovmf` package on Debian/Ubuntu,
+   `edk2-ovmf` on Fedora/RHEL. VMs boot UEFI by default (libvirt picks
+   the OVMF binary itself); a domain start fails with "no firmware
+   found" without it. Only `firmware: bios` images skip this.
+6. Optionally, **outbound internet from the VMs**: a vendor-data script
+   installs `qemu-guest-agent` from the distro mirrors on first boot.
+   Without it the VM still works — the operator falls back to the host's
+   ARP table to find the IP, which fills in once the guest sends traffic —
+   but the ARP table only sees traffic to the host itself, so on a plain
+   bridge the agent is in practice required.
+
+Check from the operator's point of view before wiring it in:
+
+```sh
+ssh virt@nas1.home.lan virsh -c qemu:///system list --all   # user can drive libvirt
+ssh virt@nas1.home.lan virsh -c qemu:///system pool-info default
+ssh virt@nas1.home.lan ip link show br0
+```
+
+## Cluster-side setup
+
+1. Create the SSH Secret from a key pair dedicated to the operator and a
+   `known_hosts` file holding every host's key. The worked example below
+   uses `~/.ssh/proxy-operator_ed25519` and `~/.ssh/kvm_known_hosts`,
+   the same paths the integration test and `make kvm-ssh-secret` default
+   to.
+
+   ```sh
+   # once: a key pair for the operator; the .pub goes on every host
+   ssh-keygen -t ed25519 -f ~/.ssh/proxy-operator_ed25519 -N '' -C proxy-operator
+   ssh-copy-id -i ~/.ssh/proxy-operator_ed25519.pub virt@nas1.home.lan
+
+   # once per host: its host key, under the address the *operator* will
+   # dial (hosts[].address), which may differ from what your laptop uses
+   ssh-keyscan -H nas1.home.lan >> ~/.ssh/kvm_known_hosts
+
+   # the Secret: the PRIVATE key (not the .pub) plus known_hosts
+   kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
+     --from-file=id_ed25519=$HOME/.ssh/proxy-operator_ed25519 \
+     --from-file=known_hosts=$HOME/.ssh/kvm_known_hosts
+   ```
+
+   `make kvm-ssh-secret` runs that last command as an apply, so it also
+   updates an existing Secret; override the paths with
+   `KVM_SSH_KEY=... KVM_KNOWN_HOSTS=...`. Putting the `.pub` in the
+   Secret is the classic mistake: the operator then fails at startup
+   with a "not a private key" error. Without `known_hosts`, set
+   `insecureIgnoreHostKey: true` and accept that a MITM could hand you a
+   fake hypervisor.
+
+   `config/manager/manager.yaml` mounts the Secret (optional) at
+   `/etc/proxy-operator/kvm` as two files, `id_ed25519` and
+   `known_hosts`, which is what `sshKeyFile` and `knownHostsFile` in the
+   provider config point at. The deployment starts fine without the
+   Secret, even with a `kvm` provider configured: only that provider's
+   proxies fail, and they recover once the Secret exists. Rotating the
+   key: re-run the command (or the make target) and restart the manager;
+   both files are read once, on the first connection, and cached.
+
+2. Add a provider entry to the providers config
+   (`config/manager/providers_config.yaml` in-cluster; full sample in
+   `config/samples/providers-config.yaml`):
+
+   ```yaml
+   - name: kvm-home
+     type: kvm
+     kvm:
+       sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+       knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+       images:
+         debian-13:
+           url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+       machineTypes:
+         small: {vcpus: 1, memoryMiB: 1024}
+       hosts:
+         - name: nas1
+           address: nas1.home.lan:22
+           user: virt
+           bridge: br0
+   ```
+
+   The operator checks both files at startup and logs a `WARNING:` line
+   when either is unreadable, but it does not refuse to start: the
+   provider set is built all-or-nothing, so exiting here would crash-loop
+   the manager and take the `kubernetes` and `gcp` providers with it.
+   The files are read for real on the first connection. A missing file is
+   a transient failure, so proxies heal by themselves once the Secret
+   lands; a key that does not parse is permanent and needs a fix.
+
+3. Apply a Proxy (`config/samples/proxy_kvm.yaml`) and watch it:
+
+   ```sh
+   kubectl get px -w
+   # NAME               MODE      PROVIDER   PHASE          IP             HEALTHY
+   # proxy-kvm-sample   Managed   kvm-home   Provisioning
+   # proxy-kvm-sample   Managed   kvm-home   Running        192.168.1.73
+   # proxy-kvm-sample   Managed   kvm-home   Ready          192.168.1.73   True
+   ```
+
+   The first Proxy for a given image on a given host sits in
+   `Provisioning` for the length of the image download; the operator log
+   shows `fetching base image into host pool` and `base image cached`
+   with the size and duration.
+
+## Config reference
+
+| Field | Required | Default | Meaning |
+|---|---|---|---|
+| `sshKeyFile` | yes | | private key used for every host |
+| `knownHostsFile` | unless `insecureIgnoreHostKey` | | OpenSSH `known_hosts` to verify host keys against |
+| `insecureIgnoreHostKey` | | `false` | skip host-key verification entirely |
+| `diskSizeGb` | | `10` | capacity of each VM's overlay disk; cloud-init's growpart expands the root fs into it |
+| `vendorData` | | | extra cloud-init vendor-data placed on every seed next to the Proxy's user-data. Beware that cloud-init drops any top-level key of a vendor-data **cloud-config** that the user-data also defines. Shipped as one part of a MIME multipart alongside the agent script when `installGuestAgent` is on |
+| `installGuestAgent` | | `true` | add the vendor-data script that installs and starts `qemu-guest-agent`. Turn off for images that ship it already or cannot reach a package mirror; IP discovery then depends on the host's ARP table |
+| `images.<key>` | | | the key names the image in `placement.image`. Letters, digits, dots, dashes and underscores only — it becomes the cached volume name `base-<key>.<format>` |
+| `images.<key>.url` | yes per entry | | http(s) URL of a cloud image (qcow2 or raw) |
+| `images.<key>.sha256` | | | expected digest; verified after download, a change triggers a re-download |
+| `images.<key>.format` | | `qcow2` | `qcow2` or `raw` |
+| `images.<key>.firmware` | | `uefi` | `uefi` (OVMF, secure boot off) or `bios` (SeaBIOS) for images without an EFI system partition; Debian 13 hangs in GRUB under BIOS on a headless VM, see *Known gaps* |
+| `machineTypes.<key>` | ≥ 1 | | `{vcpus, memoryMiB}` size profile |
+| `hosts[].name` | yes | | the `placement.zone` value that selects this host; unique. Letters, digits, dots, dashes and underscores only — it becomes part of the providerID `hosts/<name>/domains/<vm>` |
+| `hosts[].address` | yes | | SSH endpoint, `host:port` |
+| `hosts[].user` | yes | | SSH user, member of the `libvirt` group |
+| `hosts[].socket` | | `/var/run/libvirt/libvirt-sock` | libvirt socket path on the host |
+| `hosts[].pool` | | `default` | storage pool for base images, overlays and seeds |
+| `hosts[].bridge` | yes | | host bridge for the VM's NIC |
+
+## Placement
+
+| Proxy field | Meaning for kvm |
+|---|---|
+| `placement.zone` | host name from `hosts`. May be omitted when exactly one host is configured; otherwise required. Unknown → `Failed` |
+| `placement.machineType` | key in `machineTypes`. Unknown → `Failed` |
+| `placement.image` | key in `images`, or an `http(s)://` URL used directly. Anything else → `Failed` |
+| `placement.region` | ignored |
+
+`Failed` here means the reconciler stops retrying and the message names
+the offending value; fix the spec (which replaces the VM) or the config.
+
+## What lands on the host
+
+For a Proxy whose deterministic name is `proxy-abc…`, on the chosen
+host's pool:
+
+| Object | Name | Lifetime |
+|---|---|---|
+| base image | `base-<key>.qcow2` (or `base-<url-hash>.qcow2` for a URL) | cached; survives Proxy deletion |
+| image marker | `base-<key>.qcow2.sha256` | with the base image |
+| overlay disk | `proxy-abc….qcow2`, qcow2 backed by the base | with the Proxy |
+| seed ISO | `proxy-abc…-seed.iso` | with the Proxy |
+| domain | `proxy-abc…` | with the Proxy |
+
+The domain is a q35 machine with host-passthrough CPU, the overlay on
+virtio, the seed on a SATA cdrom, one virtio NIC on the bridge, a serial
+console, the guest-agent channel, a virtio RNG and no graphics. Its
+`<metadata>` carries the operator's element (managed flag, owning UID,
+creation time) — that is how orphan GC recognises its own VMs, since
+libvirt has no labels.
+
+The seed's `user-data` is the Proxy's `cloudInit`, **byte for byte**. Any
+format cloud-init accepts works, including a Jinja template, a MIME
+archive, a shell script or a gzipped payload; the operator never parses or
+re-renders it.
+
+The guest agent rides along in `vendor-data`, as a **shell script** rather
+than a cloud-config. That is the only shape cloud-init cannot drop:
+vendor-data cloud-config is merged under the user's own document with
+mergers that keep the user-data's value for every key both define and never
+append lists, so a Proxy carrying its own `packages:` silently lost the
+agent and sat in `Provisioning` forever. The strategy cannot be overridden
+from either document — `merge_how` in user-data governs only the first
+source, and `merge_how` in vendor-data is consumed and discarded before the
+document is stored. A vendor-data script skips all of it: cloud-init gives
+vendor-data its own handler chain, writes scripts to a separate directory
+and runs them from `cc_scripts_vendor`, after the user's packages are
+installed and before the user's `runcmd`. The script is idempotent, picks
+whichever of `apt-get`, `dnf`, `yum`, `zypper` or `apk` the image has, and
+logs into `/var/log/cloud-init-output.log`. Set `installGuestAgent: false`
+to leave it out. When `vendorData` is also configured, the two travel as
+separate parts of a MIME multipart vendor-data.
+
+## The image cache
+
+The base image is downloaded **by the operator, through libvirt**, into
+the host's pool the first time a Proxy asks for it on that host — nothing
+is pre-placed on the hypervisor, and the operator needs no filesystem
+access there. The bytes stream from the URL through the operator into
+the pool with a running sha256; nothing is buffered. The marker volume is
+written last and holds the digest, so a crash mid-download leaves a base
+volume without a marker, which the next attempt deletes and redoes.
+
+A re-download happens when: the marker is missing, or `sha256` is
+configured and differs from the marker's content. Otherwise the cached
+copy is used for as long as it exists, even if the URL's content changed
+upstream (a `latest` URL is pinned at the moment of first fetch, per
+host).
+
+To evict manually:
+
+```sh
+ssh virt@nas1 virsh -c qemu:///system vol-delete --pool default base-debian-13.qcow2.sha256
+ssh virt@nas1 virsh -c qemu:///system vol-delete --pool default base-debian-13.qcow2
+```
+
+⚠️ **Do not change an image entry's `url` or `sha256`, and do not evict
+its base volume, while Proxies using that image exist on the host.** The
+overlays back onto the base file; deleting or replacing it corrupts every
+VM that uses it. Delete those Proxies first, or add the new image under a
+new key and move Proxies over by changing their `placement.image` (which
+replaces the VM).
+
+## Logging into a VM
+
+- Put your key in the Proxy's cloud-init (`ssh_authorized_keys`) and
+  `ssh debian@<ip>` (Debian) / `ssh ubuntu@<ip>` (Ubuntu).
+- Or on the host: `virsh console proxy-abc…` attaches to the serial
+  console; cloud-init's output scrolls by there on first boot.
+- `virsh domifaddr proxy-abc… --source arp` / `--source agent` shows what
+  the operator sees when it looks for the IP.
+
+## Logging
+
+The provider logs through controller-runtime's logger, named `kvm`, with
+`provider`, `host` and `name` (the domain) on every line.
+
+- `--zap-log-level=debug` (logr V(1)): one line per provider operation
+  (`domain defined`, `KVM domain ensured running`, ...) **and one
+  `libvirt call` line per libvirt RPC**, named `kvm.libvirt`, with
+  `rpc` (the libvirt API function, e.g. `virDomainLookupByName`), the
+  identifiers it was given (`domain`, `pool`, `volume`), curated results
+  (`state`, `path`, `interfaces`), `duration`, and on failure `error`
+  plus `libvirtError` (the code name, e.g. `ErrNoDomain`) and
+  `libvirtCode`. Connection events (`libvirt host connected`, dial
+  failures, `libvirt connection dropped` with a reason) log at the same
+  level. A Create is about ten RPCs; a Get is four.
+- `--zap-log-level=2`: the same lines also carry the domain and volume
+  `xml`, the `metadata` element, the interface `addresses` and the
+  domain `names` from a listing.
+- Cloud-init is never logged at any level: the seed ISO upload reports
+  only `length` and `duration`. There is no raw wire tier like the GCP
+  provider's V(5); go-libvirt exposes no hook for it.
+
+## Health check
+
+The operator checks every configured host every `--provider-check-interval`
+(default 1m). It publishes the result in two places: the `ProviderHealth`
+object named after the provider (`kubectl get ph kvm-home -o yaml`), and the
+`proxy_operator_provider_target_up{provider,target}` gauge.
+
+All hosts are checked concurrently, each over the same pooled connection
+that Create and Get use:
+
+1. The SSH and libvirt handshakes. These only happen on the first check, or
+   after the link dropped.
+2. `virConnectGetLibVersion` and `virConnectGetHostname`.
+3. `virStoragePoolLookupByName` and `virStoragePoolGetInfo` on the host's
+   `pool`.
+
+A passing target's `details` carry `hostname`, `libvirtVersion`, `pool`,
+`poolState`, `poolCapacity` and `poolAvailable` (in GiB). The free space is
+the closest thing this provider has to a capacity signal. Every target also
+shows its SSH `user`. A failing target says where the check broke:
+
+| Reason | Meaning | Look at |
+|---|---|---|
+| `SSHMaterialUnavailable` | the key or known_hosts file is missing or does not parse | the `kvm-ssh` Secret (see *Cluster-side setup*) |
+| `Unreachable` | the TCP connect failed, or the link died mid-check | DNS, Tailscale or a firewall between the cluster and `address` |
+| `SSHHandshakeFailed` | the host key is missing from known_hosts or has changed, or `user` rejected the key | the known_hosts entry; `user`'s `authorized_keys` |
+| `LibvirtConnectFailed` | SSH worked, the libvirt socket did not | `user` in the `libvirt` group, `libvirtd` running, the `socket` path |
+| `Timeout` | no answer within `--provider-check-timeout` (default 15s), or the previous check of the host is still waiting | a hung `libvirtd`, a saturated link, or a reconcile stuck redialing the host |
+| `PoolNotFound` | there is no pool named `pool` | `virsh pool-list --all` |
+| `PoolNotActive` | the pool exists but is not running | `virsh pool-start <pool>` and `virsh pool-autostart <pool>` |
+| `LibvirtError` | any other libvirt error | the message carries libvirt's own text |
+
+Sharing the connection has two consequences:
+
+- The leader now keeps one SSH connection per host open from startup, with a
+  keepalive every 15s. Before, the first connection opened with the first KVM
+  reconcile.
+- A check that runs out of time gives up on its RPCs but leaves the
+  connection alone, so an image upload or a Create running on it is not cut
+  off. At most one abandoned check per host keeps waiting; until it
+  returns, the next check reports `Timeout` straight away. A link that is
+  really dead is closed by the keepalive within about two keepalive
+  intervals.
+- A check never waits longer than its timeout for the host's connection,
+  even while a reconcile holds it to redial a host that is down.
+
+## Known gaps
+
+- **No quota or capacity detection.** libvirt reports an out-of-space
+  pool or an over-committed host as an ordinary error; the reconciler
+  treats it as transient and retries with backoff rather than the slower
+  quota backoff. The health check does report each pool's free space
+  (`poolAvailable`), so a pool that is filling up shows before Creates
+  start failing.
+- **A rotated `kvm-ssh` Secret needs a manager restart.** The key and
+  known_hosts files are read on first use and cached for the life of the
+  process. Replacing them in the Secret therefore changes nothing for
+  later connections, health checks included. After a host key change, the
+  check keeps reporting `SSHHandshakeFailed` until the manager restarts.
+- **Orphan volumes without a domain are invisible to GC.** A crash
+  between creating the volumes and defining the domain leaves an overlay
+  and seed that `ListByTag` cannot see (it lists domains). They are
+  cleaned when the same Proxy reconciles again (Delete removes volumes by
+  name even with no domain), but a Proxy CR that is gone for good leaves
+  them for `virsh vol-list` to find.
+- **A host removed from the config takes its VMs with it.** `Delete` for a
+  providerID naming an unconfigured host is a no-op success, so the Proxy
+  finishes deleting instead of pinning its finalizer, but the domain and its
+  volumes stay on the hypervisor and orphan GC cannot see them either (it
+  only sweeps configured hosts). Delete the Proxies before retiring a host,
+  or clean up afterwards with `virsh undefine --nvram` plus `virsh vol-delete`.
+  `Get` still fails permanently for such a providerID on purpose: a typo in
+  `hosts` must not look like a missing VM and trigger a rebuild elsewhere.
+- **The first image fetch is synchronous.** It holds a reconcile worker
+  for the download (LAN speed, a few hundred MB, once per image per
+  host). Other Proxies keep reconciling on the remaining workers.
+- **Replacing an image in use breaks running VMs** — see the warning
+  under *The image cache*.
+- **IP discovery needs either guest traffic or the agent.** A VM on a
+  bridge with no DHCP never gets an address and stays `Provisioning`
+  forever; check `virsh console`. The same symptom with a booted, addressed
+  VM means the guest agent never came up: check `systemctl status
+  qemu-guest-agent`, `cloud-init status --long` and the vendor script's
+  output in `/var/log/cloud-init-output.log` (a VM without outbound
+  internet cannot install it). A user-data that disables vendor-data
+  (`vendor_data: {enabled: false}`) also removes the agent, by design.
+- **Debian 13 cloud images hang in GRUB under BIOS on a headless VM**
+  ([Debian bug #1111240](https://bugs.debian.org/1111240), open, in
+  `grub2`): trixie's image ships `terminal_output gfxterm serial` in its
+  GRUB config, and with a serial console but no video device (exactly
+  the domain this provider defines) GRUB prints "Booting" and never
+  hands over to the kernel. Not host-specific; reproduced by the Debian
+  cloud team. The same image boots under UEFI with the serial console
+  intact, which is why `firmware` defaults to `uefi`. Debian 12
+  (bookworm) images are unaffected and boot either way. Images that
+  need `firmware: bios` are the exception, not the rule.
+
+## Testing
+
+Unit tests run over an in-memory fake of the libvirt API. The optional
+integration test against a real host is described in
+[testing.md](testing.md#the-kvm-integration-test-real-hypervisor); its
+`-v` output includes the `libvirt call` trace above, at the verbosity
+`KVM_TEST_LOG_LEVEL` selects (default 1). `TestKVMCheck_realHost` runs the
+health check against the same host and creates nothing:
+`go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/`.

docs/plans-executions/2026-08-07-1747-proxy-operator.md

@@ -10,13 +10,13 @@ Pairs with [docs/plans/2026-08-07-1747-proxy-operator.md](../plans/2026-08-07-17
 - [x] Step 3 — Kubernetes pod provider (`internal/provider/kubernetes/`; first built as an in-memory mock, then replaced — see the two Step 3 sections below)
 - [x] Step 4 — Reconciler (`internal/controller/`)
 - [x] Step 5 — Health engine (`internal/health/`)
-- [ ] Step 6 — Lease store (`internal/lease/`)
-- [ ] Step 7 — Discovery API (`internal/discovery/`)
-- [ ] Step 8 — GCP provider (`internal/provider/gcp/`)
-- [ ] Step 9 — Orphan GC + metrics
-- [ ] Step 10 — Wiring, config, docs
-- [ ] Step 11 — Tests
-- [ ] Verification (vet/test/kind e2e) + commit, push, open MR
+- [x] Step 6 — Lease store (`internal/lease/`)
+- [x] Step 7 — Discovery API (`internal/discovery/`)
+- [x] Step 8 — GCP provider (`internal/provider/gcp/`)
+- [x] Step 9 — Orphan GC + metrics
+- [x] Step 10 — Wiring, config, docs
+- [x] Step 11 — Tests
+- [x] Verification (vet/test/kind e2e) + commit, push, open MR
 
 ## Step 0 — Branch and scaffold
 
@@ -694,3 +694,438 @@ engine deliberately knows nothing about conditions except reading one at
 seed time, keeping the state/representation split honest. The
 `hint`-driven `wg.Go` idiom (Go 1.25+) replaced the classic
 `wg.Add/defer wg.Done` in the worker pool.
+
+## Step 6 — Lease store (`internal/lease/`)
+
+Implemented `store.go` per the plan: `Acquire` takes the whole candidate
+set so selection and insertion happen under the one store mutex (no
+overcommit between concurrent requests), selection is a linear scan +
+`slices.SortFunc` on `(activeLeases asc, latency asc, name asc)`,
+`AcquireStats{Considered, AtCapacity, InCooldown}` feeds Step 7's 409
+body, cooldowns live in a `map[{proxy, target}]time.Time` (empty target =
+global pool), and expired leases are retained for `CooldownWindow` past
+their TTL so a late `Report` — arriving exactly when a proxy is being
+rate-limited — still resolves and records its cooldown.
+
+Semantics pinned against the spec (§8) rather than guessed:
+
+- Report results are exactly `ok | rate_limited | banned` (`ParseResult`
+  gives the API layer its 400 check). `rate_limited` and `banned` both
+  record a cooldown for the same window; `ok` records nothing.
+  Distinguishing ban duration from rate-limit duration would be a second
+  knob the spec doesn't ask for — noted for the Decisions section.
+- Cooldown scoping: the global cooldown (empty target) always applies; a
+  target-scoped cooldown additionally blocks acquisitions for that target;
+  acquisitions without a target see only the global pool ("a proxy
+  rate-limited by one site is still fine for everyone else").
+- A `Report` without a target falls back to the lease's own target before
+  falling back to global — so a client that leased with a target doesn't
+  accidentally poison the whole proxy by omitting it in the report.
+
+Design notes:
+
+- **Correctness never depends on the sweep.** Every read path
+  (`Acquire`/`ActiveCount`/`Counts`) compares `ExpiresAt` against the
+  injected clock, so TTL expiry frees capacity immediately even if the
+  background loop hasn't run; the sweep is purely garbage collection. The
+  plan's `ExpireLoop` became `Start(ctx)` + `NeedLeaderElection() false`
+  so the store satisfies `manager.Runnable` directly — Step 10 just
+  `mgr.Add(store)`s it. Not leader-elected because lease state is
+  per-process and must expire wherever the discovery API is serving.
+- The store knows nothing about Proxy objects — `Candidate` carries the
+  opaque key, `MaxLeases`, and latency; the discovery layer does the
+  health/attribute filtering. The spec's `LeaseStore` interface will be
+  defined consumer-side in `internal/discovery` (Step 7), per Go idiom;
+  this package exports only the concrete in-memory `*Store`.
+- Lease IDs come from `crypto/rand.Text()` (Go 1.24+); returned `Lease`
+  values are copies so callers can't mutate store internals.
+
+Tests (94.8% coverage, `-race -count=2` clean): capacity + release
+freeing slots, `MaxLeases=0` unleasable, least-loaded/latency/name
+selection order, target-scoped vs global cooldown scoping, cooldown
+expiry via the injected fake clock, TTL freeing capacity with no sweep,
+report-on-expired-but-retained lease (then `ErrUnknownLease` after
+retention), `ok` recording nothing, idempotent release, `ParseResult`,
+40 concurrent acquires against `MaxLeases=5` granting exactly 5, and the
+`Start` loop sweeping then stopping cleanly on cancel.
+
+```bash
+go test -race -count=2 ./internal/lease/
+make test    # whole repo green, other packages' coverage unchanged
+```
+
+Worth noting: `docs/architecture.md` was not extended this step — the
+lease store is HTTP-driven, not cluster-event-driven, so its diagram
+belongs with the discovery API and lands in Step 7 (banner updated to say
+so).
+
+## Step 7 — Discovery API (`internal/discovery/`)
+
+Implemented `server.go` (Runnable + middleware chain) and `handlers.go`
+(the four endpoints + `proxyView` wire shape) per the plan: stdlib
+`http.ServeMux` method+wildcard routing (no third-party router — see the
+plan clarification commit: this is a stdlib feature since Go 1.22, the
+project stays on the pinned Go 1.26), middleware outermost-first recover →
+request-log → `MaxBytesReader(64KiB)` → constant-time bearer auth with
+`/healthz` exempt, empty `DISCOVERY_TOKEN` serving unauthenticated with a
+loud startup warning, `NeedLeaderElection() = false` with the plan's
+runnable-ordering rationale in the doc comment, and graceful `Shutdown`
+with a 10 s grace on ctx cancel.
+
+The `LeaseStore` interface landed consumer-side in this package (spec §8
+wants handlers swappable to a CRD/Redis store); `internal/lease.*Store`
+satisfies it without modification.
+
+Judgment calls the plan/spec left open:
+
+- **409 arithmetic:** the store only ever sees healthy candidates, so its
+  `Considered` excludes unhealthy matches. The handler counts unhealthy
+  selector-matches itself and reports `considered = healthy + unhealthy`,
+  keeping the plan's example arithmetic (7 = 2+2+3) consistent.
+- **TTL handling:** omitted/zero `ttlSeconds` → 300 s default; negative or
+  above `MaxLeaseTTL` (default 1h, flag in Step 10) → 400 `invalid_ttl`
+  rather than silent clamping — a client asking for a week-long lease
+  should find out, not get an hour quietly.
+- **Grant response includes the fresh `activeLeases`** (the just-granted
+  lease counted), read back via `Store.Counts()` after the acquire.
+- Proxies with a deletionTimestamp are filtered out of both list and
+  candidate selection — a proxy mid-teardown shouldn't be advertised.
+
+Tests (87.3% coverage, `-race -count=2` clean, green on first run):
+httptest over the real handler chain with a fake cache reader and a real
+`lease.Store` — auth on/off/wrong-token/healthz-exempt, list filtering
+(attributes, healthy, combined, empty-is-200), grant shape (201, default
+TTL, lowest-latency pick, RFC3339 expiresAt, activeLeases=1), the full
+409 body arithmetic, invalid TTL/body/result, idempotent 204 release,
+report→cooldown→409 round-trip, 404 on unknown lease, and a real
+`Start` on `127.0.0.1:0` (via the new `BoundAddr()` accessor) serving
+healthz then shutting down cleanly on cancel.
+
+```bash
+go test -race -count=2 ./internal/discovery/
+make test    # whole repo green
+```
+
+Worth noting: `go mod tidy` promoted `github.com/go-logr/logr` from
+indirect to direct (the server holds a `logr.Logger` field). The
+`--discovery-addr`, `--max-lease-ttl` flags and the `DISCOVERY_TOKEN`
+Secret mount arrive with `cmd/main.go` in Step 10. `docs/architecture.md`
+gained §7 covering the whole HTTP path and the store's sweep Runnable.
+
+## Step 8 — GCP provider (`internal/provider/gcp/`)
+
+Implemented per the plan: `gcp.go` (Provider + the flattened `instancesAPI`
+test seam + providerID handling + state mapping), `insert.go` (pure
+`buildInsertRequest` + config defaults), `errors.go` (HTTP-code → taxonomy
+classification). Only the four calls the spec allows — instances Insert /
+Get / Delete / AggregatedList — and `Operation.Wait` is never called:
+Create/Delete return once the operation is submitted, `409 alreadyExists`
+on Insert and `404` on Delete both count as success, which is what makes
+repeat calls after a crash correct.
+
+Dependency added (the plan's environment check pinned it):
+
+```bash
+go get cloud.google.com/go/compute@v1.65.0 google.golang.org/api@latest
+# resolved google.golang.org/api v0.292.0; go mod tidy pulled the auth/gax chain
+```
+
+Key shapes, all straight from the plan:
+
+- **providerID `zones/<zone>/instances/<name>`** — Get/Delete parse the
+  zone out of the ID instead of re-reading `spec.placement.zone`, which is
+  wrong exactly when a zone edit is the replacement being processed.
+- **The seam is not an SDK mirror** — verified the plan's premise against
+  the vendored source before designing around it:
+  `InstancesScopedListPairIterator` has an unexported `nextFunc`, so a
+  fake cannot construct one. The seam flattens `AggregatedList` to
+  `[]*computepb.Instance` and returns operations as just their name.
+- `AggregatedList` sets `ReturnPartialSuccess: true` (one unreachable
+  zone must not fail a GC sweep) and filters by
+  `labels.proxy-operator-managed = true`.
+- `RUNNING` without a `NatIP` maps to `Provisioning` — never publish an
+  empty IP. Unknown/new GCP statuses map to `Stopped`: the reconciler's
+  response is delete-and-recreate, always safe for cattle.
+- Classification: 404→NotFound; 429 and 403-with-
+  `quotaExceeded`/`rateLimitExceeded`→Quota; 400/401/403-other→Permanent;
+  everything else (408, 5xx, network, unknown)→Transient.
+
+Judgment call: `placement.zone`/`machineType`/`image` are all required at
+`Create` — missing values fail as `ErrPermanent` with a message naming
+the empty fields, rather than inventing defaults the spec doesn't define.
+A wrong guess here would silently create billable VMs of an arbitrary
+shape; a Failed condition telling the user what to set is strictly better.
+
+Tests (75.2%, `-race -count=2` clean, green on first run): the plan's
+primary field-by-field `buildInsertRequest` assertion (machine-type URL,
+boot disk, the exact `{External NAT, ONE_TO_ONE_NAT}` access config,
+user-data metadata, GC labels, network tag) plus config overrides and
+no-metadata-without-cloud-init; the full classification table including
+`errors.Is` AND `errors.As` through the multi-unwrap; and fake-seam tests
+for zone-qualified IDs, 409-is-success, permanent-on-bad-placement (no
+API call made), the nine-row state-mapping table, 404 paths, malformed
+providerIDs, and the ListByTag filter/partial-success assertions. The
+uncovered remainder is `New()` (dials real Google with ADC) and the
+`realInstances` adapter — the same deliberately-untested posture as the
+kubernetes provider's `New()`.
+
+Worth noting: gopls suggested replacing `proto.String(x)` with Go 1.26's
+`new(x)` expression; left as `proto.String` — it is the universal
+protobuf-construction idiom and matches every example in the SDK docs.
+Registry wiring (`"gcp": gcp.New`) happens at the composition root in
+Step 10, as designed in Step 2. `docs/architecture.md` §5 now shows both
+providers' call mappings.
+
+## Step 9 — Orphan GC + metrics
+
+Implemented `internal/gc/gc.go` (the `Sweeper` Runnable) and
+`internal/metrics/metrics.go` (explicit-registration metric set), plus the
+`provider.WithMetrics` decorator deferred from Step 2 into
+`internal/provider/metrics.go`, and the observation hooks in the health
+engine and discovery server.
+
+**GC**, per the plan: `NeedLeaderElection() = true` (destructive ⇒ single
+writer), 10 min ticker with the first sweep one full interval after start,
+per-provider `ListByTag` with log-and-continue on provider errors, and a
+kill requires all three of: our UID label present, older than `MinAge`
+(10 min), and the UID matching no existing CR — where a CR with a
+`deletionTimestamp` still counts as live (its finalizer owns that
+deletion; GC racing it would double-delete). Two safety behaviors worth
+naming: a failed Proxy `List` skips the whole sweep (an unreadable live
+set proves nothing orphaned), and the namespace-scope guard makes `Start`
+refuse to run against a namespace-restricted cache unless
+`--gc-allow-namespaced` is explicit, with the flag named in the error.
+One deviation of record: the plan says kills log "at Warn", but logr has
+no Warn level — kills log at Info with a `WARNING:` prefix carrying
+provider, providerID, and UID, same convention as the discovery server's
+empty-token warning.
+
+**Metrics**, per the plan: no `init()` — `Metrics.Register(reg, phases,
+leases)` is called explicitly by the composition root (Step 10), which is
+also what lets every test use a fresh registry (asserted by a test that
+registers two sets on two registries). `proxy_operator_proxies{phase}`
+and `proxy_operator_leases_active` are scrape-time collectors fed by
+closures — a reconcile-incremented gauge drifts and leaks series on
+delete; reading the source of truth at scrape time cannot. The
+probe vectors observe **every** probe (status stays transition-only;
+metrics carry the high-frequency signal), and the health engine calls
+`ForgetProxy` when it prunes a state entry so per-proxy series don't leak.
+`provider_requests_total{provider,op,result}` comes from the
+`WithMetrics` decorator — the one place `Class()` is called purely for
+observability, with results labelled ok / not_found / quota_exceeded /
+permanent / transient.
+
+**Decoupling shape:** each consuming package defines its own small
+recorder interface (`health.ProbeMetrics`, `discovery.LeaseMetrics`,
+`provider.RequestRecorder`); `metrics.Metrics` satisfies all of them
+structurally. Only `cmd/main.go` will import `internal/metrics`.
+
+Tests (`-race -count=2` clean): GC's true-orphan matrix in one sweep
+(live kept, deleting-CR kept, young kept, unlabelled kept, orphan
+deleted), broken-provider isolation, list-failure skips sweep, the
+namespace guard both ways, and the Start loop sweeping then stopping;
+metrics via `prometheus/testutil` — `GatherAndCompare` on the scrape-time
+collectors, ForgetProxy dropping series, outcome/result label counts;
+the decorator's five-way classification table with error passthrough
+asserted. Coverage: gc 86.1%, metrics 95.0%, provider up to 97.1%;
+health/discovery re-ran green with the hooks in place.
+
+```bash
+go test -race -count=2 ./internal/gc/ ./internal/metrics/ ./internal/provider/ ./internal/health/ ./internal/discovery/
+make test    # whole repo green
+```
+
+Worth noting: `prometheus/client_golang` was already in the module via
+controller-runtime's metrics server, so no new dependency — `go mod tidy`
+just promoted it to direct. `docs/architecture.md` gained §8 (GC sweep)
+and §9 (metrics shape).
+
+## Step 10 — Wiring, config, docs
+
+The composition root and everything around it. `cmd/main.go` now: parses
+the plan's flag set (plus `--gc-allow-namespaced` from Step 9), loads the
+provider config first and fails fast, builds the registry with
+`{"kubernetes": kubernetes.New, "gcp": gcp.New}`, wraps every provider in
+`provider.WithMetrics`, then adds the lease store, health engine,
+discovery server, and GC sweeper to one manager and hands the reconciler
+its providers + health snapshotter + events channel. Metrics register on
+controller-runtime's global registry with scrape-time closures (phase
+counts from the cache, active leases summed from `store.Counts()`).
+
+Two cache decisions became concrete here:
+
+- The Secret cache is restricted to Secrets labelled
+  `crawl.example.com/cloud-init=true` (new constant
+  `v1alpha1.LabelCloudInit`) — the operator holds cluster-wide Secret
+  read RBAC, and an unrestricted cache would hold every Secret in scope.
+  Consequence documented in the README: an unlabelled referenced Secret
+  is invisible → `CloudInitError`.
+- `--proxy-namespace` restricts the whole cache via `DefaultNamespaces`
+  and flips the GC sweeper's `NamespaceRestricted` guard.
+
+Manifests: `config/manager/manager.yaml` gained the
+`--providers-config` arg, the optional `DISCOVERY_TOKEN` secretKeyRef
+(`optional: true` — without the Secret the API runs unauthenticated with
+its loud warning), the ConfigMap volume mount, and containerPort 8090;
+new `config/manager/providers_config.yaml` (kubernetes-only default) and
+`config/default/discovery_service.yaml`. RBAC: the pods marker landed in
+the controller RBAC block (cluster-scoped role — the kubernetes
+provider's ListByTag spans namespaces). The plan's events RBAC was
+deliberately omitted: nothing wires an EventRecorder, and unused verbs
+are lint noise — recorded in Decisions.
+
+Samples: `proxy_kubernetes.yaml` / `proxy_gcp.yaml` (with a working
+Squid-installing cloud-init) / `proxy_external.yaml` replace the scaffold
+placeholder; `providers-config.yaml` documents both provider blocks;
+`hack/providers-dev.yaml` + a new `run-dev` Makefile target run locally
+with plain-HTTP metrics:
+
+```bash
+make run-dev
+# go run ./cmd/main.go --providers-config hack/providers-dev.yaml \
+#   --metrics-bind-address :8080 --metrics-secure=false
+```
+
+Docs: README rewritten per the plan (60-second architecture, kind
+quickstart, in-cluster deploy incl. token Secret creation, GCP setup with
+the IAM roles, the two prominent caveats, the no-substitutions version
+pins note). `docs/architecture.md` gained the components table and the
+full Decisions section — the plan's listed decisions plus everything
+accumulated in this log (RequeueNow, quota≠Failed, 409 arithmetic,
+TTL-400-not-clamp, GCP required placement, unknown-status→Stopped,
+banned==rate_limited window, GC logging convention, the Secret label
+contract, events-RBAC omission, logr-not-slog). `CHANGELOG.md` got its
+first entry with a real timestamp.
+
+Verified: `make test` green across the repo (coverage unchanged),
+`bin/kustomize build config/default` and `config/samples` render clean,
+`make build` produces the binary, e2e-tagged build + vet clean. The kind
+end-to-end run is deliberately still ahead — it is the Verification
+step's job, after Step 11 closes the remaining test gaps.
+
+Worth noting: `make run-dev` passes `--metrics-secure=false` because the
+scaffold's secure-serving default requires authn/authz reachability that
+a local process doesn't have; in-cluster deployments keep the secure
+default from the kustomize patch. The `providers` map wrapping happens
+*before* any consumer sees it, so the reconciler and GC only ever hold
+instrumented providers.
+
+## Step 11 — Tests
+
+Most of the plan's Step 11 list was deliberately front-loaded into the
+step that built each component (the action-table suite, computePhase and
+SpecHash tables, lease-store matrix incl. the concurrent `-race` case,
+discovery httptest suite, health threshold/CONNECT tests, GCP
+`buildInsertRequest` + classification, name-derivation tests from
+Step 2). This step closed what remained — the envtest-only coverage —
+and audited the list item by item.
+
+Added to `internal/controller/proxy_controller_test.go`:
+
+- **The CEL cases only a real API server can test** (fake clients run
+  neither CEL nor structural defaulting): six invalid-create rejections
+  (Managed-without-provider, External-with-provider,
+  External-without-endpoint, Managed-with-endpoint, cloudInit
+  both/neither), mode-mutation rejection, provider mutation *and removal*
+  rejection (the `has(self.x)==has(oldSelf.x)` form exists exactly for
+  the removal case), and the `+kubebuilder:default={}` assertion — a
+  Proxy created with no `healthCheck` comes back with every nested
+  default materialized, plus port and maxLeases defaults.
+- **Ready-through-health**: Managed proxy walks to Running (phase still
+  Provisioning — "no health verdict yet must not be Ready"), then a fake
+  `HealthSnapshotter` supplies a healthy snapshot and the phase flips to
+  Ready with latency in status.
+- **Quota + permanent, envtest edition**: quota → condition
+  QuotaExceeded, `RequeueAfter = QuotaRetry`, nil error, phase *not*
+  Failed; then permanent → Failed and the generation latch provably stops
+  further provider calls.
+- **Adopt**: strip the spec-hash annotation off a Running proxy (as an
+  operator upgrade with a changed hash-input struct would), reconcile,
+  and assert the annotation is restored byte-identical, the providerID
+  unchanged, and zero provider deletes.
+
+One repo-wide change: `make test` now runs with `-race` (the plan's
+"everything runs with -race" was previously only true of the manual
+verification runs, not the canonical target):
+
+```make
+go test -race $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+```
+
+Everything green on the first run of the new specs; full suite ~9 s for
+the controller package under race, `-short` still skips envtest in 0.6 s.
+
+Worth noting: the provider-removal CEL test has a subtlety worth keeping —
+removing `provider` alone would also trip the required-iff rule, so the
+test flips mode and adds an endpoint in the same update to isolate the
+immutability rules as the thing that rejects. The plan's remaining
+checklist item is Verification: the throwaway-kind-cluster run of the
+README quickstart, then push + MR.
+
+## Verification — kind end-to-end
+
+Static checks first (`go vet ./...`, `make build`, full `make test` with
+`-race`): all green. Then the real thing, per the spec's §13 "run the kind
+quickstart yourself and fix what breaks" — and two things broke, both now
+fixed.
+
+**Finding 1 — `make run-dev` cannot produce a Ready proxy on kind.** The
+operator on the host provisions the pod fine (Provisioned=True, IP
+published), but the health probe originates on the host, and kind pod IPs
+(10.244.x.x) are not host-routable — every probe fails by construction
+and the proxy latches `Unhealthy`:
+
+```text
+Healthy=False: Get "https://www.gstatic.com/generate_204":
+  proxyconnect tcp: dial tcp 10.244.0.5:3128: connect: connection refused
+```
+
+Everything around the failure worked exactly as designed (thresholds,
+condition, phase, and the finalizer delete ran clean from the host). Fix:
+the README quickstart now deploys the operator **in-cluster**
+(docker-build → kind load → deploy → port-forward 8090), with the
+run-dev limitation documented in both the quickstart and the Development
+section.
+
+**Finding 2 — Squid was OOM-killed at startup in-cluster.** With the
+operator deployed in-cluster the pod crash-looped (`OOMKilled`, empty
+logs). Root cause: squid sizes its file-descriptor tables from
+`RLIMIT_NOFILE`, and containerd under kind sets that effectively
+unlimited (~10^9) — squid allocates gigabytes before it ever listens.
+Fix in the generated config (`internal/provider/kubernetes/pod.go`):
+`max_filedescriptors 1024` (the load-bearing line) plus `cache_mem 16 MB`
+(a crawling forward proxy gains nothing from squid's 256 MB default),
+with a regression assertion added to `pod_test.go`.
+
+**With both fixes, the full pass:**
+
+```bash
+kind create cluster --name proxy-operator-demo
+make install
+make docker-build IMG=egress-proxies-operator:dev
+kind load docker-image egress-proxies-operator:dev --name proxy-operator-demo
+make deploy IMG=egress-proxies-operator:dev
+kubectl apply -f config/samples/proxy_kubernetes.yaml
+# → Ready 10.244.0.9 lat=89ms Provisioned=True Healthy=True   (~30 s)
+kubectl -n egress-proxies-operator-system port-forward svc/...-discovery-service 8090:8090 &
+curl -s 'localhost:8090/v1/proxies?healthy=true'          # count:1, latencyMillis:89
+curl -s -XPOST localhost:8090/v1/leases -d '{"selector":{"geo":"local"},"ttlSeconds":300}'
+#   → 201 {leaseID, proxy(activeLeases:1), expiresAt, ttlSeconds:300}
+curl -XPOST .../report -d '{"result":"rate_limited","target":"example.com"}'   # 204
+curl -XPOST /v1/leases -d '{...,"target":"example.com"}'  # 409 {inCooldown:1} ✓
+curl -XDELETE /v1/leases/<id>   # 204, and 204 again ✓
+kubectl delete -f config/samples/proxy_kubernetes.yaml    # finalizer: pod Terminating, CR gone
+kind delete cluster --name proxy-operator-demo
+```
+
+A real Squid pod went Ready through a real CONNECT probe, a lease was
+held on it, the cooldown machinery answered a 409 with correct
+arithmetic, and the finalizer cleaned up — the plan's success bar, met
+with the actual product.
+
+Worth noting: `make deploy` runs `kustomize edit set image` and mutates
+`config/manager/kustomization.yaml` in the working tree — reverted before
+committing (the repo keeps the pinned stanza). The health probe's ~89 ms
+latency is gstatic-through-squid from a kind pod on this machine;
+metrics-side observations were not separately checked in-cluster (covered
+by unit tests).

docs/plans-executions/2026-08-11-1742-gcp-provider-verbose-logging.md

@@ -0,0 +1,42 @@
+# Execution: Verbose (V-level) logging in the GCP provider
+
+Plan: `docs/plans/2026-08-11-1742-gcp-provider-verbose-logging.md`
+
+- [x] Step 0 — Save and commit the plan
+- [x] Step 1 — V(1)/V(2) logging in `internal/provider/gcp` (gcp.go, errors.go)
+- [x] Step 2 — Tests (verbosity tiers, error detail, cloud-init leak guard)
+- [ ] Step 3 — CHANGELOG entry (after the user confirms it works live)
+
+## Step 1 — logging in the provider
+
+Went as planned: context-carried logger (`logf.FromContext(ctx).WithName("gcp")`),
+V(1) one line per API call, V(2) request/list detail, opNames captured from the
+`instancesAPI` seam instead of being discarded. `logAPIError` lives in
+`errors.go` (next to `classify`, whose imports it shares) rather than `gcp.go`
+as loosely implied by the plan — same package, so no behavioural difference.
+These are the first `.V(n)` calls and the first logging import anywhere under
+`internal/provider/`.
+
+Worth noting: the gopls `errorsastype` suggestion fired on the new
+`errors.As` in `logAPIError` (Go's newer `errors.AsType`); kept `errors.As`
+for consistency with the three existing uses in the same file. Same for the
+`newexpr` (`proto.String` → `new`) suggestions — the codebase consistently
+uses `proto.String`.
+
+## Step 2 — tests
+
+`funcr.New` as the capturing sink, injected via `logr.NewContext`, exactly the
+seam the plan predicted. One deviation: instead of a single
+`TestLogging_verbosity` table, it split into three tests — `_verbosityTiers`
+(table over V=0/1/2, incl. the cloud-init sentinel leak assertion),
+`_apiErrorKeepsHTTPDetail` (403 quotaExceeded keeps `httpStatus`/reason at
+V(1)), and `_treatedAsSuccessPathsAreExplicit` (409-on-create /
+404-on-delete each log their "treated as success" line) — the last two
+exercise fake error wiring that didn't fit the tier table cleanly.
+
+Verified with:
+
+```bash
+go test -race ./internal/provider/gcp/
+go test ./...
+```

docs/plans-executions/2026-08-11-1802-bake-commit-version.md

@@ -0,0 +1,61 @@
+# Execution: Bake the git commit into the operator binary and log it at startup
+
+Plan: `docs/plans/2026-08-11-1802-bake-commit-version.md`
+
+- [x] Step 0 — Save and commit the plan
+- [x] Step 1 — `internal/version` package + tests
+- [x] Step 2 — `cmd/main.go`: `--version` flag + startup log line
+- [x] Step 3 — Makefile `GIT_COMMIT` + `--build-arg` wiring
+- [x] Step 4 — Dockerfile `-ldflags` stamp + OCI revision label
+- [ ] Step 5 — CHANGELOG entry (after the user confirms it works live)
+
+## Steps 1–4
+
+Mostly as planned. One deviation worth recording: the plan claimed
+`build`/`run` targets need no changes because Go's automatic VCS stamp covers
+host builds — that turned out to be only half true. Go skips VCS stamping
+when the build target is a *file argument* rather than a package pattern, and
+the Makefile's `build` target used `go build -o bin/manager cmd/main.go`.
+Verified empirically:
+
+```bash
+go build -o bin/manager cmd/main.go && ./bin/manager --version   # unknown (no vcs settings)
+go build -o bin/manager ./cmd       && ./bin/manager --version   # e4d2a191d0c2-dirty
+```
+
+So `build:` now uses `go build -o bin/manager ./cmd`. `go run` never stamps
+VCS info regardless of invocation form — `make run`/`run-dev` print
+`commit=unknown`, which is acceptable for dev loops (the Dockerfile path uses
+the explicit ldflags stamp and is unaffected; it kept `cmd/main.go`).
+
+The ldflags path was verified independently:
+
+```bash
+go build -ldflags "-X gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version.Commit=deadbeef1234" \
+  -o bin/manager-stamped cmd/main.go
+./bin/manager-stamped --version   # deadbeef1234
+```
+
+Worth noting: `cmd/main.go` imports k8s apimachinery as `runtime`, so the
+stdlib runtime needed an alias (`goruntime "runtime"`) for
+`goruntime.Version()` in the startup line. The `--version` check happens
+right after `flag.Parse()`, before logger and manager setup, so it works
+without a kubeconfig.
+
+## Verification
+
+```bash
+go vet ./... && go test ./...           # all green, incl. new resolve() table tests
+go build -o bin/manager ./cmd && ./bin/manager --version   # e4d2a191d0c2-dirty
+```
+
+The image-level check initially failed (Docker daemon not running); after
+the daemon was started it passed in full:
+
+```bash
+make docker-build IMG=egress-proxies-operator:dev
+docker run --rm egress-proxies-operator:dev --version
+# ae434a7167ec-dirty  (matches git rev-parse --short=12 HEAD + untracked files)
+docker inspect egress-proxies-operator:dev --format '{{index .Config.Labels "org.opencontainers.image.revision"}}'
+# ae434a7167ec-dirty
+```

docs/plans-executions/2026-08-11-1838-gcp-http-wire-logging-v5.md

@@ -0,0 +1,63 @@
+# Execution: GCP HTTP wire logging at V(5)
+
+Plan: `docs/plans/2026-08-11-1838-gcp-http-wire-logging-v5.md`
+
+- [x] Step 1 — `wireLogger` + `option.WithLogger` wiring in `internal/provider/gcp/gcp.go`
+- [x] Step 2 — Tests (`TestWireLogger_gatesAtV5`, `TestWireLogger_infoLandsAtV1`)
+- [x] Step 3 — Live verification at `--zap-log-level=5` (user, on cluster)
+- [x] Step 3b — Post-verification fix: drop auth records, elide huge fields
+- [ ] Step 4 — CHANGELOG entry (after live confirmation of 3b; batch with the
+      two earlier pending entries: GCP V-logging, version stamp)
+
+## Steps 1–2
+
+Went exactly as planned — the whole feature is ~10 lines of production code
+because both halves already existed: the compute SDK logs full HTTP
+request/response records at slog Debug to an injectable logger, and
+`logr.ToSlogHandler` does the slog→logr bridging. The only real design
+content is the level shift (`base.V(1)` + slog-Debug's +4 = V(5)) and the
+startup warning line when V(5) is active (raw payloads include cloud-init
+user-data, which the curated V(2) logging deliberately hides).
+
+Worth noting for future readers:
+
+- `option.WithLogger` **disables** `GOOGLE_SDK_GO_LOGGING_LEVEL` for this
+  client (documented SDK precedence) — `--zap-log-level` is now the only knob
+  for GCP wire logs.
+- The V(5) check in `New` runs once at startup; that is sound because the zap
+  level is fixed by flags at process start.
+- Added `TestWireLogger_infoLandsAtV1` beyond the plan's table — it pins the
+  shift arithmetic from the other side (slog Info → V(1)), so a future logr
+  mapping change would fail loudly.
+
+Verified with:
+
+```bash
+go test -race ./internal/provider/gcp/
+go build ./... && go test ./...
+```
+
+## Step 3b — what live verification exposed, and the fix
+
+Live V(5) output revealed two problems the plan missed:
+
+1. **Security: the injected logger propagates into `cloud.google.com/go/auth`**,
+   which logs its own token exchange (`auth.go:571/576`) — signed JWT
+   assertion in the request, full bearer access token in the response. The
+   plan's "auth token is safe" analysis only covered the compute client's
+   request headers, not the auth library's own records. Fix: `wireLogger`
+   now wraps the handler in a filter that drops every Debug record except
+   the compute client's `"api request"`/`"api response"` (allowlist, so
+   future SDK additions fail closed); Warn/Error still pass through.
+2. **Readability: GCP responses embed multi-KB blobs** (Shielded-VM UEFI
+   dbx databases) that swamp the line. Fix: string fields >1KiB are elided
+   to `[elided N bytes]` by default, recursively through payload
+   maps/arrays. Opt-out via new manager flag
+   `--gcp-wire-log-full-payloads` (threaded through a constructor closure
+   in `cmd/main.go` → `gcp.NewWithWireOptions`; the `registry.Constructor`
+   signature stays unchanged). Chosen by the user: elision on by default,
+   verbatim available on demand. Auth records are dropped in both modes.
+
+The filter/elision logic lives in `internal/provider/gcp/wirelog.go` with
+tests covering: auth-record drop (both modes), elision marker + small-field
+preservation, verbatim mode, and the original V(5) gating.

docs/plans-executions/2026-08-11-1935-gitea-build-workflow.md

@@ -0,0 +1,53 @@
+# Execution: Distilled Gitea Actions image-build workflow
+
+Plan: `docs/plans/2026-08-11-1935-gitea-build-workflow.md`
+
+- [x] Step 1 — Create `.gitea/workflows/build.yaml`
+- [x] Step 2 — Replace CLAUDE.md CI TODO with a CI/CD subsection
+- [x] Step 3 — Push branch + open MR
+- [ ] Step 4 — CHANGELOG entry (after the first successful run is confirmed)
+
+## Steps 1–2 — workflow + CLAUDE.md
+
+The workflow distills the house pattern from 9 sibling projects (survey in the plan)
+plus improvements none of them combine: an immutable `sha-<12>` tag, a lightweight
+test gate, `:latest` moving only on real tag pushes, and a `concurrency` group.
+
+Work happened in a git worktree off `origin/main` so the main checkout (which had
+unrelated uncommitted changes) stayed untouched:
+
+```bash
+git worktree add -b feat/gitea-build-workflow \
+  "$SCRATCH/wt-build" origin/main
+```
+
+Deviation from the plan's assumptions: while planning, `feat/proxy-operator` was
+still unmerged and the plan noted `main` lacked `internal/`/`test/`. By execution
+time `origin/main` had moved (`076bc66..f7000f7` — the proxy-operator MR merged),
+so the branch and its CI gate cover the full operator code.
+
+Verified the workflow parses and the check-gate commands pass on this exact tree
+(ruby stands in for a YAML linter because the system python3 has no `yaml` module):
+
+```bash
+ruby -ryaml -e "YAML.load_file('.gitea/workflows/build.yaml'); puts 'YAML OK'"
+go vet ./... && go build ./... && go test -short ./...   # all packages ok
+```
+
+## Step 3 — push + MR
+
+Branch pushed and MR opened with `tea` (the worktree was then removed and the main
+checkout switched onto the branch so the files are visible locally):
+
+```bash
+tea pr create --title "Add Gitea Actions image-build workflow" \
+  --description "..." --base main --head feat/gitea-build-workflow
+# → https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/pulls/2
+```
+
+Worth noting: the workflow itself cannot run end-to-end until (a) the MR merges
+(it only triggers on tags / manual dispatch, not branch pushes) and (b) the
+`REGISTRY_TOKEN` secret is created in this repo's Gitea settings (PAT with
+`write:package`). First real verification = manual dispatch with tag
+`manual-test`, expecting `manual-test` + `sha-…` in Packages and `:latest`
+untouched.

docs/plans-executions/2026-08-11-2152-discovery-api-docs.md

@@ -0,0 +1,38 @@
+# Execution: Discovery API documentation
+
+Plan: [2026-08-11-2152-discovery-api-docs.md](../plans/2026-08-11-2152-discovery-api-docs.md)
+
+- [x] Step 1 — Write `docs/api.md` full API reference
+- [x] Step 2 — Add pointers in README and architecture.md
+
+## Step 1 — docs/api.md
+
+Wrote the full reference: base URL (in-cluster FQDN + port-forward), bearer
+auth, error envelope, configuration table, all five routes with schemas,
+status codes and curl examples, the selection/cooldown semantics section,
+an end-to-end curl walkthrough, and the in-memory/single-replica caveats.
+All facts were taken from the code, not from memory of prior docs.
+
+Worth noting: the doc explicitly calls out two things no earlier doc
+stated for clients — that a report **without** a target on a targetless
+lease creates a *global* cooldown (blocking the proxy for everyone), and
+that `rate_limited` and `banned` currently behave identically. Both came
+straight from `internal/lease/store.go` and are easy to trip over.
+
+## Step 2 — Pointers + verification
+
+Added one-line links to the new doc in README's quickstart (above the curl
+block) and in `docs/architecture.md` §7. Verified the documented behavior
+against the tree rather than trusting the write-up:
+
+```sh
+go build ./... && go test -short ./internal/discovery/ ./internal/lease/
+```
+
+Both pass; a grep of `server_test.go` confirmed every documented status
+code and error code (`invalid_ttl`, `invalid_query`, `invalid_result`,
+`no_match`, `unknown_lease`, 201/204/401/404/409) is asserted by tests.
+
+Worth noting: CHANGELOG entry deliberately deferred until the user
+confirms the docs read well, per the CHANGELOG convention's
+"once the user confirms it works" clause.

docs/plans-executions/2026-08-11-2220-demo-scripts.md

@@ -0,0 +1,55 @@
+# Execution: Demo scripts
+
+Plan: [2026-08-11-2220-demo-scripts.md](../plans/2026-08-11-2220-demo-scripts.md)
+
+- [x] Step 0 — Branch `feat/demo-scripts` + plan commit
+- [x] Step 1 — `docs/demo/show-egress-ips.sh`
+- [x] Extra (added iteratively, not in the original plan) — proxy-creation scripts
+
+## Step 0 + Step 1
+
+Branched off `main`, committed the plan alone, then wrote
+`docs/demo/show-egress-ips.sh`: takes `BASE_URL` as its argument, lists
+`/v1/proxies` (bearer auth via optional `TOKEN` env), probes each healthy
+proxy with `curl -x http://ip:port` against an IP-echo site
+(`IP_ECHO_URL`, default ipify JSON), banners which proxy each request goes
+through, skips unhealthy ones, and ends with a probed/skipped/failed
+summary. Per user request the script was left uncommitted for iteration
+and no verification beyond `bash -n` was run.
+
+## Extra — create-kubernetes-proxies.sh, create-gcp-proxies.sh
+
+Added on the same branch before the first commit:
+
+- `create-kubernetes-proxies.sh <count>` — creates
+  `proxy-kubernetes-demo-1..N` with the kubernetes provider, spec taken
+  from `config/samples/proxy_kubernetes.yaml`, applied via
+  `kubectl apply -f -` heredocs. Optional `NAMESPACE` env.
+- `create-gcp-proxies.sh <count>` — creates `proxy-gcp-demo-1..N` from the
+  user-supplied gcp-eu manifest (e2-micro, Ubuntu 24.04, Squid
+  cloud-init), each with a zone picked randomly from a hardcoded list of
+  27 EU zones so the fleet gets egress IPs from different locations.
+  Env overrides: `ZONES`, `GCP_PROVIDER` (default `gcp-eu`), `NAMESPACE`.
+
+## Extra — run-demo.sh, show-egress-ips-table.sh
+
+Second round of iterative additions:
+
+- `run-demo.sh` — tmux demo driver: 2x2 tiled grid with the egress-IP
+  table looping every 10s inside a netshoot pod (script `kubectl cp`'d
+  into the pod, `BASE_URL` set to the in-cluster Service FQDN),
+  `watch -n3 kubectl get px`, and both create scripts auto-running with
+  `COUNT` proxies each (default 4). Env knobs: `COUNT`, `SESSION`,
+  `NETSHOOT_POD`, `DEMO_DIR` (defaults to the script's own dir).
+- `show-egress-ips-table.sh` — condensed one-line-per-proxy variant of
+  `show-egress-ips.sh` sized for a tmux pane: PROXY / ENDPOINT /
+  LOCATION (zone→geo attribute fallback) / EGRESS-IP columns, with
+  `(unhealthy)` and `FAILED` inline instead of verbose output. The
+  verbose script stays for standalone use; the demo driver uses the
+  table variant.
+
+Worth noting: beyond the user's sample manifest, the gcp script also
+writes the picked zone into `attributes.zone`, so the discovery API
+exposes each proxy's location and leases can select on it. The zone list
+is static — if a project lacks quota in some region, `ZONES` narrows the
+pool; nothing validates zones against the live project.

docs/plans-executions/2026-08-24-1025-otel-tracing.md

@@ -0,0 +1,183 @@
+# Execution log: OpenTelemetry tracing integrated with logr/zap logging
+
+Plan: `docs/plans/2026-08-24-1025-otel-tracing.md`
+
+- [x] Step 1 — Dependencies
+- [x] Step 2 — New package `internal/tracing`
+- [x] Step 3 — `provider.WithTracing` decorator
+- [x] Step 4 — `cmd/main.go` wiring
+- [x] Step 5 — Reconciler spans
+- [x] Step 6 — Discovery server
+- [x] Step 7 — GC + health
+- [x] Step 8 — GCP wire-log enrichment
+- [x] Step 9 — Manifests + docs
+
+## Step 1 — Dependencies
+
+Aligned the pre-existing indirect skew (otel core v1.44.0 vs otlptrace exporters
+v1.40.0) and added the new direct deps in one shot:
+
+```bash
+go get go.opentelemetry.io/otel@v1.45.0 \
+  go.opentelemetry.io/otel/sdk@v1.45.0 \
+  go.opentelemetry.io/otel/trace@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/stdout/stdouttrace@v1.45.0 \
+  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0
+go mod tidy
+```
+
+MVS side-effects worth recording: `logr v1.4.3→v1.4.4`, `httpsnoop
+v1.0.4→v1.1.0`, `grpc-gateway/v2 v2.27.7→v2.29.0`, `proto/otlp v1.9.0→v1.11.0`,
+plus a `genproto/googleapis/api` pseudo-version bump. Full `make test` passed
+against the bumped graph (`google.golang.org/api v0.292.0` and k8s v0.36
+tolerate otelhttp v0.70.0).
+
+Worth noting: the first `go mod tidy` ran *before* any first-party code
+imported `otlptracehttp`/`stdouttrace`, so it silently dropped those two
+modules again; the Step 2 tidy re-added them. The plan's semconv question
+resolved to `semconv/v1.43.0` — that's what `sdk@v1.45.0/resource/builtin.go`
+imports, so first-party code uses the same version to avoid
+`ErrSchemaURLConflict` in the common path.
+
+## Step 2 — `internal/tracing` package
+
+Landed as planned: `tracing.go` (env-gated `Setup`, hand-rolled exporter
+selection), `logger.go` (`Start`/`StartSpan`/`ContextWithLogger` with the
+base-logger ctx key that prevents duplicate `traceID` zap fields on nested
+spans), `reconciler.go`, `transport.go`, `http.go`, `options.go`; tests for
+all of it (first use of `sdk/trace/tracetest` in the repo).
+
+Two deviations from the plan's letter:
+
+- **k8s transport gating is a custom RoundTripper, not `otelhttp.WithFilter`**
+  (`parentGatedTransport`): requests without a parent span bypass the otel
+  transport entirely, so the no-root-spans-from-informers guarantee doesn't
+  depend on otelhttp filter semantics for transports.
+- **`HTTPMiddleware` must copy the route pattern back.** The logger-injecting
+  inner handler wraps the request via `WithContext` (a shallow copy), so the
+  mux records the matched pattern on the copy while otelhttp's post-routing
+  span rename reads the original. Found by the middleware test (span named
+  `"GET"` instead of `"GET /v1/things/{id}"`); fixed with `r.Pattern =
+  r2.Pattern` after `next.ServeHTTP`.
+
+Also: both the transport and the middleware pass explicit W3C propagators
+instead of relying on the global, so behavior is deterministic under tests
+and when tracing is disabled. `Setup` tests reset the global provider to a
+fresh noop per case — restoring otel's own default delegate triggers a
+"Setting tracer provider to its current value" warning from the SDK.
+
+## Step 3 — `provider.WithTracing`
+
+`internal/provider/tracing.go` mirrors `metrics.go` exactly (same wrap shape,
+same `resultLabel` classification reused as the `provider.result` span
+attribute); tests reuse `staticProvider` from `metrics_test.go`. `ErrNotFound`
+maps to span status Ok as planned — the reconciler polls `Get` to NotFound
+during replacement/deletion, so it's an answer, not a failure.
+
+One small API addition to `internal/tracing` for this: exported
+`tracing.Tracer(opts...)`, since the decorator lives in package `provider`
+and couldn't reach the unexported option resolver. The `cmd/main.go` wiring
+(`WithTracing` outermost around `WithMetrics`) lands with Step 4's commit —
+same file, one commit.
+
+## Step 4 — `cmd/main.go` wiring
+
+As planned: `tracing.Setup` right after `ctrl.SetLogger`; manager rest.Config
+wrapped via `restCfg := ctrl.GetConfigOrDie(); restCfg.Wrap(...)` (the
+previous inline call left nowhere to wrap); providers decorated tracing-
+outermost; explicit trace flush after `mgr.Start` returns on both the error
+and clean paths, with a fresh 10s context since the signal ctx is already
+cancelled by then.
+
+The kubernetes provider grew `NewWithTransportWrapper(ctx, cfg, wrap)` (its
+client is built from its own `ctrl.GetConfig()`, invisible to the manager's
+wrapped config); `New` now delegates with a nil wrapper, so `newWithClient`
+tests stayed untouched, and main registers a closure — the same pattern the
+gcp constructor already used for wire-log options.
+
+Deviation: the plan put `--trace-health-probes` here, but the flag needs the
+`health.Engine.TraceProbes` field that Step 7 introduces — moved there to
+keep every commit compiling.
+
+## Step 5 — Reconciler spans
+
+`SetupWithManager` completes with `tracing.NewReconciler("Proxy", r)`;
+sub-spans `reconcile.managed` / `reconcile.replaceInstance` /
+`reconcile.delete` open at the top of each state machine, and `status.patch`
+opens inside the deferred flush closure so it stays within the root span
+while its error still folds into the recorded result. `reconcileExternal`
+left unspanned as planned (no I/O).
+
+Two small judgment calls: the sub-spans carry no extra attributes — the
+provider decorator already records `provider.id`, and the root span carries
+the object identity, so duplicating them was noise; and `status.patch` is
+emitted every reconcile even when nothing changed (the no-op compare is the
+span's content — a real PATCH shows up as its k8s HTTP child). Tests that
+call `r.Reconcile` directly bypass the wrapper; with no global tracer set
+they see no-op spans, so the existing fake-client and envtest suites run
+unchanged.
+
+## Step 6 — Discovery server
+
+Middleware chain is now recover → tracing (server span + request logger) →
+request-log → body cap → auth; the request-log middleware and the three
+handler error sites log via `logf.FromContext(r.Context())` — which is
+`s.log` enriched with the request's traceID/spanID by the tracing
+middleware, not a different logger (logr sinks can't read ctx at log time,
+so per-request values must ride on the logger instance in the ctx; user
+asked, answered in-session).
+
+Deviation from the plan's letter: `recoverMiddleware` keeps `s.log`. It sits
+*outside* the tracing layer, so its request ctx never has the enriched
+logger — switching it to `FromContext` would silently drop the `"discovery"`
+name and gain nothing.
+
+## Step 7 — GC + health
+
+`gc.sweep` runs in a root span with `gc.providers` / `gc.deleted` counters
+(a failed orphan delete no longer counts as deleted — the loop grew an
+explicit `continue`). The health engine got the deferred `--trace-health-probes`
+flag and `TraceProbes` field; the probe call moved into `runProbe`, which
+wraps it in a `health.probe` client span only when enabled, ending before
+`record` (which stays ctx-free by design). Probe transports remain
+uninstrumented so no traceparent can leak through a proxy to external
+targets. Probe→reconcile span links stay out of scope (GenericEvent carries
+no ctx; the workqueue coalesces events) — recorded in the architecture
+Decisions in Step 9.
+
+## Step 8 — GCP wire-log enrichment
+
+`wireFilterHandler.Handle` clones the record and appends `traceID`/`spanID`
+when the ctx carries a valid span — same keys as the logr enrichment, added
+before the elision pass (both values are well under the 1KiB threshold).
+The new `wirelog_test.go` covers the enrichment, that the JWT/token security
+filter still drops non-wire Debug records even with a span present, and that
+spanless records stay unchanged.
+
+## Step 9 — Manifests, docs, lint
+
+`config/manager/manager.yaml` gained the OTel env block (downward-API
+`POD_NAME`/`POD_NAMESPACE` deliberately listed *before*
+`OTEL_RESOURCE_ATTRIBUTES` — `$(VAR)` expansion only sees earlier vars),
+with the OTLP endpoint and sampler left as commented examples so deployments
+stay tracing-off by default. `docs/architecture.md` got "### 10. Tracing"
+plus five Decisions entries; README got a Tracing section.
+
+`make lint` surfaced 16 new-on-branch issues (checked with
+`golangci-lint run --new-from-rev main`; the other ~43 pre-date this work):
+
+- one gofmt slip, 13 goconst repeats — fixed with `env*`/`exporter*`
+  constants in `internal/tracing/tracing.go` shared by the tests;
+- two logcheck hits ("function takes both a context and a logger"):
+  `Setup` now reads its logger from ctx (`logf.FromContext`, caller seeds
+  via `logf.IntoContext` — the linter's sanctioned pattern), and
+  `ContextWithLogger` carries a justified `//nolint:logcheck`, since being
+  the IntoContext-analogue is its purpose.
+
+Worth noting: pre-existing lint debt (50 goconst etc. on `main`) was left
+untouched — `make lint` still fails overall; only branch-introduced issues
+were cleaned. The lint fixes changed `Setup`'s signature after Step 2/4 had
+landed, so those commits show the two-arg version; this commit is where it
+settles.

docs/plans-executions/2026-08-24-1224-tracing-e2e.md

@@ -0,0 +1,68 @@
+# Execution log: e2e test — OTel tracing against real Tempo
+
+Plan: `docs/plans/2026-08-24-1224-tracing-e2e.md`
+
+- [x] Step 1 — `test/e2e/tracing_test.go`
+- [x] Step 2 — Makefile
+- [x] Step 3 — Docs
+
+## Steps 1–3 — spec, Makefile, docs (one commit)
+
+The three steps landed together — the spec is one new file and the other
+two are its wiring. `Describe("OTel tracing", Ordered)` is fully
+self-contained (own ns create → `make install`/`make deploy` → teardown)
+because Ginkgo randomizes top-level container order, so it cannot share the
+Manager Describe's deployment. It reuses the package-level `namespace` /
+`managerImage` and the suite's idioms (`utils.Run`, curl-pod with the
+restricted-PSS overrides JSON, log-substring `Eventually`s).
+
+Judgment calls beyond the plan's letter:
+
+- The squid pre-pull (`docker pull` + `kind load`) is **best effort** — a
+  missing docker binary logs a note and continues rather than failing the
+  spec; the 5m Ready timeout still covers an in-cluster pull.
+- The OTLP preflight pod prints per-attempt HTTP codes and a final
+  `OTLP_OK`/`OTLP_UNREACHABLE` marker; the assertion quotes the pod's
+  output, so an unreachable endpoint names itself in the failure.
+- `kubectl set env` is passed the literal
+  `OTEL_RESOURCE_ATTRIBUTES=...$(POD_NAME)...` string via `exec.Command` —
+  no shell involved, kubectl stores `$()` verbatim, and the in-place update
+  keeps the var after the downward-API vars it references.
+- Tempo helpers are stdlib-only; `/api/traces/<id>` is decoded as
+  OTLP-JSON (`batches[].scopeSpans[].spans[].name`), which is Tempo's
+  actual shape (not Jaeger's).
+
+Verified so far: `go vet -tags=e2e ./...` clean.
+
+## Live run against homelab Tempo
+
+First attempt failed before the suite started — Docker Desktop wasn't
+running (`kind` could not create the cluster), and the failure was masked
+to exit 0 by a `| tail` pipe on the make invocation (no pipefail in that
+shell). Rerun with Docker started first and no pipe:
+
+```bash
+TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e
+```
+
+Result: **SUCCESS — 4/4 specs (Manager smoke + both tracing Its), 219 s**,
+kind cluster auto-deleted. Run id `e2e-1787567624494810000`; the traces are
+findable in Grafana with TraceQL
+`{resource.test.run.id="e2e-1787567624494810000"}`. Both proxies reached
+`Ready` on real squid pods; the create trace carried
+`Reconcile Proxy` / `reconcile.managed` / `provider.create` /
+`status.patch` and the expected resource attrs; the deletion trace carried
+`reconcile.delete` / `provider.delete`.
+
+Worth noting: the OTLP preflight and both Tempo polls succeeded on the
+in-cluster → LAN path (kind on macOS reaches 192.168.0.30 through Docker
+Desktop's NAT), so no extra networking setup is needed on this machine.
+
+The plan's negative verification also ran: with a deliberately wrong
+endpoint (`OTLP_ENDPOINT=http://192.168.0.30:9999`) the tracing spec
+failed in the BeforeAll preflight after ~52 s with
+`OTLP endpoint http://192.168.0.30:9999 is not reachable from inside the
+kind cluster; curl output: OTLP_UNREACHABLE` — a named, fast failure
+instead of a 2-minute opaque search timeout — and teardown still deleted
+the kind cluster (`make cleanup-test-e2e` run explicitly, since a failing
+`go test` skips the Makefile's cleanup step).

docs/plans-executions/2026-09-04-1404-kvm-provider.md

@@ -0,0 +1,673 @@
+# Execution: KVM/libvirt provider
+
+Plan: [2026-09-04-1404-kvm-provider.md](../plans/2026-09-04-1404-kvm-provider.md)
+
+- [x] Step 0 — Save plan + branch `feat/kvm-provider`
+- [x] Step 1 — Config (`KVMConfig` & friends, `validate()` case, tests)
+- [x] Step 2 — Pure builders (`domainxml.go`, `volume.go`, `seed.go`, `id.go`, `metadata.go`)
+- [x] Step 3 — Transport + errors (`conn.go`, `errors.go`)
+- [x] Step 3b — Image cache (`image.go`)
+- [x] Step 4 — Provider methods (`kvm.go`) over a fake API
+- [x] Step 5 — Wiring, manifests, samples, docs, CHANGELOG
+- [x] Step 6 — Integration test against a real hypervisor (written and compiled; first real run is part of Step 7)
+- [x] Step 7 — Manual validation + MR
+
+## Step 0 — Save plan + branch
+
+Plan committed alone on `feat/proxy-operator`, then branched:
+
+```bash
+git checkout -b feat/kvm-provider
+```
+
+## Step 1 — Config
+
+The new structs (`KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage`) and
+`validateKVM` went into their own file, `internal/provider/config_kvm.go`,
+rather than being appended to `config.go` as the plan said: the KVM block has
+far more fields and rules than the other two providers, and keeping it
+separate leaves `config.go` readable. Tests likewise live in
+`config_kvm_test.go` instead of extending the table in `config_test.go`; the
+invalid-config table covers every required field, duplicate host names, image
+URL scheme, sha256 shape, and the disk format enum.
+
+While adding the `kvm` case to `validate()`, the pairwise "type is X but a Y
+block is set" checks (which would have needed six branches with three types)
+were replaced by a `setBlocks()` helper that lists the non-nil blocks and
+rejects any that don't match `type`. The error strings are unchanged, so the
+existing GCP/kubernetes tests still pass.
+
+Dependencies were fetched up front so the API signatures in the plan could be
+checked with `go doc`; they stay `// indirect` in `go.mod` until Step 2 imports
+them:
+
+```bash
+go get github.com/digitalocean/go-libvirt@latest github.com/kdomanski/iso9660@v0.4.0
+# go-libvirt resolved to v0.0.0-20260814190004-1a83157e1858 (no tagged release)
+# side effect: golang.org/x/crypto 0.54.0 → 0.56.0, x/mod, x/text, x/tools bumped
+```
+
+Worth noting: `go doc` showed that go-libvirt's `New(net.Conn)` is deprecated
+in favour of `NewWithDialer(socket.Dialer)`, where `Dialer` is a one-method
+`Dial() (net.Conn, error)` interface. Step 3 should wrap the SSH client's unix
+forward in that interface instead of using `New`; the plan's transport design
+is otherwise unaffected.
+
+## Step 2 — Pure builders
+
+`internal/provider/kvm/` now has the five builders from the plan plus
+`kvm.go` holding the package doc and `withDefaults` (disk size 10 GiB, the
+default vendor-data, `/var/run/libvirt/libvirt-sock`, pool `default`, image
+format `qcow2`). Everything is a pure function over `encoding/xml` structs or
+strings, unit-tested by unmarshalling the output back and checking fields.
+
+Two design choices differ from the plan text:
+
+- **Domain disks reference pool volumes, not host paths.** The plan had
+  `buildDomainXML` take the overlay and seed paths (needing two
+  `StorageVolGetPath` round-trips per Create). The domain XML now uses
+  `<disk type="volume"><source pool=... volume=.../>`, so only the *base*
+  image path is ever looked up (the overlay's `<backingStore>` needs a real
+  path). Fewer RPCs and the builder's inputs are all names.
+  **Reverted in Step 7**: volume-referenced disks cannot start on an
+  AppArmor host, see there. The domain XML is back to `type="file"` with
+  the two `StorageVolGetPath` calls the plan had.
+- **Capacities are always `unit="bytes"`.** libvirt's `G` means GiB but
+  `GB` means 10^9; spelling the byte count out removes the ambiguity from
+  both the overlay and the seed/base volumes.
+
+The metadata element is written with a default namespace
+(`<instance xmlns="…/kvm">`) because `encoding/xml` cannot emit prefixes on
+marshal. libvirt may hand it back prefixed; `parseInstanceMetadata` is
+tested against both spellings since `encoding/xml` matches on the namespace
+URI. Whether libvirt accepts the default-namespace form on `DomainDefineXML`
+is confirmed in Step 6 against a real host, not here. (Step 7: it does,
+but `DomainGetMetadata` returns the element with no namespace at all; the
+parser now matches on local name only.)
+
+Checked before relying on `github.com/kdomanski/iso9660` for the seed: its
+filename mangling lowercases and keeps `-`, so `user-data`/`meta-data`/
+`vendor-data` come out as `user-data;1` etc. in the directory record, which
+the Linux isofs driver (and therefore cloud-init's NoCloud mount) presents as
+`user-data`. The seed test opens the ISO with the same library's reader and
+asserts label `cidata`, all three files, and byte-identical output for
+identical inputs. `NewWriter()` stages files in a temp directory, so
+`Cleanup()` is deferred.
+
+`go mod tidy` ran as part of this step; it promoted `kdomanski/iso9660` to a
+direct dependency and dropped `go-libvirt` again because nothing imports it
+yet — Step 3 brings it back.
+
+Worth noting: `withDefaults` copies the `Images` map and `Hosts` slice
+instead of mutating the config in place, so the registry's shared
+`ProviderConfig` stays untouched. The default vendor-data installs
+`qemu-guest-agent` via `packages:` + `runcmd`, which means the first boot
+needs network access to the distro mirrors; a Proxy on an isolated bridge
+would still work (ARP discovery), just without the agent.
+
+## Step 3 — Transport + errors
+
+`conn.go` holds the `libvirtAPI` seam (17 methods, signatures as in the
+plan), a per-host connection pool, and the SSH dialer; `errors.go` holds
+`classify`, `isLibvirtError`, `hasCode` and `logAPIError`. Both are
+unit-tested; the whole package still runs in about a second under `-race`.
+
+The one thing the plan did not anticipate: **go-libvirt RPCs take no
+context and never time out**. A hung hypervisor would pin a reconcile worker
+forever. So the pool exposes `call(ctx, host, fn)` rather than handing out
+raw clients: `fn` runs in a goroutine, and if `ctx` ends first the pool
+closes the SSH connection out from under it (which makes the blocked RPC
+return) and reports `ctx.Err()`. The same helper (`awaitOrCancel`) bounds
+`lv.Connect()` during dial. Two more liveness measures for the same reason:
+the SSH handshake gets a deadline (ctx's, or 30 s), and each connection runs
+an OpenSSH `keepalive@openssh.com` loop every 15 s that closes the client
+when a request errors or goes unanswered for one interval.
+
+Connection-drop policy in `call`: any error that is *not* a `libvirt.Error`
+(so io.EOF, "connection closed", SSH failures) drops the cached connection
+and the next call redials; a daemon-reported `libvirt.Error` keeps it. Dials
+are serialised per host (concurrent Creates on one host share a dial) but
+hosts don't block each other.
+
+As Step 1 noted, the dialer uses `libvirt.NewWithDialer` with a
+`socket.Dialer` that wraps `ssh.Client.Dial("unix", socket)` rather than the
+deprecated `New(net.Conn)`.
+
+Tests: the pool is exercised through a fake dialer (reuse, redial after
+`Disconnected`, drop on transport error, keep on libvirt error, cancel
+releases a blocked call, concurrent callers share one dial, hosts
+independent). The SSH layer is tested against an in-process
+`x/crypto/ssh` server that serves `direct-streamlocal@openssh.com` channels
+by dialing the requested unix path (an echo socket standing in for
+`libvirt-sock`), with keys generated per test and a real `known_hosts` file
+written via `knownhosts.Line`. Covered: forward round-trip and the socket
+path the server saw, unknown host key rejected with `knownhosts.KeyError`,
+`insecureIgnoreHostKey`, wrong client key, cancelled ctx, missing key or
+known_hosts file, and keepalive both closing a stalled client and leaving a
+healthy one alone. The libvirt handshake (`Connect()`) itself is *not*
+covered here — faking libvirt's XDR RPC just for that is not worth it; the
+Step 6 integration test covers it against a real host.
+
+`go mod tidy` promoted `go-libvirt` and `golang.org/x/crypto` to direct
+dependencies.
+
+Worth noting: `classify` maps `ErrOperationInvalid` (e.g. destroying a
+domain that is already off) to Transient; Delete in Step 4 must check that
+code itself via `hasCode` and treat it as success rather than relying on
+the classifier. Nothing maps to `ErrQuotaExceeded` — libvirt has no quota
+concept; an out-of-space pool surfaces as a Transient `ErrInternalError`.
+
+## Step 3b — Image cache
+
+`image.go` follows the plan closely: `resolveImage` maps `placement.image`
+to a configured key or a direct http(s) URL (else Permanent);
+`imageCache.ensure` checks base volume + marker (+ marker content when a
+checksum is configured), tears down and re-fetches when anything is off,
+streams the HTTP body through `StorageVolUpload` with a `sha256` tee,
+refreshes the pool, then writes the marker last. A per-`(host, volume)`
+mutex serialises concurrent callers; the pool itself is the only cache
+state.
+
+Errors from this file are pre-tagged with the taxonomy sentinel via
+`fmt.Errorf("%w: …", provider.ErrPermanent, …)` and `classify` now honours
+such tags before looking at libvirt codes. That keeps one `wrapErr` path in
+the provider instead of special-casing image errors.
+
+`fake_test.go` landed with this step rather than Step 4: an in-memory
+`libvirtAPI` with pools, volumes (content, format, capacity, backing path),
+domains (state, metadata, per-source interface addresses), per-method error
+injection (`fail` always / `failOnce`) and an `uploadHook` to break a stream
+mid-way. It models the libvirt error codes the provider branches on
+(`ErrStorageVolExist`, `ErrNoStorageVol`, `ErrNoDomain`,
+`ErrOperationInvalid`, `ErrNoDomainMetadata`, `ErrAgentUnresponsive`).
+
+Tests cover every case in the plan plus: no checksum configured (marker is
+still written with the computed digest; second call needs no marker read),
+403/500 alongside 404/503, missing pool → Permanent, cancelled ctx →
+Transient with nothing created, and different hosts fetching independently.
+
+One test-harness lesson worth keeping: `httptest`'s server drops
+`Content-Length` and switches to chunked encoding for bodies larger than
+its 2 KiB write buffer, so the fake mirror sets the header explicitly.
+Real distro mirrors send it; a server that doesn't is reported as
+Permanent with a message naming the header.
+
+Worth noting: the base volume is created with `capacity = Content-Length`
+and the *image's* format (`qcow2`), and the raw bytes are uploaded over it.
+That relies on libvirt's upload writing the file verbatim (which is how
+`virsh vol-upload` populates volumes); the following `StoragePoolRefresh`
+makes libvirt re-read the real virtual size from the qcow2 header. Step 6
+confirms this on a real host. Also, the whole download runs inside one
+`pool.call`, so cancellation aborts the HTTP body (ctx-bound) and the
+upload stream fails; the half-written base volume is deleted on the spot
+and, failing that, on the next attempt via the missing marker.
+
+## Step 4 — Provider methods
+
+`kvm.go` now holds `Provider`, `New` (loads the SSH key and known_hosts
+up front via `newSSHAuth`, so a bad path fails at startup), the
+`newWithDialer` seam, and `Create`/`Get`/`Delete`/`ListByTag`. Each
+method resolves its host from config, then runs the whole libvirt
+conversation inside one `hostPool.call`, so a cancelled reconcile drops
+the connection and the blocked RPC returns. Errors pass through `classify`
+in one `wrapErr`, as the Step 3b note intended.
+
+Behaviour that goes beyond the plan text, or that a future reader should
+know before touching this file:
+
+- **Create looks up the domain first.** If it exists, only the running
+  check runs and the volumes are never touched. If not, nothing can be
+  using the volumes yet: the overlay is reused when it exists (its
+  creation is atomic; `ErrStorageVolExist` → reuse) but the **seed is
+  always deleted and rebuilt**. The plan said "exists → skip upload"; the
+  reviewer pointed out that a reconcile cancelled mid-upload leaves a
+  truncated seed that the compensating delete cannot remove, because
+  `hostPool.call` has already closed the connection the delete would go
+  over. Skipping such a seed would boot the VM without cloud-init and no
+  retry would ever repair it. Rebuilding costs one ~40 KiB upload per
+  attempt before the domain exists, nothing afterwards. The table test
+  breaks Create after each step and re-runs it, and a separate test
+  starts from a leftover truncated seed.
+- **Get swallows address-source errors.** `DomainInterfaceAddresses` from
+  ARP returns nothing until the guest has sent traffic and the agent
+  source errors with `ErrAgentUnresponsive` until `qemu-guest-agent` is
+  up, so both are logged at V(2) and mean "no IP yet" → `Provisioning`.
+  Loopback, link-local and unspecified addresses are skipped when
+  choosing the first IPv4 (the agent reports `lo` too).
+- **State mapping:** `running`/`blocked` are active; `nostate` →
+  Provisioning (the reconciler answers Stopped with delete-and-recreate,
+  too drastic for a state libvirt could not determine); everything else
+  (paused, pmsuspended, shutdown, shutoff, crashed) → Stopped.
+- **Delete treats a missing pool as "nothing to clean"** rather than
+  Permanent: failing there would pin the Proxy's finalizer on a config
+  edit that renamed the pool. Domain removal is destroy (`ErrOperationInvalid`
+  on a stopped domain is fine) → undefine with managed-save, snapshot
+  metadata and nvram flags → overlay and seed deleted by name.
+- **ListByTag** reads each domain's metadata once (passed into the same
+  `inspect` that Get uses, so no second `DomainGetMetadata` round-trip)
+  and skips domains without the operator's element. A domain that cannot
+  be read at all — vanished mid-sweep, corrupt metadata, state call
+  failing — is *skipped*, not fatal: GC never deletes what it does not
+  see, so leaving one out is safe, whereas one bad domain blocking GC for
+  every other one (the first version's behaviour, caught in review) is
+  not. An unreachable host still fails the whole sweep.
+  `ConnectListAllDomains` is called with flags 0 (active and inactive).
+- A host named in a providerID but no longer in `kvm.hosts` is Permanent
+  for Get and Delete: the operator cannot reach it, and retrying forever
+  would hide the config problem.
+
+Tests run the provider over a `hostDialer` that maps host names to
+`fakeAPI` instances (an unmapped host is "unreachable"). The image is
+pre-seeded in the pool for most tests; one test runs the full first-use
+download through `httptest` and confirms a second Proxy on the same host
+does not download again.
+
+Found while running the package: `TestBuildSeedISO_deterministic` from
+Step 2 was wrong. `iso9660.ImageWriter` stamps `time.Now()` into the
+volume descriptor, so two builds in different seconds differ by six
+timestamp bytes; the test had only passed because both builds landed in
+the same second. Byte identity is nothing the provider depends on (an
+existing seed is never re-uploaded), so the test now compares label,
+files and size, and the comment on `buildSeedISO` no longer claims
+identical output.
+
+The `go-operator-reviewer` agent ran on the first version of this step
+and found the two bugs above (truncated seed, wedged GC sweep) plus the
+double metadata read and the `nostate` mapping; all were fixed before
+committing. One finding it raised is deliberately **not** fixed here
+because it lives in Step 3b's `image.go` and changes the volume naming
+the plan chose: re-fetching a base image (checksum edited in config, or
+upstream content changed) deletes `base-<key>.<fmt>` while existing
+overlays still back onto it, breaking those VMs' disks. Content-addressed
+base names would avoid it. Flagged for the user; until decided, Step 5's
+docs must warn that an image's `sha256`/`url` must not change while
+Proxies using it exist.
+
+Worth noting: `go test -cover` reports 88.2 % for the package; the
+uncovered lines are the real SSH/libvirt dial path and log-only
+branches. `go vet` and `-race` (three runs) are clean. Wiring into
+`cmd/main.go` is Step 5, so `New` is not yet reachable from a running
+operator.
+
+## Step 6 — Integration test
+
+Done before Step 5, at the user's request, after a discussion of what the
+unit tests can and cannot prove: they run over an in-memory fake of the
+libvirt calls, so every assumption about libvirt's real behaviour (domain
+XML accepted, raw qcow2 upload + pool refresh usable as a backing image,
+guest boots and reports an address) needed a test against a real host.
+The wiring in Step 5 is not a prerequisite — the test drives the provider
+directly.
+
+`internal/provider/kvm/kvm_integration_test.go` is behind
+`//go:build integration` and skips itself under `-short` or when any of
+`KVM_TEST_HOST`, `KVM_TEST_USER`, `KVM_TEST_SSH_KEY`, `KVM_TEST_BRIDGE`,
+`KVM_TEST_IMAGE_URL` is unset. Optional knobs beyond the plan's list:
+`KVM_TEST_KNOWN_HOSTS` (without it the run disables host-key checking
+and logs that), `KVM_TEST_IMAGE_SHA256`, `KVM_TEST_POOL`,
+`KVM_TEST_SOCKET`, `KVM_TEST_TIMEOUT` (default 20 m, must cover a cold
+image download) and `KVM_TEST_KEEP` to leave a failed run's VM behind for
+`virsh console`.
+
+One sequential test, `TestKVM_lifecycle`, not subtests: each stage
+depends on hypervisor state the previous one left. Create (timed, so the
+log shows the image fetch), re-Create returns the same ID, poll Get to
+Running with an IPv4 (Stopped fails fast with a hint to use the console),
+UID and CreatedAt survive the metadata round-trip, ListByTag includes it,
+Delete, Get → NotFound, second Delete is a no-op, ListByTag excludes it,
+and the pool is inspected through the provider's own connection pool
+(`p.hosts.call`, reachable because the test is in package `kvm`): overlay
+and seed gone, base image and marker kept. A TCP connect to the guest's
+port 22 is attempted but only logged, since the test machine may not
+route to the bridge network. Cleanup deletes the VM on failure with its
+own two-minute budget, because the test context may already be spent.
+
+Makefile target, so the invocation is not something to remember:
+
+```bash
+make test-integration-kvm
+# = go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/
+```
+
+`docs/testing.md` gained the kvm package rows in the suite table, a
+"live libvirt" bullet under the deliberately-not-covered list, and a
+section on running this test, with the deviation from the testcontainers
+convention called out as the plan asked.
+
+Worth noting: **the test has not been executed yet.** This session had
+no hypervisor credentials; it was verified with
+`go vet -tags=integration ./...` and by running it without the
+environment (self-skip). Its first real run is the opening act of
+Step 7's manual validation, and it is the point at which the Step 2 and
+Step 3b "confirm on a real host" notes get answered. The test's config
+is built directly rather than through the provider package's validation
+(unexported), so a typo in an env var surfaces from `New` or from
+libvirt, not from a validation message.
+
+## Step 5 — Wiring, manifests, samples, docs, CHANGELOG
+
+Done after Step 6 (see that entry for why the order flipped). Everything
+the plan listed, plus a few things it did not:
+
+- `cmd/main.go`: `"kvm": kvm.New` in the builtins map. No RBAC change —
+  the provider talks to hypervisors, not the API server.
+- `config/manager/manager.yaml`: `kvm-ssh` Secret mounted read-only at
+  `/etc/proxy-operator/kvm` with `optional: true`, so deployments without
+  a kvm provider are unaffected. It nests under the providers-config
+  mount at `/etc/proxy-operator`; kustomize renders it (`kubectl
+  kustomize config/default` checked). The container runs as uid 65532
+  and Secret files default to 0644, so the key is readable without an
+  `fsGroup`; the `kubectl create secret` recipe is in a comment next to
+  the mount, in `docs/kvm-provider.md` and in the README.
+- `config/samples/providers-config.yaml`: `kvm-home` block with Debian 13
+  and Ubuntu 24.04 official image URLs, two machine types, one host, and
+  every optional field shown commented out. Parsed through
+  `provider.LoadConfigFile` in a throwaway test to make sure the sample
+  validates. `config/samples/proxy_kvm.yaml` uses the GCP sample's squid
+  cloud-init and shows `ssh_authorized_keys` commented out rather than
+  set — a sample with a placeholder key would either fail cloud-init or
+  invite pasting a real key into a committed file. Added to the samples
+  kustomization; `config/manager/providers_config.yaml`'s comment now
+  mentions kvm.
+- `docs/kvm-provider.md`: prerequisites with a three-line preflight
+  (`virsh list`, `pool-info`, `ip link` over ssh), cluster setup, the full
+  config reference table, placement table, a table of what lands on the
+  host and its lifetime, how the image cache works and how to evict, the
+  **warning not to change or evict an image while Proxies use it** (the
+  reviewer's finding from Step 4, carried as a documented rule since the
+  content-addressed-naming alternative was not chosen), how to log in,
+  and the known gaps.
+- README: providers bullet and a "KVM setup" section mirroring the GCP
+  one. `docs/architecture.md`: components row, a §5 block for the kvm
+  call flow in the same style as the other two, and five decision-log
+  entries (SSH transport, operator-fetched images, seed always rebuilt,
+  `nostate` → Provisioning, skip-vs-fail in the GC sweep).
+- `CHANGELOG.md` entry at the top, stating explicitly that the provider
+  is not yet validated on a real host.
+
+Verification: `go build ./...`, `go vet ./...`, `go test -short` on
+`cmd` and `internal/provider/...`, and both kustomizations rendered.
+`hack/providers-dev.yaml` untouched, as planned.
+
+Worth noting: the CRD has no rule requiring all three placement fields
+(that is GCP's own check), so the single-host "zone may be omitted"
+behaviour works without an API change. And with this step the branch is
+feature-complete but **unproven**: Step 7 runs the integration test and
+the end-to-end checklist against a real hypervisor before the MR.
+
+## Step 7 — Manual validation + MR (in progress)
+
+Host: `beelink` (Ubuntu, libvirt 11.6.0, QEMU 10.1.0, pool `default` at
+`/srv/vms/images` on local ext4, bridge `br0`), reached over Tailscale as
+`beelink.tail8c69c.ts.net`, SSH user `novakj`. Not the `nas1.home.lan`
+the plan and samples name; that host does not resolve from the dev box.
+
+```bash
+export KVM_TEST_HOST=beelink.tail8c69c.ts.net:22
+export KVM_TEST_USER=novakj
+export KVM_TEST_SSH_KEY=$HOME/.ssh/proxy-operator_ed25519
+export KVM_TEST_KNOWN_HOSTS=$HOME/.ssh/kvm_known_hosts
+export KVM_TEST_BRIDGE=br0
+export KVM_TEST_IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+make test-integration-kvm
+```
+
+### Run 1 and 2: `Could not open '/srv/vms/images/base-test-image.qcow2': Permission denied`
+
+Everything up to `DomainCreate` worked on the first try against a real
+host, which answers the open notes from Steps 2 and 3b: the SSH-forwarded
+libvirt handshake, the raw qcow2 upload plus pool refresh as a backing
+image, the default-namespace metadata element, and `DomainDefineXML` for
+the generated XML were all accepted by libvirt 11.6. QEMU then exited at
+start with `Permission denied` on the base image.
+
+Ownership was ruled out the slow way: the file was `root:root 0600` (the
+dir-pool default; libvirt's DAC driver chowns the chain at start and
+restores it after a failed start, so that observation meant nothing),
+`qemu.conf` was all defaults (`libvirt-qemu`, `dynamic_ownership=1`), and
+after a manual `chown libvirt-qemu:kvm` the second run failed identically.
+
+The actual cause is the Step 2 choice of `<disk type="volume">`.
+libvirt's AppArmor helper builds each domain's profile from the domain XML
+without consulting the storage driver, and its `add_file_path` only emits a
+rule when the source has a `path`; a volume-referenced disk has pool and
+volume names instead, so neither the overlay nor its backing image gets a
+rule and QEMU is denied on the first one it opens. Every stock Debian and
+Ubuntu hypervisor confines QEMU this way. The unit tests could not have
+caught it: the fake accepts either form.
+
+Fix, in this step: `buildDomainXML` takes `DiskPath`/`SeedPath` and emits
+`type="file"` disks; `ensureOverlay` and `writeSeed` return the volume
+handles (the exists-already branch now does a `StorageVolLookupByName`),
+and Create resolves both paths with `StorageVolGetPath` before defining
+the domain. Two extra cheap RPCs per Create. `TestBuildDomainXML_fieldByField`
+now fails on any `type="volume"` or `pool=` attribute so the design cannot
+drift back, and `TestCreate_freshHost` asserts the disk sources equal the
+paths the fake reports.
+
+Worth noting: the first two runs left the base image and its marker in
+the pool as designed, so later runs skip the 435 MB download.
+
+### Run 3: `parsing domain metadata: expected element <instance> in name space … but have no name space`
+
+With file-backed disks, Create returned in 1 s (image cached) and the
+domain started. Get then failed to parse the metadata: libvirt 11.6 finds
+the element by our URI but returns it as a bare `<instance>` with no
+namespace declaration at all. Step 2 had tested the default-namespace and
+prefixed spellings and left "does libvirt accept the default-namespace
+form" for the real host; it does, it just does not give it back that way.
+
+Fix: `parseInstanceMetadata` matches on the local name only, since the
+lookup already selected the element by URI. The fake now strips the
+declaration when storing metadata, so the provider's Get and ListByTag
+tests parse the shape libvirt really returns, and a new
+`TestParseInstanceMetadata_libvirtBareForm` pins it.
+
+### Run 4: guest hangs at GRUB's "Booting" line
+
+Create and start passed; Get polled `Provisioning` with no IP while the
+VM sat at 100 % CPU after GRUB printed "Booting `Debian GNU/Linux`" and
+before the kernel wrote anything to the serial console. The domain XML
+matches a working VM on the same host (q35, BIOS, host-passthrough,
+virtio disk, SATA cdrom, br0), so the difference is the disk content:
+the cached image was downloaded without a checksum and never verified.
+Diagnosis was handed to the user (sha512 against Debian's published
+sums, `qemu-img check`, and a `virt-install --import` of the same image
+outside the operator). Because the provider's own logs were invisible in
+the test output, this run also produced a side plan:
+[2026-09-10-0805-kvm-libvirt-tracing](2026-09-10-0805-kvm-libvirt-tracing.md),
+which traces every libvirt RPC at V(1) and routes it into `-v` output.
+
+### Run 5: the guest hang is Debian 13's BIOS boot path; UEFI fixes it
+
+The cached image's sha512 matched Debian's published sum, and
+`qemu-img info` showed a sound 3 GiB qcow2, which also confirms that
+the raw upload over a libvirt-created volume plus pool refresh yields a
+byte-identical file (the Step 3b note is answered). Bisected by hand on
+beelink with `virt-install --import` on a fresh overlay of the same
+image:
+
+| variant | seed | CPU | firmware | result |
+|---|---|---|---|---|
+| operator's domain | ours | host-passthrough | SeaBIOS | hangs after GRUB "Booting", 100 % CPU |
+| bare | none | host-passthrough | SeaBIOS | same hang |
+| bare | none | host-model | SeaBIOS | same hang |
+| bare | none | host-passthrough | **OVMF (`--boot uefi`)** | **boots to login** |
+
+So neither the seed ISO, the domain XML nor the CPU model was the
+problem: the image's legacy GRUB → kernel handoff does not work on this
+host, while its EFI stub path does. Not diagnosed further; it is
+Debian's chain, not ours, and every current cloud image boots UEFI.
+
+Change: `images.<key>.firmware` (`uefi` default, `bios` escape),
+threaded through `withDefaults` → `imageSource` → `domainParams`, and
+`buildDomainXML` emits `<os firmware="efi"><firmware><feature
+enabled="no" name="secure-boot"/></firmware>` so libvirt selects the
+OVMF binary and creates the NVRAM file; the `nvram` undefine flag Delete
+already passes removes it (seen on the host: `virsh undefine` without
+`--nvram` refuses such a domain). Direct-URL images default to UEFI
+too. Docs: OVMF is a new hypervisor prerequisite, config table row,
+sample comment, known-gaps entry, decision-log entry.
+
+Worth noting: the first image download went through with no checksum
+configured, and the cache reuses a base image by key, so switching
+`KVM_TEST_IMAGE_URL` requires deleting `base-test-image.qcow2` and its
+marker by hand first. That is the reviewer's Step 4 finding showing up
+in practice; still documented rather than fixed.
+
+Follow-up, same day: the cause is known upstream. [Debian bug
+#1111240](https://bugs.debian.org/1111240) (filed 2025-08-16 against
+the cloud image, reassigned to `grub2`, still open as of 2026-09-10;
+trixie ships grub2 2.12-9+deb13u2) reports exactly this with a domain
+XML almost identical to ours: BIOS, virtio disk, SATA cdrom seed, bridge
+NIC, `<console><target type='serial'/>`, no `<graphics>`/`<video>`.
+The cloud team reproduced it: trixie's grub.cfg has `terminal_output
+gfxterm serial`, and with a serial port present but no video adapter
+GRUB never hands over to the kernel. Their fix in the image
+(`terminal_output serial`) is not something an operator can apply;
+the reporter's workaround (console `target type='virtio'`) boots but
+takes the kernel's `console=ttyS0` output with it. UEFI sidesteps the
+gfxterm path and keeps the serial console, so the decision stands, but
+the docs now name the bug rather than blaming the host, and note that
+bookworm images are unaffected. `latest` on cloud.debian.org was the
+2026-08-31 build when this was hit, so every trixie point release to
+date carries it.
+
+### Run 6: pass
+
+With UEFI the full lifecycle passed in 23 s on a warm cache: Create 2 s
+(domain start 0.9 s), guest agent answering with `192.168.0.89` after
+20 s (ARP never did; the host had not talked to the guest), TCP 22
+reachable from the dev machine, ListByTag seeing it, Delete leaving only
+the base image and marker in the pool, second Delete a no-op. Trace
+excerpt worth keeping for the record:
+
+```
+09:44:28.603 libvirt virDomainDefineXML xmlBytes=1509 domain=proxy-hk5oavoqk45n3twq duration=13.51ms
+09:44:28.603 kvm domain defined uid=9bbceb41-… firmware=uefi
+09:44:29.542 libvirt virDomainCreate domain=proxy-hk5oavoqk45n3twq duration=938.705ms
+09:44:49.573 libvirt virDomainInterfaceAddresses domain=proxy-hk5oavoqk45n3twq source=DomainInterfaceAddressesSrcAgent interfaces=2 duration=1.742ms
+09:44:49.573 kvm KVM domain fetched state=Running ip=192.168.0.89 uid=9bbceb41-…
+```
+
+Worth noting: the sweep listed a second managed domain,
+`proxy-qxge3dbm2mnpc3ih`, shut off, left behind by an earlier failed
+run whose cleanup never ran. That is exactly the orphan GC exists for;
+on the host it needs `virsh undefine --nvram` plus its two volumes by
+hand. The `ListByTag` sweep costs one `virDomainGetMetadata` per
+domain on the host (21 here, ~0.4 ms each over the tunnel), which is
+fine at this scale but is the call to watch on a busy hypervisor.
+
+### Run 7: in-cluster, two defects the integration test could not see
+
+The e2e pass ran the operator inside the kind cluster
+(`egress-proxies-operator:kvm-dev`, built with `make docker-build`,
+loaded with `kind load docker-image`, deployed via the kustomize
+`config/manager` with the `kvm-home` provider pointing at beelink) and
+two Proxies with real cloud-configs: one plain squid, one with a debug
+login user. Both defects come from the difference between the test
+harness and the real thing, not from libvirt.
+
+**Read-only root filesystem.** The manager runs with
+`readOnlyRootFilesystem: true`, and `iso9660.NewWriter` stages the seed
+in a temp directory, so the first Create failed with `creating iso
+writer: read-only file system`. Fix: a 64 MiB `emptyDir` on `/tmp` in
+`config/manager/manager.yaml`. Side finding: the error is classified
+`ErrPermanent` by `writeSeed`, yet `hostPool.call` treated any
+non-libvirt error as a transport failure and dropped the SSH connection,
+so every retry redialled. `call` now leaves the connection alone for
+errors already carrying a provider sentinel (`isClassified`), with a
+test.
+
+**The guest agent never got installed.** Both VMs booted (confirmed on
+the console by hand) but sat in `Provisioning` for 25 minutes: the
+trace showed `virDomainInterfaceAddresses` from the agent source failing
+with `ErrAgentUnresponsive` on every poll and the ARP source empty. The
+integration test had passed with `CloudInit: "#cloud-config\n"`; the
+real Proxies have `packages: [squid]` and `runcmd:`. cloud-init merges
+vendor-data under user-data with its default mergers, which keep the
+user-data's value for any key both define and never append lists, so the
+vendor-data's `packages`/`runcmd` were dropped wholesale. Confirmed
+locally with the dev box's cloud-init 26.1 before changing anything:
+
+```sh
+python3 - <<'EOF'
+from cloudinit import mergers
+user = {"packages": ["squid"], "runcmd": ["systemctl restart squid"]}
+vendor = {"packages": ["qemu-guest-agent"], "runcmd": [["systemctl", "enable", "--now", "qemu-guest-agent"]]}
+print(mergers.construct(mergers.default_mergers()).merge(dict(user), dict(vendor)))
+EOF
+# {'packages': ['squid'], 'runcmd': ['systemctl restart squid']}
+```
+
+Fix: `withGuestAgent` in `seed.go` merges the bootstrap into the Proxy's
+cloud-config itself (append to `packages` and `runcmd`, skip if present,
+re-render with `sigs.k8s.io/yaml`), so vendor-data is no longer defaulted
+and `vendorData` is plain extra data. User-data that is not a cloud-config
+document (script, `#include`, MIME) still gets the bootstrap as
+vendor-data since there is nothing to collide with; a cloud-config that
+does not parse as a mapping fails the Proxy as `ErrPermanent`. The
+rendered document for the squid Proxy passed `cloud-init schema
+--config-file`. The logging test's fixture (`#cloud-config` + a bare
+sentinel line) is not a mapping and had to become `hostname: <sentinel>`.
+
+Rebuilt, reloaded, restarted, deleted and recreated both Proxies: both
+`Ready`/`HEALTHY True` in 2 min 1 s from `kubectl apply`, IP via the
+agent 110 s after `virDomainCreate` (the guest installs squid and the
+agent from the mirrors first), and `curl -x http://<ip>:3128
+https://www.gstatic.com/generate_204` from the dev box returns 204
+through both.
+
+Worth noting: the `#cloud-config` round-trip drops comments and reorders
+keys alphabetically; semantics are preserved (numbers, booleans and
+block scalars survive, checked in tests) but a user diffing the seed
+against their CR will see a different document. Vendor-data supplied via
+`vendorData` is subject to the same cloud-init merge rule and is now
+documented as such. The kind pod also restarted twice during the session
+from "Pod sandbox changed" (the node briefly lost the API server), which
+is unrelated to the operator. Still uncommitted on purpose: the
+`kustomization.yaml` image override (`egress-proxies-operator:kvm-dev`)
+and the `beelink` host entries in `config/manager/providers_config.yaml`
+and `config/samples/providers-config.yaml`, which are this machine's dev
+setup rather than the repo's defaults.
+
+### Run 8: review fixes
+
+A `/code-review high` pass over the branch found five defects, fixed in
+[plan 2026-09-10-2222](../plans/2026-09-10-2222-kvm-review-fixes.md) with
+its own execution log. Two of them land on Run 7's conclusions above and
+supersede them:
+
+- The guest-agent **merge described in Run 7 is gone**. It could not be made
+  safe for user-data that is not a plain cloud-config, and its YAML
+  round-trip re-typed unquoted scalars. The bootstrap now ships as a
+  vendor-data shell script, which cloud-init never merges, and the Proxy's
+  cloud-init reaches the seed byte for byte.
+- The read-only-filesystem fix from Run 7 still stands, but the eager SSH
+  key load it sat next to does not: a missing `kvm-ssh` Secret aborted the
+  whole provider set and crash-looped the manager. The material is read on
+  first use now.
+
+Re-validated the same way: integration test against beelink, both Proxies
+recreated in kind, 204 through squid.
+
+### MR opened
+
+[PR #5](https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/pulls/5),
+`feat/kvm-provider` → `main`, 19 commits. `tea` is not installed on the
+dev box and the HTTPS remote has no credential helper, so the push used a
+one-off helper reading a Gitea access token from `~/.config/gitea-token`,
+and the MR was created with the API instead of `tea pr create`:
+
+```sh
+git -c credential.helper='!f() { echo username=kacerr; echo "password=$(tr -d "[:space:]" < ~/.config/gitea-token)"; }; f' \
+  push -u origin feat/kvm-provider
+curl -s -H "Authorization: token $(tr -d '[:space:]' < ~/.config/gitea-token)" \
+  -H 'Content-Type: application/json' -d @pr.json \
+  https://gitea.home.hrajfrisbee.cz/api/v1/repos/kacerr/egress-proxies-operator/pulls
+# pr.json: {"title": ..., "head": "feat/kvm-provider", "base": "main", "body": ...}
+```
+
+The push printed a libsecret "store failed" warning from the system
+credential store; harmless, the token was not persisted anywhere but the
+file.

docs/plans-executions/2026-09-10-0805-kvm-libvirt-tracing.md

@@ -0,0 +1,109 @@
+# Execution: Trace every libvirt call in the KVM provider
+
+Plan: [2026-09-10-0805-kvm-libvirt-tracing.md](../plans/2026-09-10-0805-kvm-libvirt-tracing.md)
+
+- [x] Step 0 — Save plan
+- [x] Step 1 — `errors.go`: shared error fields
+- [x] Step 2 — `trace.go`: the `tracingAPI` decorator
+- [x] Step 3 — `conn.go`: pool logging and wrapping
+- [x] Step 4 — Call sites
+- [x] Step 5 — Integration test output
+- [x] Step 6 — Unit tests
+- [x] Step 7 — Docs, execution log, CHANGELOG
+
+## Steps 1–6 — Tracer, pool logging, tests, integration wiring
+
+Done as one commit on `feat/kvm-provider`; the plan's assumption that
+this belongs on the KVM branch rather than its own feature branch was
+accepted at plan approval.
+
+The code follows the plan closely. Things that came out differently
+or are worth knowing:
+
+- **Logger names join with `/` in funcr, `.` in zap.** The trace lines
+  are `kvm/libvirt` in the unit-test capture and in `testr` output, and
+  `kvm.libvirt` under the operator's zap logger. The docs use the zap
+  spelling; the unit test asserts the funcr one. First run of the tier
+  test failed on exactly this.
+- **Cancel is not reported as a transport error.** The original `call`
+  dropped the connection twice on cancellation (once from the abort
+  closure, once from the "not a libvirt error" branch, harmlessly, since
+  drop is idempotent). With a log line on each drop that would have
+  printed both `context cancelled` and `transport error` for one event,
+  so the transport branch now skips `ctx.Err()`. Covered by
+  `TestHostPool_call_logsDropReason`.
+- **`ConnectListAllDomains` in the fake includes every defined domain**,
+  so the trace test's fixture (one running managed domain plus one
+  stopped domain for `DomainCreate` to start) lists two; the V(2)
+  assertion checks for the `names` key rather than an exact list.
+- **`libvirtErrorFields` also feeds `logAPIError`**, which therefore
+  gains `libvirtCodeName` (e.g. `ErrNoDomain`) next to the numeric code.
+- The seam-coverage guard (`reflect` count of `libvirtAPI` methods vs
+  the test table) is the only thing standing between a future 18th RPC
+  and a silently untraced call; keep it.
+
+Verification run before committing:
+
+```bash
+go build ./... && go vet ./... && go vet -tags=integration ./internal/provider/kvm/
+go test -short ./...
+go test -race -count=1 -cover ./internal/provider/kvm/   # 90.2 %, up from 88.2
+go test -tags=integration -count=1 -run TestKVM ./internal/provider/kvm/   # self-skips without env
+```
+
+Worth noting: the live run (`make test-integration-kvm` against beelink)
+had not happened when this was written; it is what Step 7 of the KVM
+provider plan is waiting on, and its output is the acceptance test for
+this feature. Expected shape is in the plan's Verification section.
+
+## Revision after the first live run — less noise, libvirt names
+
+The first live run on beelink showed the trace working but hard to read:
+`testr` printed `"ts"`, `"level"`, `"msg"` and the three per-run
+constants on every line, the error text appeared twice (`error` and
+`libvirtMessage`), `host` appeared twice on image-cache lines (the
+logger already carried it), and `method=DomainLookupByName` was
+go-libvirt's name rather than something to look up in libvirt's docs.
+The user asked for "less verbose but more informative". Changes:
+
+- The tracer's headline field is now `rpc=virDomainLookupByName`: the
+  public libvirt API function, which go-libvirt's method names are
+  exactly minus the `vir` prefix (they come from the same
+  `remote_protocol.x`). Not gRPC, for the record: libvirt's own
+  XDR-based RPC over the forwarded unix socket.
+- Errors are one `error` field plus `libvirtError=ErrNoDomain` and
+  `libvirtCode=42`; `libvirtMessage` is gone from both the tracer and
+  `logAPIError`, since it is the error text.
+- `image.go` no longer adds `host` to lines whose logger has it.
+- The integration test replaces `testr` with a small `compactSink`
+  (`logr.LogSink` plus `GetCallStackHelper`, so `t.Log` still
+  attributes lines to `trace.go`): `HH:MM:SS.mmm libvirt
+  virDomainCreate domain=… duration=… error=… libvirtError=…`. The
+  `rpc` value becomes the headline, `provider`/`host`/`name` are
+  dropped (the run's first `t.Log` line states them), and values with
+  spaces are quoted. Production zap output is unchanged apart from the
+  field renames; structured logs are meant to repeat context.
+
+Previewed by driving the sink through the fake provider with an
+injected `DomainCreate` failure (a throwaway integration-tagged test,
+not committed):
+
+```
+trace.go:55: 09:04:33.165 libvirt virDomainLookupByName domain=proxy-abc123def456ghij duration=4µs error="Domain not found: …" libvirtError=ErrNoDomain libvirtCode=42
+trace.go:55: 09:04:33.165 libvirt virStorageVolUpload pool=default volume=proxy-abc123def456ghij-seed.iso offset=0 length=0 duration=71µs
+kvm.go:262:  09:04:33.165 kvm seed volume uploaded volume=proxy-abc123def456ghij-seed.iso bytes=45056
+trace.go:55: 09:04:33.165 libvirt virDomainCreate domain=proxy-abc123def456ghij duration=0s error="internal error: process exited…" libvirtError=ErrResourceBusy libvirtCode=87
+```
+
+Worth noting: `length=0` on `virStorageVolUpload` is what the provider
+passes (libvirt reads to EOF); the following `seed volume uploaded`
+line has the byte count.
+
+## Step 7 — Live run confirmed; CHANGELOG
+
+The compact trace was accepted on the first passing live run
+(2026-09-10, beelink): one line per RPC with the libvirt function name,
+the provider's own lines between them, `Provisioning` polls readable at
+a glance (`SrcArp interfaces=0`, `SrcAgent … ErrAgentUnresponsive`
+until the agent came up at +20 s), and a full `ListByTag` sweep over 21
+domains in 15 ms. CHANGELOG entry added under the KVM validation entry.

docs/plans-executions/2026-09-10-2222-kvm-review-fixes.md

@@ -0,0 +1,229 @@
+# Execution: fix the five code-review findings on `feat/kvm-provider`
+Plan: [docs/plans/2026-09-10-2222-kvm-review-fixes.md](../plans/2026-09-10-2222-kvm-review-fixes.md)
+
+- [x] Step 0 — Save plan into the repo
+- [x] Step 1 — Unwedge deletion (finding 1)
+- [x] Step 2 — Validate names that become identifiers (finding 2)
+- [x] Step 3 — Read the SSH material lazily (finding 3)
+- [x] Step 4 — Guest agent via vendor-data script (findings 4 and 5)
+- [x] Step 5 — Validate, record, push
+
+## Step 0 — plan committed
+
+Copied out of `~/.claude/plans/` and committed alone before touching code, as
+the repo's plan rules require.
+
+## Step 1 — Unwedge deletion
+
+Three changes, in the order the bug unfolds.
+
+`reconcileDelete` calls `Get` first and only drops the finalizer when it
+answers not-found, so any permanent error there retried forever. It now
+switches on the class: a permanent `Get` still calls `Delete`, which is
+idempotent, and releases the object when that succeeds. Polling a `Get`
+that cannot converge was the wedge; `Delete` is the only call that can
+still make progress.
+
+`crawl.example.com/skip-cleanup: "true"` abandons the instance and removes
+the finalizer. It is checked **before** the provider lookup, which also
+frees a Proxy whose provider was deleted from the config — a second wedge
+sitting three lines above the reported one, with the same shape. No
+`EventRecorder` exists on the reconciler and none was added, so the
+abandonment is an Info log; `docs/architecture.md` already records that
+decision for the whole controller.
+
+In the KVM provider, `Delete` for a providerID naming an unconfigured host
+now logs and returns success. The precedent is two functions further down:
+`removeVolumes` treats a missing pool the same way, with a comment saying
+outright that refusing "would pin the Proxy's finalizer".
+
+**`Get` deliberately still fails permanently for that case.** Making it
+answer not-found would have been a one-line fix for the wedge, but the live
+reconcile path reads not-found as "the VM is gone" and rebuilds it
+elsewhere. A typo in `hosts` would then quietly duplicate every Proxy on
+that host, with the originals still running where GC cannot see them, since
+`ListByTag` only sweeps configured hosts. The asymmetry is now pinned by a
+test that asserts both halves at once.
+
+## Step 2 — Validate names that become identifiers
+
+One regexp, `^[A-Za-z0-9][A-Za-z0-9_.-]*$`, applied to `hosts[].name` and to
+the `images` map keys. Dots stay legal because a host name like
+`nas1.home.lan` is ordinary and harmless; the character that actually
+breaks things is `/`.
+
+The review only reported the host name. The image key has the same defect
+and is arguably worse: it is not merely embedded in an ID but pasted into a
+file name, `base-<key>.<format>`, so `../etc/x` would be a path traversal
+into the storage pool rather than a parse failure. Both are validated in the
+same pass. Machine-type keys are left alone deliberately — they are only
+ever compared against `placement.machineType`, never embedded anywhere.
+
+Worth noting: this is validation of the operator's own config file, which
+only a cluster admin can write, so the traversal was never a privilege
+boundary. It is still cheaper to reject at load than to explain a stuck
+Proxy later. Both new table cases assert the message text, so the wording
+that names the consequence is now pinned by tests.
+
+## Step 3 — Read the SSH material lazily
+
+`kvm.New` no longer loads the key; `lazyAuth` does, on the first connection,
+caching the result once it works. `registry.Build` is all-or-nothing by
+design, so the old constructor error did not merely fail the KVM provider,
+it exited the manager before any controller started.
+
+The plan said "lazy dialer", and the first attempt was exactly that — one
+closure wrapping both the loading and the dial. It could not be tested: the
+fixture's SSH server is real, but the socket behind it is an echo server, so
+every successful dial dies in the libvirt handshake and the assertion could
+not tell "the key worked" from "the key did not". Splitting the seam in two,
+`lazyAuth` for the material and `lazyDialer` for the wiring, made both
+testable: the healing path against `lazyAuth`, and a separate test that the
+dialer really authenticates, asserted by the error being past SSH and by the
+server having seen the forward request.
+
+Error classes carry the intent. A missing file stays unclassified, so
+`provider.Class` defaults it to transient and a Proxy heals on its own when
+the Secret appears — the common case, since the Secret is usually created
+after the manager. A key that does not parse wraps `ErrPermanent`, because
+retrying cannot fix it. `newSSHAuth` distinguishes the two with
+`errors.Is(err, fs.ErrNotExist)`.
+
+Worth noting: the startup check is kept, just non-fatally, logged with the
+`WARNING:` prefix the repo already uses where logr has no warn level. That
+preserves the diagnostic the eager version was written for. The caching test
+deliberately deletes the key file after a successful load and asserts the
+cached value is reused, which pins the rotation behaviour the docs promise:
+a rotated Secret needs a manager restart. `TestNew_badKeyPathFailsFast` was
+inverted rather than deleted, so the old contract stays visible in the diff
+next to the new one.
+
+## Step 4 — Guest agent via vendor-data script
+
+The review filed two findings against `withGuestAgent` — detection too
+narrow, and YAML re-typing on the round-trip. Reading cloud-init 26.1's
+sources while planning turned both into one finding: the operator should
+not be editing the user's document at all.
+
+Vendor-data gets its own handler chain (`stages.py:_default_vendordata_handlers`),
+with separate script and cloud-config paths. A vendor-data **script** is
+therefore written to the vendor script directory and run by
+`cc_scripts_vendor`, never touching the config merger that drops colliding
+keys. On the final-stage module list it lands after
+`package_update_upgrade_install` and before the user's `runcmd`, so the
+agent now comes up *earlier* than the old `runcmd` append did.
+
+That also closed the door the previous approach had left ajar. The merge
+cannot be rescued from inside either document: `merge_how` in user-data
+governs only the first source, and `merge_how` in vendor-data is popped by
+`dict_extract_mergers` and discarded before the document is written. There
+was no fix that kept the merge.
+
+`withGuestAgent`, `isCloudConfig` and `appendUnique` are gone, along with
+seed.go's `sigs.k8s.io/yaml` dependency. `installGuestAgent` (default true)
+turns the script off. When `vendorData` is also configured the two ship as
+a MIME multipart, with the user's document declared `text/plain` — a type
+cloud-init re-sniffs from the content's own header line, so this code never
+classifies what the user wrote.
+
+Before deleting the old path, its damage was captured rather than taken on
+trust. Round-tripping a plausible cloud-config through it:
+
+```sh
+# temporary test in the package, deleted afterwards
+go test -run TestZZProbeRetyping -v ./internal/provider/kvm/
+# in:  version: 1.10
+# out: version: 1.1
+```
+
+`1.10` arriving as `1.1` is worse than the reviewer's `passwd` example,
+since nothing about it looks wrong in the seed. Comments were dropped and
+keys reordered on the same pass.
+
+The generated vendor-data was then checked against the cloud-init actually
+installed on this machine, rather than against the docs:
+
+```sh
+python3 -c 'from cloudinit import user_data, handlers; ...'
+# vendor-script:    multipart=False  sniffed=text/x-shellscript
+# vendor-multipart: multipart=True   part 1 text/plain -> text/cloud-config
+#                                    part 2 text/x-shellscript
+```
+
+and the script itself through `sh -n`.
+
+Worth noting: the script has to choose a package manager where cloud-init's
+`packages:` key was distro-agnostic. It covers apt, dnf, yum, zypper and
+apk, and both systemd and OpenRC for the enable step, but an image outside
+that set now needs `installGuestAgent: false` and the agent baked in. The
+seed's user-data is byte-identical to the Proxy's `cloudInit` from here on,
+which is what the new `TestBuildSeedISO_userDataIsNeverRewritten` pins
+across seven formats. One consequence worth remembering: a user-data that
+sets `vendor_data: {enabled: false}` disables the agent, and cloud-init
+documents that as intended behaviour.
+
+## Step 5 — Validation
+
+Unit suite, `go vet` and `gofmt` clean across the repo. Then the two real
+runs.
+
+**Integration against beelink**, unchanged environment from the KVM plan's
+log, passed in 22 s on a warm cache, with
+`virDomainInterfaceAddresses source=DomainInterfaceAddressesSrcAgent
+interfaces=2` — the agent installed by the new vendor script answering, on
+a stock Debian 13 image.
+
+**In-cluster**, both Proxies deleted and recreated against the rebuilt
+image: Ready with an address 2 min 3 s after `kubectl apply`, 204 through
+squid from the dev box for both. Inside the debug VM:
+
+```sh
+ssh novakj@192.168.0.94 'systemctl is-active qemu-guest-agent; cloud-init status; \
+  sudo head -4 /var/lib/cloud/instance/user-data.txt; sudo ls /var/lib/cloud/instance/scripts/vendor/'
+# active
+# status: done
+# #cloud-config
+# # Login for debugging: user "novakj", password "kvm-dev", passwordless sudo.
+# ...
+# part-001
+```
+
+Two things worth having seen with your own eyes there: the user's **comment
+lines survived** into the guest, which the old merge stripped, and the
+script really did land in the vendor directory rather than being merged
+into the user's config.
+
+**Negative check for the Secret**, done by pointing the deployment's volume
+at a name that does not exist rather than deleting the real Secret:
+
+```sh
+kubectl -n egress-proxies-operator-system patch deploy egress-proxies-operator-controller-manager \
+  --type=json -p '[{"op":"replace","path":"/spec/template/spec/volumes/1/secret/secretName","value":"kvm-ssh-absent"}]'
+```
+
+The manager came up and stayed up, 0 restarts, logging
+`WARNING: kvm ssh material is not usable ... open /etc/proxy-operator/kvm/id_ed25519:
+no such file or directory`. The Kubernetes Proxies kept reconciling; the
+KVM ones failed per reconcile and retried. Before this branch the same
+config exited the process before any controller started. Patched back
+afterwards, everything Ready.
+
+Worth noting: the KVM Proxies kept reporting `Ready` during that window
+rather than degrading visibly. That is existing behaviour, not a
+regression — a transient provider error rides the workqueue and does not
+clear a healthy status — but it does mean the only signal of a missing
+Secret is the log and the reconcile errors. Also, the ListByTag sweep on
+beelink still reports `managed=2` where one domain is an orphan from an
+older failed run; unrelated to this work, and the same leftover noted in
+the KVM plan's Run 6.
+
+Worth noting: the review filed this as a KVM finding, but the fix is mostly
+in the controller, because KVM is the only provider that resolves a
+providerID segment against operator config at all. GCP and Kubernetes hand
+the lookup to the backend, which answers 404 for a retired zone or a deleted
+namespace, so they reach not-found and release on their own. Anything else
+that classifies permanently on the delete path — revoked RBAC on the
+Kubernetes provider is the realistic one — was equally stuck before this
+step and is what the annotation is really for. The KVM table case
+`"host not configured"` moved out of `TestDelete_errors` into its own named
+test, since it is no longer an error.

docs/plans-executions/2026-09-11-1526-provider-health-checks.md

@@ -0,0 +1,378 @@
+# Execution: provider health checks
+Plan: [docs/plans/2026-09-11-1526-provider-health-checks.md](../plans/2026-09-11-1526-provider-health-checks.md)
+
+- [x] Step 0 — Save the plan
+- [x] Step 1 — ProviderHealth API and CRD
+- [x] Step 2 — Contract plus the kubernetes and GCP checks
+- [x] Step 3 — KVM check
+- [x] Step 4 — Monitor
+- [x] Step 5 — Metrics and wiring
+- [x] Step 6 — Docs
+- [x] Step 7 — Validate, record, push, open the PR
+
+## Step 0 — Save the plan
+
+Branched off `main` (at the PR #5 merge, `eb8f075`). The plan was copied out of
+`~/.claude/plans/` and committed on its own, before any code, as the repo's plan rules
+require:
+
+```bash
+git checkout -b feat/provider-health main
+cp ~/.claude/plans/now-i-would-like-flickering-wall.md docs/plans/2026-09-11-1526-provider-health-checks.md
+```
+
+The uncommitted `.claude/settings.json` change was left unstaged and stays out of this
+branch.
+
+## Step 1 — ProviderHealth API and CRD
+
+`kubebuilder` isn't installed here (only `bin/controller-gen`, `bin/kustomize` and
+`bin/setup-envtest` are), so `api/v1alpha1/providerhealth_types.go` and the `PROJECT`
+entry were written by hand instead of with `kubebuilder create api`. For a cluster-scoped
+resource `PROJECT` simply omits `namespaced`, which is what kubebuilder itself writes.
+
+The new types are registered in the existing `AddKnownTypes` call in `proxy_types.go`,
+not in a second `init()`.
+
+The deepcopy code, the CRD and the RBAC come from the usual generators:
+
+```bash
+make generate manifests
+go build ./... && go vet ./api/... && go test -short ./api/...
+```
+
+To confirm the new CRD and the viewer role actually reach the rendered install, I built
+the kustomization and grepped for them:
+
+```bash
+bin/kustomize build config/default | grep -n providerhealth
+# providerhealths.crawl.example.com (scope: Cluster, shortName ph)
+# egress-proxies-operator-providerhealth-viewer-role
+```
+
+Worth noting:
+- The only role added is the viewer role. The operator is the object's only writer, so
+  the admin and editor roles kubebuilder scaffolds for Proxy were not added.
+- The type file has no unit test, since it has no logic. The schema, including the
+  required `ready`/`lastTransitionTime` and the list-map keys, is proven by the envtest
+  round-trip in Step 4.
+- `config/rbac/role.yaml` is unchanged so far. The manager's own permissions arrive in
+  Step 4 with the Monitor's RBAC markers.
+
+## Step 2 — Contract plus the kubernetes and GCP checks
+
+`provider.Checker` is an optional interface in `internal/provider/check.go`, kept out of
+`Provider`. Each implementation has a compile-time assertion, because the monitor's
+type assertion would otherwise fail silently.
+
+- **kubernetes:** lists at most one managed Pod cluster-wide, using the same client, verb
+  and scope as `ListByTag`. The target is `apiserver`, at the `restCfg.Host` now stored
+  in the new `apiHost` field.
+- **GCP:** calls a new seam method, `Probe`. `realInstances.Probe` lives in `check.go`,
+  next to its only caller. `ListByTag`'s `Sprintf` filter became the shared
+  `managedFilter` constant, so the check and the GC sweep can't drift apart.
+
+Before relying on "MaxResults is only a page size", I checked the SDK's REST
+`AggregatedList` in the module cache:
+
+```bash
+grep -n "func (c \*instancesRESTClient) AggregatedList" -A80 \
+  "$(go list -m -f '{{.Dir}}' cloud.google.com/go/compute)"/apiv1/instances_client.go
+# InternalFetch: req.MaxResults = pageSize ... return elems, resp.GetNextPageToken(), nil
+```
+
+```bash
+go build ./... && go vet ./internal/provider/... && go test -race -count=1 ./internal/provider/...
+```
+
+Worth noting:
+- The planned `TestRealInstances_Probe_fetchesOnePage` became a three-case
+  `TestRealInstances_Probe` table.
+  - It keeps the first page's `nextPageToken`, and asserts exactly one request with
+    `maxResults=1` for each case.
+  - "empty project" covers `iterator.Done`.
+  - "api error keeps its status" proves the real client returns a `*googleapi.Error`, so
+    a disabled Compute API is reported as `Forbidden` rather than `Unreachable`.
+- Once a context has ended, both checks report `Timeout` whatever the error says. The
+  monitor's timeout is the usual cause, and the underlying error only repeats it.
+
+## Step 3 — KVM check
+
+`conn.go` now wraps each of the four dial stages in a `dialError`: ssh-material, tcp,
+ssh-handshake and libvirt-connect. `Error()` returns the inner message unchanged and
+`Unwrap()` returns the cause, so `classify`/`isClassified` and every existing
+message-matching test behave as before. `TestDialError_keepsClassification` pins this
+down.
+
+The three new RPCs went into `libvirtAPI`, `tracingAPI` and `fakeAPI`. The method-count
+guard in `TestTracingAPI_logsEveryMethod` fails until a traced row exists for each new
+method, which is what forced the three new rows.
+
+`Check` runs one goroutine per host, over the shared connection, through
+`hostPool.call`. The pool verdict is decided only after `call` returns, because
+returning it as an error from inside the callback would make `call` drop a perfectly
+good connection.
+
+Build, vet (also with the integration tag, since that file changed) and race tests:
+
+```bash
+go build ./internal/provider/... && go vet ./internal/provider/kvm/ \
+  && go vet -tags=integration ./internal/provider/kvm/ \
+  && go test -race -count=1 ./internal/provider/kvm/
+```
+
+Worth noting:
+- **Pool state wording:** `poolState` uses virsh's words (`running`, `inactive`, …)
+  rather than go-libvirt's constant names, so it reads like `virsh pool-list` output.
+- **Details:** each host also reports its SSH `user`. That is the first thing to check
+  when the reason is `SSHHandshakeFailed`.
+- **Tests renamed or merged:**
+  - The planned `TestSSHAuth_dial_stages` is `TestLazySSH_dial_tagsTheFailingStage`. It
+    dials through `lazySSH`, so the ssh-material stage is covered as well as the three
+    stages the in-process SSH server can trigger.
+  - The pool-inactive and pool-missing cases are merged into `TestCheck_libvirtVerdicts`,
+    which also covers a daemon error. Each case runs two checks and asserts a single
+    dial.
+- **Tests added beyond the plan:**
+  - `TestCheck_timeout`: a daemon that stops answering gives `Timeout`.
+  - `TestDialError_keepsClassification`.
+- **Not run yet:** `TestKVMCheck_realHost` in the integration file only compiles here.
+  It needs beelink, so it goes on the Step 7 checklist for the user to run.
+
+## Step 4 — Monitor
+
+`internal/providerhealth` has the `Monitor` runnable plus the pure `buildStatus`.
+
+- **Scheduling:** the Monitor is leader-only. It cleans up once at start, checks
+  immediately, then checks on a ticker.
+- **Each round:** providers are checked concurrently under one timeout, then logged and
+  published one after another. A failed write is logged, and the next round writes
+  again.
+- **Reads:** objects are read through the reader it is given. `main.go` passes the
+  manager's API reader (Step 5).
+
+The first `make manifests` left `config/rbac/role.yaml` unchanged, with no error.
+I had put the `+kubebuilder:rbac` markers inside `Monitor`'s doc comment, and
+controller-gen only reads RBAC markers from package-level comment groups. So they were
+silently attached to the type and ignored. An empty `git diff config/rbac/role.yaml` was
+the only sign. Moving them into their own comment block fixed it:
+
+```bash
+make manifests && git diff config/rbac/role.yaml
+# + providerhealths: create, delete, get, list
+# + providerhealths/status: get, patch
+```
+
+Vet, race tests and the envtest round-trip (using the `bin/k8s/1.36.2-linux-amd64`
+binaries `make setup-envtest` had already fetched):
+
+```bash
+go vet ./internal/providerhealth/ && go test -race -count=1 -v ./internal/providerhealth/
+# --- PASS: TestMonitor_envtestRoundTrip (6.45s) ... ok
+```
+
+Worth noting:
+- **Fake client scope:** the fake client needs a RESTMapper that registers
+  ProviderHealth as `RESTScopeRoot`. Its default mapper treats every kind as namespaced.
+- **Why the envtest calls `publish` directly:** `checkOnce` only logs write errors, so a
+  schema rejection would look like a missing object. Calling `publish` makes the API
+  server's answer the test failure.
+  - Its first round writes a 2000-character message; the stored message is truncated to
+    1024.
+  - Its second round proves that only the target that flipped gets a new
+    `lastTransitionTime`.
+- **No guard against a hung checker:** the Monitor trusts `Checker`'s contract to return
+  once the context ends, and doesn't abandon a checker that ignores it. All three
+  checkers stop when the context ends: KVM through `awaitOrCancel`, and kubernetes and
+  GCP through their clients.
+- **Test renamed:** the planned `TestMonitor_timeoutBoundsBlockingChecker` is
+  `TestMonitor_checkOnce_timeoutBoundsEachRound`. It also asserts that a fast provider
+  still publishes next to a hung one.
+
+## Step 5 — Metrics and wiring
+
+`metrics.RegisterProviderTargets` registers a scrape-time collector for three gauges,
+each labelled `{provider,target}`: `up`, check duration and check timestamp.
+`Register`'s signature is unchanged.
+
+`cmd/main.go`:
+- gains `--provider-check-interval` (1m) and `--provider-check-timeout` (15s);
+- validates them before even loading the providers config;
+- collects the `Checker`s from the unwrapped providers in config order, just before the
+  metrics and tracing wrap loop;
+- adds the Monitor with the manager's API reader;
+- registers the collector through a small `providerTargetSamples` closure next to
+  `proxyPhaseCounts`.
+
+Build, vet and metrics tests:
+
+```bash
+go build ./... && go vet ./cmd/... ./internal/metrics/ && go test -race -count=1 ./internal/metrics/
+```
+
+To confirm bad flags fail fast, before anything touches the cluster, I ran the binary
+with a 5s interval. zap's development mode prints a stack trace after the error, hence
+the grep:
+
+```bash
+go run ./cmd --providers-config=/dev/null --provider-check-interval=5s 2>&1 | grep -m1 "Invalid provider check flags"
+# ERROR setup Invalid provider check flags {"error": "--provider-check-interval must be 0 (off) or at least 10s, got 5s"}
+go vet -tags=e2e ./test/e2e/
+```
+
+The e2e suite gets a new ordered spec, "should publish provider health". It waits for
+`providerhealth kubernetes` to report `Ready=True`. It runs before the existing metrics
+spec, which now also asserts
+`proxy_operator_provider_target_up{provider="kubernetes",target="apiserver"} 1`. By then
+the first round is known to be done, so the series is certain to exist.
+
+Worth noting:
+- **Main-only wiring:** `providerTargetSamples` is an 8-line loop in `package main`,
+  which has no test file. The e2e metrics assertion is its only coverage.
+- **The KVM entry isn't asserted in e2e:** the default providers config also ships
+  `kvm-home` (beelink), which would report `SSHMaterialUnavailable` in kind. That entry
+  is the user's own host, and asserting it would break the suite as soon as the default
+  config changes.
+- **Manifest:** `config/manager/manager.yaml` only gained a comment about the two flags.
+  The defaults need no args.
+
+## Step 6 — Docs
+
+- **`docs/architecture.md`:**
+  - component rows for the CRD, the `Checker` contract and the monitor;
+  - a new §11 covering the provider table, the loop diagram, the write, read and
+    decorator choices, the gauges, and sample `kubectl get ph` output;
+  - cross-references from the metrics (§9) and tracing (§10) sections;
+  - four Decisions entries.
+- **`docs/kvm-provider.md`:**
+  - a "Health check" section with the reason → meaning → where-to-look table and the
+    two consequences of sharing the connection;
+  - under Known gaps, the capacity bullet now points at `poolAvailable`, and a new
+    bullet says a rotated `kvm-ssh` Secret needs a restart;
+  - the Testing section names `TestKVMCheck_realHost`.
+- **`docs/testing.md`:**
+  - suite rows for the three `check_test.go` files, `internal/providerhealth` (unit and
+    envtest) and the metrics gauges;
+  - "17 libvirt calls" is now 20;
+  - the integration and e2e sections describe the new tests.
+- **`CLAUDE.md`:**
+  - the Architecture TODO now points at `docs/architecture.md`;
+  - the stale "only `go.mod` exists" gotcha is replaced by a note that the "What this
+    is" and "Commands" sections predate the code, plus Step 4's RBAC-marker gotcha.
+
+Worth noting:
+- `docs/api.md` is untouched. It documents the discovery HTTP API, which this change
+  doesn't alter.
+- `CLAUDE.md`'s "What this is" and "Commands" sections were left stale on purpose.
+  Rewriting them is outside this plan, so the new gotcha flags them instead of leaving
+  them silently wrong.
+
+## Step 7 — Validate, record, push, open the PR
+
+The full suite runs codegen, `go fmt` and `go vet`, then `go test -race` on every
+package, with both envtest suites included:
+
+```bash
+make test
+# every package ok; internal/providerhealth 92.5 %, internal/provider/kvm 92.0 % coverage
+```
+
+`go fmt` rewrote nothing, and `git status` showed only the Step 6 docs.
+
+`make lint` could not run, for a reason unrelated to this branch. The Makefile rebuilt
+the custom golangci-lint, which reports "built with go1.27.0", and its bundled
+staticcheck (honnef.co/go/tools v0.7.0) panics on the standard library before it
+reaches any repo package:
+
+```bash
+make lint
+# buildir: package "poll" (isInitialPkg: false, needAnalyzeSource: true): unexpected expr: *ast.KeyValueExpr
+# make: *** [Makefile:107: lint] Error 3
+```
+
+The local toolchain is go1.27.0 while `go.mod` says 1.26.0; the otel-tracing plan's
+log shows `make lint` running normally before the toolchain moved. Retrying with
+`GOTOOLCHAIN=go1.26.0` would have meant downloading a toolchain, and the user declined
+that. `go vet` is clean in all three configurations: plain inside `make test`, plus
+`-tags=integration` and `-tags=e2e`.
+
+The kind e2e suite creates and deletes its own `egress-proxies-operator-test-e2e`
+cluster. I checked `KIND_CLUSTER` first, because the two clusters already on the
+machine (`kind`, `proxy-operator-demo`) must not be torn down by `cleanup-test-e2e`:
+
+```bash
+make test-e2e > <scratch>/e2e.log 2>&1
+# Ran 3 of 5 Specs in 86.210 seconds
+# SUCCESS! -- 3 Passed | 0 Failed | 0 Pending | 2 Skipped
+```
+
+In the deployed manager, the `kubernetes` ProviderHealth reported `Ready=True` about a
+second after the pod was up (20:30:05.6 → 20:30:06.7). The metrics scrape that followed
+contained `proxy_operator_provider_target_up{provider="kubernetes",target="apiserver"} 1`.
+That exercises the generated RBAC, the API-reader path and the `main.go` gauge wiring
+end to end. The two skipped specs are the Tempo-gated tracing specs; `TEMPO_URL` was not
+set.
+
+### Follow-up — the check must not cost the reconciles their connection
+
+Reviewing the shared-connection decision after the automated validation turned up two
+ways the health check could hurt the work it shares the link with, both of which the
+plan's "a check against a hung host drops the shared connection" line accepted too
+easily:
+
+- A check that timed out went through `hostPool.call`, whose cancel path drops the
+  connection. A `libvirtd` slow to answer one `ConnectGetLibVersion` would therefore
+  cut off an image upload or a `Create` running on the same link.
+- `hostEntry` was guarded by a `sync.Mutex`, so waiting for a host's slot could not be
+  interrupted. A reconcile redialing a host that is down holds the slot for as long as
+  the SSH dial takes — minutes on a blackholed address — and the whole check round
+  would wait behind it, past its own timeout.
+
+So the check now calls `hostPool.probe` instead of `call`: on `ctx` ending it abandons
+the RPC and leaves the connection alone (a link that is really dead is closed by the
+keepalive anyway, which ends the abandoned `fn` too). One abandoned probe per host is
+allowed to stay waiting — `hostEntry.probing` is an `atomic.Bool`, not a counter, so a
+daemon that never answers cannot pile up goroutines; further probes get the new
+`errProbeBusy`, which `checkReason` maps to `Timeout`. `hostEntry.mu` became a one-slot
+channel with an `acquire(ctx)` that only bounds *waiting*: an uncontended caller takes a
+free slot even with an expired context, so `call`'s behaviour is unchanged where nothing
+is contended.
+
+The second change came out of the same pass: `--provider-check-timeout` defaulted to a
+flat 15s while the flag validation rejected a timeout longer than the interval, so
+`--provider-check-interval=12s` alone failed to start. The flag now defaults to 0 and
+`Monitor.setDefaults` derives `min(15s, Interval)`; `ValidateSchedule` accepts 0 and
+rejects only negatives and timeouts past the interval.
+
+Re-verified with the same commands as above:
+
+```bash
+go vet ./...
+go test -race -count=1 ./internal/provider/kvm/ ./internal/providerhealth/
+make test
+# every package ok; internal/provider/kvm 92.2 %, internal/providerhealth 93.7 % coverage
+```
+
+Worth noting:
+- `make lint` still cannot run, for the toolchain reason recorded above; unchanged by
+  this follow-up.
+- The two new `kvm` tests are timing-shaped by necessity — they assert that a caller
+  with a 50ms deadline does not wait out a dial that is blocked on a channel. The
+  assertions are one-sided (`> 1s` fails) so a slow machine does not make them flaky.
+- `docs/architecture.md`, `docs/kvm-provider.md` and `docs/testing.md` were corrected in
+  the same commit: the old text promised the drop-on-timeout behaviour that is now gone.
+- The real-hypervisor commands from the plan's Verification section are still the user's
+  to run; nothing here replaces them.
+
+### Real-hypervisor validation and PR
+
+The user ran the plan's real-hypervisor checks against beelink (the `TestKVMCheck`
+integration test plus the manual `ProviderHealth` and metrics checks) and confirmed
+they passed. After that the CHANGELOG entry went in, and the branch was pushed with
+the token-file credential helper. The PR against `main` was opened through the Gitea
+API because `tea` isn't installed.
+
+Worth noting:
+- The branch also carries `docs/plans/2026-09-16-0841-provider-blueprint.md`, on
+  purpose: that plan reaches `main` with this merge, and the blueprint work starts on a
+  new branch afterwards.

docs/plans/2026-08-07-1747-proxy-operator.md

@@ -391,7 +391,10 @@ pods would refuse connections while still being Service endpoints. The 1-replica
 constraint comes from lease state being per-process, which the spec already accepts —
 both facts go in the README caveats.
 
-stdlib `http.ServeMux` with Go 1.22 method+wildcard patterns:
+stdlib `http.ServeMux` using its method+wildcard patterns (`"GET /path"`,
+`"/{id}"` + `r.PathValue`) — a stdlib feature available since Go 1.22, used
+here so no third-party router is needed; the project itself stays on the
+pinned Go 1.26:
 `GET /v1/proxies`, `POST /v1/leases`, `DELETE /v1/leases/{id}`,
 `POST /v1/leases/{id}/report`, plus unauthenticated `GET /healthz`.
 Middleware outermost-first: recover → request-log → `MaxBytesReader(64KiB)` → bearer

docs/plans/2026-08-11-1742-gcp-provider-verbose-logging.md

@@ -0,0 +1,87 @@
+# Plan: Verbose (V-level) logging in the GCP provider
+
+**Created:** 2026-08-11 17:42
+
+## Context
+
+Debugging GCP provisioning is currently blind: the operator has zero `.V(n)` calls
+anywhere, so `--zap-log-level=debug` (or any numeric level) reveals nothing about
+what the GCP provider is doing — which API calls it makes, with what parameters,
+and what came back. The goal: with debug/V-level logging enabled, see the details
+of every GCP Compute API call (Insert/Get/Delete/AggregatedList) including a
+summary of the response; with default `info` level, the provider stays as quiet
+as today.
+
+## Approach (the "how")
+
+**Logger source — context-carried, not injected.** Provider methods all take
+`ctx`, and the reconciler already builds a per-request logger
+(`logf.FromContext(ctx)` in `internal/controller/proxy_controller.go:117`) that
+carries the proxy's name/namespace. The GCP provider will do
+`log := logf.FromContext(ctx).WithName("gcp").WithValues("provider", p.name)` at
+the top of each public method. Zero wiring changes (no registry/constructor/struct
+changes), and every provider log line automatically inherits the reconcile
+context (which Proxy triggered it). Calls from the GC sweeper inherit its
+`orphan-gc` logger name the same way.
+
+**Verbosity scheme** (logr convention: `.Info()` = V(0), `debug` flag = V(1)):
+
+- **V(1)** — one line per GCP API call, after it returns: operation, identifying
+  params, outcome. Examples:
+  - `Create`: `"GCP insert instance"` with `zone`, `name`, `machineType`,
+    `image`, `opName` (currently discarded at gcp.go:117 — capture it, it's the
+    only handle for correlating with GCP's operation log), plus a line for the
+    409-already-exists path.
+  - `Get`: `"GCP get instance"` with `zone`, `name`, `status`, mapped `state`, `ip`.
+  - `Delete`: `"GCP delete instance"` with `zone`, `name`, `opName`, and the
+    404-treated-as-success path.
+  - `ListByTag`: `"GCP aggregated list"` with `filter`, `count`.
+  - Error paths at V(1) too: log the raw classification (HTTP status / reason
+    from `googleapi.Error`) before it's wrapped, since the wrapped error the
+    reconciler sees is coarser.
+- **V(2)** — request/response detail: full curated insert-request summary
+  (network, networkTag, diskSizeGB, port, labels, `cloudInitBytes` = `len`),
+  per-instance lines in `ListByTag` (id, state, uid, age).
+
+**Curated fields, never raw proto dumps.** `CreateRequest.CloudInit` is resolved
+user-data possibly sourced from a Secret, and it lands in the insert request's
+metadata — so logging the request proto wholesale would leak it. Log named safe
+fields only; for cloud-init, log only its byte length. This is a hard rule, and
+a test asserts it.
+
+**Where the calls live: the `Provider` methods in
+`internal/provider/gcp/gcp.go`** (Create/Get/Delete/ListByTag), not in
+`realInstances` (deliberately untested by design, gcp.go:86) and not an HTTP
+round-tripper (would log auth headers/user-data, unredactable). The
+`instancesAPI` fake seam (`newWithAPI`, gcp.go:96) keeps everything testable.
+To surface `opName`, change `Provider.Create`/`Delete` to capture the string
+their `instancesAPI` calls already return instead of discarding it.
+
+## Files to change
+
+- `internal/provider/gcp/gcp.go` — add `logf` import; V(1)/V(2) logging in
+  `Create`, `Get`, `Delete`, `ListByTag`; capture opNames. Only file with
+  production changes.
+- `internal/provider/gcp/gcp_test.go` — new table-driven test
+  `TestLogging_verbosity` (name TBD per house `Test<Function>_<scenario>`
+  style): inject a capturing logger via `logf.IntoContext(ctx, funcr.New(...))`
+  (`github.com/go-logr/logr/funcr`, logr already a direct dep), assert:
+  - at V(1): expected message + keys per operation (incl. opName),
+  - at V(0): nothing logged,
+  - **cloud-init content never appears in any log output** (grep the captured
+    lines for a sentinel string placed in `CloudInit`).
+- No changes to `provider.Provider` interface, registry, `cmd/main.go`,
+  manifests, or the kubernetes provider (it can copy this pattern later).
+
+## Verification
+
+```bash
+go test -race ./internal/provider/gcp/...
+go build ./...
+```
+
+Optional live check: run the manager with `--zap-log-level=2` against the GCP
+project and confirm insert/get lines appear during a Proxy reconcile, and that
+`--zap-log-level=info` stays quiet.
+
+Also append a CHANGELOG.md entry per house convention once confirmed working.

docs/plans/2026-08-11-1802-bake-commit-version.md

@@ -0,0 +1,75 @@
+# Plan: Bake the git commit into the operator binary and log it at startup
+
+**Created:** 2026-08-11 18:02
+
+## Context
+
+There is no versioning yet, and the image tag (`egress-proxies-operator:dev`,
+`imagePullPolicy: IfNotPresent`) says nothing about what code is actually
+running. The user wants the commit hash baked into the image at build time,
+and — the key requirement — the binary itself must know it and print it into
+the log stream during initialization, so `kubectl logs | head` answers "which
+version is running". Docker builds cannot use Go's automatic VCS stamp because
+`.dockerignore` excludes `.git` (correctly — re-including it would bust layer
+caching), so the hash must travel git → Makefile → `--build-arg` → `-ldflags -X`.
+
+## Implementation
+
+**1. New package `internal/version`** (`version.go` + `version_test.go`):
+
+- `var Commit string` — stamped at link time via
+  `-ldflags "-X gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version.Commit=<hash>"`.
+- `func Resolve() string` — returns `Commit` if non-empty; otherwise falls
+  back to `debug.ReadBuildInfo()` VCS settings (`vcs.revision` truncated to
+  12 chars, `-dirty` suffix when `vcs.modified=true`), so plain host builds
+  (`make build`, `make run`, `go run`) are stamped for free since `.git` is
+  present there; `"unknown"` when neither source is available (e.g. `go test`).
+- Internal `resolve(ldflagsCommit string, readBuildInfo func() (*debug.BuildInfo, bool)) string`
+  so the fallback logic is table-testable with a fake build-info func
+  (house style: stdlib testing, `t.Parallel()`, subtests,
+  `Test<Function>_<scenario>` names).
+
+**2. `cmd/main.go`**:
+
+- Add `--version` bool flag; immediately after the existing `flag.Parse()`
+  (main.go:132), if set: print `version.Resolve()` to stdout and exit 0
+  (before logger/manager setup).
+- Right after `ctrl.SetLogger(...)` (main.go:134):
+  `setupLog.Info("Starting egress-proxies-operator", "commit", version.Resolve(), "goVersion", runtime.Version())`
+  — first line of every run, plain V(0) so it appears at any log level.
+
+**3. `Makefile`**:
+
+- Near the other variables:
+  `GIT_COMMIT ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)$(shell test -z "$$(git status --porcelain 2>/dev/null)" || echo -dirty)`
+  (`git status --porcelain` catches staged and untracked changes, which
+  `git diff --quiet` misses).
+- `docker-build`: add `--build-arg GIT_COMMIT=$(GIT_COMMIT)`.
+- `docker-buildx`: add the same `--build-arg` to the `buildx build` line.
+- `build`/`run`/`run-dev` stay untouched — the ReadBuildInfo fallback covers them.
+
+**4. `Dockerfile`**:
+
+- `ARG GIT_COMMIT=unknown` in the builder stage; extend the existing
+  `go build` with `-ldflags "-X <module>/internal/version.Commit=${GIT_COMMIT}"`.
+- Re-declare `ARG GIT_COMMIT` in the distroless stage and add
+  `LABEL org.opencontainers.image.revision="${GIT_COMMIT}"` so the hash is
+  also visible via `docker inspect` without running the binary.
+
+No changes to deploy manifests, providers, or the reconciler. CHANGELOG entry
+after the user confirms it works (house convention).
+
+## Verification
+
+```bash
+go test ./internal/version/            # resolve() table tests
+go build ./... && go test ./...        # nothing else broke
+go run ./cmd/main.go --version         # host build: VCS-stamped hash (+ -dirty), exit 0
+make docker-build IMG=egress-proxies-operator:dev
+docker run --rm egress-proxies-operator:dev --version          # prints the baked commit
+docker inspect egress-proxies-operator:dev \
+  --format '{{index .Config.Labels "org.opencontainers.image.revision"}}'
+```
+
+Live check: redeploy on the cluster and confirm the first log line carries
+`"commit"`.

docs/plans/2026-08-11-1838-gcp-http-wire-logging-v5.md

@@ -0,0 +1,85 @@
+# Plan: GCP HTTP wire logging at V(5)
+
+**Created:** 2026-08-11 18:38
+
+## Context
+
+The GCP provider logs curated call summaries at V(1)/V(2), but when debugging
+against the real API the user wants ground truth: the actual HTTP requests and
+responses ("gory details") — visible at `--zap-log-level=5`, in the same log
+stream as everything else. The compute SDK already produces exactly this:
+`cloud.google.com/go/compute@v1.65.0/apiv1/helpers.go:60,70` logs
+`"api request"`/`"api response"` (method, URL, headers, full JSON payloads,
+lazily via `internallog.HTTPRequest/HTTPResponse`) to an injectable
+`*slog.Logger` at slog Debug level. We inject one bridged to the operator's
+zap sink, level-shifted so those Debug records surface only at V(5).
+
+Level scheme after this change: V(1) call outcomes, V(2) curated detail,
+V(5) raw HTTP traffic. V(3)/V(4) reserved.
+
+## Mechanism (verified in module sources)
+
+- `option.WithLogger(*slog.Logger)` exists in `google.golang.org/api@v0.292.0`
+  (option.go:529) and **takes precedence over `GOOGLE_SDK_GO_LOGGING_LEVEL`**
+  — after this change, V(5) is the single knob for this client; document that.
+- `logr.ToSlogHandler` (go-logr/logr v1.4.3, already a direct dep) maps
+  slog Debug → logr V(4), plus the base logger's V-bias. logr's own docs
+  (sloghandler.go:180-184): `slog.New(ToSlogHandler(logrV2)).Debug()` ≈ V(6).
+  So a base of `.V(1)` lands Debug at exactly V(5).
+- Gating is cheap: the slog handler's `Enabled()` consults the zap sink, so
+  below level 5 the SDK's lazy `LogValuer`s are never evaluated.
+
+## Implementation
+
+**`internal/provider/gcp/gcp.go`** (only production file):
+
+1. New pure function:
+   ```go
+   // wireLogger returns the slog logger handed to the SDK: its Debug-level
+   // "api request"/"api response" records (slog Debug = +4 on the logr
+   // scale) land at V(5) on top of the base's V(1) shift.
+   func wireLogger(base logr.Logger) *slog.Logger {
+       return slog.New(logr.ToSlogHandler(base.V(1)))
+   }
+   ```
+2. In `New` (gcp.go:96): pass it to the client —
+   `compute.NewInstancesRESTClient(ctx, option.WithLogger(wireLogger(logf.Log.WithName("gcp").WithName("http"))))`.
+   Base is the process-root `logf.Log` (client is built once at startup;
+   `ctrl.SetLogger` runs before `registry.Build` in cmd/main.go, so it
+   resolves to the real zap logger).
+3. One-time notice in `New`: if `logf.Log.V(5).Enabled()`, log at Info:
+   `"GCP HTTP wire logging active — request payloads include cloud-init user-data"`
+   (the secret-leak warning our curated V(2) logging exists to avoid; at V(5)
+   the user has explicitly opted into raw payloads).
+4. New imports: `log/slog`, `google.golang.org/api/option` (module already in
+   go.mod as a direct dep; `option` package is a first-time import in the repo).
+
+**`internal/provider/gcp/gcp_test.go`**:
+
+- `TestWireLogger_gatesAtV5`: table over funcr sink verbosities
+  (`funcr.Options{Verbosity: N}`, pattern already used by `captureContext`):
+  at 5 a `Debug("api request", ...)` through `wireLogger` emits (message and
+  attrs present); at 4 it emits nothing; an `Info` record through the same
+  logger lands at V(1) (sanity-check of the shift).
+- `New` itself stays untested by design (dials real Google endpoints —
+  existing convention, gcp.go:86-87).
+
+No changes to manifests, Makefile, other providers, or the reconciler.
+CHANGELOG entry after the user confirms it works (house convention) — this
+plus the two earlier pending entries (GCP V-logging, version stamp).
+
+## Verification
+
+```bash
+go test -race ./internal/provider/gcp/
+go build ./... && go test ./...
+```
+
+Live (the real proof, needs the cluster):
+
+```bash
+# rebuild + load image, set --zap-log-level=5, restart, then:
+kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager -f \
+  | grep -m2 'api request\|api response'   # full URL/headers/payload visible
+# and at --zap-log-level=2: the same grep stays silent while V(2) lines still appear
+```

docs/plans/2026-08-11-1935-gitea-build-workflow.md

@@ -0,0 +1,135 @@
+# Plan: Distilled Gitea Actions image-build workflow
+**Created:** 2026-08-11 19:35
+
+## Context
+
+This repo (`egress-proxies-operator`) has a Dockerfile, a Makefile with `docker-build`/`docker-push` targets, and a Gitea remote — but no CI workflow (CLAUDE.md flags this as a TODO). A survey of all projects under `/Users/jan.novak/srv` found 9 image-build workflows, all variations of one lineage: trigger on `workflow_dispatch` + tag push, `docker login` to `gitea.home.hrajfrisbee.cz` with `secrets.REGISTRY_TOKEN`, raw `docker build`/`docker push`, `runs-on: ubuntu-latest`, `permissions: {contents: read, packages: write}`.
+
+The best individual ideas are scattered:
+- **aviso_v2**: quality-gate job before build; computes `sha-<short>` as a second immutable tag via `$GITHUB_OUTPUT`.
+- **gateway-helper-operator** (closest sibling — same kubebuilder shape): passes build args (`GIT_COMMIT` etc.), tags `:latest` alongside the version tag.
+- **psmf-data-sync test.yaml**: `actions/setup-go@v5` with `go-version-file: go.mod` + module cache (proven to work on the act_runner).
+
+Goal: distill these into one `build.yaml` for this repo. User decisions: triggers = **tags + manual dispatch only** (house convention, no builds from main); build tool = **raw docker CLI** (the runner bind-mounts docker.sock, so this just works); **lightweight test gate** (`go vet` + `go build` + `go test -short`, no envtest download); **amd64 only**.
+
+## The workflow
+
+Create `.gitea/workflows/build.yaml`:
+
+```yaml
+name: Build and Push
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Image tag'
+        required: true
+        default: 'latest'
+  push:
+    tags:
+      - '*'
+
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Vet
+        run: go vet ./...
+      - name: Build
+        run: go build ./...
+      - name: Test (short)
+        run: go test -short ./...
+
+  build:
+    needs: check
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Compute image tags
+        id: meta
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            TAG="${{ inputs.tag }}"
+          else
+            TAG="${{ github.ref_name }}"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "sha=sha-$(echo '${{ github.sha }}' | cut -c1-12)" >> "$GITHUB_OUTPUT"
+
+      - name: Login to Gitea registry
+        run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login -u ${{ github.actor }} --password-stdin gitea.home.hrajfrisbee.cz
+
+      - name: Build and push
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker build \
+            --build-arg GIT_COMMIT=$(echo '${{ github.sha }}' | cut -c1-12) \
+            --label org.opencontainers.image.source=https://gitea.home.hrajfrisbee.cz/${{ github.repository }} \
+            --label org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
+            -t "$IMAGE:${{ steps.meta.outputs.tag }}" \
+            -t "$IMAGE:${{ steps.meta.outputs.sha }}" \
+            .
+          docker push "$IMAGE:${{ steps.meta.outputs.tag }}"
+          docker push "$IMAGE:${{ steps.meta.outputs.sha }}"
+
+      - name: Push latest (tag builds only)
+        if: github.event_name == 'push'
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker tag "$IMAGE:${{ steps.meta.outputs.tag }}" "$IMAGE:latest"
+          docker push "$IMAGE:latest"
+```
+
+### What's distilled vs. improved over the existing workflows
+
+Distilled (house patterns kept as-is): triggers, `REGISTRY_TOKEN` + `github.actor` login, image name `gitea.home.hrajfrisbee.cz/${{ github.repository }}`, `ubuntu-latest`, raw docker CLI, `permissions` block.
+
+Improvements none of the existing workflows have all of:
+1. **`sha-<12>` immutable tag** alongside the human tag (aviso_v2 had this; nobody else) — lets deployments pin exactly what was built.
+2. **Test gate** (aviso_v2 had one; the Go projects don't) — lightweight variant per user choice; uses `go-version-file: go.mod` so the Go version never drifts from the module.
+3. **`GIT_COMMIT` build arg** matches the Makefile/Dockerfile contract — the binary's `internal/version.Commit` and the `org.opencontainers.image.revision` label get the real commit (12-char, same width as the Makefile's `git rev-parse --short=12`; no `-dirty` needed since CI checkouts are clean).
+4. **`:latest` only on real tag pushes**, not manual dispatch — gateway-helper pushed `latest` unconditionally, which lets an ad-hoc dispatch of an old ref clobber `latest`.
+5. **`concurrency` group** — cancels a superseded run of the same ref (none of the 20 surveyed workflows have this).
+6. **OCI `source`/`created` labels** added at build time (revision label already comes from the Dockerfile).
+
+## Files
+
+- **Create** `.gitea/workflows/build.yaml` — content above.
+- **Update** `CLAUDE.md` — replace the `TODO: no .gitea/workflows/ CI pipeline exists yet` note in the Git Commits section with a short CI/CD subsection describing the workflow (triggers, secret, tags produced).
+- **Update** `CHANGELOG.md` — new top entry (after user confirms it works, per convention; timestamp via `date "+%Y-%m-%d %H:%M %Z"`).
+- Copy this plan to `docs/plans/YYYY-MM-DD-HHMM-gitea-build-workflow.md` (timestamp via `date "+%Y-%m-%d-%H%M"`) and commit it first, per CLAUDE.md ordering rule.
+
+## Branch & MR
+
+House convention: feature → own branch + MR. Dockerfile and `cmd/` already exist on `main`, so:
+
+1. `git checkout -b feat/gitea-build-workflow origin/main` (do not touch the current `feat/proxy-operator` branch's uncommitted `.claude/settings.json` change — leave it be).
+2. Commit plan file, then the workflow + CLAUDE.md update (with `Co-Authored-By: Claude <noreply@anthropic.com>`).
+3. `git push -u origin feat/gitea-build-workflow`, open MR with `tea pr create --base main --head feat/gitea-build-workflow`. Do not merge.
+
+Note: `main` has no `internal/`/`test/` dirs yet (those are on `feat/proxy-operator`), which is fine — the workflow only fires on tags/dispatch, and by then the operator branch will be merged. `go build ./...` / `go test -short ./...` work on both branch states.
+
+## Prerequisite (user action)
+
+`REGISTRY_TOKEN` secret must exist in this repo's Gitea settings (Settings → Actions → Secrets): a personal access token with `write:package` scope — same as every other project uses. Flag this in the MR description.
+
+## Verification
+
+The workflow doesn't trigger on branch pushes, so end-to-end verification happens after merge:
+1. Local sanity: `docker build --build-arg GIT_COMMIT=test -t scratch-check .` (confirms the build args/labels line is valid) — or at minimum a YAML parse check.
+2. After the MR merges: run the workflow manually via Gitea UI (Actions → Build and Push → Run workflow, tag `manual-test`), confirm both `manual-test` and `sha-…` tags appear under Packages, and that `:latest` was NOT updated.
+3. Then push a real version tag (e.g. `v0.1.0`) and confirm `v0.1.0`, `sha-…`, and `latest` all appear.

docs/plans/2026-08-11-2152-discovery-api-docs.md

@@ -0,0 +1,58 @@
+# Plan: Discovery API documentation (docs/api.md)
+**Created:** 2026-08-11 21:52
+
+
+## Context
+
+The operator serves an HTTP discovery/lease API on `:8090` ([internal/discovery/](internal/discovery/)) that crawler clients use to list proxies, acquire TTL leases, release them, and report rate-limiting. There is no dedicated API reference today: README has four quickstart curls (no auth header, no schemas), and `docs/architecture.md` §7 has an ASCII route map. The user wants full documentation with curl examples for every feature.
+
+**Decisions made with user:** doc lives in a new `docs/api.md`; commit straight to `main` (no MR).
+
+## Deliverable
+
+### 1. New file `docs/api.md` — full API reference
+
+Content (all facts verified against code during planning):
+
+- **Overview & base URL** — what the API is; in-cluster FQDN `http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090` (Service: `config/default/discovery_service.yaml`); local access via `kubectl port-forward svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090`.
+- **Authentication** — static bearer token from `DISCOVERY_TOKEN` env var (populated from the optional `discovery-token` Secret, key `token`; `config/manager/manager.yaml`). Empty token ⇒ auth disabled with startup warning. Curl: `-H "Authorization: Bearer $TOKEN"` on every example. `/healthz` always exempt.
+- **Conventions** — JSON everywhere; error envelope `{"error":"<code>","message":"<text>"}`; request bodies capped at 64 KiB; proxies with a deletion timestamp are excluded from all responses.
+- **Configuration table** — `--discovery-addr` (default `:8090`), `--max-lease-ttl` (default 1h), `--lease-cooldown` (default 15m), `DISCOVERY_TOKEN`. Note the shipped Deployment passes none of these flags, so defaults apply.
+- **Endpoints**, each with request/response schema, status codes, and a copy-pasteable curl example:
+  - `GET /healthz` — liveness, unauthenticated.
+  - `GET /v1/proxies` — filters `healthy=true|false` (else 400 `invalid_query`) and repeatable `attr.<key>=<value>` (verbatim equality on `spec.attributes`, all pairs must match). Response `{"proxies":[proxyView...],"count":N}` sorted by id. Full `proxyView` field table: `id` (ns/name), `ip`, `port`, `attributes`, `phase` (Pending/Provisioning/Ready/Unhealthy/Deleting/Failed), `healthy` (condition `Healthy` == True), `latencyMillis`, `activeLeases`, `maxLeases` (default 5; explicit 0 = unleasable).
+  - `POST /v1/leases` — body `{selector, ttlSeconds, target}` all optional; TTL defaults 5m, capped at max-lease-ttl (else 400 `invalid_ttl`). 201 `{leaseID, proxy, expiresAt, ttlSeconds}`; 409 `no_match` with `considered/atCapacity/inCooldown/unhealthy` counts (documented meanings).
+  - `DELETE /v1/leases/{id}` — early release; always 204, idempotent.
+  - `POST /v1/leases/{id}/report` — body `{result: ok|rate_limited|banned, target}`; 204, 404 `unknown_lease`, 400 `invalid_result`. `ok` is a pure ack; `rate_limited` and `banned` behave identically today (both start one cooldown window).
+- **Selection & cooldown semantics** (short section — this is the non-obvious part clients need):
+  - Selection order: fewest active leases → lowest latency → lexicographic id; deterministic; only healthy, non-deleting proxies with free capacity are candidates.
+  - Cooldown: 15m default (`--lease-cooldown`), keyed `{proxy, target}`. Report with a target blocks only leases requesting that target; report without a target (and lease without one) creates a **global** cooldown blocking all acquisitions of that proxy — call this footgun out explicitly.
+  - Expired leases stay reportable for one cooldown window past TTL.
+- **End-to-end workflow example** — numbered curl walkthrough: acquire → use `proxy.ip:port` as HTTP proxy (`curl -x`) → report `rate_limited` on 429 → release. Using a `jq`-extracted `leaseID`.
+- **Caveats** — lease/cooldown state is in-memory and per-process: single replica only, operator restart drops all leases and cooldowns.
+
+### 2. Pointers to the new doc (small edits)
+
+- `README.md`: one-line link near the quickstart curl section ("full reference: docs/api.md").
+- `docs/architecture.md` §7: one-line link to `docs/api.md` as the detailed reference.
+
+### 3. Housekeeping per CLAUDE.md
+
+- First action post-approval: copy this plan to `docs/plans/<timestamp>-discovery-api-docs.md` (timestamp from `date "+%Y-%m-%d-%H%M"`), commit it alone.
+- Then write the docs, commit to `main` with `Co-Authored-By: Claude <noreply@anthropic.com>` trailer, push.
+- Append execution summary + status checklist to `docs/plans-executions/<same-timestamp>-discovery-api-docs.md` in the docs commit.
+- Add `CHANGELOG.md` entry (timestamp via `date "+%Y-%m-%d %H:%M %Z"`) once the user confirms.
+
+## Key source files (facts source of truth)
+
+- [internal/discovery/handlers.go](internal/discovery/handlers.go), [internal/discovery/server.go](internal/discovery/server.go) — routes, schemas, status codes, auth, limits.
+- [internal/lease/store.go](internal/lease/store.go) — selection order, cooldown/retention, stats.
+- [api/v1alpha1/proxy_types.go](api/v1alpha1/proxy_types.go), [api/v1alpha1/helpers.go](api/v1alpha1/helpers.go) — defaults (port 3128, maxLeases 5), phases, conditions.
+- [cmd/main.go](cmd/main.go) — flags/env defaults.
+- [config/default/discovery_service.yaml](config/default/discovery_service.yaml), [config/manager/manager.yaml](config/manager/manager.yaml) — service DNS, token secret.
+
+## Verification
+
+- Cross-check every documented status code / field name against `internal/discovery/server_test.go` expectations.
+- Sanity-run `go build ./... && go test -short ./internal/discovery/ ./internal/lease/` (no code changes expected — confirms docs match current behavior, not a stale tree).
+- Optionally lint the curl JSON bodies by piping each through `jq .` locally.

docs/plans/2026-08-11-2220-demo-scripts.md

@@ -0,0 +1,33 @@
+# Plan: Demo script — egress IP check through each healthy proxy
+**Created:** 2026-08-11 22:20
+
+
+## Context
+
+First of a planned series of demo scripts under `docs/demo/`. This one showcases the discovery API end to end without leases: list proxies from `$BASE_URL/v1/proxies`, and for each **healthy** one, call an IP-echo site through it (`curl -x`) to show the egress IP that proxy provides — clearly labeling which proxy each request goes through. The user explicitly asked to start by switching to a new branch. The script will be iterated on: **write it but do not commit it** — the user wants to add things to it before anything is committed.
+
+## Steps
+
+### Step 0 — Branch
+
+Create `feat/demo-scripts` off `main`. Commit only the plan copy (`docs/plans/<timestamp>-demo-scripts.md`, timestamp via `date "+%Y-%m-%d-%H%M"`) per CLAUDE.md — nothing else gets committed this round.
+
+### Step 1 — `docs/demo/show-egress-ips.sh` (new file, executable, left uncommitted)
+
+Style: match [.devcontainer/post-install.sh](.devcontainer/post-install.sh) — `#!/bin/bash`, `set -euo pipefail`, `ERROR:`/`WARNING:` messages, `${VAR}` braces.
+
+Behavior:
+
+1. **Args/env:** `BASE_URL` is `$1` (required; missing → usage text + exit 1, e.g. `usage: show-egress-ips.sh <BASE_URL>  (e.g. localhost:8090)`). Optional env: `TOKEN` (bearer token, same name docs/api.md uses; when set, send `Authorization: Bearer $TOKEN`), `IP_ECHO_URL` (default `https://api.ipify.org?format=json` — returns `{"ip":"..."}`).
+2. **Dependency check:** `command -v curl`, `command -v jq` → `ERROR` + exit 1 if missing.
+3. **Fetch** `"$BASE_URL/v1/proxies"` once (no server-side `healthy` filter — fetch all so unhealthy ones can be shown as skipped, which makes the demo more informative). Fail with a clear error if curl or JSON parsing fails.
+4. **Iterate** proxies with `jq -c '.proxies[]'`; for each, extract `id`, `ip`, `port`, `healthy`:
+   - unhealthy → print `--- skipping <id> (unhealthy) ---`
+   - healthy → print a clear banner naming the proxy before the request, e.g. `=== via <id> — http://<ip>:<port> ===`, then `curl -sS --max-time 10 -x "http://${ip}:${port}" "$IP_ECHO_URL"`; print the JSON response. A failed probe prints `WARNING: request through <id> failed` and continues (guard so `set -e` doesn't kill the loop).
+5. Finish with a one-line summary: N proxies, M probed, K skipped/failed.
+
+Reference for API shapes: [docs/api.md](docs/api.md) (`proxies[].id/ip/port/healthy`; `curl -x http://ip:port` usage is already documented there and in README).
+
+### Deliberately deferred (user will iterate on the script first)
+
+- No commit of the script, no push beyond the plan commit, no MR, no execution summary, no CHANGELOG — all wait until the user says the script (or script set) is ready.

docs/plans/2026-08-24-1025-otel-tracing.md

@@ -0,0 +1,107 @@
+# Plan: OpenTelemetry tracing integrated with the existing logr/zap logging
+**Created:** 2026-08-24 10:25
+
+## Context
+
+The operator has structured logging (controller-runtime zap → logr, `logf.FromContext(ctx)` everywhere) and Prometheus metrics, but no tracing. A single reconcile fans out into k8s API calls, provider calls (GCP VMs / squid Pods), and status patches; the discovery API serves lease requests; GC and health engines run on tickers. Today correlating one operation across those hops means grepping for `reconcileID` or `providerID`. Goal: proper OTel traces — a new trace per unit of work (reconcile, HTTP request, GC sweep, optionally health probe), child spans for k8s/provider/HTTP calls — **and every log line inside a traced operation enriched with the trace context**, without disturbing the existing logr/zap system.
+
+User decisions (confirmed):
+- **Enablement:** auto from standard `OTEL_*` env vars — no collector configured ⇒ tracing fully off, zero new flags for the common case.
+- **Health probes:** not traced by default; new `--trace-health-probes` flag enables one root span per probe.
+- **Backend:** none exists yet — verification uses the console exporter and a throwaway Jaeger all-in-one in kind; manifests ship commented-out OTLP env examples.
+
+## Key design facts (verified, non-obvious)
+
+- **logr can't read ctx:** `logr.LogSink` never sees `context.Context`, so trace IDs cannot be added inside the zap sink. They must be attached as `WithValues` at span-creation points via `logf.IntoContext`. zap does **not** dedupe repeated keys, so enrichment must happen exactly once per ctx chain (see `tracing.Start` below).
+- Log keys: **`traceID` / `spanID`** (lowerCamel matches `reconcileID`/`providerID` house style; note in docs that Grafana/Loki derived-field regexes must match `traceID`, not the `trace_id` default).
+- **GCP calls trace themselves:** `google.golang.org/api/transport/http` already wraps its transport in `otelhttp.NewTransport` — once a global TracerProvider is set, every Compute call emits an HTTP client span. Do not add HTTP-level spans in the gcp provider; the `provider.*` decorator span becomes their parent.
+- **kube-apiserver ignores incoming `traceparent`** by design; our k8s-API client spans are leaves. Don't promise cross-process traces into the apiserver.
+- The Go SDK does **not** implement `OTEL_SDK_DISABLED` — handle it ourselves.
+- `contrib/exporters/autoexport` drags in metric/log/prometheus exporters (binary + deps on a 128Mi pod) — hand-roll trace-exporter selection (~40 lines) instead.
+- Span events are deprecated (OTel spec, Mar 2026): use `span.SetStatus` + `error.type`-style attributes, not `RecordError`/`AddEvent`.
+- `rest.Config.Wrap` puts our transport innermost (after auth), so spans see the final request — correct; but informer list/watch + leader election run outside any span ⇒ the transport must **filter out requests with no parent span** or every watch becomes a long-lived root span.
+- Manager runnables receive no logger in ctx (only HTTP servers do) — `logf.FromContext` falls back to the global logger there, which is fine; the discovery server must keep seeding from `s.log`.
+
+## Steps
+
+### Step 1 — Dependencies
+
+`go get` (aligning the existing indirect skew: otel core v1.44 / otlptrace v1.40):
+`go.opentelemetry.io/otel@v1.45.0`, `otel/sdk@v1.45.0`, `otel/trace@v1.45.0`,
+`otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0`, `.../otlptracehttp@v1.45.0`,
+`otel/exporters/stdout/stdouttrace@v1.45.0`,
+`go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0`, plus the
+`semconv` version matching the SDK's `resource.Default()` (treat `ErrSchemaURLConflict` from `resource.New` as non-fatal if versions drift).
+`go mod tidy`; full build + `make test` to confirm `google.golang.org/api v0.292.0` / k8s v0.36 tolerate the bump (MVS picks the higher otelhttp).
+
+### Step 2 — New package `internal/tracing`
+
+- **`tracing.go` — `Setup(ctx, log logr.Logger, service, version string) (shutdown func(context.Context) error, err error)`**
+  - Disabled (returns no-op shutdown, installs nothing) when `OTEL_SDK_DISABLED=true`, or when neither `OTEL_TRACES_EXPORTER` nor `OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is set, or `OTEL_TRACES_EXPORTER=none`. One startup Info line either way stating the effective state/exporter.
+  - Exporter selection (hand-rolled, no autoexport): `OTEL_TRACES_EXPORTER` ∈ `otlp` (default) | `console`; for otlp, `OTEL_EXPORTER_OTLP_(TRACES_)PROTOCOL` ∈ `http/protobuf` (default) | `grpc` → `otlptracehttp` / `otlptracegrpc` (endpoint/TLS/headers via their native env handling).
+  - `sdktrace.NewTracerProvider` with batch processor (`WithMaxQueueSize(512)` — bounded on the 128Mi pod; sampler stays the SDK default so `OTEL_TRACES_SAMPLER(_ARG)` works), resource = `resource.New(WithAttributes(service.name, service.version=commit), WithFromEnv())` with env last so it wins.
+  - Globals: `otel.SetTracerProvider`, `otel.SetTextMapPropagator(TraceContext+Baggage)`, `otel.SetLogger(log.WithName("otel"))`, `otel.SetErrorHandler` → logr at `V(1)` (an unreachable collector errors every few seconds; Error level would spam).
+- **`logger.go` — the one logger-enrichment mechanism** (used by everything; prevents duplicate `traceID` keys):
+  - private ctx key holding the *base* (pre-enrichment) logr.Logger.
+  - `Start(ctx, spanName, opts...) (context.Context, trace.Span)`: starts a span from the global tracer; if the resulting span context is valid, capture base = stored base or `logf.FromContext(ctx)` on first call, then `logf.IntoContext(ctx, base.WithValues("traceID", ..., "spanID", ...))`. Nested `Start` re-derives from base — never stacks. Documented trade-off: `WithValues` pushed via `logf.IntoContext` *between* two `Start` calls is dropped (nothing first-party does that; controller-runtime's `reconcileID` logger is the base and survives).
+  - `ContextWithLogger(ctx, base)` seeds the base explicitly (discovery middleware).
+- **`reconciler.go` — `NewReconciler(kind string, inner reconcile.Reconciler, opts...) reconcile.Reconciler`**: root span `Reconcile <kind>` (SpanKind Internal) via `Start`; attrs `k8s.namespace.name`, `k8s.object.name`, `reconcile.id` (`controller.ReconcileIDFromContext`); on return records `reconcile.requeue_after`, sets Error status from err. `WithTracerProvider` option for tests.
+- **`transport.go` — `RestConfigWrapper() transport.WrapperFunc`**: `otelhttp.NewTransport` with `WithFilter` requiring `trace.SpanContextFromContext(r.Context()).IsValid()` — k8s API spans only under an existing trace (excludes informers, leader election, metrics authn).
+- **`http.go` — `HTTPMiddleware(operation string, base logr.Logger) func(http.Handler) http.Handler`**: `otelhttp.NewHandler` (filter `/healthz`; otelhttp renames the span from `r.Pattern` after routing — no custom naming needed) + inner layer calling `ContextWithLogger`-then-enrich so handler logs carry `traceID`. Accepts incoming `traceparent` (clients continue their traces).
+
+Tests (all with `sdktrace` + `tracetest.NewSpanRecorder` via the `WithTracerProvider` options, parallel; `Setup` env tests use `t.Setenv`, not parallel):
+- `Start`: nested spans → child span recorded; **funcr-captured log line contains `traceID` exactly once** (pattern: `internal/provider/gcp/gcp_test.go:277`); disabled tracer → no `traceID` in logs.
+- `Setup`: table over env combinations (unset→disabled, `none`→disabled, endpoint set→otlp, `console`, `OTEL_SDK_DISABLED`).
+- Reconciler decorator: fake inner returns result/err → span name/attrs/status.
+- Transport filter: request without parent span produces no span.
+- Middleware: `httptest` request → server span named from route pattern; `/healthz` unspanned; handler logger carries `traceID`.
+
+### Step 3 — `provider.WithTracing` decorator
+
+`internal/provider/tracing.go`, mirroring `metrics.go`: `WithTracing(name string, p Provider, opts...) Provider` — one Client span per call (`provider.create|get|delete|list`), attrs `provider.name`, `provider.id` (where present), result class attr from `resultLabel(err)`; status Error only for quota/permanent/transient — `ErrNotFound` stays Ok (expected during deletion polling). Returns the inner error **unmodified**. Uses `tracing.Start` so provider-internal logs (gcp `p.logger(ctx)`) inherit `traceID`. Tests in `internal/provider/tracing_test.go` reusing `staticProvider` from `metrics_test.go:19`.
+
+Wiring in `cmd/main.go:174-177` — tracing outermost: `provider.WithTracing(name, provider.WithMetrics(name, p, m))`.
+
+### Step 4 — `cmd/main.go` wiring
+
+- `tracing.Setup(...)` immediately after `ctrl.SetLogger` (cmd/main.go:148); on error: log + `os.Exit(1)`.
+- Explicit `shutdown(context.WithTimeout(context.Background(), 10*time.Second))` (fresh ctx — signal ctx is already cancelled) after `mgr.Start` returns, on **both** the error path before `os.Exit(1)` and the clean path. Earlier `os.Exit` sites have nothing to flush.
+- `cfg := ctrl.GetConfigOrDie(); cfg.Wrap(tracing.RestConfigWrapper())` before `ctrl.NewManager` (cmd/main.go:239).
+- kubernetes provider: add `kubernetes.NewWithTransportWrapper(ctx, cfg, wrap transport.WrapperFunc)` beside `New` (internal/provider/kubernetes/kubernetes.go:42) and register a closure in main's constructor map (same pattern as gcp at cmd/main.go:166) passing `tracing.RestConfigWrapper()` — keeps `internal/tracing` out of provider packages and `newWithClient` tests untouched.
+- New flag `--trace-health-probes` (default false) → `engine.TraceProbes`.
+
+### Step 5 — Reconciler spans
+
+- `SetupWithManager` (proxy_controller.go:412): `b.Complete(tracing.NewReconciler("Proxy", r))`.
+- Sub-spans via `tracing.Start`: `reconcile.managed` (:116), `reconcile.replaceInstance` (:226), `reconcile.delete` (:254), and `status.patch` opened **inside the deferred flush closure** (:98-104, runs within the root span; its error already folds into `err`). `reconcileExternal` stays unspanned — no I/O, pure noise.
+- Useful attrs where cheap: phase transitions, `providerID`.
+
+### Step 6 — Discovery server
+
+- `handler()` (internal/discovery/server.go:151-156): chain becomes recover → `tracing.HTTPMiddleware("discovery", s.log)` → log → maxBytes → auth (otelhttp must stay outside `maxBytesMiddleware`).
+- Handlers and `logMiddleware`/`recoverMiddleware` switch `s.log` → `logf.FromContext(r.Context())` (handlers.go:85, :141, :234; server.go:163, :191) so request logs carry `traceID`. `s.log` remains for startup lines.
+- k8s `List` calls in handlers already use `r.Context()` → child spans appear via the wrapped rest transport.
+
+### Step 7 — GC + health
+
+- `gc.Sweeper.sweep` (internal/gc/gc.go:83): wrap in root span `gc.sweep` via `tracing.Start` (before the `.WithName("orphan-gc")` derivation so it inherits), attrs: providers swept, orphans deleted. Provider/k8s child spans come free from Steps 3/4.
+- `health.Engine`: new field `TraceProbes bool`. In the worker loop (engine.go:158) when enabled: root span `health.probe` (SpanKind Client) around `probeFn`, attrs proxy key/ok/latency/error class, ended before `record` (which stays ctx-free). No transport wrap and no propagation — never inject `traceparent` toward external targets through the proxies. Probe→reconcile span links are **out of scope**: `GenericEvent` (engine.go:330) carries no ctx and the workqueue coalesces events; documented in architecture Decisions.
+- Lease store sweep: not traced (in-memory only).
+
+### Step 8 — GCP wire-log enrichment
+
+`wireFilterHandler.Handle` (internal/provider/gcp/wirelog.go:46) receives the real request ctx from the SDK's slog calls: append `traceID`/`spanID` attrs when `trace.SpanContextFromContext(ctx)` is valid — V(5) wire logs become correlatable with the `provider.*` span. Add `wirelog_test.go` case.
+
+### Step 9 — Manifests + docs
+
+- `config/manager/manager.yaml` env (after `DISCOVERY_TOKEN`): `POD_NAME`/`POD_NAMESPACE` via downward API **first**, then `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)` ($(VAR) expansion only sees earlier vars); commented-out `OTEL_EXPORTER_OTLP_ENDPOINT` + `OTEL_TRACES_SAMPLER` examples with a "tracing is off until an endpoint is set" comment. Note `--trace-health-probes` beside the args list.
+- `docs/architecture.md`: new "### 10. Tracing" section after Metrics (:281) — span topology, enablement, `traceID` log-key choice; Decisions entries (:303): env-gated enablement, probes off by default, no probe→reconcile links (why), `traceID` naming vs Grafana defaults, no traceparent into apiserver.
+- README ops note; CHANGELOG entry (dated via `date`) once the user confirms it works.
+- Run `make lint` (CI has no lint step; `lll`/`revive` will hit new files).
+
+## Verification
+
+1. `make test` (unit, includes all new `tracetest` assertions) and `make lint`.
+2. Console smoke: `OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml ...` against kind — create a Proxy, confirm (a) `Reconcile Proxy` span JSON on stdout with nested `provider.create` + k8s PATCH spans, (b) the reconcile log lines carry the same `traceID` as the span, (c) no spans from informer watches or leader election.
+3. End-to-end in kind: deploy Jaeger all-in-one (`jaegertracing/all-in-one`, OTLP 4318), set `OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318` on the manager Deployment; exercise a reconcile + a `POST /v1/leases` (with and without a client `traceparent`) + wait one GC interval; in the Jaeger UI: reconcile trace shows provider+k8s children; lease trace shows route-named server span with k8s child; grep manager logs for a `traceID` and resolve it in Jaeger.
+4. Negative: run with no `OTEL_*` env — startup line says tracing disabled, no export-error spam, logs unchanged (no `traceID` keys).

docs/plans/2026-08-24-1224-tracing-e2e.md

@@ -0,0 +1,150 @@
+# Plan: e2e test — OTel tracing against real Tempo
+**Created:** 2026-08-24 12:24
+
+## Context
+
+The tracing feature (MR #4, branch `feat/otel-tracing`) is covered by unit
+tests with an in-memory span recorder, but nothing proves the full pipeline:
+operator in a real cluster → OTLP export → Tempo ingest → queryable traces
+with the documented span topology. The homelab Tempo lives at
+`http://192.168.0.30:3200` (query API — verified live from this host:
+`/api/echo`, TraceQL `GET /api/search?q=…`, `GET /api/traces/<id>`), with
+OTLP ingest on `:4318` (HTTP, verified 200) and `:4317` (gRPC, open). User
+decisions: **Go Ginkgo e2e test** (not a shell script), **structural
+assertions** (span tree, not just trace-exists), flow per the user's sketch:
+deploy operator in kind with tracing on → create kubernetes-provider proxies
+→ delete them → shut down — with traces tagged for easy discovery per test
+run. This work continues on `feat/otel-tracing`; MR #4 stays the vehicle.
+
+## Design decisions (from review, with evidence)
+
+- **Proxy CRs go in the `default` namespace, not the operator namespace.**
+  The squid pods the kubernetes provider creates carry no securityContext
+  (internal/provider/kubernetes/pod.go:33-49) and are created in the CR's
+  namespace; the operator ns is labeled `pod-security enforce=restricted`
+  (test/e2e/e2e_test.go:61-62) and would reject them. Pod RBAC is
+  cluster-scoped (config/rbac/role.yaml:10), so `default` works. All
+  kubectl calls use explicit `-n`.
+- **Anchor assertions on `provider.create`, not the root-span search.** A
+  `name="Reconcile Proxy"` hit may be the finalizer-add or a drift
+  reconcile (no provider call inside). TraceQL matches span names anywhere
+  in a trace, so search `{resource.test.run.id="<runID>" && name="provider.create"}`,
+  fetch *that* traceID, then assert companion span names.
+- **Run identification with zero code changes:** OTel resource attribute
+  `test.run.id=<runID>` via `OTEL_RESOURCE_ATTRIBUTES` (env wins —
+  `resource.WithFromEnv()` merges last, internal/tracing/tracing.go). Tempo
+  vParquet searches arbitrary resource attrs without config; fallback query
+  documented: `{resource.service.name="egress-proxies-operator" && name="provider.create"}`
+  + start/end window (Unix **seconds**).
+- **OTLP preflight from inside the cluster:** host-reachability of
+  192.168.0.30 doesn't prove pod-reachability from kind on macOS, and
+  export failures log only at V(1) (invisible) — without a preflight, a
+  broken path is a 2-minute opaque timeout. Reuse the suite's curl-pod
+  pattern (e2e_test.go:218-248, incl. the restricted-PSS overrides JSON)
+  to POST to `<OTLP_ENDPOINT>/v1/traces` and fail fast.
+- **No homelab defaults baked into the Makefile.** The spec Skips unless
+  `TEMPO_URL`/`OTLP_ENDPOINT` are set (otherwise everyone without that LAN
+  host inherits a 2-minute failure). The copy-pasteable invocation lives in
+  docs/testing.md.
+- **Ginkgo ordering:** top-level container order is randomized, so the
+  tracing Describe is fully self-contained (own ns create → make install →
+  make deploy → teardown), same shape as the Manager Describe. `kubectl
+  delete ns` blocks until termination, so no create/delete race between
+  the two Describes.
+
+## Steps
+
+### Step 1 — `test/e2e/tracing_test.go` (new; `//go:build e2e`, package e2e)
+
+`Describe("OTel tracing", Ordered)`:
+
+- **BeforeAll:**
+  1. Read `TEMPO_URL` + `OTLP_ENDPOINT`; `Skip("TEMPO_URL/OTLP_ENDPOINT not set")` when empty.
+  2. `runID := "e2e-" + strconv.FormatInt(time.Now().UnixNano(), 10)`;
+     print it to `GinkgoWriter` so the run is findable in Grafana by hand.
+  3. Create operator ns + restricted-PSS label; `make install`;
+     `make deploy IMG=<managerImage>` (same commands as e2e_test.go:54-75,
+     via `utils.Run`).
+  4. Pre-pull the proxy image to kill the biggest flake source:
+     `docker pull ubuntu/squid:6.6-24.04_edge` + `kind load docker-image`
+     (via `utils.Run`, honoring `KIND`/`KIND_CLUSTER` env like
+     utils.LoadImageToKindClusterWithName).
+  5. OTLP preflight: curl pod POSTing `{}` to `<OTLP_ENDPOINT>/v1/traces`,
+     assert HTTP 200 in its logs (fail message names the endpoint).
+  6. `kubectl set env deployment/egress-proxies-operator-controller-manager
+     -n <ns> OTEL_EXPORTER_OTLP_ENDPOINT=<OTLP_ENDPOINT>
+     "OTEL_RESOURCE_ATTRIBUTES=k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE),test.run.id=<runID>"`
+     — in-place update keeps the var after the downward-API vars ($(VAR)
+     expansion needs that ordering); `exec.Command` means no shell mangling
+     of `$()`. Then `kubectl rollout status`.
+  7. `Eventually`: newest controller pod (sorted by creationTimestamp — a
+     terminating old pod may otherwise be picked) logs contain
+     `"tracing enabled"`.
+- **It "creates proxies and reports reconcile traces":**
+  - Write 2 Proxy manifests (names `proxy-tracing-e2e-1/2`,
+    `provider: kubernetes`, modeled on config/samples/proxy_kubernetes.yaml)
+    to `GinkgoT().TempDir()` (absolute paths — `utils.Run` chdirs the
+    process); `kubectl apply -n default`.
+  - `Eventually` (5m — first squid start) `.status.phase == "Ready"`
+    (confirmed field, api/v1alpha1/proxy_types.go:41).
+  - `Eventually`: Tempo search
+    `{resource.test.run.id="<runID>" && name="provider.create"}` ≥1 hit
+    (BSP flushes every ~5s; 2m default timeout is ample).
+  - Fetch that traceID via `/api/traces/<id>`; assert span names include
+    `Reconcile Proxy`, `reconcile.managed`, `provider.create`,
+    `status.patch`, and resource attr `service.name=egress-proxies-operator`.
+- **It "traces deletion":**
+  - `kubectl delete proxy -n default …`; `Eventually` CRs gone (finalizer
+    → `provider.delete`, DeletionPoll 10s).
+  - `Eventually`: Tempo search finds `provider.delete` for the runID; fetch
+    and assert `reconcile.delete` in the same trace.
+- **AfterAll** (= the sketch's "shut down operator"): delete leftover
+  proxies in `default`; `make undeploy` (SIGTERM → `tracingShutdown`
+  flushes); `make uninstall`; delete operator ns — errors discarded
+  (`_, _ = utils.Run(...)`), mirroring e2e_test.go:79-95.
+- **Tempo client helpers** (same file, stdlib only — the test process runs
+  on the host, which reaches Tempo directly):
+  - `tempoSearch(traceql string)` → `GET {TEMPO_URL}/api/search?q=…&start=…&end=…`
+    (Unix seconds, window = suite start − 5m → now); minimal struct
+    `{Traces []struct{ TraceID string }}`.
+  - `tempoTrace(id string)` → `/api/traces/<id>`, decoded as **OTLP-JSON**
+    (`batches[].scopeSpans[].spans[].name`, resource attrs as
+    `{key, value:{stringValue}}`) — not Jaeger's shape. Helper flattens to
+    a span-name set + resource attr map.
+
+### Step 2 — Makefile
+
+- `test-e2e`: pass `TEMPO_URL`/`OTLP_ENDPOINT` through to `go test` env
+  (no defaults) and add `-timeout 30m` — the suite already runs
+  docker-build + kind-load in BeforeSuite plus 3–5m Eventuallys, and this
+  adds a second full deploy cycle + image pulls; the 10m default will be
+  exceeded on cold caches.
+
+### Step 3 — Docs
+
+- `docs/testing.md` (§ e2e, currently "scaffold… only asserts manager runs
+  and serves metrics"): document the tracing spec, its Skip gate, and the
+  invocation:
+  `TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`.
+- Execution-log entries per step in `docs/plans-executions/2026-08-24-1224-tracing-e2e.md`.
+- CHANGELOG entry only after the user confirms a green run (house rule).
+
+## Not doing (and why)
+
+- No dedicated flush assertion on shutdown — teardown exercises the flush
+  path, but attributing a specific late span to it is guesswork.
+- No gRPC (4317) variant — http/protobuf is the operator default and the
+  verified path; a variant run is a one-env-var change if ever wanted.
+- No changes to `internal/` — the whole test works through public surface
+  (env vars, kubectl, Tempo API), which is the point of an e2e test.
+
+## Verification
+
+1. `TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`
+   → tracing spec passes (both Its); without the env vars → spec reports
+   Skipped, remainder of suite unaffected.
+2. In Grafana: TraceQL `{resource.test.run.id="<runID printed in test log>"}`
+   shows the run's traces with the expected tree.
+3. `go vet -tags=e2e ./...` clean (documented routine).
+4. A deliberately wrong `OTLP_ENDPOINT` (e.g. port 9) fails fast in the
+   preflight step with a clear message, not a 2-minute search timeout.

docs/plans/2026-09-04-1404-kvm-provider.md

@@ -0,0 +1,313 @@
+# Plan: KVM/libvirt provider
+
+**Created:** 2026-09-04 14:04
+
+## Context
+
+The operator provisions egress proxies through named provider instances (today
+`gcp` and `kubernetes`). The user has several KVM hypervisors on the home
+network and wants proxy VMs created there, using stock Debian/Ubuntu cloud
+images so the VM stays a normal machine they can customise through their own
+cloud-init user-data. Longer term the operator may become a generic "run a VM
+or container with a set of scripts" tool; the KVM provider should not
+foreclose that.
+
+The provider contract (`internal/provider/provider.go`: `Create`/`Get`/`Delete`/
+`ListByTag`, keyed by `NameFromUID`, cloud-init passed verbatim, GC-by-tag) fits
+libvirt with **no CRD or reconciler changes**. All work is a new provider
+package plus config/wiring/docs, following the GCP provider's shape.
+
+Decisions taken with the user (2026-09-04):
+
+- **Transport:** SSH to each hypervisor, forwarding the libvirt unix socket
+  (`ssh.Client.Dial("unix", "/var/run/libvirt/libvirt-sock")` → `libvirt.New(conn)`).
+  Pure Go, no cgo, no libvirt libs in the image. Verified: `x/crypto/ssh`
+  `Client.Dial` has a `"unix"` case (`dialStreamLocal`), go-libvirt `New(net.Conn)`.
+- **Topology:** one provider instance can span several hosts from day one
+  (config has a `hosts` list; providerID encodes the host), so adding hosts
+  later is a config edit with no providerID migration. With one host configured
+  `placement.zone` may be omitted.
+- **Networking:** VM NIC on a host bridge to the LAN, IP from the router's
+  DHCP. IP discovery via `DomainInterfaceAddresses` (ARP source first, then
+  guest agent). Provider-injected `vendor-data` installs `qemu-guest-agent`
+  so agent discovery works on stock images without touching user-data.
+- **Base image:** a stock Debian/Ubuntu cloud image (qcow2, cloud-init
+  preinstalled) that **the operator fetches itself** on first use per host:
+  `placement.image` names an entry in the provider's `images:` map (url +
+  optional sha256) or is an `https://` URL directly. The provider streams the
+  download through libvirt's `StorageVolUpload` into the host's pool, so
+  nothing is pre-placed and the operator needs no filesystem access on the
+  host. Per-VM disk is a qcow2 overlay with the cached base as backing store.
+- **Seed:** NoCloud ISO (`user-data`, `meta-data`, `vendor-data`) built in
+  pure Go with `github.com/kdomanski/iso9660` (`WriteTo(w, "cidata")`) and
+  streamed to the pool with `StorageVolUpload(vol, io.Reader, ...)`. Nothing
+  to install on the hypervisors.
+
+## Design
+
+### Config (`internal/provider/config.go`)
+
+```yaml
+- name: kvm-home
+  type: kvm
+  kvm:
+    sshKeyFile: /etc/proxy-operator/kvm/id_ed25519        # required
+    knownHostsFile: /etc/proxy-operator/kvm/known_hosts   # required unless insecureIgnoreHostKey
+    insecureIgnoreHostKey: false
+    diskSizeGb: 10                    # overlay capacity; cloud-init growpart expands root fs
+    vendorData: |                     # optional override; default installs qemu-guest-agent
+      #cloud-config
+      ...
+    images:                           # optional; placement.image names a key (or is an https URL)
+      debian-13:
+        url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+        sha256: "…"                   # optional; verified after download, re-fetch when it changes
+        format: qcow2                 # default qcow2
+      ubuntu-24.04:
+        url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+    machineTypes:                     # required, ≥1; placement.machineType names a key
+      small:  {vcpus: 1, memoryMiB: 1024}
+      medium: {vcpus: 2, memoryMiB: 2048}
+    hosts:                            # required, ≥1, unique names
+      - name: nas1                    # placement.zone value
+        address: nas1.home.lan:22     # required
+        user: virt                    # required; must be in the libvirt group
+        socket: /var/run/libvirt/libvirt-sock   # default
+        pool: default                 # default "default"
+        bridge: br0                   # required
+```
+
+Add `KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage` structs; a `"kvm"`
+case in `validate()` (cross-block checks like the others, required fields
+above, unique host names, image URLs must be `http(s)`, sha256 if set must be
+64 hex chars). Tests in `config_test.go` extend the existing table.
+
+### Placement mapping
+
+| Placement field | KVM meaning |
+|---|---|
+| `zone` | host name from `hosts`; empty → the single configured host, else `ErrPermanent` |
+| `machineType` | key in `machineTypes`; unknown → `ErrPermanent` |
+| `image` | key in `images:`, or an `https://` URL; anything else → `ErrPermanent`. Fetched into the host's pool on first use (see *Image cache*) |
+| `region` | ignored |
+
+### providerID
+
+`hosts/<host>/domains/<name>` with `formatProviderID`/`parseProviderID`
+(same self-contained style as GCP's `zones/<zone>/instances/<name>`).
+
+### GC contract without labels
+
+Domain XML gets a `<metadata>` child in a private namespace, e.g.
+`xmlns="https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm"`,
+carrying `<managed>true</managed><uid>…</uid><createdAt>RFC3339</createdAt>`.
+`Get` reads it via `DomainGetMetadata(dom, DomainMetadataElement, uri)`;
+`ListByTag` does `ConnectListAllDomains` on every host and keeps domains where
+that call succeeds and `managed == "true"`. `CreatedAt` comes from the metadata
+(libvirt has no creation time). A host that is unreachable makes `ListByTag`
+return an error (GC skips the sweep that round rather than acting on a partial view).
+
+### Connection management (`internal/provider/kvm/conn.go`)
+
+Per-host lazily dialed `*ssh.Client` + `*libvirt.Libvirt`, cached under a
+mutex, re-dialed when `Disconnected()` is closed or a call fails with a
+transport error. SSH auth: private key from `sshKeyFile`; host key check via
+`knownhosts.New(knownHostsFile)` unless `insecureIgnoreHostKey`. Dial timeout
+from `ctx`. The provider calls libvirt through a narrow interface:
+
+```go
+type libvirtAPI interface {
+    DomainLookupByName(name string) (libvirt.Domain, error)
+    DomainDefineXML(xml string) (libvirt.Domain, error)
+    DomainCreate(d libvirt.Domain) error
+    DomainDestroy(d libvirt.Domain) error
+    DomainUndefineFlags(d libvirt.Domain, f libvirt.DomainUndefineFlagsValues) error
+    DomainGetState(d libvirt.Domain, flags uint32) (int32, int32, error)
+    DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, f libvirt.DomainModificationImpact) (string, error)
+    DomainInterfaceAddresses(d libvirt.Domain, src uint32, flags uint32) ([]libvirt.DomainInterface, error)
+    ConnectListAllDomains(need int32, f libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error)
+    StoragePoolLookupByName(name string) (libvirt.StoragePool, error)
+    StorageVolLookupByName(p libvirt.StoragePool, name string) (libvirt.StorageVol, error)
+    StorageVolGetPath(v libvirt.StorageVol) (string, error)
+    StorageVolCreateXML(p libvirt.StoragePool, xml string, f libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error)
+    StorageVolUpload(v libvirt.StorageVol, r io.Reader, off, n uint64, f libvirt.StorageVolUploadFlags) error
+    StorageVolDownload(v libvirt.StorageVol, w io.Writer, off, n uint64, f libvirt.StorageVolDownloadFlags) error
+    StorageVolDelete(v libvirt.StorageVol, f libvirt.StorageVolDeleteFlags) error
+    StoragePoolRefresh(p libvirt.StoragePool, flags uint32) error
+}
+```
+
+(All signatures verified against go-libvirt `main` on 2026-09-04.) A
+`hostDialer func(ctx, KVMHost) (libvirtAPI, io.Closer, error)` is the test seam;
+`newWithDialer(cfg, dialer)` mirrors GCP's `newWithAPI`.
+
+### Image cache (`internal/provider/kvm/image.go`)
+
+`ensureImage(ctx, host, api, img) (basePath string, err error)` makes the base
+image present in the host's pool, once per host per image:
+
+- Volume names: `base-<key>.<format>` for an `images:` key, or
+  `base-<sha256(url)[:16]>.<format>` for a URL given directly in
+  `placement.image`. Marker volume `<base>.sha256` (raw, 64 bytes) holds the
+  digest of the verified upload.
+- **Ready check:** base volume exists *and* marker exists *and* (no `sha256`
+  configured, or marker content == configured `sha256`). Read the marker with
+  `StorageVolDownload`. Ready → return `StorageVolGetPath(base)`.
+- **Not ready:** delete both volumes if present (partial upload from a crash,
+  or a changed checksum), then `GET url` with the ctx, require a
+  `Content-Length` (`ErrPermanent` otherwise), `StorageVolCreateXML` the base
+  with `<capacity unit="bytes">Content-Length</capacity>` and the configured
+  format, `StorageVolUpload(base, io.TeeReader(body, sha256), 0, 0, 0)` (length
+  0 = until EOF, so nothing is buffered in the operator), then compare the
+  digest with the configured one (mismatch → delete base, `ErrPermanent`),
+  `StoragePoolRefresh` so libvirt re-reads the real qcow2 capacity, create the
+  marker volume and upload the digest last.
+- HTTP 4xx → `ErrPermanent`; 5xx, network, or upload transport errors →
+  `ErrTransient` (partial volumes are cleaned on the next attempt).
+- A per-`(host, image)` `sync.Mutex` map serialises concurrent Creates for the
+  same image on the same host; the second caller finds it ready.
+- **Judgment call:** the fetch runs synchronously inside `Create`, so the first
+  Proxy for a given image on a given host holds a reconcile worker for the
+  download (LAN speed, a few hundred MB, once). This keeps `Create` idempotent
+  and needs no background state. Async fetch with a `Provisioning`-style
+  progress status is a possible later improvement, noted in the docs.
+
+### Create (idempotent, fire-and-forget after the image is cached)
+
+1. Resolve host, machine type, pool, and image source (`images:` key or URL;
+   else `ErrPermanent`). `ensureImage` → base path.
+2. Overlay volume `<name>.qcow2`: `StorageVolCreateXML` with
+   `<capacity unit="G">diskSizeGb</capacity>`, `<target><format type="qcow2"/></target>`,
+   `<backingStore><path>base</path><format type="qcow2"/></backingStore>`.
+   `ErrStorageVolExist` → reuse.
+3. Seed volume `<name>-seed.iso`: build ISO in memory (`user-data` = req.CloudInit,
+   `meta-data` = `instance-id: <name>\nlocal-hostname: <name>\n`, `vendor-data` =
+   cfg or default), `StorageVolCreateXML` with exact capacity + `<format type="raw"/>`,
+   then `StorageVolUpload`. Exists → skip upload (content is deterministic per spec hash;
+   spec changes recreate the whole instance anyway).
+4. `DomainLookupByName(name)`; if `ErrNoDomain` → `DomainDefineXML(buildDomainXML(...))`.
+5. `DomainGetState`; if not running → `DomainCreate`. Return providerID.
+
+`buildDomainXML` is a pure function over `encoding/xml` structs: `type="kvm"`,
+q35 machine, `host-passthrough` CPU, vcpus/memory from the profile, virtio disk
+(overlay) + sata cdrom (seed), `interface type="bridge"` with virtio model,
+`org.qemu.guest_agent.0` virtio channel, serial console, virtio rng, no
+graphics, and the `<metadata>` block. Unit-tested field by field like
+`buildInsertRequest`.
+
+### Get
+
+`parseProviderID` → host → `DomainLookupByName` (`ErrNoDomain` → `ErrNotFound`)
+→ `DomainGetState` + `DomainGetMetadata` + `DomainInterfaceAddresses`
+(ARP, then agent; first IPv4 wins; agent errors while the agent isn't up yet
+are swallowed). Mapping:
+
+| libvirt state | IP | Instance.State |
+|---|---|---|
+| running | yes | Running |
+| running | no | Provisioning |
+| shutoff / crashed / paused / pmsuspended | – | Stopped |
+| no domain | – | `ErrNotFound` |
+
+### Delete (idempotent)
+
+Destroy if running (`ErrOperationInvalid` on a stopped domain is fine) →
+`DomainUndefineFlags(managed-save | snapshots-metadata | nvram)` → delete
+`<name>.qcow2` and `<name>-seed.iso` (`ErrNoStorageVol` fine). Volumes are
+deleted by name even when the domain is already gone, so a crash between
+volume creation and define still gets cleaned up on the next reconcile.
+Known gap (documented, not solved now): volumes orphaned with no domain *and*
+no CR are invisible to `ListByTag`.
+
+### Errors (`internal/provider/kvm/errors.go`)
+
+`classify(err)` on `libvirt.Error.Code`: `ErrNoDomain`, `ErrNoStorageVol` →
+`ErrNotFound`; `ErrNoStoragePool`, `ErrXMLError`, `ErrConfigUnsupported`,
+`ErrNoSupport`, `ErrAuthFailed`, `ErrInvalidArg` → `ErrPermanent`; SSH dial /
+host-key / transport errors and everything else → `ErrTransient`. Image
+download: HTTP 4xx, missing Content-Length, checksum mismatch → `ErrPermanent`;
+5xx and network errors → `ErrTransient`. Quota is not detectable from libvirt;
+left as transient (revisit with a capacity check later).
+
+### Wiring & manifests
+
+- `cmd/main.go`: add `"kvm": kvm.New` to the builtins map.
+- `config/manager/manager.yaml`: mount Secret `kvm-ssh` (optional) at
+  `/etc/proxy-operator/kvm` (keys `id_ed25519`, `known_hosts`); document the
+  `kubectl create secret generic kvm-ssh --from-file=...` command.
+- `config/samples/providers-config.yaml`: add the `kvm-home` block with the
+  `images:` map (Debian 13 and Ubuntu 24.04 official cloud-image URLs);
+  `config/samples/proxy_kvm.yaml`: Proxy with `placement: {zone: nas1,
+  machineType: small, image: debian-13}` and the same squid cloud-init as the
+  GCP sample plus `ssh_authorized_keys`.
+- `hack/providers-dev.yaml` stays kubernetes-only (needs no secrets).
+- Deps: `github.com/digitalocean/go-libvirt` (latest pseudo-version),
+  `github.com/kdomanski/iso9660 v0.4.0`, promote `golang.org/x/crypto` to direct.
+
+### Docs
+
+- New `docs/kvm-provider.md`: hypervisor prerequisites (user in `libvirt`
+  group, a bridge to the LAN, a storage pool with enough space), the config
+  reference including `images:`, how the image cache works (`base-*` and
+  `*.sha256` volumes in the pool, when a re-download happens, how to evict
+  with `virsh vol-delete`), the placement table, how to log into a VM
+  (`ssh_authorized_keys` in user-data, `virsh console` on the host), and the
+  known gaps (no quota detection, orphan volumes without a domain,
+  synchronous first-time image fetch).
+- Update the provider tables in `README.md`, `docs/architecture.md` §5,
+  `docs/testing.md` (integration test section).
+
+### Towards the generic runner (not built now, just kept open)
+
+Provider knows nothing about squid. Future hooks: `vendor-data` for injecting an
+agent, `QEMUDomainAgentCommand` for reading exit status/output, and a future
+`Placement`-independent "job" mode in the CRD. None of this changes the KVM
+provider's shape.
+
+## Steps
+
+0. **Save plan + branch.** Copy this file to `docs/plans/<ts>-kvm-provider.md`,
+   commit alone, then `git checkout -b feat/kvm-provider` off `feat/proxy-operator`
+   (the working main for this project).
+1. **Config.** `KVMConfig` & friends, `validate()` case, table tests.
+2. **Pure builders.** `internal/provider/kvm/{domainxml.go,volume.go,seed.go,id.go,metadata.go}`
+   + tests: domain XML fields, volume XML, seed contents (unpack the ISO with
+   the same library's reader and assert the three files + label `cidata`),
+   providerID round-trip, metadata marshal/parse.
+3. **Transport + errors.** `conn.go` (SSH dial, host-key policy, per-host cache,
+   reconnect), `errors.go` + classification table test. SSH dialer tested
+   against an in-process `x/crypto/ssh` server that serves a unix socket
+   forward (cheap, keeps the dial code covered).
+3b. **Image cache.** `image.go` + `image_test.go`: `httptest.Server` serving a
+   small fake image over the `fakeAPI` (which records uploaded bytes);
+   cases: fresh fetch, already ready, marker missing (partial upload → redo),
+   checksum changed → redo, checksum mismatch → Permanent + cleanup, 404 →
+   Permanent, 503 → Transient, two concurrent callers → one download.
+4. **Provider methods.** `kvm.go`: `New`, `newWithDialer`, `Create/Get/Delete/ListByTag`
+   over `libvirtAPI`; `kvm_test.go` with a hand-written `fakeAPI` (domains and
+   volumes in maps, per-call error injection) covering idempotent re-Create
+   after each partial step, state/IP mapping, delete-when-gone, ListByTag
+   across two hosts and with one host down.
+5. **Wiring, manifests, samples, docs, CHANGELOG.**
+6. **Integration test** (`//go:build integration`, `kvm_integration_test.go`):
+   skipped unless `KVM_TEST_HOST`, `KVM_TEST_USER`, `KVM_TEST_SSH_KEY`,
+   `KVM_TEST_BRIDGE`, `KVM_TEST_IMAGE_URL` are set; fetches the image (or finds
+   it cached), creates, waits for Running+IP, deletes, asserts per-VM volumes
+   gone and the base image still cached. Deviation from the testcontainers convention,
+   called out in `docs/testing.md`: libvirt+KVM can't run inside a container
+   without `/dev/kvm`, so this targets a real hypervisor.
+7. **Manual validation** on one real hypervisor, results in
+   `docs/plans-executions/` (like `docs/gcp-vm-validation.md`), then MR via `tea`.
+
+## Verification
+
+- `make test` green (`-race`, envtest suite untouched but must still pass).
+- `go test -short ./internal/provider/...` covers builders, fake-API provider, errors, config.
+- Integration: `KVM_TEST_HOST=nas1.home.lan … go test -tags=integration -run TestKVM ./internal/provider/kvm/`.
+- End to end: deploy with the `kvm-ssh` Secret and `kvm-home` config against a
+  host with no cached image, apply `config/samples/proxy_kvm.yaml`, watch the
+  operator log the image fetch and `virsh vol-list` show `base-debian-13.qcow2`
+  plus its `.sha256` marker, watch `kubectl get px -w` reach `Ready` with a LAN
+  IP, apply a second Proxy and confirm no second download, confirm the VM shows up in `GET /v1/proxies` and works with
+  `hack/` demo egress-IP check, delete the Proxy and confirm `virsh list --all`
+  and `virsh vol-list default` are clean.

docs/plans/2026-09-10-0805-kvm-libvirt-tracing.md

@@ -0,0 +1,186 @@
+# Plan: Trace every libvirt call in the KVM provider
+**Created:** 2026-09-10 08:05
+
+## Context
+
+Step 7 of the KVM provider plan (live validation on `beelink`) has gone through
+three fixes found only on the real host, and the current failure (guest hangs at
+GRUB's "Booting" line) is being diagnosed blind: the provider logs at V(1)/V(2)
+but nothing shows them. The integration test never installs a logger, so
+controller-runtime prints "log.SetLogger(...) was never called" and swallows
+everything; and even with a logger, the provider only logs per *operation*
+(Create, Get…), not per libvirt RPC, so the sequence and timing of what was
+actually sent to the hypervisor is invisible.
+
+Goal: one log line per libvirt call at debug level (`--zap-log-level=debug`, logr
+V(1) in this repo's scheme: V(1) = outcome per operation, V(2) = curated detail,
+V(5) = raw wire, per the GCP verbose-logging plans), with method, host, domain or
+volume, duration and the libvirt error code on failure; and those lines visible in
+`make test-integration-kvm` output.
+
+Assumption: this lands on the current `feat/kvm-provider` branch, since it exists
+to finish that branch's validation, and the plan file plus execution log follow
+CLAUDE.md conventions (plan committed alone first, execution summary alongside
+each step). Say so if you want a separate branch instead.
+
+## Design
+
+- **Decorator, not call-site logging.** New `tracingAPI` implements the 17-method
+  `libvirtAPI` seam (`internal/provider/kvm/conn.go:26-44`), wrapping the real
+  client. Every method logs one V(1) line after the inner call returns. Applied in
+  `hostPool.call`, so every provider method, `image.go`'s cache and the
+  integration test's own `p.hosts.call` are traced with no per-site changes.
+- **Logger passed explicitly.** `call(ctx, log, host, fn)` and `get(ctx, log, host)`
+  gain a `logr.Logger` parameter, matching how this package already threads `log`
+  into every helper. The five call sites (`kvm.go:169, 322, 458, 530`,
+  `kvm_integration_test.go:220`) already hold the right logger with
+  provider/host/name values. `call` does `log = log.WithName("libvirt")`, so trace
+  lines are named `kvm.libvirt`.
+- **Wrap only when `log.V(1).Enabled()`.** Zero cost at info level; the
+  controller-runtime unfulfilled root sink reports disabled, so unit tests and
+  `conn_test.go`'s `hostClient{api: nil}` cases are unaffected.
+- **One line per RPC at every level.** At V(2) the same line gains extra keys
+  (`xml`, `metadata`, `addresses`, `names`) instead of a second line. Upload and
+  download log offset/length/duration only, never bytes (the seed ISO carries
+  cloud-init). Domain and volume XML contain no secrets, so V(2) is fine for them.
+- **Connection events traced too** in the pool: dial with duration, dial failure,
+  dead cached client redial, and both drop reasons (context cancelled, transport
+  error). `dialSSH`/`connectLibvirt`/`keepalive` stay untouched: changing
+  `hostDialer`'s signature would ripple into every test dialer, and the pool
+  already observes every event.
+- **Rendering.** Handles by name only (`domain`, `pool`, `volume`), never UUID/Key.
+  `libvirt.DomainState` and `DomainInterfaceAddressesSource` have stringers, pass
+  them typed. Bitmask flags don't stringify, log as integers only when non-zero.
+  `libvirt.OptString` → first element or `""` under `uri`.
+- **awaitOrCancel.** `done` runs inside fn's goroutine; on the normal path it logs
+  before `call` returns. On cancel, the abandoned RPC logs one late line when the
+  closed connection makes it fail: desirable in production, and in the integration
+  test only reachable in an already-failing timeout run (testr panics on a log after
+  the test ends; note that in the header, don't engineer around it).
+
+## Steps
+
+### Step 0 — Save plan
+Copy this file to `docs/plans/2026-09-10-0805-kvm-libvirt-tracing.md`, commit alone.
+
+### Step 1 — `errors.go`: shared error fields
+Factor `libvirtErrorFields(err error) []any` out of `logAPIError` (`errors.go:62-72`):
+returns `libvirtCode` (uint32), `libvirtCodeName` (`libvirt.ErrorNumber(code)`,
+e.g. `ErrNoDomain`), `libvirtMessage`; nil for non-libvirt errors; works through
+`%w` wrapping via `errors.As`. `logAPIError` keeps its message and gains
+`libvirtCodeName`. Test `TestLibvirtErrorFields` (table: libvirt.Error, wrapped,
+plain error) in `errors_test.go`.
+
+### Step 2 — new `trace.go`
+```go
+type tracingAPI struct { inner libvirtAPI; log logr.Logger }
+type tracedCall struct { t *tracingAPI; method string; start time.Time; kv []any }
+func (t *tracingAPI) begin(method string, kv ...any) *tracedCall
+func (c *tracedCall) detail(kv ...any) *tracedCall // appended only if V(2) enabled
+func (c *tracedCall) done(err error, kv ...any)    // one V(1) line
+```
+`done` emits `log.V(1).Info("libvirt call", ...)` with keys in order: `method`,
+begin args, result kv, `duration` (rounded to µs), then on error `error` plus
+`libvirtErrorFields`. Per-method keys:
+
+| method | V(1) args | V(1) results | V(2) extra |
+|---|---|---|---|
+| DomainLookupByName | domain | | |
+| DomainDefineXML | xmlBytes | domain | xml |
+| DomainCreate, DomainDestroy | domain | | |
+| DomainUndefineFlags | domain, flags | | |
+| DomainGetState | domain | state, reason | |
+| DomainGetMetadata | domain, type, uri | metadataBytes | metadata |
+| DomainInterfaceAddresses | domain, source | interfaces (count) | addresses |
+| ConnectListAllDomains | | count | names |
+| StoragePoolLookupByName, StoragePoolRefresh | pool | | |
+| StorageVolLookupByName | pool, volume | | |
+| StorageVolGetPath | pool, volume | path | |
+| StorageVolCreateXML | pool, xmlBytes | volume | xml |
+| StorageVolUpload, StorageVolDownload | pool, volume, offset, length | | (never content) |
+| StorageVolDelete | pool, volume | | |
+
+Return values pass through untouched.
+
+### Step 3 — `conn.go`: pool logging and wrapping
+- `get(ctx, log, host)`: dead cached client → `"libvirt connection lost, redialing"`;
+  time the dial; `"libvirt host connected"` with `address`, `duration`; failure →
+  `"libvirt host dial failed"` with `address`, `duration`, `error`.
+- `call(ctx, log, host, fn)`: `log = log.WithName("libvirt")`; wrap when
+  `log.V(1).Enabled()`; abort closure logs `"libvirt connection dropped"`
+  `reason="context cancelled"` before `drop`; transport-error branch logs the same
+  message with `reason="transport error"`. Update `call`'s doc comment.
+
+### Step 4 — call sites
+`kvm.go` Create/Get/Delete pass `log`, ListByTag passes `hlog`;
+`kvm_integration_test.go:220` passes `p.logger(ctx).WithValues("host", host.Name)`;
+`conn_test.go` (12 `p.call(` sites) gets `var discard = logr.Discard()`.
+
+### Step 5 — integration test output
+- `integrationLogger(t)`: `KVM_TEST_LOG_LEVEL` (default `1`, `strconv.Atoi`,
+  `t.Fatalf` on garbage) → `testr.NewWithOptions(t, testr.Options{LogTimestamp:
+  true, Verbosity: n})` (`github.com/go-logr/logr/testr`, present in logr v1.4.3,
+  no go.mod change); `logf.SetLogger(log)` once so nothing falls back to the
+  unfulfilled root.
+- `TestKVM_lifecycle`: `ctx = logr.NewContext(ctx, log)` after `WithTimeout`; the
+  cleanup ctx gets the same logger.
+- Header comment: document `KVM_TEST_LOG_LEVEL` (1 = one line per RPC, 2 adds
+  XML/metadata/addresses, 0 = only `t.Log`) and the late-line-after-timeout caveat.
+
+### Step 6 — unit tests
+- `kvm_test.go`: `captureContext(verbosity)` like `gcp_test.go:277-286` but with a
+  mutex around the slice; `TestLogging_verbosityTiers` (mirror `gcp_test.go:322-399`):
+  V0 → no lines; V1 → `"msg"="libvirt call"`, `"method"="DomainDefineXML"`,
+  `"method"="StorageVolUpload"`, `"msg"="libvirt host connected"`, provider/host/name
+  values, `"libvirtCodeName"="ErrNoDomain"` from the first lookup, no `"xml"=`;
+  V2 → `"xml"="<domain` and `"metadata"=`; a cloud-init sentinel absent at every tier.
+- `trace_test.go`: `TestTracingAPI_logsEveryMethod` (table of 17 calls over a
+  `readyHost()` fake, exactly one line each, plus
+  `reflect.TypeOf((*libvirtAPI)(nil)).Elem().NumMethod() == len(cases)` so a new
+  interface method without tracing fails); `TestTracingAPI_passesResultsThrough`;
+  `TestTracingAPI_errorFields` (libvirt error → code/name/message; `io.EOF` →
+  `"error"="EOF"`, no code); `TestTracingAPI_detailOnlyAtV2`;
+  `TestTracingAPI_uploadLogsSizeNotContent`.
+- `conn_test.go`: `fakeDialer` gains an `api` field; `TestHostPool_call_tracesAtV1`
+  (V1 → connected + method lines; V0 → no lines and fn receives the bare
+  `*fakeAPI`); `TestHostPool_call_logsDropReason` (transport error, cancel).
+
+### Step 7 — docs, execution log, CHANGELOG
+- `docs/kvm-provider.md`: `## Logging` section before "Known gaps":
+  `--zap-log-level=debug` = one `libvirt call` line per RPC plus connect/drop lines;
+  `--zap-log-level=2` adds XML, metadata, addresses; cloud-init never logged; no
+  V(5) wire tier because go-libvirt exposes no hook. Mention `KVM_TEST_LOG_LEVEL`
+  in its Testing paragraph.
+- `docs/testing.md`: `KVM_TEST_LOG_LEVEL` in the optional vars; `-v` now shows every
+  RPC; extend the `kvm_test.go` table row.
+- `docs/plans-executions/2026-09-10-0805-kvm-libvirt-tracing.md` with the checklist
+  and per-step entries; also a short cross-reference line in the KVM provider
+  execution log's Step 7 section.
+- `CHANGELOG.md` entry once the user confirms the live run shows the trace.
+
+## Verification
+
+```bash
+go build ./... && go vet ./... && go vet -tags=integration ./internal/provider/kvm/
+go test -race -count=1 ./internal/provider/kvm/
+go test -short ./...
+```
+Live, with the usual env: `make test-integration-kvm`. Expect, interleaved with
+`t.Log` lines and attributed to `trace.go:<line>`, lines like
+
+```
+kvm.libvirt: "level"=1 "msg"="libvirt call" "provider"="kvm-integration" "host"="kvm-test" "name"="proxy-…" "method"="DomainLookupByName" "domain"="proxy-…" "duration"="2.1ms" "error"="…" "libvirtCode"=42 "libvirtCodeName"="ErrNoDomain" "libvirtMessage"="…"
+```
+
+plus one `libvirt host connected` line with `duration`, a `StorageVolUpload` line
+whose duration is the seed upload, and no "log.SetLogger(...) was never called"
+text. `KVM_TEST_LOG_LEVEL=2` shows `"xml"="<domain` and `"addresses"=`;
+`KVM_TEST_LOG_LEVEL=0` shows only `t.Log` output. No line at any level contains
+`#cloud-config`.
+
+## Critical files
+- `internal/provider/kvm/conn.go`, `trace.go` (new), `trace_test.go` (new)
+- `internal/provider/kvm/errors.go`, `errors_test.go`
+- `internal/provider/kvm/kvm.go`, `kvm_test.go`, `conn_test.go`
+- `internal/provider/kvm/kvm_integration_test.go`
+- `docs/kvm-provider.md`, `docs/testing.md`

docs/plans/2026-09-10-2222-kvm-review-fixes.md

@@ -0,0 +1,195 @@
+# Plan: Fix the five code-review findings on `feat/kvm-provider`
+**Created:** 2026-09-10 22:22 UTC
+
+## Context
+
+PR #5 (`feat/kvm-provider` → `main`, 19 commits) is open and awaiting merge. A
+`/code-review high` pass over the branch returned five confirmed findings: two that can
+pin a Proxy's finalizer forever, one startup crash-loop, and two in the guest-agent merge
+committed earlier today. All five are fixed on the same branch so the PR updates in place,
+then re-validated against the real hypervisor before merging.
+
+Four decisions, made by the user during planning:
+
+1. **Wedged deletes** — attempt `Delete` even when `Get` fails permanently, make an
+   unconfigured KVM host a no-op delete, and add an annotation that abandons cleanup for
+   whatever still cannot finish.
+2. **Missing `kvm-ssh` Secret** — read the SSH material lazily, so only KVM Proxies fail.
+3. **Guest agent** — stop rewriting the user's cloud-config. Ship the bootstrap as a
+   vendor-data shell script instead.
+4. Validation is unit tests, the integration test against beelink, and the kind e2e. The
+   uncommitted dev-only files stay exactly as they are and out of the PR.
+
+## Why the guest-agent fix is a rewrite, not a patch
+
+Both cloud-init findings come from the same root cause: this operator edits the user's
+document. Reading cloud-init 26.1's sources settles that it never has to.
+
+Vendor-data is consumed by its **own** handler chain with separate script and config paths
+(`stages.py:_default_vendordata_handlers`). A vendor-data **shell script** is therefore
+written to the vendor script directory and run by `cc_scripts_vendor`, never passing
+through the config merger that drops colliding keys. On the final-stage module list it
+runs after `package_update_upgrade_install` and before the user's own `runcmd`, so the
+agent comes up early regardless of what the user's config does.
+
+That also closes the door the previous approach left open. The merge cannot be rescued
+from inside the documents: `merge_how` in the user-data governs only the first source, and
+`merge_how` in vendor-data is popped and discarded by `dict_extract_mergers` before the
+document is ever written. There is no supported way to make vendor-data cloud-config keys
+survive a user-data that defines the same key.
+
+Consequences: user-data reaches the seed byte for byte in any format, so Jinja templates,
+MIME archives, gzip payloads and shell scripts stop being special cases (finding 4), and
+nothing re-serialises YAML, so unquoted scalars keep their type (finding 5). The cost is
+that the script picks the package manager itself where cloud-init's `packages` key was
+distro-agnostic.
+
+## Why the delete fix is shaped this way
+
+KVM is the only provider that resolves a providerID segment against operator config. GCP
+and Kubernetes let the backend answer "not found", so retiring a location yields
+`ErrNotFound` and the finalizer drops. `Provider.host()`
+([kvm.go:118-139](internal/provider/kvm/kvm.go#L118-L139)) returns `ErrPermanent`, and
+`reconcileDelete` ([proxy_controller.go:281-291](internal/controller/proxy_controller.go#L281-L291))
+drops the finalizer only on `ErrNotFound`. `deletionFailure`
+([:327-336](internal/controller/proxy_controller.go#L327-L336)) says in its own comment
+that it retries "until an operator intervenes", and no intervention mechanism exists. The
+annotation is that mechanism, and it also rescues the unconfigured-provider wedge at
+[:276-279](internal/controller/proxy_controller.go#L276-L279).
+
+`Get` keeps returning `ErrPermanent` for an unknown host deliberately. Turning it into
+`ErrNotFound` would make a typo in the host list look like a missing VM on the live path,
+and the reconciler would recreate every affected Proxy elsewhere while the originals keep
+running where garbage collection cannot see them.
+
+## Steps
+
+One commit per step on `feat/kvm-provider`, each ending with
+`Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>`.
+
+### Step 0 — Save this plan into the repo
+Copy to `docs/plans/2026-09-10-2222-kvm-review-fixes.md` with the same header timestamp and
+commit it alone, before any code change. Start
+`docs/plans-executions/2026-09-10-2222-kvm-review-fixes.md` with the status checklist when
+Step 1 lands.
+
+### Step 1 — Unwedge deletion (finding 1)
+- `api/v1alpha1/proxy_types.go`: add `AnnotationSkipCleanup = "crawl.example.com/skip-cleanup"`
+  beside `AnnotationSpecHash` ([:59-63](api/v1alpha1/proxy_types.go#L59-L63)), documented as
+  abandoning the backend instance.
+- `internal/controller/proxy_controller.go`, `reconcileDelete`: honour the annotation after
+  the empty-providerID branch and **before** the provider lookup, so it also covers a
+  provider that is no longer configured. Log the abandoned providerID at Info.
+  `ProxyReconciler` has no `EventRecorder`; do not add one for this.
+- Same function: replace the two `if`s after `Get` with a switch. On
+  `provider.Class(err) == provider.ErrPermanent`, call `Delete` anyway, since it is
+  idempotent, and remove the finalizer when it succeeds rather than requeueing into a `Get`
+  that can never converge. On failure, `deletionFailure(errors.Join(getErr, delErr))`.
+  Guard on `err != nil`, since `Class(nil)` is nil.
+- `internal/provider/kvm/kvm.go`, `Delete`: an unconfigured host segment means there is
+  nothing this operator can clean, so log it and return success, mirroring `removeVolumes`
+  ([kvm.go:506-514](internal/provider/kvm/kvm.go#L506-L514)) for a missing pool. `Get` is
+  unchanged.
+- Tests: extend the deletion table in `internal/controller/reconcile_test.go`
+  ([:357-393](internal/controller/reconcile_test.go#L357-L393)) with permanent-Get plus
+  succeeding Delete, permanent-Get plus failing Delete, the annotation with the provider
+  never called, and the annotation on an unconfigured provider. In `kvm_test.go`, the
+  `"host not configured"` case at [:737](internal/provider/kvm/kvm_test.go#L737) flips from
+  `ErrPermanent` to success; add a named test beside `TestDelete_missingPoolIsNotAnError`
+  ([:715](internal/provider/kvm/kvm_test.go#L715)).
+- Docs: the annotation in `README.md` (~line 163) and `docs/architecture.md` (~line 487);
+  a *Known gaps* entry in `docs/kvm-provider.md` saying a domain on a host removed from
+  config can no longer be cleaned up by the operator, the Proxy still finishes deleting,
+  and the VM must be removed by hand because garbage collection only sweeps configured
+  hosts.
+
+### Step 2 — Validate names that become identifiers (finding 2)
+- `internal/provider/config_kvm.go`: add a name regexp beside `sha256Hex`
+  ([:6](internal/provider/config_kvm.go#L6)) allowing letters, digits, dot, dash and
+  underscore. Apply it to `hosts[].name` after the existing empty check
+  ([:129-131](internal/provider/config_kvm.go#L129-L131)) and to each `images` map key,
+  which becomes the cached volume name `base-<key>.<format>`
+  ([image.go:49-55](internal/provider/kvm/image.go#L49-L55)) and half of the image-cache
+  lock key ([image.go:96](internal/provider/kvm/image.go#L96)). `machineTypes` keys are only
+  compared, never embedded, so leave them.
+- Keep the file's first-error-wins style and `fmt.Sprintf` field paths. The host message
+  should name the consequence: the value becomes part of `hosts/<name>/domains/<vm>`, which
+  `parseProviderID` splits on `/` ([id.go:11-21](internal/provider/kvm/id.go#L11-L21)).
+- Tests: two entries in the `TestLoadConfig_kvmInvalid` table
+  ([config_kvm_test.go:76](internal/provider/config_kvm_test.go#L76)), existing
+  name/yaml/wantErrSub shape.
+
+### Step 3 — Read the SSH material lazily (finding 3)
+- `internal/provider/kvm/conn.go`: keep `newSSHAuth`, and add a lazy dialer that builds it
+  on first dial and caches it once it succeeds, so a `kvm-ssh` Secret created after the
+  manager starts heals on the next reconcile with no restart.
+- `internal/provider/kvm/kvm.go`, `New`: stop calling `newSSHAuth`; pass the lazy dialer.
+  Use the context parameter it currently ignores to log a warning when either file is
+  unreadable at startup, keeping the diagnostic without the exit.
+- Error classes: a missing or unreadable file stays unclassified, so `Class` defaults it to
+  transient and the Proxy retries until the Secret appears; an unparseable key or malformed
+  known_hosts wraps `ErrPermanent`. Distinguish with `errors.Is(err, fs.ErrNotExist)`.
+- Fix the two doc comments promising startup failure
+  ([kvm.go:79-81](internal/provider/kvm/kvm.go#L79-L81),
+  [conn.go:182-185](internal/provider/kvm/conn.go#L182-L185)) and the matching claim in
+  `docs/kvm-provider.md` (~lines 107-109). `DISCOVERY_TOKEN` already sets the house
+  convention for an optional mount: degrade with a loud warning.
+- Tests in `conn_test.go`, which already writes key files: `New` succeeds with no key
+  present, the first dial fails transiently, a dial after the file appears succeeds, and a
+  corrupt key is permanent.
+
+### Step 4 — Guest agent via vendor-data script (findings 4 and 5)
+- `internal/provider/kvm/seed.go`: delete `withGuestAgent`, `isCloudConfig`, `appendUnique`
+  and the `sigs.k8s.io/yaml` import. User-data goes onto the seed exactly as the Proxy
+  supplied it.
+- Add the bootstrap as a `#!/bin/sh` vendor-data script: skip when the agent is already
+  present, otherwise install `qemu-guest-agent` through whichever of `apt-get`, `dnf`,
+  `yum`, `zypper` or `apk` exists, then `systemctl enable --now`. Run `apt-get update`
+  before installing, since nothing else guarantees a fresh index when the user's config has
+  no `packages` key. Keep it idempotent and let it log to stdout, which lands in
+  `cloud-init-output.log`.
+- When the provider config also sets `vendorData`, wrap both into a MIME multipart
+  vendor-data: the user's document as one part, the script as another. Cloud-init detects
+  multipart by finding `mime-version:` in the first 4096 bytes, so the header must lead.
+- `internal/provider/config_kvm.go`: add `installGuestAgent`, defaulting to on. Off means
+  the seed carries only the user's own vendor-data and IP discovery falls back to the host
+  ARP table. Nothing fails permanently any more, so the error path in `writeSeed`
+  ([kvm.go:252-255](internal/provider/kvm/kvm.go#L252-L255)) narrows to MIME assembly.
+- Tests in `seed_test.go`: user-data byte-identical for cloud-config, Jinja, MIME, gzip and
+  script inputs; the script present in vendor-data; the opt-out; the multipart case
+  including the `mime-version:` position; keep the existing ISO tests.
+- Docs: rewrite the *What lands on the host* passage added earlier today, the `vendorData`
+  row and a new `installGuestAgent` row in the config table, and the ARP note in
+  *Known gaps*. Cite the cloud-init behaviour that forced the change.
+
+### Step 5 — Validate, record, push
+- `gofmt -l internal/`, `go vet ./...`, `go test -short ./...`.
+- `make test-integration-kvm` against beelink, environment as recorded in the KVM execution
+  log.
+- Rebuild the image, `kind load`, restart the manager, delete and recreate both Proxies,
+  curl a test URL through each.
+- `CHANGELOG.md` entry at the top with a real `date` timestamp, noting it supersedes the
+  merge approach described in today's earlier entry. Add a Run 8 section to
+  `docs/plans-executions/2026-09-04-1404-kvm-provider.md` plus this plan's own execution
+  file. Push with the token helper; PR #5 updates in place.
+
+## Verification
+- Unit: the controller cases prove a permanent `Get` no longer pins the finalizer and that
+  the annotation short-circuits before the provider is touched. `TestDelete_*` proves an
+  unconfigured host is a clean delete while `TestGet_*` stays permanent, which is what keeps
+  the live path from recreating VMs.
+- Seed: assert the uploaded user-data equals the Proxy's `cloudInit` byte for byte, which is
+  the property the old merge could not offer.
+- Integration on beelink: Create, IP via the guest agent, ListByTag, Delete. In the guest,
+  `cloud-init status --long` clean and `systemctl is-active qemu-guest-agent` active.
+- In-cluster: both Proxies reach Ready with an IP and return 204 through squid. Their
+  cloud-config defines its own `packages`, which is the exact case that failed in Run 7.
+- Negative check in kind: rename the `kvm-ssh` Secret away and restart. The pod must stay
+  up, the Kubernetes Proxies must stay Ready, and only the KVM Proxies degrade. Restore the
+  Secret and confirm they recover without a restart.
+
+## Out of scope
+- The two items the review noted but did not report: deleting a cached base image while
+  running VMs back onto it, and an ad-hoc image URL always being treated as qcow2. Both are
+  documented known gaps and neither is a regression from this branch.
+- The uncommitted dev-only files, by the user's decision.

docs/plans/2026-09-11-1526-provider-health-checks.md

@@ -0,0 +1,354 @@
+# Plan: Provider health checks — see whether the cluster can reach each provider
+**Created:** 2026-09-11 15:26 UTC
+
+## Context
+
+Today the operator can't say whether a provider's backend is reachable. The `Provider`
+interface ([provider.go:89-109](internal/provider/provider.go#L89-L109)) only has
+Create/Get/Delete/ListByTag, and `/healthz` and `/readyz` are bare `healthz.Ping`. A
+broken KVM path only shows up later: as a Proxy stuck in Provisioning, or as an orphan-GC
+sweep logging an error. Several failures stay invisible until a Proxy needs that host:
+
+- a missing `kvm-ssh` Secret;
+- a rotated host key;
+- a Tailscale route that is down;
+- a storage pool that is inactive or full.
+
+The goal is a periodic sanity check of every configured provider from inside the cluster,
+with results you can see through `kubectl` and Prometheus.
+
+Decisions made by the user during planning:
+
+1. **Surface:** a new operator-owned, read-only, cluster-scoped **`ProviderHealth`** CRD,
+   with one object per configured provider, a `Ready` condition and per-target detail.
+   Also Prometheus gauges, and a log line whenever a target goes up or down. No
+   EventRecorder, matching the earlier decision.
+2. **Scope:** real checks for all three providers.
+   - **kvm:** checked per host.
+   - **kubernetes:** confirms the API server and RBAC work.
+   - **gcp:** confirms credentials and the API work.
+3. **KVM depth:** reachability plus the storage pool.
+   - Over the existing pooled SSH+libvirt connection, call the libvirt version and
+     hostname RPCs.
+   - Confirm the configured pool is running and report its free space.
+   - No bridge check.
+
+Not affected: the Proxy reconcile behaviour, and the manager's `/healthz` and `/readyz`.
+A dead hypervisor must never make the manager unready.
+
+## Design
+
+### Contract: `internal/provider/check.go`
+
+```go
+type TargetStatus struct {
+    Name, Address   string
+    Ready           bool
+    Reason, Message string
+    Latency         time.Duration
+    Details         map[string]string
+}
+// Check never errors; each target carries its own verdict. Must return promptly once ctx ends.
+type Checker interface{ Check(ctx context.Context) []TargetStatus }
+```
+
+- **Shared reasons:** `Reachable`, `Timeout`, `Unreachable`, `Unauthorized`, `Forbidden`,
+  `APIError`.
+- **Compile-time checks:** each provider's `check.go` has `var _ provider.Checker = (*Provider)(nil)`.
+- **Bypass the wrappers.** `cmd/main.go:197-201` wraps every provider in
+  `WithTracing(WithMetrics(...))`, so the Checkers are collected from the **raw** providers
+  map, before that loop. As a result, checks don't inflate `provider_requests_total`.
+
+### The checks
+
+**kvm** (`internal/provider/kvm/check.go`)
+- **Concurrency:** one goroutine per `cfg.Hosts` entry (`wg.Go`), writing to `out[i]` so
+  results stay in config order.
+- **Inside `p.hosts.call`:** `ConnectGetLibVersion`, `ConnectGetHostname`,
+  `StoragePoolLookupByName(host.Pool)`, then `StoragePoolGetInfo`.
+- **Rules for `fn`:**
+  - It returns only raw RPC errors. `call` drops the shared connection on any other error
+    ([conn.go:160](internal/provider/kvm/conn.go#L160)), so "pool not running" is decided
+    after `call` returns, never returned from `fn`.
+  - It fills a captured result struct only after all four calls succeed. That struct is
+    read only when `call` returned nil, which avoids a race with an abandoned goroutine.
+- **Details:** `hostname`, `libvirtVersion` (decoded as major.minor.patch), `pool`,
+  `poolState`, `poolCapacity` and `poolAvailable` (in GiB).
+- **New RPCs:** add the three to `libvirtAPI` ([conn.go:29-47](internal/provider/kvm/conn.go#L29-L47)),
+  `tracingAPI` (`trace.go`) and `fakeAPI` (`fake_test.go`). `fakeAPI` gets a hostname, a
+  fixed version, and a per-pool state map that defaults to Running.
+- **Telling failure stages apart without matching strings:** add
+  `type dialError struct{ stage dialStage; err error }` in `conn.go`, with `Error()`
+  returning the inner message unchanged and `Unwrap()`. It wraps at four places:
+  - `lazySSH.dial`, when loading the key material fails (`ssh-material`)
+  - `dialSSH`, at the dial step (`tcp`)
+  - `dialSSH`, at the handshake (`ssh-handshake`)
+  - `connectLibvirt` (`libvirt-connect`)
+
+  Because `Unwrap` is kept, `classify` and `isClassified` still see `ErrPermanent` and
+  `fs.ErrNotExist`, so reconcile behaviour doesn't change.
+- **`checkReason(err)`, tested in order:**
+
+  | Condition | Reason |
+  |---|---|
+  | stage ssh-material | `SSHMaterialUnavailable` |
+  | stage tcp | `Unreachable` |
+  | stage ssh-handshake | `SSHHandshakeFailed` |
+  | stage libvirt-connect | `LibvirtConnectFailed` |
+  | ctx deadline or cancel | `Timeout` |
+  | `ErrNoStoragePool` | `PoolNotFound` |
+  | any other libvirt error | `LibvirtError` |
+  | anything else | `Unreachable` |
+  | success, pool not running | `PoolNotActive` |
+  | success | `Reachable` |
+- **Accepted behaviour:**
+  - A check that hits its timeout drops the shared connection. That already happens for
+    any cancelled reconcile context, and it only fires when the host isn't answering.
+  - The leader now keeps one SSH connection per host open from startup, with keepalives
+    every 15s. Before, the connection only opened with the first KVM reconcile.
+  - No forced periodic redial. `lazySSH` caches the key material forever, so a redial
+    would not pick up a rotated Secret anyway; that gap goes into the docs under
+    "Known gaps".
+
+**kubernetes** (`internal/provider/kubernetes/check.go`)
+- **Call:** `List` pods with the managed-label selector and `client.Limit(1)`,
+  cluster-wide. It uses the same uncached client, verb and RBAC as `ListByTag` and GC.
+  There is no namespace setting in the kubernetes config.
+- **Target:** one target named `apiserver`, with the address from a new `apiHost` field
+  set in `NewWithTransportWrapper`.
+- **Reasons:**
+  - `Timeout`: the context ended, or the error is `IsTimeout`/`IsServerTimeout`.
+  - `Unauthorized`, `Forbidden`.
+  - `APIError`: any other API status.
+  - `Unreachable`: anything else.
+
+**gcp** (`internal/provider/gcp/check.go`, plus a seam change in `gcp.go`)
+- **Why a new seam method:** the existing `AggregatedList` seam reads every page
+  ([gcp.go:63-78](internal/provider/gcp/gcp.go#L63-L78)), and MaxResults is only the page
+  size. So add `Probe(ctx, *computepb.AggregatedListInstancesRequest) error`, which makes
+  one `it.Next()` call and treats `iterator.Done` as nil.
+- **Request:** `Project`, `MaxResults: 1`, `ReturnPartialSuccess: true`, and the same
+  managed-label filter `ListByTag` uses.
+- **Target:** one per project, `Name: cfg.Project`, `Address: compute.googleapis.com`.
+  GCP config has no zones, and reading them from Proxy placements would tie the check to
+  the Proxy cache.
+- **Reasons:**
+  - `Timeout`: the context ended.
+  - For `*googleapi.Error`: 401 is `Unauthorized`, 403 is `Forbidden` (this also covers a
+    disabled API), anything else is `APIError`.
+  - `Unreachable`: anything else, including a failure to get a token.
+
+### The `ProviderHealth` CRD: `api/v1alpha1/providerhealth_types.go`
+
+- **Written by hand**, since kubebuilder isn't installed; `PROJECT` is edited by hand too.
+- **Scope and markers:** cluster-scoped, `shortName=ph`, status subresource, no spec.
+- **Scheme registration:** add the type to the existing `AddKnownTypes` call in
+  [proxy_types.go:362](api/v1alpha1/proxy_types.go#L362), not a second `init()`.
+- **Status:**
+  - `type`
+  - `conditions` (`Ready`; `ConditionReady` constant)
+  - `targets[]`, as a list-map keyed by `name`, each with: `name`, `address`, `ready`,
+    `reason`, `message` (at most 1024 characters), `latencyMillis`,
+    `lastTransitionTime`, `details`
+  - `targetsReady` (for example `"1/2"`, used by a printer column)
+  - `lastCheckTime`
+- **Printer columns:** Type, Ready, Targets, Last Check, Age.
+- **Viewer role:** `config/rbac/providerhealth_viewer_role.yaml`. No editor role, since the
+  operator owns the object.
+
+### Monitor: `internal/providerhealth`
+
+- **Fields:** `Monitor{Client, Reader, Targets []Target{Name, Type, Checker}, Interval, Timeout}`.
+- **Scheduling:**
+  - `NeedLeaderElection() true`.
+  - Checks once immediately, then on a ticker. Copy the ticker loop from
+    [gc.go:54-80](internal/gc/gc.go#L54-L80).
+- **Each cycle:**
+  - Runs every provider concurrently under `context.WithTimeout(Timeout)`, inside one
+    `tracing.Start` span.
+  - Stores the snapshot under a mutex; `Snapshot()` returns a deep copy.
+  - Logs changes. Going down logs `WARNING: provider target not ready` with provider,
+    target, address, reason and message. Coming back logs `Provider target ready`. A
+    target that is ready on first sight logs at V(1) only.
+- **Writes:**
+  - Get through `mgr.GetAPIReader()`, not the cache. That avoids a cluster-wide informer
+    and the watch permission it would need, whatever `--proxy-namespace` is set to.
+  - Create the object if it is missing, labelled
+    `app.kubernetes.io/managed-by=egress-proxies-operator`.
+  - Then `Status().Patch(MergeFrom(base))` when the status changed, following
+    [status.go:114-121](internal/controller/status.go#L114-L121).
+  - This means one write per provider per interval, because `lastCheckTime` moves each
+    time. That's acceptable given the 10s minimum interval.
+- **`buildStatus(prev, result, now)` is a pure function:**
+  - A target keeps its `lastTransitionTime` unless `ready` flipped.
+  - The `Ready` condition is set through `apimeta.SetStatusCondition`:
+    - `True/AllTargetsReady` when every target is ready
+    - `False/TargetsNotReady` otherwise, with the failing targets in the message
+    - `Unknown/ChecksNotImplemented` when the provider has no Checker
+    - `Unknown/NoTargets` when there are zero targets
+- **Cleanup, once at `Start`:** delete labelled objects whose name isn't in the config.
+  The config is loaded once per process, and unlabelled objects are never touched.
+- **Interval 0 (disabled):** run the cleanup with nothing kept, then return nil, so no
+  object is left showing a stale `Ready=True`.
+- **Invalid provider names:** provider names aren't validated as DNS-1123 names. For an
+  invalid name, warn once and skip the object writes; metrics are still emitted.
+- **`ValidateSchedule(interval, timeout)`** is called in `main.go` before the manager
+  starts. Interval must be 0 or at least 10s; timeout must be in (0, interval].
+- **RBAC markers on `Monitor`:**
+  - `providerhealths`: get, list, create, delete
+  - `providerhealths/status`: get, patch
+  - `make manifests` regenerates `config/rbac/role.yaml`.
+
+### Metrics and wiring
+
+- **New function:** `metrics.RegisterProviderTargets(reg, read func() []ProviderTarget) error`.
+  `Register`'s signature is left alone.
+- **Gauges:** a scrape-time collector, in the style of `constCollector`, reads the Monitor
+  snapshot and emits three gauges labelled `{provider,target}`:
+  - `proxy_operator_provider_target_up`
+  - `proxy_operator_provider_check_duration_seconds`
+  - `proxy_operator_provider_check_timestamp_seconds`
+- **Conversion closure:** the one that turns the Monitor snapshot into gauges lives in
+  `main.go`, next to `proxyPhaseCounts`. So `providerhealth` doesn't import `metrics`.
+- **No pool-bytes gauge:** that value is in status, and node-exporter is the right source
+  for disk alerts.
+- **`cmd/main.go`:**
+  - New flags `--provider-check-interval` (1m) and `--provider-check-timeout` (15s).
+  - Build the `Targets` from `cfg.Providers` in config order, with
+    `c, _ := raw[pc.Name].(provider.Checker)`.
+  - `mgr.Add(&providerhealth.Monitor{Client: mgr.GetClient(), Reader: mgr.GetAPIReader(), ...})`,
+    then register the collector.
+
+## Steps (one commit each, execution-log entry per step)
+
+**Step 0 — Save the plan.**
+- Create branch `feat/provider-health` off `main`.
+- Copy this plan to `docs/plans/2026-09-11-1526-provider-health-checks.md` and commit it
+  alone.
+- The uncommitted `.claude/settings.json` change stays out of every commit.
+
+**Step 1 — ProviderHealth API and CRD.**
+- `api/v1alpha1/providerhealth_types.go`
+- `zz_generated.deepcopy.go`, from `make generate`
+- `config/crd/bases/crawl.example.com_providerhealths.yaml`, from `make manifests`
+- the CRD and RBAC `kustomization.yaml` files, plus the viewer role
+- `PROJECT`
+
+**Step 2 — Contract plus the kubernetes and GCP checks.**
+- `internal/provider/check.go`
+- `internal/provider/kubernetes/{kubernetes.go,check.go,check_test.go}`
+- `internal/provider/gcp/{gcp.go,check.go,check_test.go,gcp_test.go}`
+
+**Step 3 — KVM check.**
+- `internal/provider/kvm/{conn.go,trace.go,check.go,check_test.go,fake_test.go,conn_test.go,trace_test.go,kvm_integration_test.go}`
+
+**Step 4 — Monitor.**
+- `internal/providerhealth/{monitor.go,status.go,monitor_test.go,status_test.go,envtest_test.go}`
+- `config/rbac/role.yaml`, from `make manifests`
+
+**Step 5 — Metrics and wiring.**
+- `internal/metrics/{metrics.go,metrics_test.go}`
+- `cmd/main.go`
+- `config/manager/manager.yaml`: a comment for the new flags
+- `test/e2e/e2e_test.go`
+
+**Step 6 — Docs.**
+- `docs/architecture.md`: a component row plus a provider-health section
+- `docs/kvm-provider.md`: the reason table, the shared-connection behaviour, and the
+  Secret-rotation gap under "Known gaps"
+- `docs/testing.md`: the new tests and the commands
+- `CLAUDE.md`: its Architecture and Gotchas sections still say "empty scaffold"; point
+  them at `docs/architecture.md`
+
+**Step 7 — Validate, record, push, open the PR.**
+- Run `go vet ./...`, `go test -short ./...`, `make test` and `make test-e2e`.
+- Hand the user the KVM commands below, and record their results in the execution log.
+- Add a CHANGELOG entry once the user confirms.
+- Push, using the token-file credential helper from memory.
+- Open the PR against `main` through the Gitea API (`tea` isn't installed).
+- Don't merge.
+
+## Tests
+
+**Provider checks** (table-driven, `Test<Func>_<scenario>`):
+- kubernetes:
+  - reachable
+  - list options: `Limit==1`, the label selector, no namespace
+  - failure reasons: forbidden, unauthorized, timeout, cancelled, internal, and a
+    network `*url.Error`
+- gcp:
+  - reachable, asserting the request fields
+  - failure reasons: 401, 403, 500, deadline, network
+  - `TestRealInstances_Probe_fetchesOnePage` against `httptest`: exactly one request with
+    `maxResults=1`, even though the page has a `nextPageToken`
+- kvm (using `hostFakes`/`newTestProvider` from `kvm_test.go`):
+  - reachable, with details filled in
+  - per-host verdicts, order preserved
+  - pool inactive, pool missing
+  - hosts checked concurrently (a sequential check would deadlock)
+  - a pool verdict keeps the connection (the dial count stays 1)
+  - a `checkReason` table
+  - `TestSSHAuth_dial_stages`, using the in-process SSH server in `conn_test.go`
+  - three new rows in the tracing table
+
+**Monitor:**
+- Setup: the fake client is built with a status subresource and a RESTMapper that
+  registers ProviderHealth as cluster-scoped.
+- Tests:
+  - `buildStatus` table
+  - objects are created with the label, and existing ones are patched
+  - the timeout bounds a checker that blocks
+  - a nil checker gives Unknown
+  - only transitions are logged (`funcr` logger)
+  - cleanup deletes only labelled objects that aren't configured
+  - the disabled path cleans up and returns
+  - an invalid name skips the writes
+  - `ValidateSchedule` table
+- An envtest round-trip, skipped under `-short`, which proves the generated schema is
+  accepted by a real apiserver.
+
+**Metrics:** `TestRegisterProviderTargets_scrape`, using a fresh registry and
+`GatherAndCompare`; an empty snapshot produces no series.
+
+**e2e (kind):** `providerhealth kubernetes` reaches `Ready=True`, which proves the
+generated RBAC works end to end.
+
+## Verification
+
+**Automated (Claude runs):** `go vet ./...`, `go test -short ./...`, `make test` (envtest
+with `-race`) and `make test-e2e`.
+
+**Against the real hypervisor (handed to the user, not run by Claude)**, with beelink as
+the host:
+
+```bash
+KVM_TEST_HOST=beelink.tail8c69c.ts.net:22 KVM_TEST_USER=novakj KVM_TEST_SSH_KEY=~/.ssh/id_ed25519 \
+KVM_TEST_KNOWN_HOSTS=~/.ssh/known_hosts KVM_TEST_BRIDGE=<bridge> KVM_TEST_IMAGE_URL=<image-url> \
+  go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/
+
+make install && go run ./cmd/main.go --providers-config <scratch copy of kvm-home config, local key paths> \
+  --metrics-bind-address :8080 --metrics-secure=false --provider-check-interval=15s
+kubectl get ph -w
+kubectl get ph kvm-home -o yaml
+curl -s localhost:8080/metrics | grep -E 'proxy_operator_provider_(target_up|check_)'
+```
+
+Each negative case is a one-line edit to the scratch config; nothing changes on the host.
+
+| Edit | Expected reason |
+|---|---|
+| `pool: nope` | `PoolNotFound` |
+| `address: …:2` | `Unreachable` |
+| `knownHostsFile: /dev/null` or `user: nobody` | `SSHHandshakeFailed` |
+| `sshKeyFile: /nonexistent` | `SSHMaterialUnavailable` |
+| `socket: /nonexistent` | `LibvirtConnectFailed` |
+
+## Out of scope
+
+- Gating Proxy creates on a host that is known to be down.
+- Tying `/readyz` to provider health.
+- Bridge checks.
+- SelfSubjectAccessReview checks of create/delete rights.
+- Hot-reloading the providers config.
+- Picking up a rotated `kvm-ssh` Secret without a restart. This is an existing gap; it
+  gets documented.

docs/plans/2026-09-16-0841-provider-blueprint.md

@@ -0,0 +1,119 @@
+# Plan: Provider blueprint (docs/providers.md)
+**Created:** 2026-09-16 08:41
+
+## Context
+
+The provider contract is spread across `docs/prompts/__initial-prompt.md` §5,
+the comments in `internal/provider/{provider,errors,check}.go`, `docs/architecture.md`
+§5 and §11, `docs/kvm-provider.md`, and plan Step 2/3/8. Many rules exist only in
+code: providerID must be self-contained, UID/CreatedAt must be read back, Create
+must survive a crash partway through, calls must stop when ctx is cancelled,
+decorators are wired in `cmd/main.go`, and so on. With "~5 providers soon", the
+next provider author needs one blueprint. It should say what a provider MUST,
+SHOULD or MAY do, show which features each existing provider (kubernetes, gcp,
+kvm) actually implements (with file:line evidence), give a checklist for adding
+one, and list the gaps.
+
+Decisions (user): store in `docs/providers.md`; scope = current state plus a
+separate "Gaps and proposed features" section (nothing implemented); wait for
+`feat/provider-health` to merge, then branch off `main`.
+
+## Step 0 — Save the plan; wait for the merge
+
+- Right after ExitPlanMode, copy this plan to `docs/plans/2026-09-16-0841-provider-blueprint.md`
+  and commit it on its own. The current branch is `feat/provider-health`, so the
+  plan reaches `main` with that merge. CLAUDE.md requires the plan commit to
+  come before any branching.
+- Stop and wait for the user to confirm that `feat/provider-health` is merged.
+  Then `git switch main && git pull`, and `git switch -c feat/provider-blueprint`.
+
+## Step 1 — Collect evidence (read-only, in parallel)
+
+Run 3 Explore agents at the same time, one each for `internal/provider/kubernetes`,
+`internal/provider/gcp` and `internal/provider/kvm`, including their tests. Each
+answers the **same questionnaire**, citing `file:line` for every answer and writing
+"not implemented" explicitly when that's the case:
+
+1. **Identity**: providerID format and parsing; what it carries (zone, host, namespace).
+2. **Create**: how it stays idempotent (name lookup, 409, DomainLookupByName);
+   whether it returns before the VM is up or waits; what happens when an earlier
+   attempt crashed partway; which Placement fields are used or ignored; how
+   cloud-init is delivered; how the port is used; how GC labels/UID are attached.
+3. **Get**: native state → InstanceState mapping; IP discovery rules; UID and
+   CreatedAt readback; how NotFound is detected.
+4. **Delete**: idempotency; what gets removed (disks, volumes, seeds); returns
+   before or after the resource is gone; unknown/malformed providerID behaviour.
+5. **ListByTag**: scope (namespaces, zones, hosts); partial-failure behaviour; how
+   managed resources are recognised.
+6. **Error classification**: table of backend error → NotFound/Quota/Transient/
+   Permanent; whether quota is detected at all.
+7. **Health check** (`Checker`): targets, provider-specific Reasons, Details keys.
+8. **Config**: YAML section, defaults, validation (`internal/provider/config*.go`),
+   credentials/Secrets, which settings are read once per process.
+9. **Connections and cancellation**: pooling, reconnects, timeouts, ctx honouring.
+10. **Observability**: log keys and verbosity levels, spans (`kvm/trace.go`, gcp
+    `wirelog.go`), anything beyond the shared metrics/tracing decorators.
+11. **Testing**: fakes, envtest/kind, `//go:build integration` tests, what is
+    not covered.
+12. **Known gaps**: TODOs, stale comments, documented limitations.
+
+Meanwhile, read the shared layer myself: `internal/provider/{provider,errors,check,
+name,config,metrics,tracing}.go`, `internal/provider/registry/registry.go`,
+`cmd/main.go` (registry wiring, decorator order, Checker collection), and the
+places that consume the contract, so every MUST rule traces to a caller that depends on it:
+`internal/controller` (error-class branching, ErrNotFound → replace/adopt),
+`internal/gc` (UID, CreatedAt age skip, ListByTag errors),
+`internal/providerhealth`. Then cross-check against the existing docs listed in Context.
+
+## Step 2 — Write `docs/providers.md`
+
+Sections:
+
+1. **What a provider is**: a few sentences plus links to architecture.md §5/§11.
+2. **The contract**: per method (Create/Get/Delete/ListByTag), MUST/SHOULD/MAY
+   rules, each followed by *why*: the reconciler or GC behaviour it protects,
+   linked to code.
+3. **Error taxonomy**: what each class makes the reconciler do; how to map backend
+   errors; "unclassified = transient" pitfall.
+4. **Identity and GC labels**: providerID guidance, `LabelManaged`/`LabelUID`,
+   CreatedAt, deterministic `NameFromUID`.
+5. **Optional interfaces**: `Checker` (shared reasons, never fails as a whole,
+   collected before decorators).
+6. **Feature matrix**: one row per questionnaire item, columns kubernetes/gcp/kvm,
+   cells `yes`/`partial`/`no` plus a short note. Put the file:line evidence in a
+   compact per-provider notes subsection under the table, not in the cells.
+7. **Adding a provider: checklist**: package layout, config section and validation,
+   registry constructor, `cmd/main.go` wiring, Checker, error mapping with tests,
+   `name`/label tests, integration test behind a build tag, provider doc page,
+   architecture.md rows, demo script (optional).
+8. **Gaps and proposed features**: current-state gaps per provider (e.g. no kvm
+   quota detection, orphan volumes invisible to GC, Secret rotation needs a
+   restart, ListByTag partial-failure inconsistency, stale `mock` references in
+   `provider.go` comments), plus proposals that are clearly marked as not
+   implemented. Only list them; don't design them.
+
+Also: link it from `docs/architecture.md` (Components table → provider contract row)
+and from the CLAUDE.md "Architecture" section. Leave per-provider docs where they are.
+
+Follow the plan-execution convention: create `docs/plans-executions/2026-09-16-0841-provider-blueprint.md`
+with the status checklist and step entries, and commit it with each step.
+
+## Step 3 — Verify, then open the MR
+
+- A verification agent (Explore) re-checks each matrix cell and each MUST rule
+  against the cited file:line and reports any mismatch. Fix what it finds.
+- Check that every relative link in the new and edited docs resolves (grep the
+  `](` targets and `ls` each one).
+- `go build ./... && go test -short ./...` as a sanity check (docs-only change,
+  expected no-op).
+- Push `feat/provider-blueprint` and open the MR against `main` via the Gitea API,
+  following the project's push/MR memory note. After the user confirms, add a CHANGELOG.md entry.
+
+## Critical files
+
+- New: `docs/providers.md`, `docs/plans/2026-09-16-0841-provider-blueprint.md`,
+  `docs/plans-executions/2026-09-16-0841-provider-blueprint.md`
+- Edited: `docs/architecture.md` (link), `CLAUDE.md` (link)
+- Read-only sources: `internal/provider/**`, `cmd/main.go`, `internal/controller`,
+  `internal/gc`, `internal/providerhealth`, `docs/prompts/__initial-prompt.md`,
+  `docs/kvm-provider.md`, `docs/gcp-vm-validation.md`

docs/reviews/2026-08-10-1134-proxy-operator-pr-review.md

@@ -0,0 +1,131 @@
+# PR review findings: feat/proxy-operator
+
+**Created:** 2026-08-10 11:34
+**Scope:** `origin/main...feat/proxy-operator` (merge-base 076bc66, 25 commits, ~80 files)
+**Reviewers:** `go-operator-reviewer` + `operator-reviewer` agents; findings consolidated, most severe first. Check off items as they're processed.
+
+Both reviewers rated the core reconcile architecture sound: single status writer with one
+deferred patch, finalizer added before any provider call, Get-before-RemoveFinalizer on
+delete, CEL immutability rules correctly split to avoid the oldSelf-on-CREATE trap,
+leader-election gating on destructive runnables, GC tombstone rules (MinAge, UID-less
+instances never deleted).
+
+## Merge-blockers
+
+- [ ] **Discovery leases proxies with an empty IP** — found independently by both reviewers.
+  `internal/discovery/handlers.go:151`, `internal/controller/proxy_controller.go:226`
+  During instance replacement (and the Get→NotFound recovery path) the reconciler clears
+  `status.ip` but only the create branch removes the `Healthy` condition, and the health
+  engine prunes state for empty-host proxies so nothing refreshes it. For the whole
+  delete→recreate window (minutes on GCP), `isHealthy` still returns true and
+  `handleAcquireLease` grants `201 Created` with `"ip": ""`, burning a `MaxLeases` slot.
+  **Fix:** add `EffectiveHost() != ""` to `isHealthy` (covers list + acquire), and
+  remove/downgrade `Healthy` wherever `status.IP` is cleared.
+
+- [ ] **Orphan GC deletes other installations' fleets in a shared GCP project.**
+  `internal/provider/gcp/insert.go:57`, `internal/gc/gc.go:93`
+  Instances are tagged only `proxy-operator-managed=true` + CR UID; the sweeper deletes any
+  tagged instance whose UID isn't in *its own cluster's* Proxy list. Two clusters sharing a
+  GCP project delete each other's VMs every GC interval in a permanent loop.
+  **Fix:** add an installation-identity label (cluster/deployment ID) set by both providers
+  and filtered on in `ListByTag`.
+
+- [ ] **Permanent-error latch wedges proxies on failures that aren't spec-caused.**
+  `internal/controller/proxy_controller.go:126`
+  Latch keys on `observedGeneration == generation`, but two failure inputs live outside the
+  spec: an unconfigured provider (config fix + restart doesn't bump generation, and
+  `spec.provider` is CEL-immutable → stuck `Failed` short of deleting the CR) and resolved
+  Secret content (Secret fix enqueues a reconcile that short-circuits at the latch before
+  re-resolving cloud-init).
+  **Fix:** latch should also consider current spec-hash / provider availability.
+
+## Worth fixing
+
+- [ ] **Deletion-path failures invisible in status** — flagged by both reviewers.
+  `internal/controller/proxy_controller.go:319`, `:266`
+  `deletionFailure` swallows `ErrQuotaExceeded` (nil error, no status write); unconfigured
+  provider returns a bare error forever. A Proxy wedged in `Deleting` shows nothing in
+  `kubectl describe`. Stage `setProvisioned(p, False, ReasonDeleting, ...)` before returning.
+  Also: `Delete` is resubmitted on every `DeletionPoll` pass, churning GCP quota — a state
+  check on the `Get` result would avoid it.
+
+- [ ] **Lost providerID on `setSpecHash` conflict.**
+  `internal/controller/proxy_controller.go:161`
+  On Update conflict the function returns before `p.Status.ProviderID = id`, so the deferred
+  patch persists an empty providerID for a just-created instance. Self-heals via GC.
+  **Fix:** set `p.Status.ProviderID = id` before returning the error (one line).
+
+- [ ] **Terminating pods still report `StateRunning`.**
+  `internal/provider/kubernetes/kubernetes.go:151`
+  A pod with a deletionTimestamp keeps `phase=Running` + `PodIP` while terminating, so drift
+  reconcile republishes `Provisioned=True` and discovery keeps leasing a dying pod.
+  **Fix:** map non-zero `pod.DeletionTimestamp` to `StateTerminated` in `instanceFromPod`.
+
+- [ ] **Stale-cache spec-hash race deletes the freshly created replacement instance.**
+  `internal/controller/proxy_controller.go:176`
+  Instance name derives from CR UID, so old and new instances share a providerID. A reconcile
+  served a cached object from before a just-completed replacement re-enters `replaceInstance`
+  and deletes the *new* healthy instance. Converges, but destroys a good instance.
+  **Fix:** re-read uncached before the destructive branch, or compare `inst.CreatedAt`
+  against the annotation-update time.
+
+- [ ] **`observedGeneration` written before the generation is actually processed.**
+  `internal/controller/status.go:114`
+  Set unconditionally in `patchStatusIfChanged`, including on the finalizer-add pass and
+  `resolveCloudInit` failures — misleads kstatus-style tooling. Set it only once the state
+  machine has genuinely evaluated the spec.
+
+- [ ] **No event filtering on the Proxy watch.**
+  `internal/controller/proxy_controller.go:402`
+  Every self-inflicted status patch triggers a follow-up reconcile with an extra cloud `Get`,
+  roughly doubling provider read traffic. Caution: a plain `GenerationChangedPredicate`
+  breaks the finalizer flow (relies on its own Update event to re-enter) — needs a
+  status-only/resourceVersion-only filter or an explicit requeue in the finalizer pass.
+
+- [ ] **Unlabelled cloud-init Secrets produce a misleading NotFound with endless backoff.**
+  `internal/controller/proxy_controller.go:344`, `cmd/main.go:210`
+  The label-restricted cache turns "exists but missing `crawl.example.com/cloud-init=true`"
+  into `CloudInitError: not found`. Mention the label requirement in the condition message,
+  or read via uncached `APIReader` and validate the label explicitly.
+
+- [ ] **RBAC over-grant.**
+  `config/rbac/role.yaml:25`
+  `create;delete` on `proxies` is scaffold residue (controller never creates/deletes CRs);
+  cluster-wide `pods create/delete` and `secrets get/list/watch` apply even when only the GCP
+  provider is configured — pod rules belong in an optional kustomize component.
+
+## Simplifications
+
+- [ ] **Delete `internal/provider/registry`** — 14 lines of logic, one caller
+  (`cmd/main.go:148`); fold `Build`/`Constructor` into the composition root. Also fixes the
+  two-sources-of-truth problem: `internal/provider/config.go:84` hardcodes
+  `"kubernetes"`/`"gcp"` while `registry.Build` dispatches through a caller-supplied map —
+  validate against the constructor map instead. Net −1 package, −44 lines, −92 test lines.
+
+- [ ] **Collapse `LeaseStore` interface to `*lease.Store`.**
+  `internal/discovery/server.go:28`
+  Single implementation and not a test seam (tests wire the real `lease.NewStore`).
+  Keep `HealthSnapshotter` and `instancesAPI` — those are genuine seams.
+
+- [ ] **Replace metrics nil-guards with no-op defaults.**
+  `internal/health/engine.go:68`, `internal/discovery/server.go:38`,
+  `internal/provider/metrics.go:8`
+  Keep the interfaces (legit "no prometheus in domain packages" rationale) but default the
+  fields to a no-op impl — `provider.WithMetrics` already dereferences unconditionally, so
+  the guards are inconsistent anyway.
+
+## Nice-to-have
+
+- [ ] Add an `OwnerReference` to provider pods (`internal/provider/kubernetes/pod.go:24`) —
+  free cascading deletion if the finalizer is ever bypassed; `CreateRequest` already carries
+  Namespace/ProxyName/UID.
+- [ ] `Close()` the GCP `*compute.InstancesClient` (`internal/provider/gcp/gcp.go:89`) —
+  harmless today, a leak the moment providers are rebuilt on config reload.
+- [ ] Fix `.golangci` config: it references a missing `logcheck` plugin, so the linter only
+  runs with the project config disabled.
+- [ ] External-mode endpoint edits don't reset health-engine counters
+  (`internal/health/engine.go:206` keys on name+UID): flipping `endpoint.host` keeps the old
+  host's `Healthy=True` for `failureThreshold × interval`. Arguably a replacement, not a flap.
+- [ ] `init()` funcs at `api/v1alpha1/proxy_types.go:353` and `cmd/main.go:67` conflict with
+  the repo's "no `init()`" convention; kubebuilder-idiomatic, but scheme registration could
+  use the scaffold's `SchemeBuilder.Register` at package var scope.

docs/testing.md

@@ -0,0 +1,233 @@
+# Testing
+
+What tests exist, how to run them, and where the boundaries of the
+automated suites are. The philosophy throughout: **test against the most
+real double available** — a real envtest API server over a fake client, a
+real CONNECT-capable proxy stub over a mocked HTTP client, the real lease
+store under the discovery handlers — and leave the gaps that only real
+infrastructure can close to the kind verification run, documented at the
+bottom.
+
+## Running
+
+```sh
+make test              # THE canonical run: codegen + fmt + vet + envtest setup,
+                       # then `go test -race` on every package, with coverage
+go test -short ./...   # skip the envtest suite (runs in <1s per package)
+go tool cover -html=cover.out   # browse coverage from the last make test
+```
+
+Targeted runs:
+
+```sh
+go test -race ./internal/lease/                  # one package
+go test -race -run TestAcquire ./internal/lease/ # one test (or a prefix)
+go test -race -count=2 ./internal/health/        # flake-shaking: run twice
+```
+
+**Gotcha — plain `go test ./...` fails the controller package** with an
+error like `/usr/local/kubebuilder/bin/etcd: no such file`. That is not a
+code problem: the envtest suite needs `KUBEBUILDER_ASSETS` pointing at the
+API-server/etcd binaries, which only `make test` sets up (via
+`setup-envtest`). Either use `make test`, or export it once:
+
+```sh
+KUBEBUILDER_ASSETS="$PWD/bin/k8s/<version>-<os>-<arch>" go test -race ./...
+```
+
+Everything runs with `-race` — `make test` enforces it; keep the flag on
+manual runs too.
+
+## The suites, package by package
+
+| Package | Files | What is covered | Test double / technique |
+|---|---|---|---|
+| `api/v1alpha1` | `helpers_test.go` | `EffectivePort/EffectiveHost/HealthCheckOrDefault/MaxLeasesOrDefault` | pure table-driven; CEL rules are **not** unit-testable — see envtest below |
+| `internal/controller` | `reconcile_test.go` | every row of the reconciler's action table: create, poll, publish IP, replace, adopt, NotFound recovery, delete/finalizer, quota/permanent/transient errors, cloud-init resolution | `controller-runtime/pkg/client/fake` + an in-test `stubProvider` |
+| | `status_test.go` | `computePhase` truth table (10 rows) | pure |
+| | `spechash_test.go` | hash stability, nil/empty normalization, sensitivity to every replacement-triggering field | pure |
+| | `health_test.go` | Healthy-condition representation: Ready/Unhealthy phases, no-verdict, stale-verdict cleanup on replacement, nil snapshotter | fake client + `fakeSnapshotter` |
+| | `proxy_controller_test.go` (envtest, ginkgo) | full lifecycles against a **real API server**: provision→Running, spec-change replacement, finalizer deletion, External tracking, Ready-through-health, quota-vs-permanent, adoption; **all CEL rules** (six invalid creates, mode/provider immutability incl. removal) and the `default={}` materialization | envtest apiserver + stub provider; the only place CEL and structural defaulting actually execute |
+| `internal/provider` | `name_test.go` | deterministic naming: idempotency, `^proxy-[a-z2-7]{16}$`, 10k-UID distinctness | pure |
+| | `errors_test.go` | taxonomy: `Class()` mapping, `errors.Is` **and** `errors.As` through the multi-unwrap | pure |
+| | `config_test.go` | providers-config parsing + fail-fast validation | pure |
+| | `metrics_test.go` | `WithMetrics` decorator: classified result labels, error passthrough | fake recorder + static provider |
+| `internal/provider/kubernetes` | `kubernetes_test.go`, `pod_test.go` | Create/Get/Delete/ListByTag over real Pod objects; pure `buildPod`/`squidConf` incl. the `max_filedescriptors` OOM-regression assertion | `client/fake` (real `corev1.Pod`s, no kubelet) |
+| | `check_test.go` | health check: a single cluster-wide list with `Limit=1` and the managed-label selector; the reason for every API error class | `client/fake` + interceptor |
+| `internal/provider/gcp` | `insert_test.go`, `errors_test.go`, `gcp_test.go` | field-by-field `buildInsertRequest`; HTTP-code classification table; zone-qualified IDs, 409-is-success, 9-row state mapping, ListByTag filter + `ReturnPartialSuccess` | pure builder needs **no fake**; the rest uses the flattened `instancesAPI` seam |
+| | `check_test.go` | health check: request shape and the reason per HTTP code or network error; the **real SDK client** makes exactly one request with `maxResults=1`, even when a next page exists | fake seam + `httptest` |
+| `internal/provider/kvm` | `domainxml_test.go`, `volume_test.go`, `seed_test.go`, `id_test.go`, `metadata_test.go` | pure builders: domain/volume XML unmarshalled back and checked field by field, seed ISO reopened with the same library, providerID round-trip, metadata in both namespace spellings | pure |
+| | `errors_test.go`, `conn_test.go` | libvirt-code classification table; connection pool (reuse, redial, drop on transport error, cancel releases a blocked RPC, one dial per host under concurrency); SSH dial, host-key policy, keepalive | fake dialer; a **real in-process `x/crypto/ssh` server** forwarding a unix socket |
+| | `image_test.go`, `kvm_test.go` | image cache (fresh fetch, ready, marker missing, checksum change/mismatch, 4xx/5xx, concurrent callers); Create resume after every partial step, leftover truncated seed, state/IP mapping, every Delete already-gone case, ListByTag across hosts / one down / unreadable domain; logging tiers (V0 silent, V1 one line per RPC, V2 XML) with cloud-init never logged | `httptest` image server + in-memory `fakeAPI` behind the `libvirtAPI` seam |
+| | `trace_test.go`, `conn_test.go`, `errors_test.go` | every `libvirtAPI` method traced exactly once (guarded by reflection on the interface), results and errors pass through, libvirt error fields, detail keys only at V2, upload content never logged; pool connect/drop lines, no wrapping below V1 | `fakeAPI` + `funcr` capture |
+| | `check_test.go`, `conn_test.go` | health check: details, per-host verdicts in config order, hosts checked concurrently, a timeout that keeps the connection and refuses a second check while the first still waits, no waiting behind a reconcile's dial, the reason table; `probe` abandons without closing, but still drops on a transport error; waiting for a host's slot ends with the caller's deadline; an inactive or missing pool and daemon errors keep the connection; every dial stage tagged through the real SSH server, with classification unchanged | `fakeAPI`, fake dialers, in-process SSH server |
+| | `kvm_integration_test.go` (`-tags=integration`) | one real VM: image fetch, Create, idempotent re-Create, Running with IP, ListByTag, Delete, pool state afterwards; `TestKVMCheck_realHost` runs the health check against the host and creates nothing | a **real hypervisor** — see below |
+| `internal/health` | `probe_test.go` | probes through a **real CONNECT-capable proxy stub** to a real TLS server: tunnel success, refused CONNECT, wrong status, dead proxy, plain-http forward | `httptest` + hijacked bidirectional tunnel |
+| | `engine_test.go` | thresholds, first-verdict, latency suppression matrix, dropped-event retry, stale-probe UID guard, tick scheduling/seeding/pruning, `Start` end-to-end | fake `client.Reader` + injected `probeFn` |
+| `internal/lease` | `store_test.go` | capacity, `MaxLeases=0`, all three selection tie-breaks, target-scoped vs global cooldowns, expiry via fake clock, report-on-retained-lease, idempotent release, **40 concurrent acquires never exceeding capacity** | injectable clock; the concurrency case is why `-race` matters |
+| `internal/discovery` | `server_test.go` | auth matrix (incl. `/healthz` exemption), list filtering, grant shape, full 409 arithmetic, invalid TTL/body/result, idempotent release, report→cooldown→409 round trip, `Start`/shutdown | `httptest` over the real handler chain, fake cache reader, **real** `lease.Store` |
+| `internal/gc` | `gc_test.go` | the true-orphan matrix (live/deleting/young/unlabelled all kept), broken-provider isolation, list-failure skips sweep, namespace guard, sweep loop | fake reader + canned-list provider |
+| `internal/providerhealth` | `status_test.go`, `monitor_test.go` | `buildStatus` (the Ready condition, per-target transition times, truncation); objects created with the label and patched, the round timeout, no-checker Unknown, invalid names, transition-only logging, cleanup of labelled unconfigured objects, the checks-off path, `ValidateSchedule` | fake client with a cluster-scoped RESTMapper + `funcr` capture |
+| | `envtest_test.go` | the generated `ProviderHealth` schema against a real API server: cluster scope, status subresource, required fields, list-map keys, the 1024-character message limit | envtest (skipped under `-short`) |
+| `internal/metrics` | `metrics_test.go` | scrape-time collectors (`GatherAndCompare`) incl. the provider-target gauges, per-proxy series cleanup via `ForgetProxy`, label counts, fresh-registry-per-test property | `prometheus/client_golang/testutil` |
+
+Conventions (from `CLAUDE.md`): stdlib `testing`, table-driven by default,
+`t.Parallel()` where safe, tests next to source, ginkgo only in the
+envtest suite the scaffold generated.
+
+## What the automated suites deliberately do NOT cover
+
+- **`New()` constructors that dial real infrastructure**: the kubernetes
+  provider's `New` (connects to whatever your kubeconfig points at), the
+  GCP provider's `New` and its `realInstances` SDK adapter (ADC + real
+  Google endpoints). Both are thin; both are exercised by the kind run /
+  real deployments. Their 0% coverage is by design — do not "fix" it.
+- **CEL and structural defaulting under the fake client**: the fake
+  client runs neither. Every unit test that relies on defaults calls the
+  `*OrDefault` helpers; every CEL rule is asserted in the envtest suite
+  instead.
+- **A container actually starting and serving**: envtest has no kubelet,
+  so a Pod created there sits Pending forever. Whether Squid really comes
+  up and tunnels CONNECT is provable only on a real cluster — that is the
+  kind verification's job, and it is exactly what caught the Squid OOM
+  bug (below).
+- **Live GCP**: no test talks to Google. The seam boundary
+  (`buildInsertRequest` + classification) is tested exhaustively instead.
+- **Live libvirt, by default**: the KVM unit tests run over an in-memory
+  fake of the 20 libvirt calls the provider uses. Whether libvirt accepts
+  the domain XML, whether a raw qcow2 upload plus pool refresh yields a
+  usable backing image, and whether the guest actually boots and reports
+  an address are provable only on a real host — that is the opt-in
+  integration test below.
+
+## The KVM integration test (real hypervisor)
+
+`internal/provider/kvm/kvm_integration_test.go` compiles only under
+`-tags=integration` and skips itself unless the environment names a
+host. It deviates from the "testcontainers for real infra" convention on
+purpose: libvirt needs `/dev/kvm` and a bridge to a DHCP network, and a
+container has neither. So it targets whichever hypervisor you point it
+at and leaves that host's pool with one more cached base image than it
+had.
+
+```sh
+export KVM_TEST_HOST=nas1.home.lan:22          # SSH endpoint
+export KVM_TEST_USER=virt                      # must be in the libvirt group
+export KVM_TEST_SSH_KEY=$HOME/.ssh/id_ed25519  # private key for that user
+export KVM_TEST_KNOWN_HOSTS=$HOME/.ssh/kvm_known_hosts  # the hypervisors' host keys
+export KVM_TEST_BRIDGE=br0                     # bridge to a network with DHCP
+export KVM_TEST_IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+make test-integration-kvm
+```
+
+`KVM_TEST_KNOWN_HOSTS` is technically optional, but leaving it out turns
+host-key verification **off** for the run (the log says so); keep a
+`kvm_known_hosts` holding just the hypervisors rather than pointing the
+test at your everyday `known_hosts`, so a re-keyed hypervisor is a
+one-line fix in a file nothing else uses.
+
+Also optional: `KVM_TEST_IMAGE_SHA256`, `KVM_TEST_POOL`
+(default `default`), `KVM_TEST_SOCKET`, `KVM_TEST_TIMEOUT` (default
+`20m`, must cover the first-time image download), `KVM_TEST_KEEP`
+to leave the VM behind for `virsh console` when something goes wrong,
+and `KVM_TEST_LOG_LEVEL` (default `1`) for how much of the provider's
+logging the `-v` output shows: at `1` every libvirt RPC is one compact
+line (`HH:MM:SS.mmm libvirt virDomainLookupByName domain=... duration=...
+error=... libvirtError=ErrNoDomain`), interleaved with the test's own
+`t.Log` lines; `2` adds the domain and volume XML, metadata and
+addresses; `0` leaves only `t.Log`. The per-run constants (provider,
+host, domain name) are left off each line. See the Logging section of
+[kvm-provider.md](kvm-provider.md#logging).
+
+What one run asserts, in order: `New` dials the host; `Create` returns a
+`hosts/<host>/domains/<name>` ID (downloading the image on a cold host —
+the log prints how long it took); a second `Create` returns the same ID;
+`Get` reaches `Running` with an IPv4 within the budget, and the UID and
+`CreatedAt` survive the metadata round-trip through libvirt; `ListByTag`
+lists it; `Delete` removes it, `Get` says `ErrNotFound`, a second
+`Delete` is a no-op, and the pool holds the base image and its marker
+but no per-VM overlay or seed. It also tries a TCP connect to port 22 on
+the guest, but only logs the outcome — the machine running the test may
+not route to the bridge network.
+
+`TestKVMCheck_realHost` runs the provider health check
+([kvm-provider.md](kvm-provider.md#health-check)) against the same host.
+It must find the host reachable and the pool running, and it creates
+nothing. It needs the same variables; to run only this test:
+
+```sh
+go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/
+```
+
+The test is not part of `make test` and not run in CI. Keep it compiling
+with `go vet -tags=integration ./...` when the provider's internals move.
+
+## The scaffolded `make test-e2e` suite
+
+`test/e2e/` is the kubebuilder-generated smoke suite (build image → kind
+cluster → deploy → assert the manager pod runs, publishes a Ready
+`ProviderHealth` for the `kubernetes` provider, and serves metrics that
+include that provider's `proxy_operator_provider_target_up` series). It
+compiles only under `-tags=e2e`, manages its own kind cluster
+(`make test-e2e` / `make cleanup-test-e2e`), and has been kept compiling
+(`go vet -tags=e2e ./...` is part of the routine) but is **not part of
+`make test` and was not used for the release verification** — the manual
+kind run below covers strictly more.
+
+### The OTel tracing spec (Tempo-gated)
+
+`test/e2e/tracing_test.go` proves the full tracing pipeline against a
+**real Tempo**: deploy in kind with tracing enabled, create two
+kubernetes-provider proxies, delete them, and assert in Tempo that the
+traces exist with the documented span topology (`Reconcile Proxy` →
+`reconcile.managed` / `provider.create` / `status.patch`, and
+`reconcile.delete` / `provider.delete` on the way out).
+
+It **skips unless both env vars are set** (so the rest of the suite runs
+anywhere). Homelab invocation:
+
+```bash
+TEMPO_URL=http://192.168.0.30:3200 \
+OTLP_ENDPOINT=http://192.168.0.30:4318 \
+make test-e2e
+```
+
+Worth knowing:
+
+- Every span of a run carries the resource attribute
+  `test.run.id=e2e-<nanos>` (injected via `OTEL_RESOURCE_ATTRIBUTES`, no
+  code involved); the run ID is printed in the test log, and
+  `{resource.test.run.id="<id>"}` in Grafana shows exactly that run.
+- The spec preflights the OTLP endpoint **from inside the cluster** with a
+  curl pod and fails fast with a clear message if it's unreachable —
+  export failures are otherwise only visible at `-zap-log-level=1`.
+- Proxy CRs are created in `default`, not the operator namespace: the
+  squid pods carry no securityContext and the operator namespace enforces
+  restricted PSS.
+
+## The kind verification run (the real end-to-end)
+
+The README quickstart **is** the e2e test, run by hand before the MR:
+deploy the operator in-cluster on a throwaway kind cluster, watch a real
+Squid pod reach `Ready` via a real CONNECT probe, then exercise the whole
+lease lifecycle through the discovery API (list → lease → report
+`rate_limited` → same-target lease answered 409 `inCooldown:1` → release
+204 twice) and delete the CR to watch the finalizer remove the pod.
+
+Worth knowing about that run (full detail in
+[plans-executions/2026-08-07-1747-proxy-operator.md](plans-executions/2026-08-07-1747-proxy-operator.md),
+"Verification" section): it was not a clean pass-through. It caught two
+real bugs the automated suites structurally could not —
+
+1. `make run-dev` on a laptop can never produce a `Ready`
+   kubernetes-provider proxy: health probes originate on the host, which
+   cannot route to kind's pod network. The quickstart was rewritten to
+   deploy in-cluster.
+2. Squid was OOM-killed at startup: it sizes file-descriptor tables from
+   `RLIMIT_NOFILE`, which containerd under kind sets effectively
+   unlimited. Fixed with `max_filedescriptors 1024` in the generated
+   config, plus a regression assertion in `pod_test.go`.
+
+That is the pattern to keep: when the kind run finds something, the fix
+lands **with a unit-level regression test**, so the manual run stays a
+discovery tool rather than a recurring gate.

go.mod

@@ -3,8 +3,23 @@ module gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator
 go 1.26.0
 
 require (
+	cloud.google.com/go/compute v1.65.0
+	github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858
+	github.com/go-logr/logr v1.4.4
+	github.com/kdomanski/iso9660 v0.4.0
 	github.com/onsi/ginkgo/v2 v2.27.4
 	github.com/onsi/gomega v1.39.0
+	github.com/prometheus/client_golang v1.23.2
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
+	go.opentelemetry.io/otel v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0
+	go.opentelemetry.io/otel/sdk v1.45.0
+	go.opentelemetry.io/otel/trace v1.45.0
+	golang.org/x/crypto v0.54.0
+	google.golang.org/api v0.292.0
+	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
 	k8s.io/api v0.36.0
 	k8s.io/apimachinery v0.36.0
 	k8s.io/client-go v0.36.0
@@ -13,7 +28,10 @@ require (
 )
 
 require (
-	cel.dev/expr v0.25.1 // indirect
+	cel.dev/expr v0.25.2 // indirect
+	cloud.google.com/go/auth v0.22.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -23,10 +41,9 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/felixge/httpsnoop v1.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
-	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
@@ -37,17 +54,20 @@ require (
 	github.com/google/gnostic-models v0.7.0 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect
+	github.com/googleapis/gax-go/v2 v2.23.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.19.2 // indirect
@@ -56,33 +76,28 @@ require (
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
-	go.opentelemetry.io/otel v1.41.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.41.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.40.0 // indirect
-	go.opentelemetry.io/otel/trace v1.41.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
+	go.opentelemetry.io/otel/metric v1.45.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
-	golang.org/x/mod v0.32.0 // indirect
-	golang.org/x/net v0.49.0 // indirect
-	golang.org/x/oauth2 v0.34.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.40.0 // indirect
-	golang.org/x/term v0.39.0 // indirect
-	golang.org/x/text v0.33.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
-	golang.org/x/tools v0.41.0 // indirect
+	golang.org/x/mod v0.37.0 // indirect
+	golang.org/x/net v0.57.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/term v0.45.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	golang.org/x/tools v0.47.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
-	google.golang.org/grpc v1.79.3 // indirect
-	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
+	google.golang.org/grpc v1.83.0 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -1,5 +1,15 @@
-cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
-cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
+cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs=
+cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
+cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
+cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
+cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ=
+cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s=
+cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
+cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
+cloud.google.com/go/compute v1.65.0 h1:K0a3NRvazE7sZn5qswwI6BtlaZv1fgR5wFop5LZCLz8=
+cloud.google.com/go/compute v1.65.0/go.mod h1:vFq+Ztj9Rzhc8zf1t6hGp/6NdrEVG1GakkyVRQPRgKc=
+cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
+cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
@@ -18,14 +28,16 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858 h1:8xCFt73OddCD5WXxoYidJBWp3bJMC+CpE7Zwf5tcwk8=
+github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858/go.mod h1:qb0Ofa71d3oXARQf633h2tNaeBxLsVxuDp+jcsVO2+4=
 github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
 github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
 github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
 github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
-github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
-github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
+github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
@@ -37,8 +49,8 @@ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6O
 github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
 github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
-github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
+github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
@@ -68,10 +80,16 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
+github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
+github.com/googleapis/enterprise-certificate-proxy v0.3.19 h1:mMOE7DN2+p76/EdIrmAy9B9bH+yC4563vmnJ34QR8i4=
+github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k=
+github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE=
+github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -80,6 +98,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE
 github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg=
+github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -154,24 +174,28 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0=
-go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
-go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
-go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
-go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
-go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
-go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
-go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
-go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
-go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
-go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
-go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
-go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
+go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
+go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=
+go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
+go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
+go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
+go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
+go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
+go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
+go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
+go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
+go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
+go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -182,36 +206,42 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
 go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=
 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
-golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
-golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
-golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
-golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
-golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
-golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
-golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
-golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
-golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
-golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
-golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
-golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
-golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
-gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
-google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
-google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
-google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
+google.golang.org/api v0.292.0 h1:Ewiwo/GTtiaPZSNAZQUcWLh8AYDEoPmIXyJfeoTSMHU=
+google.golang.org/api v0.292.0/go.mod h1:07kjmMnFGm2RQuCza2EZM/5N68G/fVvFb1xKjWqoFA0=
+google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
+google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
+google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

hack/providers-dev.yaml

@@ -0,0 +1,6 @@
+# Providers config for `make run-dev`: local development against the
+# current kubeconfig context (e.g. a kind cluster). Only the
+# kubernetes-pod provider — no cloud credentials needed.
+providers:
+  - name: kubernetes
+    type: kubernetes

internal/controller/proxy_controller.go

@@ -41,6 +41,7 @@ import (
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // HealthSnapshotter provides the current probe verdict for a proxy. The
@@ -82,6 +83,9 @@ type ProxyReconciler struct {
 // +kubebuilder:rbac:groups=crawl.example.com,resources=proxies/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=crawl.example.com,resources=proxies/finalizers,verbs=update
 // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
+// The pod verbs are for the kubernetes-pod provider; cluster-scoped, since
+// its ListByTag enumerates the operator's Pods across all namespaces.
+// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;delete
 
 // Reconcile fetches the Proxy named by req into p (r.Get fills the struct
 // through the pointer), dispatches to the delete/external/managed state
@@ -93,9 +97,14 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 	}
 	base := p.DeepCopy()
 	defer func() {
+		// Runs inside the root reconcile span (this defer fires before the
+		// tracing.NewReconciler wrapper sees the return), and its error is
+		// folded into err, which that wrapper records.
+		pctx, span := tracing.Start(ctx, "status.patch")
+		defer span.End()
 		// NotFound is expected when this reconcile just removed the last
 		// finalizer and the object is already gone.
-		if perr := r.patchStatusIfChanged(ctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
+		if perr := r.patchStatusIfChanged(pctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
 			err = errors.Join(err, perr)
 		}
 	}()
@@ -111,6 +120,8 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 }
 
 func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.managed")
+	defer span.End()
 	log := logf.FromContext(ctx)
 
 	if controllerutil.AddFinalizer(p, crawlv1alpha1.FinalizerName) {
@@ -221,6 +232,8 @@ func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1
 // exists" — hence: delete, poll to NotFound, only then advance the hash and
 // let the create branch run.
 func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.Proxy, prov provider.Provider, hash string) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.replaceInstance")
+	defer span.End()
 	_, err := prov.Get(ctx, p.Status.ProviderID)
 	if provider.Class(err) == provider.ErrNotFound {
 		// Old instance is gone. The Update inside setSpecHash refreshes p
@@ -249,14 +262,22 @@ func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.
 }
 
 func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.delete")
+	defer span.End()
 	if !controllerutil.ContainsFinalizer(p, crawlv1alpha1.FinalizerName) {
 		return ctrl.Result{}, nil
 	}
 	if p.Status.ProviderID == "" {
 		// Nothing was ever recorded as created; orphan GC reaps any stray
 		// instance a crashed create might have left behind.
-		controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
-		return ctrl.Result{}, r.Update(ctx, p)
+		return r.releaseFinalizer(ctx, p)
+	}
+	if p.Annotations[crawlv1alpha1.AnnotationSkipCleanup] == "true" {
+		// Checked before the provider lookup so it also releases a Proxy whose
+		// provider is gone from the config. The instance is left running.
+		logf.FromContext(ctx).Info("skip-cleanup annotation set; abandoning the instance",
+			"providerID", p.Status.ProviderID, "provider", p.Spec.Provider)
+		return r.releaseFinalizer(ctx, p)
 	}
 	prov, ok := r.Providers[p.Spec.Provider]
 	if !ok {
@@ -264,11 +285,22 @@ func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.
 			"provider %q is not configured; cannot clean up instance %s", p.Spec.Provider, p.Status.ProviderID)
 	}
 	_, err := prov.Get(ctx, p.Status.ProviderID)
-	if provider.Class(err) == provider.ErrNotFound {
-		controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
-		return ctrl.Result{}, r.Update(ctx, p)
-	}
-	if err != nil {
+	switch {
+	case err == nil:
+	case provider.Class(err) == provider.ErrNotFound:
+		return r.releaseFinalizer(ctx, p)
+	case provider.Class(err) == provider.ErrPermanent:
+		// Get will never answer for this instance — a host retired from the
+		// config, a providerID the provider can no longer resolve. Delete is
+		// idempotent and may still finish, so try it instead of polling a Get
+		// that cannot converge, and release the object when it does.
+		if delErr := prov.Delete(ctx, p.Status.ProviderID); delErr != nil {
+			return r.deletionFailure(errors.Join(err, delErr))
+		}
+		logf.FromContext(ctx).Info("instance unreachable but cleanup succeeded; removing the finalizer",
+			"providerID", p.Status.ProviderID, "error", err.Error())
+		return r.releaseFinalizer(ctx, p)
+	default:
 		return r.deletionFailure(err)
 	}
 	if err := prov.Delete(ctx, p.Status.ProviderID); err != nil {
@@ -279,6 +311,11 @@ func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.
 	return ctrl.Result{RequeueAfter: r.DeletionPoll}, nil
 }
 
+func (r *ProxyReconciler) releaseFinalizer(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
+	controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
+	return ctrl.Result{}, r.Update(ctx, p)
+}
+
 func (r *ProxyReconciler) reconcileExternal(_ context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
 	if p.Spec.Endpoint == nil {
 		// CEL guarantees an endpoint on any object that went through the API
@@ -406,7 +443,10 @@ func (r *ProxyReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	if r.HealthEvents != nil {
 		b = b.WatchesRawSource(source.Channel(r.HealthEvents, &handler.EnqueueRequestForObject{}))
 	}
-	return b.Complete(r)
+	// Root span per reconcile; sub-reconcilers and the status flush hang
+	// their spans off it. Tests calling r.Reconcile directly bypass the
+	// wrapper and see no-op spans — the global tracer is never set there.
+	return b.Complete(tracing.NewReconciler("Proxy", r))
 }
 
 func (r *ProxyReconciler) applyDefaults() {

internal/controller/proxy_controller_test.go

@@ -28,6 +28,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
 )
 
@@ -233,6 +234,105 @@ var _ = Describe("Proxy controller", func() {
 		Expect(apierrors.IsNotFound(err)).To(BeTrue(), "proxy should be fully deleted")
 	})
 
+	It("reaches Ready once the health engine has a verdict", func() {
+		const name = "e2e-ready"
+		stub := &stubProvider{createID: "inst-rdy"}
+		r := newEnvtestReconciler(stub)
+		DeferCleanup(func() { cleanup(r, stub, name) })
+
+		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec:       managedSpec(),
+		})).To(Succeed())
+		_, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		stub.getInst = &provider.Instance{ID: "inst-rdy", IP: "10.3.3.3", State: provider.StateRunning}
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseProvisioning),
+			"no health verdict yet — must not be Ready")
+
+		By("supplying a healthy snapshot")
+		r.Health = fakeSnapshotter{ok: true, snap: health.Snapshot{
+			Healthy: true, Latency: 21 * time.Millisecond, LastProbe: time.Now(),
+		}}
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		p := fetch(name)
+		Expect(p.Status.Phase).To(Equal(crawlv1alpha1.PhaseReady))
+		Expect(p.Status.LatencyMillis).To(Equal(int64(21)))
+	})
+
+	It("treats quota exhaustion as a wait and a permanent error as Failed", func() {
+		const name = "e2e-errors"
+		stub := &stubProvider{
+			createErr: provider.Wrap(provider.ErrQuotaExceeded, "create", "stub", "", nil),
+		}
+		r := newEnvtestReconciler(stub)
+		DeferCleanup(func() { cleanup(r, stub, name) })
+
+		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec:       managedSpec(),
+		})).To(Succeed())
+		_, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("quota: condition set, slow requeue, phase NOT Failed")
+		res, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred(), "quota must not count as an error (stays off the backoff curve)")
+		Expect(res.RequeueAfter).To(Equal(r.QuotaRetry))
+		p := fetch(name)
+		cond := apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionProvisioned)
+		Expect(cond.Reason).To(Equal(ReasonQuotaExceeded))
+		Expect(p.Status.Phase).NotTo(Equal(crawlv1alpha1.PhaseFailed))
+
+		By("permanent: phase Failed and no further provider calls")
+		stub.createErr = provider.Wrap(provider.ErrPermanent, "create", "stub", "", nil)
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseFailed))
+		callsAfterLatch := stub.createCalls
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(stub.createCalls).To(Equal(callsAfterLatch), "the latch must stop provider calls")
+	})
+
+	It("adopts an instance when the spec-hash annotation is stripped", func() {
+		const name = "e2e-adopt"
+		stub := &stubProvider{createID: "inst-adopt"}
+		r := newEnvtestReconciler(stub)
+		DeferCleanup(func() { cleanup(r, stub, name) })
+
+		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec:       managedSpec(),
+		})).To(Succeed())
+		_, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		stub.getInst = &provider.Instance{ID: "inst-adopt", IP: "10.4.4.4", State: provider.StateRunning}
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		originalHash := fetch(name).Annotations[crawlv1alpha1.AnnotationSpecHash]
+		Expect(originalHash).NotTo(BeEmpty())
+
+		By("stripping the annotation, as an operator-version upgrade with a changed hash input would")
+		p := fetch(name)
+		delete(p.Annotations, crawlv1alpha1.AnnotationSpecHash)
+		Expect(k8sClient.Update(ctx, p)).To(Succeed())
+
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		p = fetch(name)
+		Expect(p.Annotations[crawlv1alpha1.AnnotationSpecHash]).To(Equal(originalHash), "hash must be restored")
+		Expect(p.Status.ProviderID).To(Equal("inst-adopt"), "adoption must keep the instance")
+		Expect(stub.deleteCalls).To(BeZero(), "adoption must never replace")
+	})
+
 	It("tracks an External proxy without touching providers", func() {
 		const name = "e2e-external"
 		stub := &stubProvider{}
@@ -264,3 +364,157 @@ var _ = Describe("Proxy controller", func() {
 		Expect(apierrors.IsNotFound(err)).To(BeTrue())
 	})
 })
+
+// These specs assert the CRD's CEL rules and structural defaulting against
+// the real envtest API server — the fake client runs neither, which is the
+// documented caveat on the action-table unit tests.
+var _ = Describe("Proxy CRD validation (CEL)", func() {
+	const ns = "default"
+
+	managed := func(name string) *crawlv1alpha1.Proxy {
+		return &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec: crawlv1alpha1.ProxySpec{
+				Mode:     crawlv1alpha1.ModeManaged,
+				Provider: "stub",
+			},
+		}
+	}
+	external := func(name string) *crawlv1alpha1.Proxy {
+		return &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec: crawlv1alpha1.ProxySpec{
+				Mode:     crawlv1alpha1.ModeExternal,
+				Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
+			},
+		}
+	}
+	mustCreate := func(p *crawlv1alpha1.Proxy) {
+		GinkgoHelper()
+		Expect(k8sClient.Create(ctx, p)).To(Succeed())
+		DeferCleanup(func() { _ = k8sClient.Delete(ctx, p) })
+	}
+
+	It("rejects invalid creates", func() {
+		invalid := []struct {
+			about string
+			spec  crawlv1alpha1.ProxySpec
+			want  string
+		}{
+			{
+				about: "Managed without provider",
+				spec:  crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged},
+				want:  "provider is required when mode is Managed",
+			},
+			{
+				about: "External with provider",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeExternal, Provider: "stub",
+					Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
+				},
+				want: "provider must not be set when mode is External",
+			},
+			{
+				about: "External without endpoint",
+				spec:  crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeExternal},
+				want:  "endpoint is required when mode is External",
+			},
+			{
+				about: "Managed with endpoint",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
+					Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
+				},
+				want: "endpoint must not be set when mode is Managed",
+			},
+			{
+				about: "cloudInit with both inline and secretRef",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
+					CloudInit: &crawlv1alpha1.CloudInitSpec{
+						Inline:    "#cloud-config",
+						SecretRef: &crawlv1alpha1.SecretKeySelector{Name: "s"},
+					},
+				},
+				want: "exactly one of inline or secretRef",
+			},
+			{
+				about: "cloudInit with neither inline nor secretRef",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
+					CloudInit: &crawlv1alpha1.CloudInitSpec{},
+				},
+				want: "exactly one of inline or secretRef",
+			},
+		}
+		for _, tc := range invalid {
+			p := &crawlv1alpha1.Proxy{
+				ObjectMeta: metav1.ObjectMeta{Name: "cel-invalid", Namespace: ns},
+				Spec:       tc.spec,
+			}
+			err := k8sClient.Create(ctx, p)
+			Expect(err).To(HaveOccurred(), tc.about)
+			Expect(err.Error()).To(ContainSubstring(tc.want), tc.about)
+		}
+	})
+
+	It("rejects mode mutation", func() {
+		p := external("cel-mode-immutable")
+		mustCreate(p)
+		p.Spec = crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged, Provider: "stub"}
+		err := k8sClient.Update(ctx, p)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("mode is immutable"))
+	})
+
+	It("rejects provider mutation and removal", func() {
+		p := managed("cel-provider-immutable")
+		mustCreate(p)
+
+		p.Spec.Provider = "other"
+		err := k8sClient.Update(ctx, p)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("provider is immutable"))
+
+		// Removal must also be rejected — the has()==has() form exists
+		// exactly because a field-level rule would not fire on absence.
+		// (Dropping provider alone would also trip the required-iff rule,
+		// so flip mode too and check the immutability rules win.)
+		fresh := fetchProxy(ns, "cel-provider-immutable")
+		fresh.Spec.Provider = ""
+		fresh.Spec.Endpoint = &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"}
+		fresh.Spec.Mode = crawlv1alpha1.ModeExternal
+		err = k8sClient.Update(ctx, fresh)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("immutable"))
+	})
+
+	It("materializes every nested healthCheck default when healthCheck is omitted", func() {
+		p := managed("cel-defaults")
+		mustCreate(p)
+
+		got := fetchProxy(ns, "cel-defaults")
+		// The +kubebuilder:default={} assertion: structural defaulting only
+		// descends into values that exist, so without it a nil healthCheck
+		// would get none of these.
+		hc := got.Spec.HealthCheck
+		Expect(hc).NotTo(BeNil())
+		Expect(hc.ProbeURL).To(Equal(crawlv1alpha1.DefaultProbeURL))
+		Expect(hc.IntervalSeconds).To(Equal(crawlv1alpha1.DefaultHealthCheckIntervalSeconds))
+		Expect(hc.TimeoutSeconds).To(Equal(crawlv1alpha1.DefaultHealthCheckTimeoutSeconds))
+		Expect(hc.FailureThreshold).To(Equal(crawlv1alpha1.DefaultFailureThreshold))
+		Expect(hc.SuccessThreshold).To(Equal(crawlv1alpha1.DefaultSuccessThreshold))
+		Expect(hc.ExpectedStatusCodes).To(Equal(crawlv1alpha1.DefaultExpectedStatusCodes))
+
+		Expect(got.Spec.Port).To(Equal(crawlv1alpha1.DefaultPort))
+		Expect(got.Spec.MaxLeases).NotTo(BeNil())
+		Expect(*got.Spec.MaxLeases).To(Equal(crawlv1alpha1.DefaultMaxLeases))
+	})
+})
+
+func fetchProxy(ns, name string) *crawlv1alpha1.Proxy {
+	GinkgoHelper()
+	p := &crawlv1alpha1.Proxy{}
+	Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, p)).To(Succeed())
+	return p
+}

internal/controller/reconcile_test.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"context"
+	"slices"
 	"testing"
 	"time"
 
@@ -78,6 +79,10 @@ func notFoundErr() error {
 	return provider.Wrap(provider.ErrNotFound, "get", "stub", "some-id", nil)
 }
 
+func permanentErr() error {
+	return provider.Wrap(provider.ErrPermanent, "get", "stub", "some-id", nil)
+}
+
 func testScheme(t *testing.T) *runtime.Scheme {
 	t.Helper()
 	s := runtime.NewScheme()
@@ -125,6 +130,15 @@ func withSpecHashAnnotation(hash string) func(*crawlv1alpha1.Proxy) {
 	}
 }
 
+func withSkipCleanup() func(*crawlv1alpha1.Proxy) {
+	return func(p *crawlv1alpha1.Proxy) {
+		if p.Annotations == nil {
+			p.Annotations = map[string]string{}
+		}
+		p.Annotations[crawlv1alpha1.AnnotationSkipCleanup] = "true"
+	}
+}
+
 func deleting() func(*crawlv1alpha1.Proxy) {
 	return func(p *crawlv1alpha1.Proxy) {
 		now := metav1.Now()
@@ -375,6 +389,59 @@ func TestReconcile_actionTable(t *testing.T) {
 				assertProxyGone(t, r)
 			},
 		},
+		{
+			// A permanent Get (a retired KVM host, a providerID the provider
+			// cannot resolve) used to retry forever with the finalizer on.
+			name:       "deletion with an unreachable instance still deletes and releases",
+			proxy:      managedProxy(deleting(), withProviderID("stub-id-1")),
+			stub:       &stubProvider{getErr: permanentErr()},
+			wantResult: ctrl.Result{},
+			verify: func(t *testing.T, r *ProxyReconciler, stub *stubProvider) {
+				assertProxyGone(t, r)
+				if stub.deleteCalls != 1 {
+					t.Errorf("deleteCalls = %d, want 1 — Delete must be tried when Get cannot answer", stub.deleteCalls)
+				}
+			},
+		},
+		{
+			name:  "deletion keeps the finalizer when cleanup itself fails",
+			proxy: managedProxy(deleting(), withProviderID("stub-id-1")),
+			stub: &stubProvider{
+				getErr:    permanentErr(),
+				deleteErr: provider.Wrap(provider.ErrPermanent, "delete", "stub", "stub-id-1", nil),
+			},
+			wantResult: ctrl.Result{},
+			wantErr:    true,
+			verify: func(t *testing.T, r *ProxyReconciler, _ *stubProvider) {
+				// getProxy fails the test if the object is gone, which is what
+				// finalizer removal would have caused.
+				if p := getProxy(t, r); !slices.Contains(p.Finalizers, crawlv1alpha1.FinalizerName) {
+					t.Error("finalizer removed although cleanup failed; the instance would leak silently")
+				}
+			},
+		},
+		{
+			name:       "skip-cleanup annotation abandons the instance",
+			proxy:      managedProxy(deleting(), withProviderID("stub-id-1"), withSkipCleanup()),
+			stub:       &stubProvider{getInst: &provider.Instance{ID: "stub-id-1", State: provider.StateRunning}},
+			wantResult: ctrl.Result{},
+			verify: func(t *testing.T, r *ProxyReconciler, stub *stubProvider) {
+				assertProxyGone(t, r)
+				if stub.getCalls != 0 || stub.deleteCalls != 0 {
+					t.Errorf("getCalls/deleteCalls = %d/%d, want 0/0 — the provider must not be asked", stub.getCalls, stub.deleteCalls)
+				}
+			},
+		},
+		{
+			name: "skip-cleanup annotation releases a proxy whose provider is gone",
+			proxy: managedProxy(deleting(), withProviderID("stub-id-1"), withSkipCleanup(),
+				func(p *crawlv1alpha1.Proxy) { p.Spec.Provider = "retired" }),
+			stub:       &stubProvider{},
+			wantResult: ctrl.Result{},
+			verify: func(t *testing.T, r *ProxyReconciler, _ *stubProvider) {
+				assertProxyGone(t, r)
+			},
+		},
 		{
 			name:  "deletion deletes the instance and polls",
 			proxy: managedProxy(deleting(), withProviderID("stub-id-1")),

internal/discovery/handlers.go

@@ -0,0 +1,245 @@
+package discovery
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"slices"
+	"strings"
+	"time"
+
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+)
+
+// proxyView is the wire shape of one proxy in list and lease responses.
+type proxyView struct {
+	ID            string            `json:"id"` // namespace/name
+	IP            string            `json:"ip"`
+	Port          int32             `json:"port"`
+	Attributes    map[string]string `json:"attributes,omitempty"`
+	Phase         string            `json:"phase"`
+	Healthy       bool              `json:"healthy"`
+	LatencyMillis int64             `json:"latencyMillis"`
+	ActiveLeases  int               `json:"activeLeases"`
+	MaxLeases     int32             `json:"maxLeases"`
+}
+
+func viewOf(p *crawlv1alpha1.Proxy, activeLeases int) proxyView {
+	return proxyView{
+		ID:            client.ObjectKeyFromObject(p).String(),
+		IP:            p.EffectiveHost(),
+		Port:          p.EffectivePort(),
+		Attributes:    p.Spec.Attributes,
+		Phase:         string(p.Status.Phase),
+		Healthy:       isHealthy(p),
+		LatencyMillis: p.Status.LatencyMillis,
+		ActiveLeases:  activeLeases,
+		MaxLeases:     p.MaxLeasesOrDefault(),
+	}
+}
+
+func isHealthy(p *crawlv1alpha1.Proxy) bool {
+	return apimeta.IsStatusConditionTrue(p.Status.Conditions, crawlv1alpha1.ConditionHealthy)
+}
+
+// matchesAttributes is spec.attributes equality: every selector pair must
+// be present verbatim.
+func matchesAttributes(p *crawlv1alpha1.Proxy, selector map[string]string) bool {
+	for k, v := range selector {
+		if p.Spec.Attributes[k] != v {
+			return false
+		}
+	}
+	return true
+}
+
+// GET /v1/proxies?attr.<key>=<value>&healthy=true|false
+func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
+	selector := map[string]string{}
+	var healthyFilter *bool
+	for key, values := range r.URL.Query() {
+		switch {
+		case key == "healthy":
+			switch values[0] {
+			case "true":
+				healthyFilter = ptr(true)
+			case "false":
+				healthyFilter = ptr(false)
+			default:
+				writeError(w, http.StatusBadRequest, "invalid_query",
+					fmt.Sprintf("healthy must be true or false, got %q", values[0]))
+				return
+			}
+		case strings.HasPrefix(key, "attr."):
+			selector[strings.TrimPrefix(key, "attr.")] = values[0]
+		}
+	}
+
+	var list crawlv1alpha1.ProxyList
+	if err := s.Reader.List(r.Context(), &list); err != nil {
+		// The ctx logger is s.log enriched with this request's
+		// traceID/spanID by the tracing middleware (logr sinks can't read
+		// ctx at log time, so per-request values ride on the logger).
+		logf.FromContext(r.Context()).Error(err, "listing proxies")
+		writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
+		return
+	}
+	counts := s.Store.Counts()
+
+	views := []proxyView{}
+	for i := range list.Items {
+		p := &list.Items[i]
+		if !p.DeletionTimestamp.IsZero() || !matchesAttributes(p, selector) {
+			continue
+		}
+		v := viewOf(p, counts[client.ObjectKeyFromObject(p).String()])
+		if healthyFilter != nil && v.Healthy != *healthyFilter {
+			continue
+		}
+		views = append(views, v)
+	}
+	slices.SortFunc(views, func(a, b proxyView) int { return strings.Compare(a.ID, b.ID) })
+
+	writeJSON(w, http.StatusOK, map[string]any{"proxies": views, "count": len(views)})
+}
+
+type leaseRequest struct {
+	Selector   map[string]string `json:"selector"`
+	TTLSeconds int64             `json:"ttlSeconds"`
+	Target     string            `json:"target"`
+}
+
+type leaseResponse struct {
+	LeaseID    string    `json:"leaseID"`
+	Proxy      proxyView `json:"proxy"`
+	ExpiresAt  time.Time `json:"expiresAt"`
+	TTLSeconds int64     `json:"ttlSeconds"`
+}
+
+// POST /v1/leases
+func (s *Server) handleAcquireLease(w http.ResponseWriter, r *http.Request) {
+	var req leaseRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid_body", err.Error())
+		return
+	}
+
+	ttl := time.Duration(req.TTLSeconds) * time.Second
+	if req.TTLSeconds == 0 {
+		ttl = defaultTTL
+	}
+	if ttl < 0 || ttl > s.MaxLeaseTTL {
+		writeError(w, http.StatusBadRequest, "invalid_ttl",
+			fmt.Sprintf("ttlSeconds must be between 1 and %d", int64(s.MaxLeaseTTL.Seconds())))
+		return
+	}
+
+	var list crawlv1alpha1.ProxyList
+	if err := s.Reader.List(r.Context(), &list); err != nil {
+		logf.FromContext(r.Context()).Error(err, "listing proxies for lease")
+		writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
+		return
+	}
+
+	// The store gets only healthy, live candidates; unhealthy matches are
+	// counted here because the store never sees them.
+	var candidates []lease.Candidate
+	byKey := map[string]*crawlv1alpha1.Proxy{}
+	unhealthy := 0
+	for i := range list.Items {
+		p := &list.Items[i]
+		if !p.DeletionTimestamp.IsZero() || !matchesAttributes(p, req.Selector) {
+			continue
+		}
+		if !isHealthy(p) {
+			unhealthy++
+			continue
+		}
+		key := client.ObjectKeyFromObject(p).String()
+		byKey[key] = p
+		candidates = append(candidates, lease.Candidate{
+			Proxy:     key,
+			MaxLeases: p.MaxLeasesOrDefault(),
+			Latency:   time.Duration(p.Status.LatencyMillis) * time.Millisecond,
+		})
+	}
+
+	granted, stats, err := s.Store.Acquire(r.Context(), lease.AcquireRequest{
+		Candidates: candidates,
+		Target:     req.Target,
+		TTL:        ttl,
+	})
+	if errors.Is(err, lease.ErrNoMatch) {
+		if s.Metrics != nil {
+			s.Metrics.LeaseRequest("no_match")
+		}
+		writeJSON(w, http.StatusConflict, map[string]any{
+			"error":      "no_match",
+			"message":    "no healthy proxy with free capacity matched the selector",
+			"considered": stats.Considered + unhealthy,
+			"atCapacity": stats.AtCapacity,
+			"inCooldown": stats.InCooldown,
+			"unhealthy":  unhealthy,
+		})
+		return
+	}
+	if err != nil {
+		writeError(w, http.StatusBadRequest, "invalid_request", err.Error())
+		return
+	}
+
+	if s.Metrics != nil {
+		s.Metrics.LeaseRequest("granted")
+	}
+	writeJSON(w, http.StatusCreated, leaseResponse{
+		LeaseID:    granted.ID,
+		Proxy:      viewOf(byKey[granted.Proxy], s.Store.Counts()[granted.Proxy]),
+		ExpiresAt:  granted.ExpiresAt,
+		TTLSeconds: int64(ttl.Seconds()),
+	})
+}
+
+// DELETE /v1/leases/{id} — early release, always 204: releasing an unknown
+// or already expired lease is not an error.
+func (s *Server) handleReleaseLease(w http.ResponseWriter, r *http.Request) {
+	s.Store.Release(r.Context(), r.PathValue("id"))
+	w.WriteHeader(http.StatusNoContent)
+}
+
+type reportRequest struct {
+	Result string `json:"result"`
+	Target string `json:"target"`
+}
+
+// POST /v1/leases/{id}/report
+func (s *Server) handleReportLease(w http.ResponseWriter, r *http.Request) {
+	var req reportRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid_body", err.Error())
+		return
+	}
+	result, ok := lease.ParseResult(req.Result)
+	if !ok {
+		writeError(w, http.StatusBadRequest, "invalid_result",
+			fmt.Sprintf("result must be one of ok, rate_limited, banned; got %q", req.Result))
+		return
+	}
+	if err := s.Store.Report(r.Context(), r.PathValue("id"), result, req.Target); err != nil {
+		if errors.Is(err, lease.ErrUnknownLease) {
+			writeError(w, http.StatusNotFound, "unknown_lease", "no such lease")
+			return
+		}
+		logf.FromContext(r.Context()).Error(err, "reporting lease", "leaseID", r.PathValue("id"))
+		writeError(w, http.StatusInternalServerError, "internal", "report failed")
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func ptr[T any](v T) *T { return &v }

internal/discovery/server.go

@@ -0,0 +1,238 @@
+// Package discovery implements the HTTP API crawler clients use to find
+// and lease proxies: list healthy proxies filtered by attributes, acquire a
+// TTL-based lease, release it early, and report how a target treated the
+// proxy. Reads go through the manager's informer cache; lease state lives
+// in the injected store.
+package discovery
+
+import (
+	"context"
+	"crypto/subtle"
+	"encoding/json"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/go-logr/logr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+// LeaseStore is what the handlers need from a lease backend. Defined here,
+// consumer-side, so a CRD- or Redis-backed store can replace the in-memory
+// one (which internal/lease's *Store satisfies) without touching handlers.
+type LeaseStore interface {
+	Acquire(ctx context.Context, req lease.AcquireRequest) (*lease.Lease, lease.AcquireStats, error)
+	Release(ctx context.Context, id string)
+	Report(ctx context.Context, id string, result lease.Result, target string) error
+	Counts() map[string]int
+}
+
+// LeaseMetrics counts lease acquisitions by outcome. Implemented by
+// internal/metrics; defined here so this package carries no metrics
+// dependency.
+type LeaseMetrics interface {
+	LeaseRequest(outcome string)
+}
+
+const (
+	defaultAddr     = ":8090"
+	defaultTTL      = 5 * time.Minute
+	defaultMaxTTL   = time.Hour
+	maxBodyBytes    = 64 << 10
+	shutdownGrace   = 10 * time.Second
+	readHeadTimeout = 5 * time.Second
+)
+
+// Server serves the discovery API as a manager Runnable.
+type Server struct {
+	// Reader lists Proxies from the manager's cache.
+	Reader client.Reader
+	// Store is the lease backend.
+	Store LeaseStore
+	// Addr is the listen address (default ":8090"; --discovery-addr).
+	Addr string
+	// Token is the static bearer token from DISCOVERY_TOKEN. Empty
+	// disables auth — allowed for the prototype, but loudly warned about
+	// at startup, because in-cluster that is a silent security hole.
+	Token string
+	// MaxLeaseTTL caps requested lease TTLs (default 1h; --max-lease-ttl).
+	MaxLeaseTTL time.Duration
+	// Metrics, when non-nil, counts lease requests by outcome.
+	Metrics LeaseMetrics
+
+	log logr.Logger
+
+	mu        sync.Mutex
+	boundAddr string
+}
+
+// NeedLeaderElection is false, and the deployment ships replicas: 1.
+// Verified against controller-runtime's runnable ordering: caches start and
+// sync before non-leader-election runnables, so cache reads here are safe.
+// If this were leader-elected, non-leader replicas would refuse connections
+// while still being Service endpoints. The 1-replica constraint comes from
+// lease state being per-process — both facts are README caveats.
+func (s *Server) NeedLeaderElection() bool { return false }
+
+// BoundAddr returns the actual listen address once Start has bound it —
+// meaningful when Addr uses port 0 (tests).
+func (s *Server) BoundAddr() string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.boundAddr
+}
+
+// Start listens and serves until ctx ends, then shuts down gracefully with
+// a 10-second grace period.
+func (s *Server) Start(ctx context.Context) error {
+	if s.Addr == "" {
+		s.Addr = defaultAddr
+	}
+	if s.MaxLeaseTTL == 0 {
+		s.MaxLeaseTTL = defaultMaxTTL
+	}
+	s.log = logf.FromContext(ctx).WithName("discovery")
+	if s.Token == "" {
+		s.log.Info("WARNING: DISCOVERY_TOKEN is empty — the discovery API is served without authentication")
+	}
+
+	ln, err := net.Listen("tcp", s.Addr)
+	if err != nil {
+		return err
+	}
+	s.mu.Lock()
+	s.boundAddr = ln.Addr().String()
+	s.mu.Unlock()
+
+	srv := &http.Server{
+		Handler:           s.handler(),
+		ReadHeaderTimeout: readHeadTimeout,
+		ReadTimeout:       10 * time.Second,
+		WriteTimeout:      10 * time.Second,
+		IdleTimeout:       60 * time.Second,
+	}
+
+	errCh := make(chan error, 1)
+	go func() { errCh <- srv.Serve(ln) }()
+	s.log.Info("discovery API listening", "addr", s.boundAddr)
+
+	select {
+	case err := <-errCh:
+		return err
+	case <-ctx.Done():
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownGrace)
+		defer cancel()
+		if err := srv.Shutdown(shutdownCtx); err != nil {
+			return err
+		}
+		<-errCh // always http.ErrServerClosed after a clean Shutdown
+		return nil
+	}
+}
+
+// handler assembles the mux and the middleware chain, outermost first:
+// recover → tracing (server span + request logger) → request-log →
+// body-size cap → bearer auth. Everything inside the tracing layer logs via
+// logf.FromContext(r.Context()) and so carries traceID/spanID.
+func (s *Server) handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok\n"))
+	})
+	mux.HandleFunc("GET /v1/proxies", s.handleListProxies)
+	mux.HandleFunc("POST /v1/leases", s.handleAcquireLease)
+	mux.HandleFunc("DELETE /v1/leases/{id}", s.handleReleaseLease)
+	mux.HandleFunc("POST /v1/leases/{id}/report", s.handleReportLease)
+
+	var h http.Handler = mux
+	h = s.authMiddleware(h)
+	h = maxBytesMiddleware(h)
+	h = s.logMiddleware(h)
+	h = tracing.HTTPMiddleware("discovery", s.log)(h)
+	h = s.recoverMiddleware(h)
+	return h
+}
+
+func (s *Server) recoverMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if p := recover(); p != nil {
+				s.log.Error(nil, "panic in discovery handler", "panic", p, "path", r.URL.Path)
+				writeError(w, http.StatusInternalServerError, "internal", "internal server error")
+			}
+		}()
+		next.ServeHTTP(w, r)
+	})
+}
+
+// statusRecorder captures the response code for the request log.
+type statusRecorder struct {
+	http.ResponseWriter
+	status int
+}
+
+func (r *statusRecorder) WriteHeader(code int) {
+	r.status = code
+	r.ResponseWriter.WriteHeader(code)
+}
+
+func (s *Server) logMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/healthz" {
+			next.ServeHTTP(w, r) // probes are noise
+			return
+		}
+		rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
+		start := time.Now()
+		next.ServeHTTP(rec, r)
+		logf.FromContext(r.Context()).Info("request",
+			"method", r.Method, "path", r.URL.Path,
+			"status", rec.status, "duration", time.Since(start).String())
+	})
+}
+
+func maxBytesMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
+		next.ServeHTTP(w, r)
+	})
+}
+
+func (s *Server) authMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if s.Token == "" || r.URL.Path == "/healthz" {
+			next.ServeHTTP(w, r)
+			return
+		}
+		token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if !ok || subtle.ConstantTimeCompare([]byte(token), []byte(s.Token)) != 1 {
+			writeError(w, http.StatusUnauthorized, "unauthorized", "missing or invalid bearer token")
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+// errorBody is the shared error shape:
+// {"error":"<machine_code>","message":"<human>"}.
+type errorBody struct {
+	Error   string `json:"error"`
+	Message string `json:"message"`
+}
+
+func writeError(w http.ResponseWriter, status int, code, message string) {
+	writeJSON(w, status, errorBody{Error: code, Message: message})
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(v)
+}

internal/discovery/server_test.go

@@ -0,0 +1,386 @@
+package discovery
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+)
+
+func testProxy(name string, attrs map[string]string, healthy bool, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy {
+	p := &crawlv1alpha1.Proxy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name, Namespace: "default", UID: types.UID("uid-" + name),
+		},
+		Spec: crawlv1alpha1.ProxySpec{
+			Mode:       crawlv1alpha1.ModeExternal,
+			Endpoint:   &crawlv1alpha1.EndpointSpec{Host: "10.0.0.1", Port: 3128},
+			Attributes: attrs,
+		},
+	}
+	p.Status.IP = "10.0.0.1"
+	p.Status.Phase = crawlv1alpha1.PhaseReady
+	status := metav1.ConditionFalse
+	if healthy {
+		status = metav1.ConditionTrue
+	}
+	p.Status.Conditions = []metav1.Condition{{
+		Type: crawlv1alpha1.ConditionHealthy, Status: status,
+		Reason: "Probing", LastTransitionTime: metav1.Now(),
+	}}
+	for _, m := range mut {
+		m(p)
+	}
+	return p
+}
+
+func withMaxLeases(n int32) func(*crawlv1alpha1.Proxy) {
+	return func(p *crawlv1alpha1.Proxy) { p.Spec.MaxLeases = &n }
+}
+
+func withLatency(ms int64) func(*crawlv1alpha1.Proxy) {
+	return func(p *crawlv1alpha1.Proxy) { p.Status.LatencyMillis = ms }
+}
+
+// newTestServer wires the handler chain to a fake cache reader and a real
+// lease store, served over httptest.
+func newTestServer(t *testing.T, token string, proxies ...*crawlv1alpha1.Proxy) (*httptest.Server, *Server) {
+	t.Helper()
+	s := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("scheme: %v", err)
+	}
+	builder := fake.NewClientBuilder().WithScheme(s)
+	for _, p := range proxies {
+		builder = builder.WithObjects(p)
+	}
+	srv := &Server{
+		Reader:      builder.Build(),
+		Store:       lease.NewStore(15 * time.Minute),
+		Token:       token,
+		MaxLeaseTTL: time.Hour,
+	}
+	ts := httptest.NewServer(srv.handler())
+	t.Cleanup(ts.Close)
+	return ts, srv
+}
+
+type response struct {
+	status int
+	body   map[string]any
+}
+
+func do(t *testing.T, ts *httptest.Server, method, path, token string, body any) response {
+	t.Helper()
+	var reader io.Reader
+	if body != nil {
+		if s, ok := body.(string); ok {
+			reader = bytes.NewBufferString(s)
+		} else {
+			b, err := json.Marshal(body)
+			if err != nil {
+				t.Fatalf("marshaling request body: %v", err)
+			}
+			reader = bytes.NewBuffer(b)
+		}
+	}
+	req, err := http.NewRequestWithContext(context.Background(), method, ts.URL+path, reader)
+	if err != nil {
+		t.Fatalf("building request: %v", err)
+	}
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
+	}
+	resp, err := ts.Client().Do(req)
+	if err != nil {
+		t.Fatalf("%s %s: %v", method, path, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	out := response{status: resp.StatusCode}
+	raw, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("reading response: %v", err)
+	}
+	if len(raw) > 0 && resp.Header.Get("Content-Type") == "application/json" {
+		if err := json.Unmarshal(raw, &out.body); err != nil {
+			t.Fatalf("decoding response %q: %v", raw, err)
+		}
+	}
+	return out
+}
+
+func TestAuth(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "sekrit", testProxy("p1", nil, true))
+
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "", nil); got.status != http.StatusUnauthorized {
+		t.Errorf("no token: status %d, want 401", got.status)
+	}
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "wrong", nil); got.status != http.StatusUnauthorized {
+		t.Errorf("wrong token: status %d, want 401", got.status)
+	}
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "sekrit", nil); got.status != http.StatusOK {
+		t.Errorf("correct token: status %d, want 200", got.status)
+	}
+	if got := do(t, ts, http.MethodGet, "/healthz", "", nil); got.status != http.StatusOK {
+		t.Errorf("healthz without token: status %d, want 200 (always unauthenticated)", got.status)
+	}
+}
+
+func TestAuth_disabledWithEmptyToken(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "", nil); got.status != http.StatusOK {
+		t.Errorf("status %d, want 200 with auth disabled", got.status)
+	}
+}
+
+func TestListProxies(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "",
+		testProxy("eu-healthy", map[string]string{"geo": "eu", "purpose": "crawl"}, true),
+		testProxy("eu-sick", map[string]string{"geo": "eu"}, false),
+		testProxy("us-healthy", map[string]string{"geo": "us"}, true),
+	)
+
+	tests := []struct {
+		name      string
+		query     string
+		wantCount int
+		wantFirst string
+	}{
+		{name: "no filter returns everything", query: "", wantCount: 3, wantFirst: "default/eu-healthy"},
+		{name: "healthy filter", query: "?healthy=true", wantCount: 2},
+		{name: "unhealthy filter", query: "?healthy=false", wantCount: 1, wantFirst: "default/eu-sick"},
+		{name: "attribute filter", query: "?attr.geo=eu", wantCount: 2},
+		{name: "attribute and health combined", query: "?attr.geo=eu&healthy=true", wantCount: 1, wantFirst: "default/eu-healthy"},
+		{name: "two attributes must both match", query: "?attr.geo=eu&attr.purpose=crawl", wantCount: 1, wantFirst: "default/eu-healthy"},
+		{name: "no matches is 200 with count 0", query: "?attr.geo=mars", wantCount: 0},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got := do(t, ts, http.MethodGet, "/v1/proxies"+tc.query, "", nil)
+			if got.status != http.StatusOK {
+				t.Fatalf("status %d, want 200", got.status)
+			}
+			count := int(got.body["count"].(float64))
+			proxies := got.body["proxies"].([]any)
+			if count != tc.wantCount || len(proxies) != tc.wantCount {
+				t.Fatalf("count = %d (len %d), want %d", count, len(proxies), tc.wantCount)
+			}
+			if tc.wantFirst != "" {
+				first := proxies[0].(map[string]any)
+				if first["id"] != tc.wantFirst {
+					t.Errorf("first id = %v, want %s", first["id"], tc.wantFirst)
+				}
+			}
+		})
+	}
+
+	t.Run("invalid healthy value is 400", func(t *testing.T) {
+		t.Parallel()
+		if got := do(t, ts, http.MethodGet, "/v1/proxies?healthy=maybe", "", nil); got.status != http.StatusBadRequest {
+			t.Errorf("status %d, want 400", got.status)
+		}
+	})
+}
+
+func TestAcquireLease_grantShape(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "",
+		testProxy("eu1", map[string]string{"geo": "eu"}, true, withLatency(30)),
+		testProxy("eu2", map[string]string{"geo": "eu"}, true, withLatency(10)),
+	)
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
+		"selector": map[string]string{"geo": "eu"},
+	})
+	if got.status != http.StatusCreated {
+		t.Fatalf("status %d (%v), want 201", got.status, got.body)
+	}
+	if got.body["leaseID"] == "" || got.body["leaseID"] == nil {
+		t.Error("empty leaseID")
+	}
+	if got.body["ttlSeconds"].(float64) != 300 {
+		t.Errorf("ttlSeconds = %v, want the 300 default", got.body["ttlSeconds"])
+	}
+	proxy := got.body["proxy"].(map[string]any)
+	if proxy["id"] != "default/eu2" {
+		t.Errorf("granted %v, want default/eu2 (lower latency at equal load)", proxy["id"])
+	}
+	if proxy["activeLeases"].(float64) != 1 {
+		t.Errorf("activeLeases = %v, want 1 (this grant included)", proxy["activeLeases"])
+	}
+	if _, err := time.Parse(time.RFC3339, got.body["expiresAt"].(string)); err != nil {
+		t.Errorf("expiresAt %v is not RFC3339: %v", got.body["expiresAt"], err)
+	}
+}
+
+func TestAcquireLease_noMatchBody(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "",
+		testProxy("eu-tiny", map[string]string{"geo": "eu"}, true, withMaxLeases(1)),
+		testProxy("eu-sick", map[string]string{"geo": "eu"}, false),
+	)
+
+	body := map[string]any{"selector": map[string]string{"geo": "eu"}}
+	if got := do(t, ts, http.MethodPost, "/v1/leases", "", body); got.status != http.StatusCreated {
+		t.Fatalf("first acquire: status %d, want 201", got.status)
+	}
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", body)
+	if got.status != http.StatusConflict {
+		t.Fatalf("second acquire: status %d, want 409", got.status)
+	}
+	want := map[string]float64{"considered": 2, "atCapacity": 1, "inCooldown": 0, "unhealthy": 1}
+	for k, v := range want {
+		if got.body[k].(float64) != v {
+			t.Errorf("%s = %v, want %v (body %v)", k, got.body[k], v, got.body)
+		}
+	}
+	if got.body["error"] != "no_match" {
+		t.Errorf("error = %v, want no_match", got.body["error"])
+	}
+}
+
+func TestAcquireLease_badRequests(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
+
+	tests := []struct {
+		name     string
+		body     any
+		wantCode string
+	}{
+		{name: "ttl above the cap", body: map[string]any{"ttlSeconds": 999999}, wantCode: "invalid_ttl"},
+		{name: "negative ttl", body: map[string]any{"ttlSeconds": -5}, wantCode: "invalid_ttl"},
+		{name: "malformed json", body: "{not json", wantCode: "invalid_body"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got := do(t, ts, http.MethodPost, "/v1/leases", "", tc.body)
+			if got.status != http.StatusBadRequest || got.body["error"] != tc.wantCode {
+				t.Errorf("= %d/%v, want 400/%s", got.status, got.body["error"], tc.wantCode)
+			}
+		})
+	}
+}
+
+func TestReleaseLease_alwaysNoContent(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{})
+	if got.status != http.StatusCreated {
+		t.Fatalf("acquire: status %d, want 201", got.status)
+	}
+	id := got.body["leaseID"].(string)
+
+	for _, path := range []string{"/v1/leases/" + id, "/v1/leases/" + id, "/v1/leases/never-existed"} {
+		if got := do(t, ts, http.MethodDelete, path, "", nil); got.status != http.StatusNoContent {
+			t.Errorf("DELETE %s: status %d, want 204", path, got.status)
+		}
+	}
+}
+
+func TestReportLease(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", map[string]string{"geo": "eu"}, true))
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
+		"selector": map[string]string{"geo": "eu"}, "target": "example.com",
+	})
+	if got.status != http.StatusCreated {
+		t.Fatalf("acquire: status %d, want 201", got.status)
+	}
+	id := got.body["leaseID"].(string)
+	reportPath := fmt.Sprintf("/v1/leases/%s/report", id)
+
+	if got := do(t, ts, http.MethodPost, reportPath, "", map[string]any{"result": "rate_limited", "target": "example.com"}); got.status != http.StatusNoContent {
+		t.Fatalf("report: status %d, want 204", got.status)
+	}
+
+	// The cooldown from the report now blocks same-target acquisition.
+	got = do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
+		"selector": map[string]string{"geo": "eu"}, "target": "example.com",
+	})
+	if got.status != http.StatusConflict || got.body["inCooldown"].(float64) != 1 {
+		t.Errorf("post-report acquire = %d/%v, want 409 with inCooldown 1", got.status, got.body)
+	}
+
+	t.Run("invalid result value", func(t *testing.T) {
+		got := do(t, ts, http.MethodPost, reportPath, "", map[string]any{"result": "throttled"})
+		if got.status != http.StatusBadRequest || got.body["error"] != "invalid_result" {
+			t.Errorf("= %d/%v, want 400/invalid_result", got.status, got.body["error"])
+		}
+	})
+	t.Run("unknown lease", func(t *testing.T) {
+		got := do(t, ts, http.MethodPost, "/v1/leases/never-existed/report", "", map[string]any{"result": "ok"})
+		if got.status != http.StatusNotFound || got.body["error"] != "unknown_lease" {
+			t.Errorf("= %d/%v, want 404/unknown_lease", got.status, got.body["error"])
+		}
+	})
+}
+
+func TestStart_servesAndShutsDown(t *testing.T) {
+	t.Parallel()
+	s := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("scheme: %v", err)
+	}
+	srv := &Server{
+		Reader: fake.NewClientBuilder().WithScheme(s).Build(),
+		Store:  lease.NewStore(time.Minute),
+		Addr:   "127.0.0.1:0",
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- srv.Start(ctx) }()
+
+	var addr string
+	deadline := time.After(5 * time.Second)
+	for addr == "" {
+		select {
+		case <-deadline:
+			t.Fatal("server never bound")
+		case <-time.After(5 * time.Millisecond):
+			addr = srv.BoundAddr()
+		}
+	}
+
+	resp, err := http.Get("http://" + addr + "/healthz")
+	if err != nil {
+		t.Fatalf("healthz: %v", err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("healthz status %d, want 200", resp.StatusCode)
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start returned %v, want nil after graceful shutdown", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}

internal/gc/gc.go

@@ -0,0 +1,137 @@
+// Package gc implements orphan garbage collection: a periodic sweep that
+// deletes provider instances tagged by this operator whose owning Proxy CR
+// no longer exists — the safety net for crashes between a provider Create
+// and the status write that records it.
+package gc
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"go.opentelemetry.io/otel/attribute"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+// Sweeper is the manager Runnable running the sweep loop.
+type Sweeper struct {
+	// Reader lists Proxies from the manager's cache to establish the live
+	// UID set.
+	Reader client.Reader
+	// Providers are the configured backends; each is swept independently.
+	Providers map[string]provider.Provider
+
+	// Interval between sweeps (default 10m). The first sweep runs one full
+	// interval after start, not immediately — right after startup the
+	// cache is coldest and an in-flight create is most likely.
+	Interval time.Duration
+	// MinAge exempts young instances (default 10m): an instance mid-create
+	// may not have its status write landed yet; deleting it would race the
+	// reconciler.
+	MinAge time.Duration
+
+	// NamespaceRestricted must be set when the manager cache is limited to
+	// one namespace. Then the live-UID set is incomplete, and a sweep
+	// would delete VMs owned by Proxies the cache cannot see — so Start
+	// refuses unless AllowNamespaced (--gc-allow-namespaced) is explicit.
+	NamespaceRestricted bool
+	AllowNamespaced     bool
+
+	now func() time.Time
+}
+
+// NeedLeaderElection is true: the sweep is destructive and must have a
+// single writer.
+func (s *Sweeper) NeedLeaderElection() bool { return true }
+
+// Start runs the sweep loop until ctx ends.
+func (s *Sweeper) Start(ctx context.Context) error {
+	if s.NamespaceRestricted && !s.AllowNamespaced {
+		return errors.New(
+			"orphan GC refuses to run against a namespace-restricted cache: proxies outside the namespace " +
+				"would count as orphans and their instances would be deleted; pass --gc-allow-namespaced to override")
+	}
+	if s.Interval == 0 {
+		s.Interval = 10 * time.Minute
+	}
+	if s.MinAge == 0 {
+		s.MinAge = 10 * time.Minute
+	}
+	if s.now == nil {
+		s.now = time.Now
+	}
+
+	ticker := time.NewTicker(s.Interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			s.sweep(ctx)
+		}
+	}
+}
+
+// sweep deletes tagged instances whose UID matches no existing Proxy CR.
+// A CR with a deletionTimestamp still counts as live: its finalizer owns
+// that deletion, and GC racing it would double-delete. A UID becomes
+// orphan-eligible only once the object is fully gone.
+func (s *Sweeper) sweep(ctx context.Context) {
+	ctx, span := tracing.Start(ctx, "gc.sweep")
+	defer span.End()
+	log := logf.FromContext(ctx).WithName("orphan-gc")
+
+	var list crawlv1alpha1.ProxyList
+	if err := s.Reader.List(ctx, &list); err != nil {
+		// Without the live set nothing can be proven orphaned; skip the
+		// whole sweep rather than guess.
+		log.Error(err, "listing proxies; skipping this sweep")
+		return
+	}
+	live := make(map[string]bool, len(list.Items))
+	for i := range list.Items {
+		live[string(list.Items[i].UID)] = true
+	}
+
+	deleted := 0
+	for name, prov := range s.Providers {
+		instances, err := prov.ListByTag(ctx)
+		if err != nil {
+			// One broken provider must not abort the sweep for the rest.
+			log.Error(err, "listing instances; skipping this provider", "provider", name)
+			continue
+		}
+		for _, inst := range instances {
+			switch {
+			case inst.UID == "":
+				// Managed label without a UID label shouldn't exist for
+				// anything this operator created; without ownership proof,
+				// never delete.
+				continue
+			case live[inst.UID]:
+				continue
+			case s.now().Sub(inst.CreatedAt) < s.MinAge:
+				continue
+			}
+			log.Info("WARNING: deleting orphaned instance",
+				"provider", name, "providerID", inst.ID, "uid", inst.UID)
+			if err := prov.Delete(ctx, inst.ID); err != nil {
+				log.Error(err, "deleting orphaned instance",
+					"provider", name, "providerID", inst.ID, "uid", inst.UID)
+				continue
+			}
+			deleted++
+		}
+	}
+	span.SetAttributes(
+		attribute.Int("gc.providers", len(s.Providers)),
+		attribute.Int("gc.deleted", deleted),
+	)
+}

internal/gc/gc_test.go

@@ -0,0 +1,214 @@
+package gc
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// listProvider serves a canned instance list and records deletions.
+type listProvider struct {
+	mu        sync.Mutex
+	instances []provider.Instance
+	listErr   error
+	deleted   []string
+}
+
+func (l *listProvider) Create(context.Context, provider.CreateRequest) (string, error) {
+	return "", errors.New("not used")
+}
+
+func (l *listProvider) Get(context.Context, string) (*provider.Instance, error) {
+	return nil, provider.ErrNotFound
+}
+
+func (l *listProvider) Delete(_ context.Context, id string) error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.deleted = append(l.deleted, id)
+	return nil
+}
+
+func (l *listProvider) ListByTag(context.Context) ([]provider.Instance, error) {
+	return l.instances, l.listErr
+}
+
+func (l *listProvider) deletedIDs() []string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return append([]string(nil), l.deleted...)
+}
+
+func proxyWithUID(name, uid string, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy {
+	p := &crawlv1alpha1.Proxy{
+		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", UID: types.UID(uid)},
+		Spec: crawlv1alpha1.ProxySpec{
+			Mode:     crawlv1alpha1.ModeManaged,
+			Provider: "stub",
+		},
+	}
+	for _, m := range mut {
+		m(p)
+	}
+	return p
+}
+
+func newReader(t *testing.T, objs ...client.Object) client.Reader {
+	t.Helper()
+	s := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("scheme: %v", err)
+	}
+	return fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build()
+}
+
+func oldInstance(id, uid string) provider.Instance {
+	return provider.Instance{ID: id, UID: uid, State: provider.StateRunning,
+		CreatedAt: time.Now().Add(-time.Hour)}
+}
+
+func newSweeper(reader client.Reader, providers map[string]provider.Provider) *Sweeper {
+	return &Sweeper{
+		Reader:    reader,
+		Providers: providers,
+		Interval:  10 * time.Minute,
+		MinAge:    10 * time.Minute,
+		now:       time.Now,
+	}
+}
+
+func TestSweep_deletesOnlyTrueOrphans(t *testing.T) {
+	t.Parallel()
+
+	deletingCR := proxyWithUID("deleting", "uid-deleting", func(p *crawlv1alpha1.Proxy) {
+		now := metav1.Now()
+		p.DeletionTimestamp = &now
+		p.Finalizers = []string{crawlv1alpha1.FinalizerName}
+	})
+	prov := &listProvider{instances: []provider.Instance{
+		oldInstance("inst-live", "uid-live"),
+		oldInstance("inst-orphan", "uid-orphan"),
+		oldInstance("inst-deleting", "uid-deleting"),
+		{ID: "inst-young", UID: "uid-young-orphan", State: provider.StateRunning,
+			CreatedAt: time.Now().Add(-time.Minute)},
+		oldInstance("inst-unlabelled", ""),
+	}}
+	s := newSweeper(
+		newReader(t, proxyWithUID("live", "uid-live"), deletingCR),
+		map[string]provider.Provider{"stub": prov},
+	)
+
+	s.sweep(context.Background())
+
+	got := prov.deletedIDs()
+	if len(got) != 1 || got[0] != "inst-orphan" {
+		t.Errorf("deleted %v, want exactly [inst-orphan]:\n"+
+			"live CR's instance must stay; a deleting CR still owns its instance (finalizer, not GC);\n"+
+			"young instances may be mid-create; unlabelled instances have no ownership proof", got)
+	}
+}
+
+func TestSweep_providerErrorDoesNotAbortOthers(t *testing.T) {
+	t.Parallel()
+	broken := &listProvider{listErr: errors.New("cloud is down")}
+	working := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
+	s := newSweeper(newReader(t), map[string]provider.Provider{
+		"broken":  broken,
+		"working": working,
+	})
+
+	s.sweep(context.Background())
+
+	if got := working.deletedIDs(); len(got) != 1 {
+		t.Errorf("working provider deletions = %v, want the orphan despite the broken provider", got)
+	}
+}
+
+// errReader fails every List: without the live set nothing can be proven
+// orphaned, so the sweep must delete nothing.
+type errReader struct{ client.Reader }
+
+func (errReader) List(context.Context, client.ObjectList, ...client.ListOption) error {
+	return errors.New("cache broken")
+}
+
+func TestSweep_listFailureSkipsSweep(t *testing.T) {
+	t.Parallel()
+	prov := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
+	s := newSweeper(errReader{}, map[string]provider.Provider{"stub": prov})
+
+	s.sweep(context.Background())
+
+	if got := prov.deletedIDs(); len(got) != 0 {
+		t.Errorf("deleted %v with an unreadable live set, want nothing", got)
+	}
+}
+
+func TestStart_namespaceGuard(t *testing.T) {
+	t.Parallel()
+
+	s := newSweeper(newReader(t), nil)
+	s.NamespaceRestricted = true
+	err := s.Start(context.Background())
+	if err == nil || !strings.Contains(err.Error(), "--gc-allow-namespaced") {
+		t.Errorf("Start with restricted cache = %v, want refusal naming the override flag", err)
+	}
+
+	s2 := newSweeper(newReader(t), nil)
+	s2.NamespaceRestricted = true
+	s2.AllowNamespaced = true
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- s2.Start(ctx) }()
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start with override = %v, want it to run until cancel", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}
+
+func TestStart_sweepsOnIntervalAndStops(t *testing.T) {
+	t.Parallel()
+	prov := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
+	s := newSweeper(newReader(t), map[string]provider.Provider{"stub": prov})
+	s.Interval = 5 * time.Millisecond
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- s.Start(ctx) }()
+
+	deadline := time.After(5 * time.Second)
+	for len(prov.deletedIDs()) == 0 {
+		select {
+		case <-deadline:
+			t.Fatal("no sweep ran")
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start = %v, want nil", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}

internal/health/engine.go

@@ -10,6 +10,9 @@ import (
 	"sync"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -18,6 +21,7 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // Snapshot is the engine's current verdict for one proxy, read by the
@@ -62,6 +66,14 @@ type probeJob struct {
 	interval time.Duration
 }
 
+// ProbeMetrics receives every probe result and the retirement of a
+// proxy's series. Implemented by internal/metrics; defined here so this
+// package carries no metrics dependency.
+type ProbeMetrics interface {
+	ObserveProbe(proxy string, latency time.Duration, success bool)
+	ForgetProxy(proxy string)
+}
+
 // Engine runs the probe scheduler and worker pool as a manager Runnable. It
 // never writes Proxy status itself — keeping the reconciler the single
 // status writer — and instead emits a GenericEvent per status-affecting
@@ -87,6 +99,14 @@ type Engine struct {
 	// ProbeTLSConfig overrides TLS verification for https probe URLs; nil
 	// means system roots. Needed for private CAs (and tests).
 	ProbeTLSConfig *tls.Config
+	// Metrics, when non-nil, is fed on every probe — the status writes are
+	// transition-only by design, so metrics are where high-frequency
+	// signal (true probe recency, every latency sample) lives.
+	Metrics ProbeMetrics
+	// TraceProbes emits one root span per probe (--trace-health-probes).
+	// Off by default: probes run about once per second per proxy and would
+	// dominate trace volume.
+	TraceProbes bool
 
 	probeFn func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult
 
@@ -143,8 +163,7 @@ func (e *Engine) Start(ctx context.Context) error {
 				case <-ctx.Done():
 					return
 				case job := <-jobs:
-					res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
-					e.record(job, res, time.Now())
+					e.record(job, e.runProbe(ctx, job), time.Now())
 				}
 			}
 		})
@@ -220,6 +239,11 @@ func (e *Engine) tick(ctx context.Context, now time.Time, jobs chan<- probeJob)
 	for key := range e.states {
 		if _, ok := probeable[key]; !ok {
 			delete(e.states, key)
+			if e.Metrics != nil {
+				// Retire the per-proxy series with the state, or series
+				// for deleted proxies leak forever.
+				e.Metrics.ForgetProxy(key.String())
+			}
 		}
 	}
 }
@@ -247,11 +271,38 @@ func newState(p *crawlv1alpha1.Proxy, now time.Time, interval time.Duration) *st
 	return st
 }
 
+// runProbe executes one probe, inside its own root span when TraceProbes is
+// on. The span ends before record folds the result (record stays ctx-free),
+// and the probe transport is deliberately uninstrumented — no traceparent
+// must ever leak through a proxy toward external targets.
+func (e *Engine) runProbe(ctx context.Context, job probeJob) probeResult {
+	if !e.TraceProbes {
+		return e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
+	}
+	ctx, span := tracing.Start(ctx, "health.probe",
+		trace.WithSpanKind(trace.SpanKindClient),
+		trace.WithAttributes(attribute.String("proxy", job.key.String())))
+	defer span.End()
+	res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
+	span.SetAttributes(
+		attribute.Bool("probe.ok", res.ok),
+		attribute.Int64("probe.latency_ms", res.latency.Milliseconds()),
+	)
+	if !res.ok {
+		span.SetStatus(codes.Error, res.err.Error())
+	}
+	return res
+}
+
 // record folds one probe result into the proxy's threshold state and emits
 // an event when the result is status-affecting: a first-ever verdict, a
 // threshold-crossing flip, or a material latency change (beyond
 // max(LatencyFloor, 50% of reported) and rate-limited by MinReportInterval).
 func (e *Engine) record(job probeJob, res probeResult, now time.Time) {
+	if e.Metrics != nil {
+		e.Metrics.ObserveProbe(job.key.String(), res.latency, res.ok)
+	}
+
 	e.mu.Lock()
 	defer e.mu.Unlock()
 

internal/lease/store.go

@@ -0,0 +1,319 @@
+// Package lease implements the in-memory lease store behind the discovery
+// API: TTL-based proxy assignment with server-side usage tracking and
+// per-(proxy, target) cooldowns. Accepted prototype limitation, documented
+// in the README: state is per-process, so an operator restart drops all
+// leases and cooldowns — clients must tolerate a lease vanishing (their
+// requests still work; they just re-lease).
+package lease
+
+import (
+	"cmp"
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Result is a client's report of how a leased proxy behaved against a
+// target. ResultRateLimited and ResultBanned record a cooldown; ResultOK is
+// an acknowledgement and records nothing.
+type Result string
+
+const (
+	ResultOK          Result = "ok"
+	ResultRateLimited Result = "rate_limited"
+	ResultBanned      Result = "banned"
+)
+
+// ParseResult maps a wire value to a Result; ok is false for anything
+// unknown, which the API layer turns into a 400.
+func ParseResult(s string) (Result, bool) {
+	switch r := Result(s); r {
+	case ResultOK, ResultRateLimited, ResultBanned:
+		return r, true
+	default:
+		return "", false
+	}
+}
+
+var (
+	// ErrNoMatch means no candidate could take a lease; AcquireStats says
+	// why, and the API layer turns both into the 409 body.
+	ErrNoMatch = errors.New("lease: no candidate available")
+	// ErrUnknownLease means the lease ID does not resolve (404). Reports on
+	// recently expired leases do NOT hit this — see the retention note on
+	// Store.
+	ErrUnknownLease = errors.New("lease: unknown lease id")
+)
+
+// Candidate is one leasable proxy as seen by the caller at selection time.
+// The store itself knows nothing about Proxy objects — the discovery layer
+// filters for health/attributes and passes what selection needs.
+type Candidate struct {
+	// Proxy is the opaque proxy key ("namespace/name").
+	Proxy string
+	// MaxLeases caps concurrent leases; 0 means unleasable.
+	MaxLeases int32
+	// Latency is the proxy's last reported latency, used as the tie-break.
+	Latency time.Duration
+}
+
+// Lease is a granted assignment. Values returned by the store are copies;
+// mutating them does not affect the store.
+type Lease struct {
+	ID        string
+	Proxy     string
+	Target    string
+	ExpiresAt time.Time
+}
+
+// AcquireRequest carries the candidate set and lease parameters. Acquire
+// deliberately takes the whole candidate set, not a pre-chosen proxy:
+// selection and insertion must happen under one lock, or two concurrent
+// requests both see "3 of 5 used" and overcommit.
+type AcquireRequest struct {
+	Candidates []Candidate
+	// Target scopes the cooldown check; empty means the global pool.
+	Target string
+	TTL    time.Duration
+}
+
+// AcquireStats explains an ErrNoMatch (and is returned on success too):
+// every candidate is either leased, at capacity, or in cooldown.
+type AcquireStats struct {
+	Considered int
+	AtCapacity int
+	InCooldown int
+}
+
+type cooldownKey struct{ proxy, target string }
+
+// Store is the in-memory lease store. One mutex guards everything: at tens
+// of proxies and human-rate QPS, sharding would be premature complexity.
+//
+// Retention: an expired lease is kept for CooldownWindow past its TTL so a
+// Report arriving just after expiry still resolves — which matters most
+// exactly when a proxy is being rate-limited. Acquire and the counts ignore
+// retained leases; only the sweep finally drops them.
+type Store struct {
+	// CooldownWindow is how long a reported proxy/target pair is excluded
+	// from selection (default 15m; --lease-cooldown in Step 10).
+	CooldownWindow time.Duration
+	// SweepInterval is how often the expiry sweep runs (default 30s).
+	SweepInterval time.Duration
+
+	now func() time.Time
+
+	mu        sync.Mutex
+	byID      map[string]*Lease
+	byProxy   map[string]map[string]*Lease
+	cooldowns map[cooldownKey]time.Time
+}
+
+// NewStore returns a ready Store. A non-positive cooldownWindow selects the
+// 15-minute default.
+func NewStore(cooldownWindow time.Duration) *Store {
+	if cooldownWindow <= 0 {
+		cooldownWindow = 15 * time.Minute
+	}
+	return &Store{
+		CooldownWindow: cooldownWindow,
+		SweepInterval:  30 * time.Second,
+		now:            time.Now,
+		byID:           map[string]*Lease{},
+		byProxy:        map[string]map[string]*Lease{},
+		cooldowns:      map[cooldownKey]time.Time{},
+	}
+}
+
+// Acquire selects the least-loaded eligible candidate (ties: lowest
+// latency, then name, so selection is deterministic and testable) and
+// grants a lease on it.
+func (s *Store) Acquire(_ context.Context, req AcquireRequest) (*Lease, AcquireStats, error) {
+	stats := AcquireStats{Considered: len(req.Candidates)}
+	if req.TTL <= 0 {
+		return nil, stats, fmt.Errorf("lease: non-positive TTL %v", req.TTL)
+	}
+	now := s.now()
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	type eligible struct {
+		cand   Candidate
+		active int
+	}
+	var elig []eligible
+	for _, c := range req.Candidates {
+		if s.inCooldownLocked(c.Proxy, req.Target, now) {
+			stats.InCooldown++
+			continue
+		}
+		active := s.activeCountLocked(c.Proxy, now)
+		if int32(active) >= c.MaxLeases {
+			stats.AtCapacity++
+			continue
+		}
+		elig = append(elig, eligible{cand: c, active: active})
+	}
+	if len(elig) == 0 {
+		return nil, stats, ErrNoMatch
+	}
+
+	slices.SortFunc(elig, func(a, b eligible) int {
+		if c := cmp.Compare(a.active, b.active); c != 0 {
+			return c
+		}
+		if c := cmp.Compare(a.cand.Latency, b.cand.Latency); c != 0 {
+			return c
+		}
+		return strings.Compare(a.cand.Proxy, b.cand.Proxy)
+	})
+
+	l := &Lease{
+		ID:        rand.Text(),
+		Proxy:     elig[0].cand.Proxy,
+		Target:    req.Target,
+		ExpiresAt: now.Add(req.TTL),
+	}
+	s.byID[l.ID] = l
+	if s.byProxy[l.Proxy] == nil {
+		s.byProxy[l.Proxy] = map[string]*Lease{}
+	}
+	s.byProxy[l.Proxy][l.ID] = l
+
+	granted := *l
+	return &granted, stats, nil
+}
+
+// Release drops a lease early. Idempotent: releasing an unknown or already
+// expired lease is a no-op, so the API's DELETE can always answer 204.
+func (s *Store) Release(_ context.Context, id string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.dropLocked(id)
+}
+
+// Report records the outcome of using a lease. Rate-limited and banned
+// results put the (proxy, target) pair in cooldown — target taken from the
+// report, falling back to the lease's own target, falling back to the
+// global pool. Reports on recently expired leases still resolve (see the
+// retention note on Store).
+func (s *Store) Report(_ context.Context, id string, result Result, target string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	l, ok := s.byID[id]
+	if !ok {
+		return ErrUnknownLease
+	}
+	if result == ResultOK {
+		return nil
+	}
+	if target == "" {
+		target = l.Target
+	}
+	s.cooldowns[cooldownKey{proxy: l.Proxy, target: target}] = s.now().Add(s.CooldownWindow)
+	return nil
+}
+
+// ActiveCount returns the number of unexpired leases held on one proxy.
+func (s *Store) ActiveCount(proxy string) int {
+	now := s.now()
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.activeCountLocked(proxy, now)
+}
+
+// Counts returns the active-lease count per proxy, for the discovery list
+// endpoint and the metrics collector. Proxies with no active leases are
+// absent from the map.
+func (s *Store) Counts() map[string]int {
+	now := s.now()
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	counts := make(map[string]int, len(s.byProxy))
+	for proxy := range s.byProxy {
+		if n := s.activeCountLocked(proxy, now); n > 0 {
+			counts[proxy] = n
+		}
+	}
+	return counts
+}
+
+// Start runs the expiry sweep until ctx ends; it satisfies
+// manager.Runnable so cmd/main.go can mgr.Add the store directly.
+func (s *Store) Start(ctx context.Context) error {
+	ticker := time.NewTicker(s.SweepInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			s.sweep(s.now())
+		}
+	}
+}
+
+// NeedLeaderElection is false: lease state is per-process and the discovery
+// API serves wherever this process runs, so the sweep must run there too.
+func (s *Store) NeedLeaderElection() bool { return false }
+
+// sweep drops leases past their retention window and elapsed cooldowns.
+// Correctness never depends on sweep timing — every read path checks
+// expiry against the clock — so this is purely garbage collection.
+func (s *Store) sweep(now time.Time) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for id, l := range s.byID {
+		if now.After(l.ExpiresAt.Add(s.CooldownWindow)) {
+			s.dropLocked(id)
+		}
+	}
+	for k, until := range s.cooldowns {
+		if now.After(until) {
+			delete(s.cooldowns, k)
+		}
+	}
+}
+
+func (s *Store) dropLocked(id string) {
+	l, ok := s.byID[id]
+	if !ok {
+		return
+	}
+	delete(s.byID, id)
+	delete(s.byProxy[l.Proxy], id)
+	if len(s.byProxy[l.Proxy]) == 0 {
+		delete(s.byProxy, l.Proxy)
+	}
+}
+
+func (s *Store) activeCountLocked(proxy string, now time.Time) int {
+	n := 0
+	for _, l := range s.byProxy[proxy] {
+		if now.Before(l.ExpiresAt) {
+			n++
+		}
+	}
+	return n
+}
+
+// inCooldownLocked: the global cooldown (empty target) always applies; a
+// target-scoped cooldown additionally applies to acquisitions for that
+// target. An acquisition without a target sees only the global pool — a
+// proxy rate-limited by one site is still fine for everyone else.
+func (s *Store) inCooldownLocked(proxy, target string, now time.Time) bool {
+	if until, ok := s.cooldowns[cooldownKey{proxy: proxy}]; ok && now.Before(until) {
+		return true
+	}
+	if target == "" {
+		return false
+	}
+	until, ok := s.cooldowns[cooldownKey{proxy: proxy, target: target}]
+	return ok && now.Before(until)
+}

internal/lease/store_test.go

@@ -0,0 +1,365 @@
+package lease
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+)
+
+// fakeClock is an injectable, manually advanced clock.
+type fakeClock struct {
+	mu  sync.Mutex
+	cur time.Time
+}
+
+func newFakeClock() *fakeClock {
+	return &fakeClock{cur: time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC)}
+}
+
+func (c *fakeClock) Now() time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.cur
+}
+
+func (c *fakeClock) Advance(d time.Duration) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.cur = c.cur.Add(d)
+}
+
+func newTestStore() (*Store, *fakeClock) {
+	s := NewStore(15 * time.Minute)
+	clock := newFakeClock()
+	s.now = clock.Now
+	return s, clock
+}
+
+func candidate(proxy string, maxLeases int32, latency time.Duration) Candidate {
+	return Candidate{Proxy: proxy, MaxLeases: maxLeases, Latency: latency}
+}
+
+func mustAcquire(t *testing.T, s *Store, req AcquireRequest) *Lease {
+	t.Helper()
+	l, _, err := s.Acquire(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Acquire: %v", err)
+	}
+	return l
+}
+
+func TestAcquire_capacity(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 2, 0)}, TTL: time.Minute}
+
+	l1 := mustAcquire(t, s, req)
+	l2 := mustAcquire(t, s, req)
+	if l1.ID == l2.ID {
+		t.Fatal("two leases share an ID")
+	}
+	if got := s.ActiveCount("ns/p1"); got != 2 {
+		t.Fatalf("ActiveCount = %d, want 2", got)
+	}
+
+	_, stats, err := s.Acquire(context.Background(), req)
+	if !errors.Is(err, ErrNoMatch) {
+		t.Fatalf("third acquire error = %v, want ErrNoMatch", err)
+	}
+	want := AcquireStats{Considered: 1, AtCapacity: 1}
+	if stats != want {
+		t.Errorf("stats = %+v, want %+v", stats, want)
+	}
+
+	// Early release frees the slot again.
+	s.Release(context.Background(), l1.ID)
+	mustAcquire(t, s, req)
+}
+
+func TestAcquire_maxLeasesZeroIsUnleasable(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+		Candidates: []Candidate{candidate("ns/p1", 0, 0)},
+		TTL:        time.Minute,
+	})
+	if !errors.Is(err, ErrNoMatch) {
+		t.Fatalf("err = %v, want ErrNoMatch", err)
+	}
+	if stats.AtCapacity != 1 {
+		t.Errorf("stats = %+v, want the unleasable proxy counted AtCapacity", stats)
+	}
+}
+
+func TestAcquire_selectionOrder(t *testing.T) {
+	t.Parallel()
+
+	t.Run("least loaded wins", func(t *testing.T) {
+		t.Parallel()
+		s, _ := newTestStore()
+		mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{candidate("ns/a", 5, 10*time.Millisecond)}, TTL: time.Minute,
+		})
+		l := mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{
+				candidate("ns/a", 5, 10*time.Millisecond), // 1 active, lower latency
+				candidate("ns/b", 5, 90*time.Millisecond), // 0 active
+			},
+			TTL: time.Minute,
+		})
+		if l.Proxy != "ns/b" {
+			t.Errorf("chose %s, want the least-loaded ns/b", l.Proxy)
+		}
+	})
+
+	t.Run("latency breaks the load tie", func(t *testing.T) {
+		t.Parallel()
+		s, _ := newTestStore()
+		l := mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{
+				candidate("ns/a", 5, 90*time.Millisecond),
+				candidate("ns/b", 5, 10*time.Millisecond),
+			},
+			TTL: time.Minute,
+		})
+		if l.Proxy != "ns/b" {
+			t.Errorf("chose %s, want the lower-latency ns/b", l.Proxy)
+		}
+	})
+
+	t.Run("name breaks a full tie deterministically", func(t *testing.T) {
+		t.Parallel()
+		s, _ := newTestStore()
+		l := mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{
+				candidate("ns/b", 5, 10*time.Millisecond),
+				candidate("ns/a", 5, 10*time.Millisecond),
+			},
+			TTL: time.Minute,
+		})
+		if l.Proxy != "ns/a" {
+			t.Errorf("chose %s, want ns/a (lexicographic tie-break)", l.Proxy)
+		}
+	})
+}
+
+func TestAcquire_cooldownScoping(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, "example.com"); err != nil {
+		t.Fatalf("Report: %v", err)
+	}
+
+	// Same target: excluded.
+	_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+		Candidates: cands, Target: "example.com", TTL: time.Minute,
+	})
+	if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
+		t.Errorf("same-target acquire = (%v, %+v), want ErrNoMatch with InCooldown=1", err, stats)
+	}
+
+	// Different target: fine.
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "other.org", TTL: time.Minute})
+
+	// No target (global pool): a target-scoped cooldown does not apply.
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+}
+
+func TestAcquire_globalCooldownBlocksEverything(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	// A lease without a target, reported banned without a target: the
+	// cooldown lands on the global pool.
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+	if err := s.Report(context.Background(), l.ID, ResultBanned, ""); err != nil {
+		t.Fatalf("Report: %v", err)
+	}
+
+	for _, target := range []string{"", "example.com"} {
+		_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+			Candidates: cands, Target: target, TTL: time.Minute,
+		})
+		if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
+			t.Errorf("acquire(target=%q) = (%v, %+v), want global cooldown to block", target, err, stats)
+		}
+	}
+}
+
+func TestAcquire_cooldownExpires(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); err != nil {
+		t.Fatalf("Report: %v", err)
+	}
+	if _, _, err := s.Acquire(context.Background(), AcquireRequest{Candidates: cands, TTL: time.Minute}); !errors.Is(err, ErrNoMatch) {
+		t.Fatal("expected cooldown to block immediately after the report")
+	}
+
+	clock.Advance(15*time.Minute + time.Second)
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+}
+
+func TestExpiry_freesCapacityWithoutSweep(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 1, 0)}, TTL: time.Minute}
+
+	mustAcquire(t, s, req)
+	if _, _, err := s.Acquire(context.Background(), req); !errors.Is(err, ErrNoMatch) {
+		t.Fatal("capacity 1 not enforced")
+	}
+
+	clock.Advance(2 * time.Minute)
+	// No sweep has run; expiry must still free capacity and zero the counts.
+	if got := s.ActiveCount("ns/p1"); got != 0 {
+		t.Fatalf("ActiveCount after TTL = %d, want 0", got)
+	}
+	if counts := s.Counts(); len(counts) != 0 {
+		t.Fatalf("Counts after TTL = %v, want empty", counts)
+	}
+	mustAcquire(t, s, req)
+}
+
+func TestReport_expiredButRetainedLease(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+
+	// TTL lapses; the report arrives late — exactly when the proxy is being
+	// rate-limited, which is when the cooldown matters most.
+	clock.Advance(5 * time.Minute)
+	s.sweep(clock.Now())
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); err != nil {
+		t.Fatalf("Report on an expired-but-retained lease: %v", err)
+	}
+	// The cooldown fell back to the lease's own target.
+	_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+		Candidates: cands, Target: "example.com", TTL: time.Minute,
+	})
+	if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
+		t.Errorf("acquire = (%v, %+v), want cooldown from the late report", err, stats)
+	}
+
+	// Past the retention window the sweep finally drops it.
+	clock.Advance(15 * time.Minute)
+	s.sweep(clock.Now())
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); !errors.Is(err, ErrUnknownLease) {
+		t.Errorf("Report after retention = %v, want ErrUnknownLease", err)
+	}
+}
+
+func TestReport_okRecordsNothing(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+
+	if err := s.Report(context.Background(), l.ID, ResultOK, "example.com"); err != nil {
+		t.Fatalf("Report(ok): %v", err)
+	}
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+}
+
+func TestRelease_isIdempotent(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	l := mustAcquire(t, s, AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 1, 0)}, TTL: time.Minute})
+
+	s.Release(context.Background(), l.ID)
+	s.Release(context.Background(), l.ID)
+	s.Release(context.Background(), "never-existed")
+	if got := s.ActiveCount("ns/p1"); got != 0 {
+		t.Errorf("ActiveCount = %d, want 0", got)
+	}
+}
+
+func TestParseResult(t *testing.T) {
+	t.Parallel()
+	for _, valid := range []string{"ok", "rate_limited", "banned"} {
+		if _, ok := ParseResult(valid); !ok {
+			t.Errorf("ParseResult(%q) rejected a valid value", valid)
+		}
+	}
+	for _, invalid := range []string{"", "OK", "throttled", "rate-limited"} {
+		if _, ok := ParseResult(invalid); ok {
+			t.Errorf("ParseResult(%q) accepted an invalid value", invalid)
+		}
+	}
+}
+
+func TestAcquire_concurrentNeverOvercommits(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 5, 0)}, TTL: time.Minute}
+
+	const attempts = 40
+	var wg sync.WaitGroup
+	granted := make(chan *Lease, attempts)
+	for range attempts {
+		wg.Go(func() {
+			if l, _, err := s.Acquire(context.Background(), req); err == nil {
+				granted <- l
+			}
+		})
+	}
+	wg.Wait()
+	close(granted)
+
+	var n int
+	for range granted {
+		n++
+	}
+	if n != 5 {
+		t.Errorf("%d of %d concurrent acquires granted, want exactly MaxLeases=5", n, attempts)
+	}
+	if got := s.ActiveCount("ns/p1"); got != 5 {
+		t.Errorf("ActiveCount = %d, want 5", got)
+	}
+}
+
+func TestStart_sweepsAndStops(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	s.SweepInterval = time.Millisecond
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 5, 0)}, TTL: time.Minute})
+	clock.Advance(20 * time.Minute) // past TTL + retention
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- s.Start(ctx) }()
+
+	deadline := time.After(5 * time.Second)
+	for {
+		if err := s.Report(context.Background(), l.ID, ResultOK, ""); errors.Is(err, ErrUnknownLease) {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("sweep never dropped the lease")
+		case <-time.After(5 * time.Millisecond):
+		}
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start returned %v, want nil", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}

internal/metrics/metrics.go

@@ -0,0 +1,167 @@
+// Package metrics defines the operator's Prometheus metrics. Nothing here
+// registers itself — no init(), per house rules — the composition root
+// calls Register explicitly, which also lets every test use a fresh
+// registry.
+package metrics
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metrics holds the vector metrics the operator's components feed. The
+// consuming packages (health, discovery, provider) each define their own
+// small recorder interface; *Metrics satisfies all of them structurally,
+// so none of them import this package's prometheus surface.
+type Metrics struct {
+	healthcheckDuration *prometheus.HistogramVec
+	healthcheckFailures *prometheus.CounterVec
+	leaseRequests       *prometheus.CounterVec
+	providerRequests    *prometheus.CounterVec
+}
+
+// New builds the metric set, unregistered.
+func New() *Metrics {
+	return &Metrics{
+		healthcheckDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "proxy_operator_healthcheck_duration_seconds",
+			Help:    "Duration of through-the-proxy health probes.",
+			Buckets: prometheus.DefBuckets,
+		}, []string{"proxy"}),
+		healthcheckFailures: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_healthcheck_failures_total",
+			Help: "Failed health probes.",
+		}, []string{"proxy"}),
+		leaseRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_lease_requests_total",
+			Help: "Lease acquisition requests by outcome.",
+		}, []string{"outcome"}),
+		providerRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_provider_requests_total",
+			Help: "Provider API calls by operation and classified result.",
+		}, []string{"provider", "op", "result"}),
+	}
+}
+
+// Register registers the vectors plus the two scrape-time collectors.
+// proxyPhases and activeLeases are read at every scrape: gauges derived
+// from reconcile-time increments inevitably drift and leak series on
+// delete; reading the source of truth cannot.
+func (m *Metrics) Register(reg prometheus.Registerer, proxyPhases func() map[string]int, activeLeases func() int) error {
+	collectors := []prometheus.Collector{
+		m.healthcheckDuration,
+		m.healthcheckFailures,
+		m.leaseRequests,
+		m.providerRequests,
+		&constCollector{
+			desc: prometheus.NewDesc("proxy_operator_proxies",
+				"Proxy objects by phase.", []string{"phase"}, nil),
+			read: proxyPhases,
+		},
+		&constCollector{
+			desc: prometheus.NewDesc("proxy_operator_leases_active",
+				"Currently active leases.", nil, nil),
+			read: func() map[string]int { return map[string]int{"": activeLeases()} },
+		},
+	}
+	for _, c := range collectors {
+		if err := reg.Register(c); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ObserveProbe records one health probe. Called on every probe — metrics
+// are the home for high-frequency signal that must never touch status.
+func (m *Metrics) ObserveProbe(proxy string, latency time.Duration, success bool) {
+	m.healthcheckDuration.WithLabelValues(proxy).Observe(latency.Seconds())
+	if !success {
+		m.healthcheckFailures.WithLabelValues(proxy).Inc()
+	}
+}
+
+// ForgetProxy drops the per-proxy series when the health engine prunes its
+// state — without this, series for deleted proxies leak forever.
+func (m *Metrics) ForgetProxy(proxy string) {
+	m.healthcheckDuration.DeleteLabelValues(proxy)
+	m.healthcheckFailures.DeleteLabelValues(proxy)
+}
+
+// LeaseRequest records a lease acquisition outcome ("granted"|"no_match").
+func (m *Metrics) LeaseRequest(outcome string) {
+	m.leaseRequests.WithLabelValues(outcome).Inc()
+}
+
+// ProviderRequest records one provider API call with its classified result.
+func (m *Metrics) ProviderRequest(provider, op, result string) {
+	m.providerRequests.WithLabelValues(provider, op, result).Inc()
+}
+
+// constCollector reads a label→value map at scrape time and emits one
+// gauge sample per entry. An empty-string label key means "no labels".
+type constCollector struct {
+	desc *prometheus.Desc
+	read func() map[string]int
+}
+
+func (c *constCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.desc }
+
+func (c *constCollector) Collect(ch chan<- prometheus.Metric) {
+	for label, value := range c.read() {
+		if label == "" {
+			ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(value))
+			continue
+		}
+		ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(value), label)
+	}
+}
+
+// ProviderTarget is one provider health-check target as the gauges see it.
+type ProviderTarget struct {
+	Provider, Target string
+	Up               bool
+	Duration         time.Duration
+	CheckedAt        time.Time
+}
+
+// RegisterProviderTargets registers the provider-check gauges. Like the
+// other scrape-time collectors they are read from the source at every
+// scrape, so a target that goes away stops being reported rather than
+// leaving a stale series behind.
+func RegisterProviderTargets(reg prometheus.Registerer, read func() []ProviderTarget) error {
+	labels := []string{"provider", "target"}
+	return reg.Register(&providerTargetCollector{
+		up: prometheus.NewDesc("proxy_operator_provider_target_up",
+			"Whether the target passed its latest provider health check (1) or not (0).", labels, nil),
+		duration: prometheus.NewDesc("proxy_operator_provider_check_duration_seconds",
+			"Duration of the target's latest provider health check.", labels, nil),
+		checked: prometheus.NewDesc("proxy_operator_provider_check_timestamp_seconds",
+			"Unix time of the target's latest provider health check.", labels, nil),
+		read: read,
+	})
+}
+
+type providerTargetCollector struct {
+	up, duration, checked *prometheus.Desc
+	read                  func() []ProviderTarget
+}
+
+func (c *providerTargetCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.up
+	ch <- c.duration
+	ch <- c.checked
+}
+
+func (c *providerTargetCollector) Collect(ch chan<- prometheus.Metric) {
+	for _, t := range c.read() {
+		up := 0.0
+		if t.Up {
+			up = 1
+		}
+		ch <- prometheus.MustNewConstMetric(c.up, prometheus.GaugeValue, up, t.Provider, t.Target)
+		ch <- prometheus.MustNewConstMetric(c.duration, prometheus.GaugeValue, t.Duration.Seconds(), t.Provider, t.Target)
+		ch <- prometheus.MustNewConstMetric(c.checked, prometheus.GaugeValue, float64(t.CheckedAt.UnixNano())/1e9, t.Provider, t.Target)
+	}
+}

internal/metrics/metrics_test.go

@@ -0,0 +1,155 @@
+package metrics
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+// register wires a fresh registry — the reason Register exists instead of
+// init()-time self-registration.
+func register(t *testing.T, m *Metrics, phases map[string]int, active int) *prometheus.Registry {
+	t.Helper()
+	reg := prometheus.NewRegistry()
+	err := m.Register(reg,
+		func() map[string]int { return phases },
+		func() int { return active },
+	)
+	if err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+	return reg
+}
+
+func TestRegister_scrapeTimeCollectors(t *testing.T) {
+	t.Parallel()
+	m := New()
+	reg := register(t, m, map[string]int{"Ready": 3, "Provisioning": 1}, 7)
+
+	expected := `
+# HELP proxy_operator_leases_active Currently active leases.
+# TYPE proxy_operator_leases_active gauge
+proxy_operator_leases_active 7
+# HELP proxy_operator_proxies Proxy objects by phase.
+# TYPE proxy_operator_proxies gauge
+proxy_operator_proxies{phase="Provisioning"} 1
+proxy_operator_proxies{phase="Ready"} 3
+`
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(expected),
+		"proxy_operator_proxies", "proxy_operator_leases_active"); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestObserveProbe_andForget(t *testing.T) {
+	t.Parallel()
+	m := New()
+	reg := register(t, m, nil, 0)
+
+	m.ObserveProbe("default/p1", 30*time.Millisecond, true)
+	m.ObserveProbe("default/p1", 40*time.Millisecond, false)
+	m.ObserveProbe("default/p2", 10*time.Millisecond, true)
+
+	if got := testutil.CollectAndCount(m.healthcheckDuration); got != 2 {
+		t.Errorf("duration series = %d, want 2 (one per proxy)", got)
+	}
+	if got := testutil.ToFloat64(m.healthcheckFailures.WithLabelValues("default/p1")); got != 1 {
+		t.Errorf("p1 failures = %v, want 1 (only the failed probe)", got)
+	}
+
+	m.ForgetProxy("default/p1")
+	if got := testutil.CollectAndCount(m.healthcheckDuration); got != 1 {
+		t.Errorf("duration series after ForgetProxy = %d, want 1 — series must not leak", got)
+	}
+	if got := testutil.CollectAndCount(m.healthcheckFailures); got != 0 {
+		t.Errorf("failure series after ForgetProxy = %d, want 0", got)
+	}
+	_ = reg
+}
+
+func TestLeaseRequest(t *testing.T) {
+	t.Parallel()
+	m := New()
+	register(t, m, nil, 0)
+
+	m.LeaseRequest("granted")
+	m.LeaseRequest("granted")
+	m.LeaseRequest("no_match")
+
+	if got := testutil.ToFloat64(m.leaseRequests.WithLabelValues("granted")); got != 2 {
+		t.Errorf("granted = %v, want 2", got)
+	}
+	if got := testutil.ToFloat64(m.leaseRequests.WithLabelValues("no_match")); got != 1 {
+		t.Errorf("no_match = %v, want 1", got)
+	}
+}
+
+func TestProviderRequest(t *testing.T) {
+	t.Parallel()
+	m := New()
+	register(t, m, nil, 0)
+
+	m.ProviderRequest("gcp-eu", "create", "ok")
+	m.ProviderRequest("gcp-eu", "create", "quota_exceeded")
+
+	if got := testutil.ToFloat64(m.providerRequests.WithLabelValues("gcp-eu", "create", "ok")); got != 1 {
+		t.Errorf("ok = %v, want 1", got)
+	}
+	if got := testutil.ToFloat64(m.providerRequests.WithLabelValues("gcp-eu", "create", "quota_exceeded")); got != 1 {
+		t.Errorf("quota_exceeded = %v, want 1", got)
+	}
+}
+
+func TestRegister_freshRegistryPerTest(t *testing.T) {
+	t.Parallel()
+	// Registering the same metric set on two registries must both succeed —
+	// the property init()-style global registration would break.
+	m1, m2 := New(), New()
+	register(t, m1, nil, 0)
+	register(t, m2, nil, 0)
+}
+
+func TestRegisterProviderTargets_scrape(t *testing.T) {
+	t.Parallel()
+	checked := time.Unix(1757602800, 0)
+	targets := []ProviderTarget{
+		{Provider: "kvm-home", Target: "nas1", Up: true, Duration: 84 * time.Millisecond, CheckedAt: checked},
+		{Provider: "kvm-home", Target: "nas2", Up: false, Duration: 15 * time.Second, CheckedAt: checked},
+	}
+	reg := prometheus.NewRegistry()
+	if err := RegisterProviderTargets(reg, func() []ProviderTarget { return targets }); err != nil {
+		t.Fatalf("RegisterProviderTargets: %v", err)
+	}
+
+	expected := `
+# HELP proxy_operator_provider_check_duration_seconds Duration of the target's latest provider health check.
+# TYPE proxy_operator_provider_check_duration_seconds gauge
+proxy_operator_provider_check_duration_seconds{provider="kvm-home",target="nas1"} 0.084
+proxy_operator_provider_check_duration_seconds{provider="kvm-home",target="nas2"} 15
+# HELP proxy_operator_provider_check_timestamp_seconds Unix time of the target's latest provider health check.
+# TYPE proxy_operator_provider_check_timestamp_seconds gauge
+proxy_operator_provider_check_timestamp_seconds{provider="kvm-home",target="nas1"} 1.7576028e+09
+proxy_operator_provider_check_timestamp_seconds{provider="kvm-home",target="nas2"} 1.7576028e+09
+# HELP proxy_operator_provider_target_up Whether the target passed its latest provider health check (1) or not (0).
+# TYPE proxy_operator_provider_target_up gauge
+proxy_operator_provider_target_up{provider="kvm-home",target="nas1"} 1
+proxy_operator_provider_target_up{provider="kvm-home",target="nas2"} 0
+`
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(expected)); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestRegisterProviderTargets_emptySnapshotHasNoSeries(t *testing.T) {
+	t.Parallel()
+	reg := prometheus.NewRegistry()
+	if err := RegisterProviderTargets(reg, func() []ProviderTarget { return nil }); err != nil {
+		t.Fatalf("RegisterProviderTargets: %v", err)
+	}
+	if n, err := testutil.GatherAndCount(reg); err != nil || n != 0 {
+		t.Errorf("series before the first check = %d (%v), want 0", n, err)
+	}
+}

internal/provider/check.go

@@ -0,0 +1,46 @@
+package provider
+
+import (
+	"context"
+	"time"
+)
+
+// Reasons shared by every Checker, so a listing reads the same across
+// providers. Providers add their own for failures only they can tell
+// apart (KVM's SSHHandshakeFailed or PoolNotActive, say).
+const (
+	ReasonReachable    = "Reachable"
+	ReasonTimeout      = "Timeout"
+	ReasonUnreachable  = "Unreachable"
+	ReasonUnauthorized = "Unauthorized"
+	ReasonForbidden    = "Forbidden"
+	ReasonAPIError     = "APIError"
+)
+
+// TargetStatus is one target's verdict from a Checker.
+type TargetStatus struct {
+	// Name identifies the target within its provider: a KVM host name,
+	// "apiserver", a GCP project.
+	Name string
+	// Address is what the check connected to, for humans.
+	Address string
+	Ready   bool
+	// Reason is ReasonReachable when Ready, otherwise why not.
+	Reason string
+	// Message is human-readable detail, usually the error.
+	Message string
+	Latency time.Duration
+	// Details are provider-specific facts gathered by the check.
+	Details map[string]string
+}
+
+// Checker is the optional interface of a provider that can verify its
+// backend is reachable and usable without touching any instance. The
+// provider health monitor type-asserts for it on the unwrapped providers,
+// so it stays out of the Provider contract.
+type Checker interface {
+	// Check returns one verdict per target, in a stable order. It never
+	// fails as a whole — a failure is a verdict — and must return promptly
+	// once ctx ends.
+	Check(ctx context.Context) []TargetStatus
+}

internal/provider/config.go

@@ -21,6 +21,23 @@ type ProviderConfig struct {
 	Type       string            `json:"type"`
 	Kubernetes *KubernetesConfig `json:"kubernetes,omitempty"`
 	GCP        *GCPConfig        `json:"gcp,omitempty"`
+	KVM        *KVMConfig        `json:"kvm,omitempty"`
+}
+
+// setBlocks names the type-specific blocks that are non-nil, so validate can
+// reject a block that doesn't match Type without a pairwise check per type.
+func (p ProviderConfig) setBlocks() []string {
+	var out []string
+	if p.Kubernetes != nil {
+		out = append(out, "kubernetes")
+	}
+	if p.GCP != nil {
+		out = append(out, "gcp")
+	}
+	if p.KVM != nil {
+		out = append(out, "kvm")
+	}
+	return out
 }
 
 // KubernetesConfig configures the kubernetes-pod provider, which creates
@@ -81,22 +98,28 @@ func (c *Config) validate() error {
 		}
 		seen[p.Name] = true
 
+		prefix := fmt.Sprintf("providers[%d] %q", i, p.Name)
 		switch p.Type {
-		case "kubernetes":
-			if p.GCP != nil {
-				return fmt.Errorf("providers[%d] %q: type is kubernetes but a gcp block is set", i, p.Name)
-			}
-		case "gcp":
-			if p.Kubernetes != nil {
-				return fmt.Errorf("providers[%d] %q: type is gcp but a kubernetes block is set", i, p.Name)
-			}
-			if p.GCP == nil || p.GCP.Project == "" {
-				return fmt.Errorf("providers[%d] %q: gcp.project is required", i, p.Name)
-			}
+		case "kubernetes", "gcp", "kvm":
 		case "":
-			return fmt.Errorf("providers[%d] %q: type is required", i, p.Name)
+			return fmt.Errorf("%s: type is required", prefix)
 		default:
-			return fmt.Errorf("providers[%d] %q: unknown provider type %q", i, p.Name, p.Type)
+			return fmt.Errorf("%s: unknown provider type %q", prefix, p.Type)
+		}
+		for _, block := range p.setBlocks() {
+			if block != p.Type {
+				return fmt.Errorf("%s: type is %s but a %s block is set", prefix, p.Type, block)
+			}
+		}
+		switch p.Type {
+		case "gcp":
+			if p.GCP == nil || p.GCP.Project == "" {
+				return fmt.Errorf("%s: gcp.project is required", prefix)
+			}
+		case "kvm":
+			if err := validateKVM(prefix, p.KVM); err != nil {
+				return err
+			}
 		}
 	}
 	return nil

internal/provider/config_kvm.go

@@ -0,0 +1,184 @@
+package provider
+
+import (
+	"fmt"
+	"net/url"
+	"regexp"
+)
+
+// KVMConfig configures a kvm provider instance: one or more libvirt/KVM
+// hypervisors reached over SSH (the libvirt unix socket is forwarded through
+// the SSH connection, so hosts need no libvirtd TCP/TLS listener).
+// placement.zone on a Proxy selects a host by name; with a single host it may
+// be omitted.
+type KVMConfig struct {
+	// SSHKeyFile is the path to the private key used for every host. Required.
+	SSHKeyFile string `json:"sshKeyFile"`
+	// KnownHostsFile is an OpenSSH known_hosts file used to verify host keys.
+	// Required unless InsecureIgnoreHostKey is set.
+	KnownHostsFile string `json:"knownHostsFile,omitempty"`
+	// InsecureIgnoreHostKey skips host-key verification entirely.
+	InsecureIgnoreHostKey bool `json:"insecureIgnoreHostKey,omitempty"`
+	// DiskSizeGB is the capacity of each VM's overlay disk. Default 10. The
+	// guest's cloud-init growpart expands the root filesystem into it.
+	DiskSizeGB int64 `json:"diskSizeGb,omitempty"`
+	// VendorData is optional cloud-init vendor-data placed on every seed next
+	// to the Proxy's own user-data. Note that cloud-init drops any top-level
+	// key of a vendor-data cloud-config that the user-data also defines.
+	// When InstallGuestAgent is on, this document and the agent script are
+	// carried as separate parts of a MIME multipart vendor-data.
+	VendorData string `json:"vendorData,omitempty"`
+	// InstallGuestAgent adds a vendor-data script that installs and starts
+	// qemu-guest-agent, which is how the provider learns a VM's address
+	// before the host's ARP table has seen traffic from it. Default true.
+	// Turn it off for images that already ship the agent, or that cannot
+	// reach a package mirror; discovery then depends on ARP alone.
+	InstallGuestAgent *bool `json:"installGuestAgent,omitempty"`
+	// Images are named base images; placement.image names a key here (or is
+	// an http(s) URL directly). The provider downloads each image into a
+	// host's storage pool on first use and caches it there.
+	Images map[string]KVMImage `json:"images,omitempty"`
+	// MachineTypes are named size profiles; placement.machineType names a
+	// key here. Required, at least one.
+	MachineTypes map[string]KVMMachineType `json:"machineTypes"`
+	// Hosts are the hypervisors. Required, at least one, unique names.
+	Hosts []KVMHost `json:"hosts"`
+}
+
+// KVMImage is one downloadable base image (a distro cloud image with
+// cloud-init preinstalled, e.g. Debian's generic qcow2).
+type KVMImage struct {
+	// URL is where to fetch the image from. Required, http or https.
+	URL string `json:"url"`
+	// SHA256 is the expected hex digest of the file. Optional; when set it is
+	// verified after download and a later change triggers a re-download.
+	SHA256 string `json:"sha256,omitempty"`
+	// Format is the image's disk format, "qcow2" (default) or "raw".
+	Format string `json:"format,omitempty"`
+	// Firmware is how VMs from this image boot: "uefi" (default; OVMF with
+	// secure boot off) or "bios" (SeaBIOS) for images with no EFI system
+	// partition. Current Debian, Ubuntu, Fedora and Rocky cloud images all
+	// boot UEFI, and Debian 13's BIOS path was found not to boot at all.
+	Firmware string `json:"firmware,omitempty"`
+}
+
+// KVMMachineType is a VM size profile.
+type KVMMachineType struct {
+	VCPUs     int32 `json:"vcpus"`
+	MemoryMiB int64 `json:"memoryMiB"`
+}
+
+// KVMHost is one hypervisor.
+type KVMHost struct {
+	// Name is the placement.zone value that selects this host. Required.
+	Name string `json:"name"`
+	// Address is the SSH endpoint, host:port. Required.
+	Address string `json:"address"`
+	// User is the SSH user; it must be allowed to use the libvirt socket
+	// (typically a member of the libvirt group). Required.
+	User string `json:"user"`
+	// Socket is the libvirt unix socket path on the host.
+	// Default /var/run/libvirt/libvirt-sock.
+	Socket string `json:"socket,omitempty"`
+	// Pool is the libvirt storage pool for base images, overlay disks and
+	// seed ISOs. Default "default".
+	Pool string `json:"pool,omitempty"`
+	// Bridge is the host bridge the VM's NIC attaches to; it must lead to a
+	// network the cluster can reach and that hands out DHCP leases. Required.
+	Bridge string `json:"bridge"`
+}
+
+// Accepted KVMImage.Format and KVMImage.Firmware values.
+const (
+	KVMFormatQcow2  = "qcow2"
+	KVMFormatRaw    = "raw"
+	KVMFirmwareUEFI = "uefi"
+	KVMFirmwareBIOS = "bios"
+)
+
+// IsHTTPURL reports whether s is an absolute http(s) URL with a host, the
+// only kind of image source the kvm provider can fetch.
+func IsHTTPURL(s string) bool {
+	u, err := url.Parse(s)
+	return err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != ""
+}
+
+var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
+
+// kvmIdentifier constrains the config names that are pasted into
+// identifiers elsewhere: a host name goes into the providerID
+// hosts/<name>/domains/<vm>, which is split on "/" to parse, and an image
+// key becomes the file name of the cached base volume. A slash in either
+// produces an ID or a path that never resolves again, so the VM leaks with
+// nothing able to address it.
+var kvmIdentifier = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`)
+
+func validateKVM(prefix string, c *KVMConfig) error {
+	if c == nil {
+		return fmt.Errorf("%s: kvm block is required", prefix)
+	}
+	if c.SSHKeyFile == "" {
+		return fmt.Errorf("%s: kvm.sshKeyFile is required", prefix)
+	}
+	if c.KnownHostsFile == "" && !c.InsecureIgnoreHostKey {
+		return fmt.Errorf("%s: kvm.knownHostsFile is required unless kvm.insecureIgnoreHostKey is true", prefix)
+	}
+	if c.DiskSizeGB < 0 {
+		return fmt.Errorf("%s: kvm.diskSizeGb must not be negative", prefix)
+	}
+	for name, img := range c.Images {
+		p := fmt.Sprintf("%s: kvm.images[%q]", prefix, name)
+		if !kvmIdentifier.MatchString(name) {
+			return fmt.Errorf("%s: key must be letters, digits, dots, dashes or underscores"+
+				" — it becomes the cached volume name base-<key>.<format>", p)
+		}
+		if !IsHTTPURL(img.URL) {
+			return fmt.Errorf("%s.url must be an http(s) URL (got %q)", p, img.URL)
+		}
+		if img.SHA256 != "" && !sha256Hex.MatchString(img.SHA256) {
+			return fmt.Errorf("%s.sha256 must be 64 hex characters", p)
+		}
+		if img.Format != "" && img.Format != KVMFormatQcow2 && img.Format != KVMFormatRaw {
+			return fmt.Errorf("%s.format must be %s or %s (got %q)", p, KVMFormatQcow2, KVMFormatRaw, img.Format)
+		}
+		if img.Firmware != "" && img.Firmware != KVMFirmwareUEFI && img.Firmware != KVMFirmwareBIOS {
+			return fmt.Errorf("%s.firmware must be %s or %s (got %q)", p, KVMFirmwareUEFI, KVMFirmwareBIOS, img.Firmware)
+		}
+	}
+	if len(c.MachineTypes) == 0 {
+		return fmt.Errorf("%s: kvm.machineTypes must define at least one profile", prefix)
+	}
+	for name, mt := range c.MachineTypes {
+		if mt.VCPUs <= 0 || mt.MemoryMiB <= 0 {
+			return fmt.Errorf("%s: kvm.machineTypes[%q] needs vcpus > 0 and memoryMiB > 0", prefix, name)
+		}
+	}
+	if len(c.Hosts) == 0 {
+		return fmt.Errorf("%s: kvm.hosts must list at least one host", prefix)
+	}
+	seen := make(map[string]bool, len(c.Hosts))
+	for i, h := range c.Hosts {
+		p := fmt.Sprintf("%s: kvm.hosts[%d]", prefix, i)
+		if h.Name == "" {
+			return fmt.Errorf("%s: name is required", p)
+		}
+		if !kvmIdentifier.MatchString(h.Name) {
+			return fmt.Errorf("%s: name %q must be letters, digits, dots, dashes or underscores"+
+				" — it becomes part of the providerID hosts/<name>/domains/<vm>", p, h.Name)
+		}
+		if seen[h.Name] {
+			return fmt.Errorf("%s: duplicate host name %q", p, h.Name)
+		}
+		seen[h.Name] = true
+		if h.Address == "" {
+			return fmt.Errorf("%s %q: address is required", p, h.Name)
+		}
+		if h.User == "" {
+			return fmt.Errorf("%s %q: user is required", p, h.Name)
+		}
+		if h.Bridge == "" {
+			return fmt.Errorf("%s %q: bridge is required", p, h.Name)
+		}
+	}
+	return nil
+}

internal/provider/config_kvm_test.go

@@ -0,0 +1,226 @@
+package provider
+
+import (
+	"strings"
+	"testing"
+)
+
+const validKVMYAML = `
+providers:
+  - name: kvm-home
+    type: kvm
+    kvm:
+      sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+      knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+      diskSizeGb: 20
+      images:
+        debian-13:
+          url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+          sha256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+        ubuntu:
+          url: http://mirror.local/noble.img
+          format: raw
+      machineTypes:
+        small: {vcpus: 1, memoryMiB: 1024}
+        medium: {vcpus: 2, memoryMiB: 2048}
+      hosts:
+        - name: nas1
+          address: nas1.home.lan:22
+          user: virt
+          bridge: br0
+        - name: lab2
+          address: 10.0.0.2:2222
+          user: root
+          socket: /run/libvirt/virtqemud-sock
+          pool: fast
+          bridge: br1
+`
+
+func TestLoadConfig_kvmValid(t *testing.T) {
+	t.Parallel()
+	cfg, err := LoadConfig([]byte(validKVMYAML))
+	if err != nil {
+		t.Fatalf("LoadConfig() error = %v, want nil", err)
+	}
+	k := cfg.Providers[0].KVM
+	if k == nil {
+		t.Fatal("providers[0].kvm = nil")
+	}
+	if k.SSHKeyFile != "/etc/proxy-operator/kvm/id_ed25519" || k.DiskSizeGB != 20 {
+		t.Errorf("kvm = %+v, want sshKeyFile and diskSizeGb parsed", k)
+	}
+	if got := k.Images["debian-13"].SHA256; !strings.HasPrefix(got, "0123") {
+		t.Errorf("images[debian-13].sha256 = %q", got)
+	}
+	if k.Images["ubuntu"].Format != "raw" {
+		t.Errorf("images[ubuntu].format = %q, want raw", k.Images["ubuntu"].Format)
+	}
+	if mt := k.MachineTypes["medium"]; mt.VCPUs != 2 || mt.MemoryMiB != 2048 {
+		t.Errorf("machineTypes[medium] = %+v, want {2 2048}", mt)
+	}
+	if len(k.Hosts) != 2 || k.Hosts[1].Socket != "/run/libvirt/virtqemud-sock" || k.Hosts[1].Pool != "fast" {
+		t.Errorf("hosts = %+v, want two hosts with lab2's socket and pool parsed", k.Hosts)
+	}
+}
+
+func TestLoadConfig_kvmInsecureHostKeyNeedsNoKnownHosts(t *testing.T) {
+	t.Parallel()
+	yaml := strings.Replace(validKVMYAML,
+		"knownHostsFile: /etc/proxy-operator/kvm/known_hosts",
+		"insecureIgnoreHostKey: true", 1)
+	if _, err := LoadConfig([]byte(yaml)); err != nil {
+		t.Fatalf("LoadConfig() error = %v, want nil", err)
+	}
+}
+
+func TestLoadConfig_kvmInvalid(t *testing.T) {
+	t.Parallel()
+	minimal := func(kvmBody string) string {
+		return "providers:\n  - name: kvm-home\n    type: kvm\n    kvm:\n" + indent(kvmBody, 6)
+	}
+	hostsAndTypes := `
+machineTypes:
+  small: {vcpus: 1, memoryMiB: 1024}
+hosts:
+  - {name: nas1, address: nas1:22, user: virt, bridge: br0}
+`
+	tests := []struct {
+		name       string
+		yaml       string
+		wantErrSub string
+	}{
+		{
+			name:       "missing kvm block",
+			yaml:       "providers:\n  - name: kvm-home\n    type: kvm\n",
+			wantErrSub: "kvm block is required",
+		},
+		{
+			name:       "missing sshKeyFile",
+			yaml:       minimal("knownHostsFile: /kh\n" + hostsAndTypes),
+			wantErrSub: "kvm.sshKeyFile is required",
+		},
+		{
+			name:       "missing knownHostsFile without insecure flag",
+			yaml:       minimal("sshKeyFile: /key\n" + hostsAndTypes),
+			wantErrSub: "kvm.knownHostsFile is required unless",
+		},
+		{
+			name:       "negative disk size",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\ndiskSizeGb: -1\n" + hostsAndTypes),
+			wantErrSub: "diskSizeGb must not be negative",
+		},
+		{
+			name:       "no machine types",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nhosts:\n  - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
+			wantErrSub: "machineTypes must define at least one",
+		},
+		{
+			name:       "machine type with zero memory",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1}\nhosts:\n  - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `machineTypes["small"] needs vcpus > 0 and memoryMiB > 0`,
+		},
+		{
+			name:       "no hosts",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\n"),
+			wantErrSub: "kvm.hosts must list at least one host",
+		},
+		{
+			name:       "duplicate host name",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, user: virt, bridge: br0}\n  - {name: nas1, address: b:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `duplicate host name "nas1"`,
+		},
+		{
+			// A slash would produce hosts/dc1/rack2/domains/<vm>, which
+			// parseProviderID can never split back, so the VM would be
+			// unreachable for Get, Delete and GC alike.
+			name:       "host name with a slash",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: dc1/rack2, address: a:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `name "dc1/rack2" must be letters, digits, dots, dashes or underscores`,
+		},
+		{
+			name:       "image key with a slash",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  ../etc/d: {url: https://x/y.qcow2}\n" + hostsAndTypes),
+			wantErrSub: `images["../etc/d"]: key must be letters, digits, dots, dashes or underscores`,
+		},
+		{
+			name:       "host missing address",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, user: virt, bridge: br0}\n"),
+			wantErrSub: "address is required",
+		},
+		{
+			name:       "host missing user",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, bridge: br0}\n"),
+			wantErrSub: "user is required",
+		},
+		{
+			name:       "host missing bridge",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, user: virt}\n"),
+			wantErrSub: "bridge is required",
+		},
+		{
+			name:       "image with non-http url",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: file:///tmp/x.qcow2}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].url must be an http(s) URL`,
+		},
+		{
+			name:       "image with bad sha256",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.qcow2, sha256: abc}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].sha256 must be 64 hex characters`,
+		},
+		{
+			name:       "image with unknown firmware",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.qcow2, firmware: coreboot}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].firmware must be uefi or bios`,
+		},
+		{
+			name:       "image with unknown format",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.vmdk, format: vmdk}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].format must be qcow2 or raw`,
+		},
+		{
+			name: "kvm type with gcp block",
+			yaml: `
+providers:
+  - name: p1
+    type: kvm
+    gcp:
+      project: my-project`,
+			wantErrSub: "type is kvm but a gcp block is set",
+		},
+		{
+			name: "gcp type with kvm block",
+			yaml: `
+providers:
+  - name: p1
+    type: gcp
+    gcp:
+      project: my-project
+    kvm:
+      sshKeyFile: /key`,
+			wantErrSub: "type is gcp but a kvm block is set",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			_, err := LoadConfig([]byte(tc.yaml))
+			if err == nil {
+				t.Fatalf("LoadConfig() error = nil, want error containing %q", tc.wantErrSub)
+			}
+			if !strings.Contains(err.Error(), tc.wantErrSub) {
+				t.Errorf("LoadConfig() error = %q, want substring %q", err.Error(), tc.wantErrSub)
+			}
+		})
+	}
+}
+
+func indent(s string, n int) string {
+	pad := strings.Repeat(" ", n)
+	lines := strings.Split(strings.TrimLeft(s, "\n"), "\n")
+	for i, l := range lines {
+		if l != "" {
+			lines[i] = pad + l
+		}
+	}
+	return strings.Join(lines, "\n")
+}

internal/provider/gcp/check.go

@@ -0,0 +1,78 @@
+package gcp
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"time"
+
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/iterator"
+	"google.golang.org/protobuf/proto"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+var _ provider.Checker = (*Provider)(nil)
+
+// computeAddress is the endpoint the SDK talks to, shown only to humans.
+const computeAddress = "compute.googleapis.com"
+
+// managedFilter selects the instances this operator created.
+const managedFilter = "labels." + provider.LabelManaged + " = " + provider.LabelManagedYes
+
+// Check asks the project for one page holding at most one managed
+// instance. That exercises the credentials, the Compute API and the
+// compute.instances.list permission orphan GC depends on, in a single
+// request whatever the project's size. The target is the project: the
+// provider config names no zones.
+func (p *Provider) Check(ctx context.Context) []provider.TargetStatus {
+	start := time.Now()
+	err := p.api.Probe(ctx, &computepb.AggregatedListInstancesRequest{
+		Project:              p.cfg.Project,
+		Filter:               proto.String(managedFilter),
+		MaxResults:           proto.Uint32(1),
+		ReturnPartialSuccess: proto.Bool(true),
+	})
+	ts := provider.TargetStatus{Name: p.cfg.Project, Address: computeAddress, Latency: time.Since(start)}
+	if err != nil {
+		logAPIError(p.logger(ctx), "check", err)
+		ts.Reason = checkReason(ctx, err)
+		ts.Message = err.Error()
+		return []provider.TargetStatus{ts}
+	}
+	ts.Ready = true
+	ts.Reason = provider.ReasonReachable
+	return []provider.TargetStatus{ts}
+}
+
+// Probe fetches the first page of an aggregated list and discards it. The
+// SDK treats MaxResults as the page size, so AggregatedList, which drains
+// the iterator, would still walk every page of the project.
+func (r *realInstances) Probe(ctx context.Context, req *computepb.AggregatedListInstancesRequest) error {
+	_, err := r.client.AggregatedList(ctx, req).Next()
+	if err == iterator.Done {
+		return nil
+	}
+	return err
+}
+
+// checkReason maps a failed probe to a reason. A token that cannot be
+// obtained never reaches the API, so it has no status code and lands on
+// Unreachable, with the detail in the message.
+func checkReason(ctx context.Context, err error) string {
+	var gerr *googleapi.Error
+	switch {
+	case ctx.Err() != nil, errors.Is(err, context.DeadlineExceeded):
+		return provider.ReasonTimeout
+	case !errors.As(err, &gerr):
+		return provider.ReasonUnreachable
+	case gerr.Code == http.StatusUnauthorized:
+		return provider.ReasonUnauthorized
+	case gerr.Code == http.StatusForbidden:
+		return provider.ReasonForbidden
+	default:
+		return provider.ReasonAPIError
+	}
+}

internal/provider/gcp/check_test.go

@@ -0,0 +1,165 @@
+package gcp
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"sync"
+	"syscall"
+	"testing"
+
+	compute "cloud.google.com/go/compute/apiv1"
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
+	"google.golang.org/protobuf/proto"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func TestCheck_reachable(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{}
+	p := newTestProvider(api)
+
+	got := p.Check(context.Background())
+
+	if len(got) != 1 {
+		t.Fatalf("Check() returned %d targets, want 1", len(got))
+	}
+	ts := got[0]
+	if !ts.Ready || ts.Reason != provider.ReasonReachable || ts.Message != "" {
+		t.Errorf("Check() = %+v, want ready and Reachable", ts)
+	}
+	if ts.Name != "my-project" || ts.Address != "compute.googleapis.com" {
+		t.Errorf("target = %q at %q, want my-project at compute.googleapis.com", ts.Name, ts.Address)
+	}
+	want := &computepb.AggregatedListInstancesRequest{
+		Project:              "my-project",
+		Filter:               proto.String("labels.proxy-operator-managed = true"),
+		MaxResults:           proto.Uint32(1),
+		ReturnPartialSuccess: proto.Bool(true),
+	}
+	if !proto.Equal(api.probeReq, want) {
+		t.Errorf("probe request = %v, want %v", api.probeReq, want)
+	}
+}
+
+func TestCheck_failureReasons(t *testing.T) {
+	t.Parallel()
+	ended, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	tests := []struct {
+		name string
+		ctx  context.Context
+		err  error
+		want string
+	}{
+		{"unauthorized", context.Background(), gerr(401), provider.ReasonUnauthorized},
+		{"forbidden", context.Background(), gerr(403, "forbidden"), provider.ReasonForbidden},
+		{"api disabled", context.Background(), gerr(403, "accessNotConfigured"), provider.ReasonForbidden},
+		{"server error", context.Background(), gerr(500), provider.ReasonAPIError},
+		{"deadline", context.Background(), &url.Error{
+			Op: "Get", URL: "https://compute.googleapis.com/", Err: context.DeadlineExceeded,
+		}, provider.ReasonTimeout},
+		{"context ended", ended, context.Canceled, provider.ReasonTimeout},
+		{"connection refused", context.Background(), &url.Error{
+			Op: "Get", URL: "https://compute.googleapis.com/", Err: syscall.ECONNREFUSED,
+		}, provider.ReasonUnreachable},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			p := newTestProvider(&fakeAPI{probeErr: tc.err})
+
+			got := p.Check(tc.ctx)
+
+			if len(got) != 1 {
+				t.Fatalf("Check() returned %d targets, want 1", len(got))
+			}
+			if got[0].Ready || got[0].Reason != tc.want || got[0].Message != tc.err.Error() {
+				t.Errorf("Check() = %+v, want not ready, reason %s, message %q", got[0], tc.want, tc.err.Error())
+			}
+		})
+	}
+}
+
+// TestRealInstances_Probe runs the real SDK client against a local server,
+// since the point of Probe is how many requests the SDK makes.
+func TestRealInstances_Probe(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name     string
+		status   int
+		body     string
+		wantCode int
+	}{
+		{
+			name:   "fetches only the first page",
+			status: http.StatusOK,
+			body:   `{"items":{"zones/europe-west1-b":{"instances":[{"name":"proxy-a"}]}},"nextPageToken":"page-2"}`,
+		},
+		{
+			name:   "empty project",
+			status: http.StatusOK,
+			body:   `{}`,
+		},
+		{
+			name:     "api error keeps its status",
+			status:   http.StatusForbidden,
+			body:     `{"error":{"code":403,"message":"Compute Engine API has not been used","errors":[{"reason":"accessNotConfigured"}]}}`,
+			wantCode: http.StatusForbidden,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			var (
+				mu   sync.Mutex
+				reqs []*url.URL
+			)
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				mu.Lock()
+				reqs = append(reqs, r.URL)
+				mu.Unlock()
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(tc.status)
+				_, _ = w.Write([]byte(tc.body))
+			}))
+			t.Cleanup(srv.Close)
+			ctx := context.Background()
+			c, err := compute.NewInstancesRESTClient(ctx, option.WithEndpoint(srv.URL), option.WithoutAuthentication())
+			if err != nil {
+				t.Fatalf("NewInstancesRESTClient: %v", err)
+			}
+			t.Cleanup(func() { _ = c.Close() })
+
+			err = (&realInstances{client: c}).Probe(ctx, &computepb.AggregatedListInstancesRequest{
+				Project:    "my-project",
+				MaxResults: proto.Uint32(1),
+			})
+
+			var gerr *googleapi.Error
+			switch {
+			case tc.wantCode == 0 && err != nil:
+				t.Fatalf("Probe() = %v, want nil", err)
+			case tc.wantCode != 0 && (!errors.As(err, &gerr) || gerr.Code != tc.wantCode):
+				t.Fatalf("Probe() = %v, want a googleapi error with code %d", err, tc.wantCode)
+			}
+			mu.Lock()
+			defer mu.Unlock()
+			if len(reqs) != 1 {
+				t.Fatalf("server saw %d requests, want 1", len(reqs))
+			}
+			if got := reqs[0].Query().Get("maxResults"); got != "1" {
+				t.Errorf("maxResults = %q, want 1", got)
+			}
+			if want := "/compute/v1/projects/my-project/aggregated/instances"; reqs[0].Path != want {
+				t.Errorf("path = %q, want %q", reqs[0].Path, want)
+			}
+		})
+	}
+}

internal/provider/gcp/errors.go

@@ -0,0 +1,81 @@
+package gcp
+
+import (
+	"errors"
+	"net/http"
+	"slices"
+
+	"github.com/go-logr/logr"
+	"google.golang.org/api/googleapi"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// classify maps a GCP API error onto the provider taxonomy:
+// 404 → NotFound; 429 and quota-flavored 403s → QuotaExceeded;
+// 400/401/other 403s → Permanent; 408/5xx and anything unrecognized
+// (network errors, context cancellation) → Transient, because retrying is
+// always safer than latching Failed on an error nobody taught this
+// function to recognize.
+func classify(err error) error {
+	var gerr *googleapi.Error
+	if !errors.As(err, &gerr) {
+		return provider.ErrTransient
+	}
+	switch {
+	case gerr.Code == http.StatusNotFound:
+		return provider.ErrNotFound
+	case gerr.Code == http.StatusTooManyRequests:
+		return provider.ErrQuotaExceeded
+	case gerr.Code == http.StatusForbidden && hasReason(gerr, "quotaExceeded", "rateLimitExceeded"):
+		return provider.ErrQuotaExceeded
+	case gerr.Code == http.StatusBadRequest,
+		gerr.Code == http.StatusUnauthorized,
+		gerr.Code == http.StatusForbidden:
+		return provider.ErrPermanent
+	default:
+		return provider.ErrTransient
+	}
+}
+
+func (p *Provider) wrapErr(op, id string, err error) error {
+	return provider.Wrap(classify(err), op, p.name, id, err)
+}
+
+// logAPIError records the raw googleapi error shape (HTTP status, reasons)
+// at V(1) — classify collapses it onto the coarser provider taxonomy, so
+// this line is the only place the original status survives.
+func logAPIError(log logr.Logger, op string, err error) {
+	if !log.V(1).Enabled() {
+		return
+	}
+	kv := []any{"op", op, "error", err.Error()}
+	var gerr *googleapi.Error
+	if errors.As(err, &gerr) {
+		reasons := make([]string, 0, len(gerr.Errors))
+		for _, item := range gerr.Errors {
+			reasons = append(reasons, item.Reason)
+		}
+		kv = append(kv, "httpStatus", gerr.Code, "reasons", reasons)
+	}
+	log.V(1).Info("GCP API call failed", kv...)
+}
+
+func hasReason(gerr *googleapi.Error, reasons ...string) bool {
+	for _, item := range gerr.Errors {
+		if slices.Contains(reasons, item.Reason) {
+			return true
+		}
+	}
+	return false
+}
+
+func isAlreadyExists(err error) bool {
+	var gerr *googleapi.Error
+	return errors.As(err, &gerr) && gerr.Code == http.StatusConflict
+}
+
+func isNotFound(err error) bool {
+	var gerr *googleapi.Error
+	return errors.As(err, &gerr) && gerr.Code == http.StatusNotFound
+}

internal/provider/gcp/errors_test.go

@@ -0,0 +1,71 @@
+package gcp
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"google.golang.org/api/googleapi"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func gerr(code int, reasons ...string) error {
+	e := &googleapi.Error{Code: code, Message: "boom"}
+	for _, r := range reasons {
+		e.Errors = append(e.Errors, googleapi.ErrorItem{Reason: r})
+	}
+	return e
+}
+
+func TestClassify(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		err  error
+		want error
+	}{
+		{name: "404 is NotFound", err: gerr(404), want: provider.ErrNotFound},
+		{name: "429 is Quota", err: gerr(429), want: provider.ErrQuotaExceeded},
+		{name: "403 quotaExceeded is Quota", err: gerr(403, "quotaExceeded"), want: provider.ErrQuotaExceeded},
+		{name: "403 rateLimitExceeded is Quota", err: gerr(403, "rateLimitExceeded"), want: provider.ErrQuotaExceeded},
+		{name: "403 plain is Permanent", err: gerr(403, "forbidden"), want: provider.ErrPermanent},
+		{name: "400 is Permanent", err: gerr(400), want: provider.ErrPermanent},
+		{name: "401 is Permanent", err: gerr(401), want: provider.ErrPermanent},
+		{name: "408 is Transient", err: gerr(408), want: provider.ErrTransient},
+		{name: "500 is Transient", err: gerr(500), want: provider.ErrTransient},
+		{name: "503 is Transient", err: gerr(503), want: provider.ErrTransient},
+		{name: "409 is Transient (alreadyExists is handled before classify)", err: gerr(409), want: provider.ErrTransient},
+		{name: "plain network error is Transient", err: errors.New("connection reset"), want: provider.ErrTransient},
+		{name: "wrapped googleapi error still classifies", err: fmt.Errorf("calling api: %w", gerr(404)), want: provider.ErrNotFound},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := classify(tc.err); got != tc.want {
+				t.Errorf("classify() = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// The wrapped error must satisfy both halves of the taxonomy contract:
+// errors.Is against the sentinel AND errors.As back to the SDK error.
+func TestWrapErr_isAndAsBothWork(t *testing.T) {
+	t.Parallel()
+	p := &Provider{name: "gcp-eu"}
+	wrapped := p.wrapErr("get", "zones/z/instances/i", gerr(404))
+
+	if !errors.Is(wrapped, provider.ErrNotFound) {
+		t.Error("errors.Is(wrapped, ErrNotFound) = false")
+	}
+	var ge *googleapi.Error
+	if !errors.As(wrapped, &ge) || ge.Code != 404 {
+		t.Error("errors.As back to *googleapi.Error failed")
+	}
+	if provider.Class(wrapped) != provider.ErrNotFound {
+		t.Errorf("Class() = %v, want ErrNotFound", provider.Class(wrapped))
+	}
+}

internal/provider/gcp/gcp.go

@@ -0,0 +1,308 @@
+// Package gcp implements the provider contract on GCP Compute Engine via
+// the modern Cloud Client Library (cloud.google.com/go/compute/apiv1),
+// deliberately restricted to four calls: instances.Insert, Get, Delete,
+// AggregatedList. Operations are fire-and-forget — Operation.Wait is never
+// called; Create/Delete return as soon as the operation is submitted and
+// the reconciler discovers progress by polling Get.
+package gcp
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	compute "cloud.google.com/go/compute/apiv1"
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"github.com/go-logr/logr"
+	"google.golang.org/api/iterator"
+	"google.golang.org/api/option"
+	"google.golang.org/protobuf/proto"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// instancesAPI is the test seam. It deliberately does not mirror the SDK:
+// the SDK's InstancesScopedListPairIterator has an unexported nextFunc, so
+// a fake cannot construct one — the seam flattens AggregatedList to a
+// slice, and returns operations as just their name (the only thing this
+// provider ever uses, since it never waits on them).
+type instancesAPI interface {
+	Insert(ctx context.Context, req *computepb.InsertInstanceRequest) (opName string, err error)
+	Get(ctx context.Context, req *computepb.GetInstanceRequest) (*computepb.Instance, error)
+	Delete(ctx context.Context, req *computepb.DeleteInstanceRequest) (opName string, err error)
+	AggregatedList(ctx context.Context, req *computepb.AggregatedListInstancesRequest) ([]*computepb.Instance, error)
+	// Probe fetches only the first page of an aggregated list; see Check.
+	Probe(ctx context.Context, req *computepb.AggregatedListInstancesRequest) error
+}
+
+// realInstances adapts *compute.InstancesClient to the seam.
+type realInstances struct {
+	client *compute.InstancesClient
+}
+
+func (r *realInstances) Insert(ctx context.Context, req *computepb.InsertInstanceRequest) (string, error) {
+	op, err := r.client.Insert(ctx, req)
+	if err != nil {
+		return "", err
+	}
+	return op.Name(), nil
+}
+
+func (r *realInstances) Get(ctx context.Context, req *computepb.GetInstanceRequest) (*computepb.Instance, error) {
+	return r.client.Get(ctx, req)
+}
+
+func (r *realInstances) Delete(ctx context.Context, req *computepb.DeleteInstanceRequest) (string, error) {
+	op, err := r.client.Delete(ctx, req)
+	if err != nil {
+		return "", err
+	}
+	return op.Name(), nil
+}
+
+func (r *realInstances) AggregatedList(ctx context.Context, req *computepb.AggregatedListInstancesRequest) ([]*computepb.Instance, error) {
+	it := r.client.AggregatedList(ctx, req)
+	var out []*computepb.Instance
+	for {
+		pair, err := it.Next()
+		if err == iterator.Done {
+			return out, nil
+		}
+		if err != nil {
+			return nil, err
+		}
+		if pair.Value != nil {
+			out = append(out, pair.Value.Instances...)
+		}
+	}
+}
+
+// Provider implements provider.Provider on GCP Compute Engine.
+type Provider struct {
+	name string
+	cfg  provider.GCPConfig
+	api  instancesAPI
+}
+
+// New builds a Provider using Application Default Credentials (workload
+// identity in-cluster, gcloud ADC locally — no key-file plumbing).
+// Deliberately untested: it dials real Google endpoints; everything below
+// it is exercised through newWithAPI.
+func New(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+	return NewWithWireOptions(ctx, pc, WireLogOptions{})
+}
+
+// NewWithWireOptions is New with explicit control over the V(5) wire
+// logging; the injected wire logger surfaces the SDK's HTTP
+// request/response records at V(5). Note option.WithLogger overrides the
+// SDK's own GOOGLE_SDK_GO_LOGGING_LEVEL env var, so --zap-log-level is
+// the only knob.
+func NewWithWireOptions(ctx context.Context, pc provider.ProviderConfig, opts WireLogOptions) (provider.Provider, error) {
+	base := logf.Log.WithName("gcp").WithName("http")
+	if base.V(5).Enabled() {
+		logf.Log.WithName("gcp").Info(
+			"GCP HTTP wire logging active — request payloads include cloud-init user-data",
+			"fullPayloads", opts.FullPayloads)
+	}
+	client, err := compute.NewInstancesRESTClient(ctx, option.WithLogger(wireLogger(base, opts)))
+	if err != nil {
+		return nil, fmt.Errorf("creating GCP instances client: %w", err)
+	}
+	return newWithAPI(pc, &realInstances{client: client}), nil
+}
+
+func newWithAPI(pc provider.ProviderConfig, api instancesAPI) *Provider {
+	cfg := provider.GCPConfig{}
+	if pc.GCP != nil {
+		cfg = *pc.GCP
+	}
+	return &Provider{name: pc.Name, cfg: withDefaults(cfg), api: api}
+}
+
+// logger derives the request-scoped logger from ctx, so provider lines
+// inherit the reconcile context (which Proxy triggered the call).
+func (p *Provider) logger(ctx context.Context) logr.Logger {
+	return logf.FromContext(ctx).WithName("gcp").WithValues("provider", p.name)
+}
+
+// Create submits the insert and returns immediately with the
+// zone-qualified providerID. A 409 alreadyExists is success — the
+// deterministic instance name means a repeat call after a crash found the
+// VM it already created, which is exactly the idempotency the contract
+// demands.
+func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
+	pl := req.Placement
+	if pl.Zone == "" || pl.MachineType == "" || pl.Image == "" {
+		return "", provider.Wrap(provider.ErrPermanent, "create", p.name, "", fmt.Errorf(
+			"gcp requires placement.zone, placement.machineType and placement.image (got zone=%q machineType=%q image=%q)",
+			pl.Zone, pl.MachineType, pl.Image))
+	}
+	log := p.logger(ctx)
+	id := formatProviderID(pl.Zone, req.Name)
+	insertReq := buildInsertRequest(p.cfg, req)
+	// Curated fields only: the request proto embeds the cloud-init
+	// user-data, which may be Secret-sourced and must never reach logs.
+	log.V(2).Info("GCP insert request built",
+		"zone", pl.Zone, "name", req.Name,
+		"network", p.cfg.Network, "networkTag", p.cfg.NetworkTag,
+		"diskSizeGb", p.cfg.DiskSizeGB, "port", req.Port,
+		"labels", insertReq.GetInstanceResource().GetLabels(),
+		"cloudInitBytes", len(req.CloudInit))
+	opName, err := p.api.Insert(ctx, insertReq)
+	switch {
+	case err == nil:
+		log.V(1).Info("GCP instance insert submitted",
+			"zone", pl.Zone, "name", req.Name,
+			"machineType", pl.MachineType, "image", pl.Image, "opName", opName)
+	case isAlreadyExists(err):
+		log.V(1).Info("GCP instance already exists, insert treated as success",
+			"zone", pl.Zone, "name", req.Name)
+	default:
+		logAPIError(log, "create", err)
+		return "", p.wrapErr("create", id, err)
+	}
+	return id, nil
+}
+
+// Get returns the instance state. The providerID carries its own zone, so
+// this stays correct even mid-replacement after a zone edit — re-reading
+// spec.placement.zone would look up the wrong zone exactly then.
+func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instance, error) {
+	zone, name, err := parseProviderID(providerID)
+	if err != nil {
+		return nil, provider.Wrap(provider.ErrPermanent, "get", p.name, providerID, err)
+	}
+	log := p.logger(ctx)
+	inst, err := p.api.Get(ctx, &computepb.GetInstanceRequest{
+		Project:  p.cfg.Project,
+		Zone:     zone,
+		Instance: name,
+	})
+	if err != nil {
+		logAPIError(log, "get", err)
+		return nil, p.wrapErr("get", providerID, err)
+	}
+	out := toInstance(inst, zone)
+	log.V(1).Info("GCP instance fetched",
+		"zone", zone, "name", name,
+		"status", inst.GetStatus(), "state", out.State, "ip", out.IP)
+	return out, nil
+}
+
+// Delete submits the delete and returns; deleting an instance that is
+// already gone is success.
+func (p *Provider) Delete(ctx context.Context, providerID string) error {
+	zone, name, err := parseProviderID(providerID)
+	if err != nil {
+		return provider.Wrap(provider.ErrPermanent, "delete", p.name, providerID, err)
+	}
+	log := p.logger(ctx)
+	opName, err := p.api.Delete(ctx, &computepb.DeleteInstanceRequest{
+		Project:  p.cfg.Project,
+		Zone:     zone,
+		Instance: name,
+	})
+	switch {
+	case err == nil:
+		log.V(1).Info("GCP instance delete submitted",
+			"zone", zone, "name", name, "opName", opName)
+	case isNotFound(err):
+		log.V(1).Info("GCP instance already gone, delete treated as success",
+			"zone", zone, "name", name)
+	default:
+		logAPIError(log, "delete", err)
+		return p.wrapErr("delete", providerID, err)
+	}
+	return nil
+}
+
+// ListByTag sweeps every zone for instances carrying the GC labels.
+// ReturnPartialSuccess matters: without it one unreachable zone fails the
+// entire GC sweep.
+func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
+	log := p.logger(ctx)
+	instances, err := p.api.AggregatedList(ctx, &computepb.AggregatedListInstancesRequest{
+		Project:              p.cfg.Project,
+		Filter:               proto.String(managedFilter),
+		ReturnPartialSuccess: proto.Bool(true),
+	})
+	if err != nil {
+		logAPIError(log, "list", err)
+		return nil, p.wrapErr("list", "", err)
+	}
+	log.V(1).Info("GCP instances listed", "filter", managedFilter, "count", len(instances))
+	out := make([]provider.Instance, 0, len(instances))
+	for _, inst := range instances {
+		conv := toInstance(inst, lastPathSegment(inst.GetZone()))
+		out = append(out, *conv)
+		log.V(2).Info("GCP listed instance",
+			"id", conv.ID, "state", conv.State, "uid", conv.UID, "createdAt", conv.CreatedAt)
+	}
+	return out, nil
+}
+
+func toInstance(inst *computepb.Instance, zone string) *provider.Instance {
+	var ip string
+	if nics := inst.GetNetworkInterfaces(); len(nics) > 0 {
+		if acs := nics[0].GetAccessConfigs(); len(acs) > 0 {
+			ip = acs[0].GetNatIP()
+		}
+	}
+	// CreationTimestamp is RFC3339; a parse failure leaves the zero time,
+	// which orphan GC treats as "old" — safe, since a malformed timestamp
+	// never protects a candidate from collection forever.
+	created, _ := time.Parse(time.RFC3339, inst.GetCreationTimestamp())
+	return &provider.Instance{
+		ID:        formatProviderID(zone, inst.GetName()),
+		IP:        ip,
+		State:     mapState(inst.GetStatus(), ip),
+		UID:       inst.GetLabels()[provider.LabelUID],
+		CreatedAt: created,
+	}
+}
+
+// mapState collapses GCP instance statuses onto the provider states. A
+// RUNNING instance without a NatIP maps to Provisioning — an empty IP must
+// never be published as Running. Anything unrecognized maps to Stopped:
+// the reconciler's answer to Stopped is delete-and-recreate, which is
+// always safe for cattle.
+func mapState(status, ip string) provider.InstanceState {
+	switch status {
+	case "PROVISIONING", "STAGING", "REPAIRING":
+		return provider.StateProvisioning
+	case "RUNNING":
+		if ip == "" {
+			return provider.StateProvisioning
+		}
+		return provider.StateRunning
+	case "STOPPING", "STOPPED", "SUSPENDING", "SUSPENDED":
+		return provider.StateStopped
+	case "TERMINATED":
+		return provider.StateTerminated
+	default:
+		return provider.StateStopped
+	}
+}
+
+func formatProviderID(zone, name string) string {
+	return fmt.Sprintf("zones/%s/instances/%s", zone, name)
+}
+
+func parseProviderID(id string) (zone, name string, err error) {
+	parts := strings.Split(id, "/")
+	if len(parts) != 4 || parts[0] != "zones" || parts[2] != "instances" || parts[1] == "" || parts[3] == "" {
+		return "", "", fmt.Errorf("malformed gcp providerID %q, want zones/<zone>/instances/<name>", id)
+	}
+	return parts[1], parts[3], nil
+}
+
+// lastPathSegment extracts the zone name from the URL-style
+// ".../zones/europe-west1-b" the API returns on instances.
+func lastPathSegment(url string) string {
+	if i := strings.LastIndexByte(url, '/'); i >= 0 {
+		return url[i+1:]
+	}
+	return url
+}

internal/provider/gcp/gcp_test.go

@@ -0,0 +1,572 @@
+package gcp
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"strings"
+	"testing"
+	"time"
+
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"github.com/go-logr/logr"
+	"github.com/go-logr/logr/funcr"
+	"google.golang.org/protobuf/proto"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// fakeAPI implements the instancesAPI seam.
+type fakeAPI struct {
+	insertReq *computepb.InsertInstanceRequest
+	insertErr error
+
+	getReq  *computepb.GetInstanceRequest
+	getInst *computepb.Instance
+	getErr  error
+
+	deleteReq *computepb.DeleteInstanceRequest
+	deleteErr error
+
+	listReq   *computepb.AggregatedListInstancesRequest
+	listInsts []*computepb.Instance
+	listErr   error
+
+	probeReq *computepb.AggregatedListInstancesRequest
+	probeErr error
+}
+
+func (f *fakeAPI) Probe(_ context.Context, req *computepb.AggregatedListInstancesRequest) error {
+	f.probeReq = req
+	return f.probeErr
+}
+
+func (f *fakeAPI) Insert(_ context.Context, req *computepb.InsertInstanceRequest) (string, error) {
+	f.insertReq = req
+	return "op-insert", f.insertErr
+}
+
+func (f *fakeAPI) Get(_ context.Context, req *computepb.GetInstanceRequest) (*computepb.Instance, error) {
+	f.getReq = req
+	return f.getInst, f.getErr
+}
+
+func (f *fakeAPI) Delete(_ context.Context, req *computepb.DeleteInstanceRequest) (string, error) {
+	f.deleteReq = req
+	return "op-delete", f.deleteErr
+}
+
+func (f *fakeAPI) AggregatedList(_ context.Context, req *computepb.AggregatedListInstancesRequest) ([]*computepb.Instance, error) {
+	f.listReq = req
+	return f.listInsts, f.listErr
+}
+
+func newTestProvider(api *fakeAPI) *Provider {
+	return newWithAPI(provider.ProviderConfig{
+		Name: "gcp-eu",
+		Type: "gcp",
+		GCP:  &provider.GCPConfig{Project: "my-project"},
+	}, api)
+}
+
+func TestCreate_returnsZoneQualifiedID(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{}
+	p := newTestProvider(api)
+
+	id, err := p.Create(context.Background(), testCreateRequest())
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if want := "zones/europe-west1-b/instances/proxy-abc123def456ghij"; id != want {
+		t.Errorf("providerID = %s, want %s", id, want)
+	}
+	if api.insertReq.Project != "my-project" || api.insertReq.Zone != "europe-west1-b" {
+		t.Errorf("insert sent to %s/%s, want my-project/europe-west1-b", api.insertReq.Project, api.insertReq.Zone)
+	}
+}
+
+func TestCreate_alreadyExistsIsSuccess(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{insertErr: gerr(409)}
+	p := newTestProvider(api)
+
+	id, err := p.Create(context.Background(), testCreateRequest())
+	if err != nil {
+		t.Fatalf("Create after crash (409): %v — alreadyExists must be success", err)
+	}
+	if want := "zones/europe-west1-b/instances/proxy-abc123def456ghij"; id != want {
+		t.Errorf("providerID = %s, want %s", id, want)
+	}
+}
+
+func TestCreate_incompletePlacementIsPermanent(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{}
+	p := newTestProvider(api)
+	req := testCreateRequest()
+	req.Placement.MachineType = ""
+
+	_, err := p.Create(context.Background(), req)
+	if provider.Class(err) != provider.ErrPermanent {
+		t.Errorf("Class = %v, want ErrPermanent for missing placement", provider.Class(err))
+	}
+	if api.insertReq != nil {
+		t.Error("Insert was called despite invalid placement")
+	}
+}
+
+func TestCreate_quotaErrorClassified(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(&fakeAPI{insertErr: gerr(403, "quotaExceeded")})
+	_, err := p.Create(context.Background(), testCreateRequest())
+	if provider.Class(err) != provider.ErrQuotaExceeded {
+		t.Errorf("Class = %v, want ErrQuotaExceeded", provider.Class(err))
+	}
+}
+
+func TestGet_stateMapping(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name      string
+		status    string
+		natIP     string
+		wantState provider.InstanceState
+		wantIP    string
+	}{
+		{name: "provisioning", status: "PROVISIONING", wantState: provider.StateProvisioning},
+		{name: "staging", status: "STAGING", wantState: provider.StateProvisioning},
+		{name: "repairing", status: "REPAIRING", wantState: provider.StateProvisioning},
+		{name: "running without NatIP stays provisioning", status: "RUNNING", wantState: provider.StateProvisioning},
+		{name: "running with NatIP", status: "RUNNING", natIP: "34.1.2.3", wantState: provider.StateRunning, wantIP: "34.1.2.3"},
+		{name: "stopped", status: "STOPPED", wantState: provider.StateStopped},
+		{name: "suspended", status: "SUSPENDED", wantState: provider.StateStopped},
+		{name: "terminated", status: "TERMINATED", wantState: provider.StateTerminated},
+		{name: "unknown status maps to stopped for recreation", status: "SOMETHING_NEW", wantState: provider.StateStopped},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			inst := &computepb.Instance{
+				Name:              proto.String("proxy-abc"),
+				Status:            proto.String(tc.status),
+				CreationTimestamp: proto.String("2026-08-09T10:00:00+02:00"),
+				Labels: map[string]string{
+					provider.LabelUID: "uid-1",
+				},
+			}
+			if tc.natIP != "" {
+				inst.NetworkInterfaces = []*computepb.NetworkInterface{{
+					AccessConfigs: []*computepb.AccessConfig{{NatIP: proto.String(tc.natIP)}},
+				}}
+			}
+			p := newTestProvider(&fakeAPI{getInst: inst})
+
+			got, err := p.Get(context.Background(), "zones/europe-west1-b/instances/proxy-abc")
+			if err != nil {
+				t.Fatalf("Get: %v", err)
+			}
+			if got.State != tc.wantState || got.IP != tc.wantIP {
+				t.Errorf("state/ip = %s/%q, want %s/%q", got.State, got.IP, tc.wantState, tc.wantIP)
+			}
+			if got.UID != "uid-1" {
+				t.Errorf("UID = %q, want uid-1 (from the GC label)", got.UID)
+			}
+			if got.ID != "zones/europe-west1-b/instances/proxy-abc" {
+				t.Errorf("ID = %s, want the zone-qualified providerID", got.ID)
+			}
+			if got.CreatedAt.IsZero() {
+				t.Error("CreatedAt not parsed from creationTimestamp")
+			}
+		})
+	}
+}
+
+func TestGet_notFound(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(&fakeAPI{getErr: gerr(404)})
+	_, err := p.Get(context.Background(), "zones/z/instances/gone")
+	if !errors.Is(err, provider.ErrNotFound) {
+		t.Errorf("err = %v, want ErrNotFound", err)
+	}
+}
+
+func TestGet_malformedProviderID(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(&fakeAPI{})
+	for _, id := range []string{"", "proxy-abc", "zones//instances/x", "zones/z/instances/", "z/zone/i/name"} {
+		if _, err := p.Get(context.Background(), id); provider.Class(err) != provider.ErrPermanent {
+			t.Errorf("Get(%q): Class = %v, want ErrPermanent", id, provider.Class(err))
+		}
+	}
+}
+
+func TestDelete_notFoundIsSuccess(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{deleteErr: gerr(404)}
+	p := newTestProvider(api)
+	if err := p.Delete(context.Background(), "zones/z/instances/gone"); err != nil {
+		t.Errorf("Delete of missing instance: %v, want nil", err)
+	}
+}
+
+func TestDelete_sendsParsedZoneAndName(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{}
+	p := newTestProvider(api)
+	if err := p.Delete(context.Background(), "zones/us-east1-c/instances/proxy-xyz"); err != nil {
+		t.Fatalf("Delete: %v", err)
+	}
+	if api.deleteReq.Zone != "us-east1-c" || api.deleteReq.Instance != "proxy-xyz" {
+		t.Errorf("delete sent %s/%s, want us-east1-c/proxy-xyz", api.deleteReq.Zone, api.deleteReq.Instance)
+	}
+}
+
+func TestListByTag(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{listInsts: []*computepb.Instance{{
+		Name:              proto.String("proxy-old"),
+		Status:            proto.String("RUNNING"),
+		Zone:              proto.String("https://www.googleapis.com/compute/v1/projects/my-project/zones/europe-west1-b"),
+		CreationTimestamp: proto.String(time.Now().Format(time.RFC3339)),
+		Labels: map[string]string{
+			provider.LabelManaged: provider.LabelManagedYes,
+			provider.LabelUID:     "uid-orphan",
+		},
+		NetworkInterfaces: []*computepb.NetworkInterface{{
+			AccessConfigs: []*computepb.AccessConfig{{NatIP: proto.String("34.9.9.9")}},
+		}},
+	}}}
+	p := newTestProvider(api)
+
+	got, err := p.ListByTag(context.Background())
+	if err != nil {
+		t.Fatalf("ListByTag: %v", err)
+	}
+
+	if want := "labels.proxy-operator-managed = true"; api.listReq.GetFilter() != want {
+		t.Errorf("filter = %q, want %q", api.listReq.GetFilter(), want)
+	}
+	if !api.listReq.GetReturnPartialSuccess() {
+		t.Error("ReturnPartialSuccess not set — one unreachable zone would fail the whole GC sweep")
+	}
+
+	if len(got) != 1 {
+		t.Fatalf("instances = %d, want 1", len(got))
+	}
+	if got[0].ID != "zones/europe-west1-b/instances/proxy-old" {
+		t.Errorf("ID = %s, want the zone parsed out of the URL-style zone field", got[0].ID)
+	}
+	if got[0].UID != "uid-orphan" || got[0].State != provider.StateRunning {
+		t.Errorf("instance = %+v, want uid-orphan/Running", got[0])
+	}
+}
+
+func TestListByTag_errorPropagates(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(&fakeAPI{listErr: gerr(500)})
+	_, err := p.ListByTag(context.Background())
+	if provider.Class(err) != provider.ErrTransient {
+		t.Errorf("Class = %v, want ErrTransient", provider.Class(err))
+	}
+}
+
+func TestParseProviderID_roundTrip(t *testing.T) {
+	t.Parallel()
+	id := formatProviderID("europe-west1-b", "proxy-abc")
+	zone, name, err := parseProviderID(id)
+	if err != nil || zone != "europe-west1-b" || name != "proxy-abc" {
+		t.Errorf("round trip = %s/%s (%v), want europe-west1-b/proxy-abc", zone, name, err)
+	}
+}
+
+// captureContext returns a ctx carrying a funcr logger that records every
+// emitted line, capped at the given verbosity — the test stand-in for
+// --zap-log-level=<verbosity>.
+func captureContext(verbosity int) (context.Context, *[]string) {
+	lines := &[]string{}
+	log := funcr.New(func(prefix, args string) {
+		*lines = append(*lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: verbosity})
+	return logr.NewContext(context.Background(), log), lines
+}
+
+func runningInstance() *computepb.Instance {
+	return &computepb.Instance{
+		Name:              proto.String("proxy-abc123def456ghij"),
+		Status:            proto.String("RUNNING"),
+		Zone:              proto.String("https://www.googleapis.com/compute/v1/projects/my-project/zones/europe-west1-b"),
+		CreationTimestamp: proto.String("2026-08-09T10:00:00+02:00"),
+		Labels: map[string]string{
+			provider.LabelManaged: provider.LabelManagedYes,
+			provider.LabelUID:     "uid-1",
+		},
+		NetworkInterfaces: []*computepb.NetworkInterface{{
+			AccessConfigs: []*computepb.AccessConfig{{NatIP: proto.String("34.1.2.3")}},
+		}},
+	}
+}
+
+func runAllOps(t *testing.T, ctx context.Context, req provider.CreateRequest) {
+	t.Helper()
+	inst := runningInstance()
+	p := newTestProvider(&fakeAPI{getInst: inst, listInsts: []*computepb.Instance{inst}})
+	if _, err := p.Create(ctx, req); err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if _, err := p.Get(ctx, "zones/europe-west1-b/instances/proxy-abc123def456ghij"); err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	if err := p.Delete(ctx, "zones/europe-west1-b/instances/proxy-abc123def456ghij"); err != nil {
+		t.Fatalf("Delete: %v", err)
+	}
+	if _, err := p.ListByTag(ctx); err != nil {
+		t.Fatalf("ListByTag: %v", err)
+	}
+}
+
+func TestLogging_verbosityTiers(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		verbosity   int
+		wantLines   []string
+		absentLines []string
+	}{
+		{
+			name:      "v0 stays silent",
+			verbosity: 0,
+			absentLines: []string{
+				"GCP instance insert submitted",
+				"GCP instance fetched",
+				"GCP instance delete submitted",
+				"GCP instances listed",
+			},
+		},
+		{
+			name:      "v1 logs one line per API call",
+			verbosity: 1,
+			wantLines: []string{
+				`"msg"="GCP instance insert submitted"`,
+				`"opName"="op-insert"`,
+				`"msg"="GCP instance fetched"`,
+				`"status"="RUNNING"`,
+				`"msg"="GCP instance delete submitted"`,
+				`"opName"="op-delete"`,
+				`"msg"="GCP instances listed"`,
+				`"provider"="gcp-eu"`,
+			},
+			absentLines: []string{
+				"GCP insert request built",
+				"GCP listed instance",
+			},
+		},
+		{
+			name:      "v2 adds request and per-instance detail",
+			verbosity: 2,
+			wantLines: []string{
+				`"msg"="GCP insert request built"`,
+				`"cloudInitBytes"=`,
+				`"msg"="GCP listed instance"`,
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			ctx, lines := captureContext(tc.verbosity)
+			req := testCreateRequest()
+			const sentinel = "SENTINEL-cloud-init-must-never-be-logged"
+			req.CloudInit = sentinel
+
+			runAllOps(t, ctx, req)
+
+			joined := strings.Join(*lines, "\n")
+			if tc.verbosity == 0 && len(*lines) != 0 {
+				t.Errorf("verbosity 0 logged %d lines:\n%s", len(*lines), joined)
+			}
+			for _, want := range tc.wantLines {
+				if !strings.Contains(joined, want) {
+					t.Errorf("output missing %q:\n%s", want, joined)
+				}
+			}
+			for _, absent := range tc.absentLines {
+				if strings.Contains(joined, absent) {
+					t.Errorf("output unexpectedly contains %q:\n%s", absent, joined)
+				}
+			}
+			if strings.Contains(joined, sentinel) {
+				t.Errorf("cloud-init content leaked into logs:\n%s", joined)
+			}
+		})
+	}
+}
+
+func TestWireLogger_gatesAtV5(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name      string
+		verbosity int
+		wantDebug bool
+	}{
+		{name: "v5 shows wire records", verbosity: 5, wantDebug: true},
+		{name: "v4 hides wire records", verbosity: 4, wantDebug: false},
+		{name: "v2 hides wire records", verbosity: 2, wantDebug: false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			lines := &[]string{}
+			base := funcr.New(func(prefix, args string) {
+				*lines = append(*lines, prefix+" "+args)
+			}, funcr.Options{Verbosity: tc.verbosity})
+
+			slogger := wireLogger(base, WireLogOptions{})
+			slogger.Debug("api request", "rpcName", "Insert")
+
+			joined := strings.Join(*lines, "\n")
+			if got := strings.Contains(joined, "api request"); got != tc.wantDebug {
+				t.Errorf("Debug record visible = %v, want %v; output:\n%s", got, tc.wantDebug, joined)
+			}
+			if tc.wantDebug && !strings.Contains(joined, "rpcName") {
+				t.Errorf("wire record lost its attrs:\n%s", joined)
+			}
+		})
+	}
+}
+
+func TestWireLogger_infoLandsAtV1(t *testing.T) {
+	t.Parallel()
+	lines := &[]string{}
+	base := funcr.New(func(prefix, args string) {
+		*lines = append(*lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: 1})
+
+	wireLogger(base, WireLogOptions{}).Info("hello")
+
+	if joined := strings.Join(*lines, "\n"); !strings.Contains(joined, "hello") {
+		t.Errorf("slog Info should land at V(1) and be visible at verbosity 1; output:\n%s", joined)
+	}
+}
+
+func captureWireLogger(verbosity int, opts WireLogOptions) (*slog.Logger, *[]string) {
+	lines := &[]string{}
+	base := funcr.New(func(prefix, args string) {
+		*lines = append(*lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: verbosity})
+	return wireLogger(base, opts), lines
+}
+
+func TestWireLogger_dropsNonAPIDebugRecords(t *testing.T) {
+	t.Parallel()
+	slogger, lines := captureWireLogger(9, WireLogOptions{})
+
+	const secret = "assertion=eyJhbGciOiJSUzI1NiJ9.SECRET"
+	slogger.Debug("2LO token request", "request", map[string]any{"payload": secret})
+	slogger.Debug("2LO token response", "response", map[string]any{"payload": "ya29.SECRET-TOKEN"})
+
+	if len(*lines) != 0 {
+		t.Errorf("auth token-exchange records must be dropped; got:\n%s", strings.Join(*lines, "\n"))
+	}
+
+	slogger.Warn("credential refresh failed")
+	if joined := strings.Join(*lines, "\n"); !strings.Contains(joined, "credential refresh failed") {
+		t.Errorf("non-debug SDK records should pass through; output:\n%s", joined)
+	}
+}
+
+func TestWireLogger_elidesLargeFields(t *testing.T) {
+	t.Parallel()
+	slogger, lines := captureWireLogger(9, WireLogOptions{})
+
+	huge := strings.Repeat("x", 4096)
+	slogger.Debug("api response", "response", map[string]any{
+		"status": "200",
+		"payload": map[string]any{
+			"name":  "proxy-abc",
+			"disks": []any{map[string]any{"content": huge}},
+		},
+	})
+
+	joined := strings.Join(*lines, "\n")
+	if strings.Contains(joined, huge[:64]) {
+		t.Errorf("large field not elided:\n%.500s", joined)
+	}
+	if !strings.Contains(joined, "[elided 4096 bytes]") {
+		t.Errorf("elision marker missing:\n%s", joined)
+	}
+	for _, keep := range []string{"proxy-abc", "200", "api response"} {
+		if !strings.Contains(joined, keep) {
+			t.Errorf("small field %q lost during elision:\n%s", keep, joined)
+		}
+	}
+}
+
+func TestWireLogger_fullPayloadsDisablesElision(t *testing.T) {
+	t.Parallel()
+	slogger, lines := captureWireLogger(9, WireLogOptions{FullPayloads: true})
+
+	huge := strings.Repeat("y", 4096)
+	slogger.Debug("api response", "response", map[string]any{"payload": huge})
+
+	joined := strings.Join(*lines, "\n")
+	if !strings.Contains(joined, huge) {
+		t.Errorf("FullPayloads should keep fields verbatim:\n%.200s", joined)
+	}
+
+	slogger.Debug("2LO token response", "response", "ya29.SECRET")
+	if joined := strings.Join(*lines, "\n"); strings.Contains(joined, "ya29.SECRET") {
+		t.Error("auth records must be dropped even with FullPayloads")
+	}
+}
+
+func TestLogging_apiErrorKeepsHTTPDetail(t *testing.T) {
+	t.Parallel()
+	ctx, lines := captureContext(1)
+	p := newTestProvider(&fakeAPI{insertErr: gerr(403, "quotaExceeded")})
+
+	if _, err := p.Create(ctx, testCreateRequest()); err == nil {
+		t.Fatal("Create: want error")
+	}
+
+	joined := strings.Join(*lines, "\n")
+	for _, want := range []string{
+		`"msg"="GCP API call failed"`,
+		`"httpStatus"=403`,
+		`"quotaExceeded"`,
+		`"op"="create"`,
+	} {
+		if !strings.Contains(joined, want) {
+			t.Errorf("output missing %q:\n%s", want, joined)
+		}
+	}
+}
+
+func TestLogging_treatedAsSuccessPathsAreExplicit(t *testing.T) {
+	t.Parallel()
+	ctx, lines := captureContext(1)
+	p := newTestProvider(&fakeAPI{insertErr: gerr(409), deleteErr: gerr(404)})
+
+	if _, err := p.Create(ctx, testCreateRequest()); err != nil {
+		t.Fatalf("Create with 409: %v", err)
+	}
+	if err := p.Delete(ctx, "zones/z/instances/gone"); err != nil {
+		t.Fatalf("Delete with 404: %v", err)
+	}
+
+	joined := strings.Join(*lines, "\n")
+	for _, want := range []string{
+		`"msg"="GCP instance already exists, insert treated as success"`,
+		`"msg"="GCP instance already gone, delete treated as success"`,
+	} {
+		if !strings.Contains(joined, want) {
+			t.Errorf("output missing %q:\n%s", want, joined)
+		}
+	}
+}

internal/provider/gcp/insert.go

@@ -0,0 +1,74 @@
+package gcp
+
+import (
+	"fmt"
+
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"google.golang.org/protobuf/proto"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+const (
+	defaultNetwork    = "default"
+	defaultNetworkTag = "proxy-operator"
+	defaultDiskSizeGB = 10
+	userDataKey       = "user-data"
+)
+
+func withDefaults(cfg provider.GCPConfig) provider.GCPConfig {
+	if cfg.Network == "" {
+		cfg.Network = defaultNetwork
+	}
+	if cfg.NetworkTag == "" {
+		cfg.NetworkTag = defaultNetworkTag
+	}
+	if cfg.DiskSizeGB == 0 {
+		cfg.DiskSizeGB = defaultDiskSizeGB
+	}
+	return cfg
+}
+
+// buildInsertRequest is pure so the field-by-field unit test needs no fake
+// at all — the plan's primary test for this provider.
+func buildInsertRequest(cfg provider.GCPConfig, req provider.CreateRequest) *computepb.InsertInstanceRequest {
+	inst := &computepb.Instance{
+		Name:        proto.String(req.Name),
+		MachineType: proto.String(fmt.Sprintf("zones/%s/machineTypes/%s", req.Placement.Zone, req.Placement.MachineType)),
+		Disks: []*computepb.AttachedDisk{{
+			Boot:       proto.Bool(true),
+			AutoDelete: proto.Bool(true),
+			InitializeParams: &computepb.AttachedDiskInitializeParams{
+				SourceImage: proto.String(req.Placement.Image),
+				DiskSizeGb:  proto.Int64(cfg.DiskSizeGB),
+			},
+		}},
+		NetworkInterfaces: []*computepb.NetworkInterface{{
+			Network: proto.String("global/networks/" + cfg.Network),
+			// An ephemeral external IP: exactly this pair, per the API's
+			// contract for one-to-one NAT.
+			AccessConfigs: []*computepb.AccessConfig{{
+				Name: proto.String("External NAT"),
+				Type: proto.String("ONE_TO_ONE_NAT"),
+			}},
+		}},
+		// The GC contract: every resource this operator creates carries
+		// these two labels, and orphan GC relies on both.
+		Labels: map[string]string{
+			provider.LabelManaged: provider.LabelManagedYes,
+			provider.LabelUID:     req.UID,
+		},
+		Tags: &computepb.Tags{Items: []string{cfg.NetworkTag}},
+	}
+	if req.CloudInit != "" {
+		inst.Metadata = &computepb.Metadata{Items: []*computepb.Items{{
+			Key:   proto.String(userDataKey),
+			Value: proto.String(req.CloudInit),
+		}}}
+	}
+	return &computepb.InsertInstanceRequest{
+		Project:          cfg.Project,
+		Zone:             req.Placement.Zone,
+		InstanceResource: inst,
+	}
+}

internal/provider/gcp/insert_test.go

@@ -0,0 +1,127 @@
+package gcp
+
+import (
+	"testing"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func testCreateRequest() provider.CreateRequest {
+	return provider.CreateRequest{
+		Name:      "proxy-abc123def456ghij",
+		UID:       "11111111-2222-3333-4444-555555555555",
+		Namespace: "default",
+		ProxyName: "eu-proxy-1",
+		Placement: provider.Placement{
+			Zone:        "europe-west1-b",
+			MachineType: "e2-micro",
+			Image:       "projects/debian-cloud/global/images/family/debian-12",
+		},
+		CloudInit: "#cloud-config\npackages: [squid]",
+		Port:      3128,
+	}
+}
+
+func TestBuildInsertRequest_fieldByField(t *testing.T) {
+	t.Parallel()
+	cfg := withDefaults(provider.GCPConfig{Project: "my-project"})
+	req := buildInsertRequest(cfg, testCreateRequest())
+
+	if req.Project != "my-project" || req.Zone != "europe-west1-b" {
+		t.Errorf("project/zone = %s/%s, want my-project/europe-west1-b", req.Project, req.Zone)
+	}
+	inst := req.InstanceResource
+	if inst.GetName() != "proxy-abc123def456ghij" {
+		t.Errorf("name = %s", inst.GetName())
+	}
+	if got, want := inst.GetMachineType(), "zones/europe-west1-b/machineTypes/e2-micro"; got != want {
+		t.Errorf("machineType = %s, want %s", got, want)
+	}
+
+	if len(inst.GetDisks()) != 1 {
+		t.Fatalf("disks = %d, want 1", len(inst.GetDisks()))
+	}
+	disk := inst.GetDisks()[0]
+	if !disk.GetBoot() || !disk.GetAutoDelete() {
+		t.Errorf("boot/autoDelete = %v/%v, want true/true", disk.GetBoot(), disk.GetAutoDelete())
+	}
+	if got, want := disk.GetInitializeParams().GetSourceImage(), "projects/debian-cloud/global/images/family/debian-12"; got != want {
+		t.Errorf("sourceImage = %s, want %s", got, want)
+	}
+	if disk.GetInitializeParams().GetDiskSizeGb() != 10 {
+		t.Errorf("diskSizeGb = %d, want the 10 default", disk.GetInitializeParams().GetDiskSizeGb())
+	}
+
+	if len(inst.GetNetworkInterfaces()) != 1 {
+		t.Fatalf("networkInterfaces = %d, want 1", len(inst.GetNetworkInterfaces()))
+	}
+	nic := inst.GetNetworkInterfaces()[0]
+	if got, want := nic.GetNetwork(), "global/networks/default"; got != want {
+		t.Errorf("network = %s, want %s", got, want)
+	}
+	if len(nic.GetAccessConfigs()) != 1 {
+		t.Fatalf("accessConfigs = %d, want 1", len(nic.GetAccessConfigs()))
+	}
+	ac := nic.GetAccessConfigs()[0]
+	if ac.GetName() != "External NAT" || ac.GetType() != "ONE_TO_ONE_NAT" {
+		t.Errorf("accessConfig = %s/%s, want External NAT/ONE_TO_ONE_NAT", ac.GetName(), ac.GetType())
+	}
+
+	wantLabels := map[string]string{
+		provider.LabelManaged: provider.LabelManagedYes,
+		provider.LabelUID:     "11111111-2222-3333-4444-555555555555",
+	}
+	labels := inst.GetLabels()
+	if len(labels) != len(wantLabels) {
+		t.Errorf("labels = %v, want %v", labels, wantLabels)
+	}
+	for k, v := range wantLabels {
+		if labels[k] != v {
+			t.Errorf("label %s = %q, want %q", k, labels[k], v)
+		}
+	}
+
+	if tags := inst.GetTags().GetItems(); len(tags) != 1 || tags[0] != "proxy-operator" {
+		t.Errorf("tags = %v, want [proxy-operator]", tags)
+	}
+
+	items := inst.GetMetadata().GetItems()
+	if len(items) != 1 || items[0].GetKey() != "user-data" {
+		t.Fatalf("metadata items = %v, want one user-data entry", items)
+	}
+	if items[0].GetValue() != "#cloud-config\npackages: [squid]" {
+		t.Errorf("user-data = %q, want the resolved cloud-init", items[0].GetValue())
+	}
+}
+
+func TestBuildInsertRequest_configOverrides(t *testing.T) {
+	t.Parallel()
+	cfg := withDefaults(provider.GCPConfig{
+		Project:    "my-project",
+		Network:    "crawl-vpc",
+		NetworkTag: "crawl-egress",
+		DiskSizeGB: 42,
+	})
+	req := buildInsertRequest(cfg, testCreateRequest())
+	inst := req.InstanceResource
+
+	if got, want := inst.GetNetworkInterfaces()[0].GetNetwork(), "global/networks/crawl-vpc"; got != want {
+		t.Errorf("network = %s, want %s", got, want)
+	}
+	if tags := inst.GetTags().GetItems(); len(tags) != 1 || tags[0] != "crawl-egress" {
+		t.Errorf("tags = %v, want [crawl-egress]", tags)
+	}
+	if got := inst.GetDisks()[0].GetInitializeParams().GetDiskSizeGb(); got != 42 {
+		t.Errorf("diskSizeGb = %d, want 42", got)
+	}
+}
+
+func TestBuildInsertRequest_noCloudInitMeansNoMetadata(t *testing.T) {
+	t.Parallel()
+	req := testCreateRequest()
+	req.CloudInit = ""
+	built := buildInsertRequest(withDefaults(provider.GCPConfig{Project: "p"}), req)
+	if built.InstanceResource.GetMetadata() != nil {
+		t.Errorf("metadata = %v, want none without cloud-init", built.InstanceResource.GetMetadata())
+	}
+}

internal/provider/gcp/wirelog.go

@@ -0,0 +1,127 @@
+package gcp
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"github.com/go-logr/logr"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// wireLogMaxFieldBytes is the elision threshold for string fields in wire
+// payloads: GCP responses embed multi-KB blobs (Shielded-VM UEFI dbx
+// databases, licenses) that swamp the log line without diagnostic value.
+const wireLogMaxFieldBytes = 1024
+
+// WireLogOptions controls the V(5) HTTP wire logging of the GCP SDK.
+type WireLogOptions struct {
+	// FullPayloads disables field elision and logs payloads verbatim.
+	FullPayloads bool
+}
+
+// wireLogger returns the slog logger handed to the SDK: its Debug-level
+// "api request"/"api response" records (slog Debug = +4 on the logr
+// scale) land at V(5) on top of the base's V(1) shift.
+//
+// Debug records other than the compute client's api request/response are
+// dropped entirely: the same logger propagates into the auth library,
+// whose token-exchange records contain the signed JWT assertion and the
+// bearer access token. Warnings and errors pass through.
+func wireLogger(base logr.Logger, opts WireLogOptions) *slog.Logger {
+	return slog.New(&wireFilterHandler{
+		inner:        logr.ToSlogHandler(base.V(1)),
+		fullPayloads: opts.FullPayloads,
+	})
+}
+
+type wireFilterHandler struct {
+	inner        slog.Handler
+	fullPayloads bool
+}
+
+func (h *wireFilterHandler) Enabled(ctx context.Context, level slog.Level) bool {
+	return h.inner.Enabled(ctx, level)
+}
+
+func (h *wireFilterHandler) Handle(ctx context.Context, rec slog.Record) error {
+	if rec.Level <= slog.LevelDebug && rec.Message != "api request" && rec.Message != "api response" {
+		return nil
+	}
+	// The SDK logs with the request ctx, so wire records can carry the
+	// surrounding provider span — the one place slog's ctx-aware handlers
+	// beat logr, and the same keys the logr enrichment uses.
+	if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
+		rec = rec.Clone()
+		rec.AddAttrs(
+			slog.String("traceID", sc.TraceID().String()),
+			slog.String("spanID", sc.SpanID().String()))
+	}
+	if h.fullPayloads {
+		return h.inner.Handle(ctx, rec)
+	}
+	elided := slog.NewRecord(rec.Time, rec.Level, rec.Message, rec.PC)
+	rec.Attrs(func(a slog.Attr) bool {
+		elided.AddAttrs(slog.Attr{Key: a.Key, Value: elideValue(a.Value)})
+		return true
+	})
+	return h.inner.Handle(ctx, elided)
+}
+
+func (h *wireFilterHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
+	return &wireFilterHandler{inner: h.inner.WithAttrs(attrs), fullPayloads: h.fullPayloads}
+}
+
+func (h *wireFilterHandler) WithGroup(name string) slog.Handler {
+	return &wireFilterHandler{inner: h.inner.WithGroup(name), fullPayloads: h.fullPayloads}
+}
+
+func elideValue(v slog.Value) slog.Value {
+	v = v.Resolve()
+	switch v.Kind() {
+	case slog.KindString:
+		if s := v.String(); len(s) > wireLogMaxFieldBytes {
+			return slog.StringValue(elisionMarker(len(s)))
+		}
+		return v
+	case slog.KindGroup:
+		attrs := v.Group()
+		out := make([]slog.Attr, 0, len(attrs))
+		for _, a := range attrs {
+			out = append(out, slog.Attr{Key: a.Key, Value: elideValue(a.Value)})
+		}
+		return slog.GroupValue(out...)
+	case slog.KindAny:
+		return slog.AnyValue(elideAny(v.Any()))
+	default:
+		return v
+	}
+}
+
+func elideAny(v any) any {
+	switch t := v.(type) {
+	case string:
+		if len(t) > wireLogMaxFieldBytes {
+			return elisionMarker(len(t))
+		}
+		return t
+	case map[string]any:
+		out := make(map[string]any, len(t))
+		for k, val := range t {
+			out[k] = elideAny(val)
+		}
+		return out
+	case []any:
+		out := make([]any, len(t))
+		for i, val := range t {
+			out[i] = elideAny(val)
+		}
+		return out
+	default:
+		return v
+	}
+}
+
+func elisionMarker(size int) string {
+	return fmt.Sprintf("[elided %d bytes]", size)
+}

internal/provider/gcp/wirelog_test.go

@@ -0,0 +1,50 @@
+package gcp
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/go-logr/logr/funcr"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+func TestWireFilterHandler_addsTraceContext(t *testing.T) {
+	t.Parallel()
+
+	var lines []string
+	base := funcr.New(func(prefix, args string) {
+		lines = append(lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: 5})
+	log := wireLogger(base, WireLogOptions{})
+
+	tp := sdktrace.NewTracerProvider()
+	t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
+	ctx, span := tp.Tracer("test").Start(context.Background(), "provider.create")
+	defer span.End()
+
+	log.DebugContext(ctx, "api request", "url", "https://compute.googleapis.com/x")
+	if len(lines) != 1 {
+		t.Fatalf("got %d lines, want 1: %v", len(lines), lines)
+	}
+	traceID := span.SpanContext().TraceID().String()
+	if !strings.Contains(lines[0], "traceID") || !strings.Contains(lines[0], traceID) {
+		t.Errorf("wire record missing trace context %s: %s", traceID, lines[0])
+	}
+
+	// The security filter must still win: non-wire Debug records are
+	// dropped even when a span is present.
+	log.DebugContext(ctx, "token exchange", "assertion", "secret-jwt")
+	if len(lines) != 1 {
+		t.Fatalf("filtered record leaked: %v", lines[1:])
+	}
+
+	// No span in ctx: record passes through without trace keys.
+	log.DebugContext(context.Background(), "api response", "status", 200)
+	if len(lines) != 2 {
+		t.Fatalf("got %d lines, want 2", len(lines))
+	}
+	if strings.Contains(lines[1], "traceID") {
+		t.Errorf("spanless record must not carry traceID: %s", lines[1])
+	}
+}