Log the provider docs split in the changelog

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

.claude/agents/operator-reviewer.md

@@ -0,0 +1,15 @@
+---
+name: operator-reviewer
+description: Reviews Kubernetes operator PRs for controller-runtime correctness, reconcile semantics, and API design
+tools: Read, Grep, Glob, Bash
+---
+You are a senior reviewer specializing in Kubernetes operators.
+Review with focus on:
+- Reconcile idempotency and requeue behavior; no state assumptions between reconciles
+- Informer cache reads vs direct API reads; stale-cache races
+- Finalizer handling, deletion flow, orphaned resources
+- CRD schema evolution, conversion webhooks, status subresource / conditions conventions
+- RBAC minimality vs what the controller actually touches
+- Leader election, watch predicates, event filtering for churn reduction
+- Go: context propagation, error wrapping, client.Object handling
+Output: findings ranked by severity, with file:line refs. No praise padding.

.claude/settings.json

@@ -63,7 +63,43 @@
       "Bash(grep -n 'func Channel' -A8 __CMDSUB_OUTPUT__/sigs.k8s.io/controller-runtime@v0.24.1/pkg/source/source.go)",
       "Bash(grep -n 'type GenericEvent' __CMDSUB_OUTPUT__/sigs.k8s.io/controller-runtime@v0.24.1/pkg/event/event.go)",
       "Bash(KUBEBUILDER_ASSETS=__TRACKED_VAR__/bin/k8s/1.36.2-darwin-arm64 go test -race ./...)",
-      "Bash(cat >> *)"
+      "Bash(cat >> *)",
+      "Bash(make run-dev *)",
+      "Bash(kubectl get *)",
+      "Bash(kubectl delete *)",
+      "Bash(make docker-build *)",
+      "Bash(kind load *)",
+      "Bash(make deploy *)",
+      "Bash(kubectl -n egress-proxies-operator-system rollout status deploy/egress-proxies-operator-controller-manager --timeout=120s)",
+      "Bash(kubectl -n egress-proxies-operator-system rollout restart deploy/egress-proxies-operator-controller-manager)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator add docs/plans/2026-08-11-1742-gcp-provider-verbose-logging.md)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator commit -m 'Add plan: verbose V-level logging in the GCP provider *)",
+      "Bash(echo \"exit: $?\")",
+      "Bash(echo \"tests exit: $?\")",
+      "Bash(./bin/manager --version)",
+      "Bash(./bin/manager-stamped --version)",
+      "Bash(./bin/manager-pkg --version)",
+      "Bash(docker run *)",
+      "Bash(kubectl -n egress-proxies-operator-system get pods -o wide)",
+      "Bash(kubectl -n egress-proxies-operator-system get deploy egress-proxies-operator-controller-manager -o jsonpath='{.spec.template.spec.containers[0].args}')",
+      "Bash(kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager)",
+      "Bash(python3 -c \"import json; d=json.load\\(open\\('docs/deploy/sa_key.json'\\)\\); print\\(d.get\\('type'\\), d.get\\('client_email'\\)\\)\")",
+      "Bash(tea pr *)",
+      "Bash(git worktree *)",
+      "Bash(python3 -c \"import yaml; yaml.safe_load\\(open\\('.gitea/workflows/build.yaml'\\)\\); print\\('YAML OK'\\)\")",
+      "Bash(ruby -ryaml -e \"YAML.load_file\\('.gitea/workflows/build.yaml'\\); puts 'YAML OK'\")",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator status --short --branch)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator log --oneline -1)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator tag 0.01)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator push origin 0.01)",
+      "Bash(chmod +x docs/demo/run-demo.sh)",
+      "Bash(bash -n docs/demo/run-demo.sh)",
+      "Bash(git add *)",
+      "Bash(git commit -q -m 'Add plan: KVM/libvirt provider *)",
+      "Bash(go get *)",
+      "Bash(go build *)",
+      "Bash(go test *)",
+      "Bash(git commit *)"
     ],
     "additionalDirectories": [
       "/Users/jan.novak/srv/go/egress-proxies-operator/.claude",

.gitea/workflows/build.yaml

@@ -0,0 +1,76 @@
+name: Build and Push
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Image tag'
+        required: true
+        default: 'latest'
+  push:
+    tags:
+      - '*'
+
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Vet
+        run: go vet ./...
+      - name: Build
+        run: go build ./...
+      - name: Test (short)
+        run: go test -short ./...
+
+  build:
+    needs: check
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Compute image tags
+        id: meta
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            TAG="${{ inputs.tag }}"
+          else
+            TAG="${{ github.ref_name }}"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "sha=sha-$(echo '${{ github.sha }}' | cut -c1-12)" >> "$GITHUB_OUTPUT"
+
+      - name: Login to Gitea registry
+        run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login -u ${{ github.actor }} --password-stdin gitea.home.hrajfrisbee.cz
+
+      - name: Build and push
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker build \
+            --build-arg GIT_COMMIT=$(echo '${{ github.sha }}' | cut -c1-12) \
+            --label org.opencontainers.image.source=https://gitea.home.hrajfrisbee.cz/${{ github.repository }} \
+            --label org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
+            -t "$IMAGE:${{ steps.meta.outputs.tag }}" \
+            -t "$IMAGE:${{ steps.meta.outputs.sha }}" \
+            .
+          docker push "$IMAGE:${{ steps.meta.outputs.tag }}"
+          docker push "$IMAGE:${{ steps.meta.outputs.sha }}"
+
+      # Only real tag pushes move :latest — an ad-hoc dispatch of an old ref must not clobber it.
+      - name: Push latest (tag builds only)
+        if: github.event_name == 'push'
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker tag "$IMAGE:${{ steps.meta.outputs.tag }}" "$IMAGE:latest"
+          docker push "$IMAGE:latest"

.gitignore

@@ -28,3 +28,6 @@ go.work
 
 # Kubeconfig might contain secrets
 *.kubeconfig
+
+# GCP service-account keys (created per docs/gcp-in-specific-project.md)
+sa_key.json

CHANGELOG.md

@@ -1 +1,152 @@
 # Changelog
+
+## 2026-09-16 11:46 UTC — Provider blueprint split into contract, spec template and appendix
+
+- **`docs/provider-contract.md`** is the generic provider specification: MUST/SHOULD/MAY
+  rules with the caller behind each one, 19 behaviour-profile axes with stable codes
+  (where providers may differ, such as blocking vs submit-and-return Create, or a
+  backend with no provisioning state), error taxonomy, credential hygiene, testing
+  obligations, the adding-a-provider checklist and the contract's known limits. It
+  names no existing provider.
+- **`docs/provider-spec-template.md`** is the fill-in questionnaire a new provider's
+  design answers before implementation; entry numbers match the contract's profile
+  table.
+- **`docs/providers-existing.md`** maps kubernetes, gcp and kvm to every profile and
+  keeps their per-provider reference notes, shared gaps and proposed features.
+- **`docs/providers.md`** is now an index; README, CLAUDE.md and architecture.md point
+  at the new files. This supersedes the single-file blueprint written earlier on the
+  same branch.
+- Two rules the callers enforce that were not written down before: orphan GC sweeps
+  providers sequentially with no deadline, so a hung `ListByTag` stalls all collection;
+  and the reconciler deletes an instance a backend reports as Stopped before start.
+
+## 2026-09-16 08:45 UTC — Provider health checks: ProviderHealth CRD, per-provider checks, gauges
+
+- **Each configured provider is now checked on a schedule** and reported in a
+  cluster-scoped `ProviderHealth` object (`kubectl get ph`), one per provider, with a
+  `Ready` condition, a reason, and per-target verdicts.
+- **Checks:** kubernetes lists one object from the API server; GCP fetches a single
+  instance page; KVM checks SSH/libvirt reachability and the storage pool on every
+  host, in parallel. Failures map to fixed reasons (`Unreachable`,
+  `SSHHandshakeFailed`, `PoolNotFound`, `Timeout`, …).
+- **The KVM check can't hurt reconciles:** it uses `hostPool.probe`, which gives up on
+  a slow RPC without dropping the shared connection, and waiting for a host slot can
+  be interrupted.
+- **Metrics:** `proxy_operator_provider_target_up`, `_check_duration_seconds` and
+  `_check_timestamp_seconds`. The flags are `--provider-check-interval` and
+  `--provider-check-timeout` (defaults to `min(15s, interval)`).
+- Validated with `make test`, `make test-e2e` and the real-hypervisor KVM run against
+  beelink. `make lint` is blocked by a Go 1.27 toolchain/staticcheck panic that has
+  nothing to do with this change.
+- Key files: `api/v1alpha1/providerhealth_types.go`, `internal/provider/check.go`,
+  `internal/provider/kvm/`, `internal/providerhealth/`, `docs/architecture.md`.
+
+## 2026-09-10 22:58 UTC — Code-review fixes on the KVM branch: deletion, validation, lazy SSH, guest agent
+
+Five findings from a review of `feat/kvm-provider`, all fixed on the branch.
+
+- **A Proxy can always finish deleting.** The delete path used to drop the finalizer
+  only when `Get` answered not-found, so a permanent error there retried forever —
+  a KVM host removed from the config was enough. It now calls `Delete` anyway, since
+  that is idempotent and can finish where `Get` cannot, and
+  `crawl.example.com/skip-cleanup: "true"` abandons the instance for whatever stays
+  stuck. In the KVM provider, deleting over an unconfigured host is a no-op success;
+  `Get` still fails permanently there so a typo cannot look like a missing VM.
+- **The guest agent ships as a vendor-data script**, superseding the cloud-config
+  merge in the entry below. Merging could never be made safe: cloud-init drops
+  vendor-data keys the user-data defines, and the strategy cannot be overridden from
+  either document. The Proxy's cloud-init now reaches the seed byte for byte in any
+  format — Jinja, MIME, gzip, scripts — where the round-trip previously re-typed
+  unquoted scalars (`version: 1.10` arrived as `1.1`), dropped comments and reordered
+  keys. New `installGuestAgent` setting, default on.
+- **A missing `kvm-ssh` Secret no longer crash-loops the manager.** The SSH material
+  is read on first use and cached, so only KVM Proxies fail and they recover when the
+  Secret appears. A missing file is transient, an unparseable key permanent, and
+  startup still logs a warning.
+- **Host names and image keys are validated.** Either can end up in a providerID or a
+  volume file name, where a slash made the VM unaddressable for good.
+- Verified end to end: unit suite, the integration test against a real hypervisor
+  (22 s, agent reporting the IP), both in-cluster Proxies Ready in 2 min with 204
+  through squid, the guest showing `qemu-guest-agent` active with its cloud-config
+  comments intact, and the manager staying up with the Secret pointed away.
+- Key files: `internal/provider/kvm/{seed.go,conn.go,kvm.go}`,
+  `internal/controller/proxy_controller.go`, `internal/provider/config_kvm.go`,
+  `api/v1alpha1/proxy_types.go`.
+
+## 2026-09-10 21:04 UTC — KVM provider passes in-cluster; guest agent merged into cloud-config
+
+- Two KVM Proxies with real cloud-configs reach `Ready` from inside the kind cluster
+  (2 min from apply, IP via the guest agent, 204 through squid from the LAN).
+- The `qemu-guest-agent` bootstrap is now merged into each Proxy's cloud-config
+  (`packages` + `runcmd`) instead of shipped as vendor-data: cloud-init's default
+  merger drops vendor-data keys the user-data also defines, so any Proxy with its own
+  `packages:` never got the agent and stayed `Provisioning` forever. `vendorData` is
+  now plain extra vendor-data with no default; non-cloud-config user-data (scripts,
+  `#include`) still gets the bootstrap as vendor-data. A cloud-config that is not a
+  YAML mapping fails the Proxy permanently instead of booting without its config.
+- The manager pod mounts a 64 MiB `emptyDir` on `/tmp`: the seed ISO writer stages
+  files on disk and the root filesystem is read-only.
+- Errors already classified with a provider sentinel no longer drop the cached
+  libvirt SSH connection; only libvirt and transport errors do.
+- Key files: `internal/provider/kvm/{seed.go,conn.go,errors.go}`,
+  `config/manager/manager.yaml`, `docs/kvm-provider.md`.
+
+## 2026-09-10 10:07 UTC — KVM provider validated on a real hypervisor; UEFI default; libvirt call tracing
+
+- `make test-integration-kvm` passes end to end against a libvirt 11.6 / QEMU 10.1
+  host (Debian 13 image, UEFI, guest agent reporting the IP after ~20 s). Four
+  defects found only on the real host, all fixed on the branch:
+  - domain disks are now `type="file"` with resolved paths: pool-volume references
+    get no AppArmor rule from `virt-aa-helper`, so QEMU was denied the disks;
+  - domain metadata is parsed by local name: libvirt returns the element without
+    its namespace declaration;
+  - guests boot **UEFI by default** (`images.<key>.firmware: uefi|bios`, OVMF now a
+    host prerequisite): Debian 13's image hangs in GRUB under SeaBIOS on a headless
+    VM with a serial console (Debian bug #1111240, open); bookworm is unaffected;
+  - the image cache's raw-upload path is confirmed to produce a byte-identical file.
+- Every libvirt RPC is logged at `--zap-log-level=debug` as one `libvirt call` line
+  (`rpc=virDomainCreate`, identifiers, duration, `libvirtError` on failure); level 2
+  adds XML, metadata and addresses; cloud-init is never logged. The integration test
+  shows the trace in `-v` output (`KVM_TEST_LOG_LEVEL`, default 1).
+- Key files: `internal/provider/kvm/{domainxml.go,metadata.go,trace.go,conn.go}`,
+  `internal/provider/config_kvm.go`, `docs/kvm-provider.md`.
+
+## 2026-09-08 11:00 UTC — KVM/libvirt provider
+
+- New `kvm` provider type: Managed proxies as VMs on your own libvirt/KVM hosts. Each
+  host is reached over SSH with the libvirt unix socket forwarded through the
+  connection (pure Go, nothing to expose on the hypervisors beyond sshd); one provider
+  instance can span several hosts, selected by `placement.zone`.
+- Base images (`images:` map or a direct URL in `placement.image`) are downloaded by the
+  operator into each host's storage pool on first use, verified by sha256 when configured,
+  and cached there. Per-VM disks are qcow2 overlays; cloud-init arrives on a NoCloud seed
+  ISO built in-process. GC identity lives in the domain's `<metadata>` element.
+- Wiring: `"kvm": kvm.New` in `cmd/main.go`; optional `kvm-ssh` Secret mounted at
+  `/etc/proxy-operator/kvm`; samples `config/samples/proxy_kvm.yaml` and a `kvm-home`
+  block in `providers-config.yaml`; `docs/kvm-provider.md`; opt-in integration test
+  (`make test-integration-kvm`) against a real hypervisor.
+- Not validated on a real host at the time of this entry; see 2026-09-10 above.
+
+## 2026-08-10 09:34 CEST — kind e2e verified; fix Squid OOM in containers; quickstart goes in-cluster
+
+- Full end-to-end pass on a throwaway kind cluster: Squid pod Ready with a real CONNECT
+  probe (89 ms), lease grant/report/cooldown-409/release through the discovery API,
+  finalizer cleanup on delete.
+- Fixed the kubernetes provider's generated squid.conf: `max_filedescriptors 1024`
+  (squid sizes FD tables from the container's effectively-unlimited RLIMIT_NOFILE and
+  was OOM-killed at startup under kind/containerd) + `cache_mem 16 MB`.
+- README quickstart now deploys the operator in-cluster: `make run-dev` on a laptop
+  cannot reach kind pod IPs, so health probes fail by construction there (documented).
+
+## 2026-08-09 17:27 CEST — Operator wired end to end: reconciler, health, leases, discovery, GC, two providers
+
+- `cmd/main.go` is now the full composition root: `--providers-config` (required, fail-fast),
+  `--discovery-addr`, `--proxy-namespace`, `--health-workers`, `--gc-interval`, `--gc-min-age`,
+  `--gc-allow-namespaced`, `--lease-cooldown`, `--max-lease-ttl`; wires the kubernetes + gcp
+  providers (metrics-instrumented), health engine, lease store, discovery API, orphan GC, and
+  Prometheus metrics onto one manager.
+- Deploy manifests: providers ConfigMap mount, optional `DISCOVERY_TOKEN` Secret env,
+  discovery port 8090 + Service; pods RBAC for the kubernetes provider.
+- Samples for all three proxy flavors + providers-config; `make run-dev` for local development.
+- README rewritten (kind quickstart, GCP setup, the two load-bearing caveats); architecture doc
+  completed with components table and the full decision log.

CLAUDE.md

@@ -25,10 +25,16 @@ entrypoint yet, so there's nothing to run locally.
 
 ## Architecture
 
-Not yet built. TODO: once the controller-runtime manager and reconcile loop exist,
-document the data-flow diagram and package/role table here, following the standard
-reconcile-loop shape: fetch → resolve → list related → compute desired state → merge
-→ patch only on diff.
+Documented in [docs/architecture.md](docs/architecture.md). It holds the component
+table, the reconcile flow, and one section per runnable: health engine, discovery and
+lease API, orphan GC, metrics, tracing, and provider health checks. The generic
+provider contract (MUST/SHOULD rules, behaviour profiles, checklist for adding a
+provider, known limits) is in [docs/provider-contract.md](docs/provider-contract.md);
+a new provider starts from [docs/provider-spec-template.md](docs/provider-spec-template.md);
+how the existing providers meet the contract, with their gaps, is in
+[docs/providers-existing.md](docs/providers-existing.md); [docs/providers.md](docs/providers.md)
+indexes them. Provider-specific detail is in `docs/kvm-provider.md` and the GCP
+docs; the test layout is in `docs/testing.md`.
 
 ## Conventions
 
@@ -199,12 +205,24 @@ Always append a `Co-Authored-By` trailer to indicate AI assistance:
 
     Co-Authored-By: Claude <noreply@anthropic.com>
 
-TODO: no `.gitea/workflows/` CI pipeline exists yet — add a CI/CD subsection here once
-one is set up.
+### CI/CD
+
+`.gitea/workflows/build.yaml` builds the manager image and pushes it to the Gitea
+registry. Triggers: any tag push, or manual `workflow_dispatch` with a `tag` input.
+A lightweight `check` job (`go vet` / `go build` / `go test -short`) gates the build.
+
+- Images: `gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator:<tag>` plus an
+  immutable `sha-<12-char-commit>` tag on every build; `:latest` moves only on real
+  tag pushes, never on manual dispatch.
+- Requires the `REGISTRY_TOKEN` repo secret (Gitea PAT with `write:package`),
+  same convention as the other projects on this Gitea instance.
+- The commit is baked into the binary via the `GIT_COMMIT` build arg (see Dockerfile).
 
 ## Gotchas
 
-- This repo currently contains only `go.mod` (module
-  `gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator`) — no controller code,
-  `main.go`, Dockerfile, k8s manifests, Makefile, or CI config exist yet. Nothing in
-  this file describes working software; it's the convention scaffold to build against.
+- The "What this is" and "Commands" sections above predate the code and are stale.
+  The Makefile targets (`make test`, `make manifests generate`, `make test-e2e`,
+  `make test-integration-kvm`) and `docs/testing.md` are authoritative.
+- `+kubebuilder:rbac` markers are package-level. Keep them in their own comment block,
+  never inside a type's or function's doc comment: controller-gen silently ignores
+  them there. After `make manifests`, check `git diff config/rbac/role.yaml`.

Dockerfile

@@ -2,6 +2,7 @@
 FROM golang:1.26 AS builder
 ARG TARGETOS
 ARG TARGETARCH
+ARG GIT_COMMIT=unknown
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -19,11 +20,15 @@ COPY . .
 # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a \
+    -ldflags "-X gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version.Commit=${GIT_COMMIT}" \
+    -o manager cmd/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
+ARG GIT_COMMIT=unknown
+LABEL org.opencontainers.image.revision="${GIT_COMMIT}"
 WORKDIR /
 COPY --from=builder /workspace/manager .
 USER 65532:65532

Makefile

@@ -2,6 +2,9 @@
 IMG ?= controller:latest
 # YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header.
 YEAR ?= $(shell date +%Y)
+# GIT_COMMIT is baked into the image (-ldflags in the Dockerfile); -dirty
+# covers staged and untracked changes too, which `git diff --quiet` misses.
+GIT_COMMIT ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)$(shell test -z "$$(git status --porcelain 2>/dev/null)" || echo -dirty)
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -61,7 +64,11 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet setup-envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test -race $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+
+.PHONY: test-integration-kvm
+test-integration-kvm: ## Run the KVM provider test against a real hypervisor. Needs KVM_TEST_HOST/USER/SSH_KEY/BRIDGE/IMAGE_URL; see docs/testing.md.
+	go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/
 
 # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
 # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
@@ -87,7 +94,8 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist
 
 .PHONY: test-e2e
 test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
-	KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v
+	KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) TEMPO_URL=$(TEMPO_URL) OTLP_ENDPOINT=$(OTLP_ENDPOINT) \
+		go test -tags=e2e ./test/e2e/ -v -ginkgo.v -timeout 30m
 	$(MAKE) cleanup-test-e2e
 
 .PHONY: cleanup-test-e2e
@@ -110,18 +118,22 @@ lint-config: golangci-lint ## Verify golangci-lint linter configuration
 
 .PHONY: build
 build: manifests generate fmt vet ## Build manager binary.
-	go build -o bin/manager cmd/main.go
+	go build -o bin/manager ./cmd
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./cmd/main.go
 
+.PHONY: run-dev
+run-dev: manifests generate fmt vet ## Run locally against the current kubeconfig with the kubernetes-pod provider.
+	go run ./cmd/main.go --providers-config hack/providers-dev.yaml --metrics-bind-address :8080 --metrics-secure=false
+
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
 docker-build: ## Build docker image with the manager.
-	$(CONTAINER_TOOL) build -t ${IMG} .
+	$(CONTAINER_TOOL) build --build-arg GIT_COMMIT=$(GIT_COMMIT) -t ${IMG} .
 
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
@@ -140,7 +152,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform
 	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
 	- $(CONTAINER_TOOL) buildx create --name egress-proxies-operator-builder
 	$(CONTAINER_TOOL) buildx use egress-proxies-operator-builder
-	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
+	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --build-arg GIT_COMMIT=$(GIT_COMMIT) --tag ${IMG} -f Dockerfile.cross .
 	- $(CONTAINER_TOOL) buildx rm egress-proxies-operator-builder
 	rm Dockerfile.cross
 
@@ -166,6 +178,20 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 	@out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \
 	if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi
 
+KVM_SSH_KEY ?= $(HOME)/.ssh/proxy-operator_ed25519
+KVM_KNOWN_HOSTS ?= $(HOME)/.ssh/kvm_known_hosts
+KVM_SSH_NAMESPACE ?= egress-proxies-operator-system
+
+.PHONY: kvm-ssh-secret
+kvm-ssh-secret: ## Create or update the kvm-ssh Secret from KVM_SSH_KEY (private key) and KVM_KNOWN_HOSTS.
+	@test -f "$(KVM_SSH_KEY)" || { echo "KVM_SSH_KEY=$(KVM_SSH_KEY) not found"; exit 1; }
+	@test -f "$(KVM_KNOWN_HOSTS)" || { echo "KVM_KNOWN_HOSTS=$(KVM_KNOWN_HOSTS) not found"; exit 1; }
+	@grep -q 'PRIVATE KEY' "$(KVM_SSH_KEY)" || { echo "KVM_SSH_KEY=$(KVM_SSH_KEY) is not a private key (did you pass the .pub?)"; exit 1; }
+	"$(KUBECTL)" -n $(KVM_SSH_NAMESPACE) create secret generic kvm-ssh \
+		--from-file=id_ed25519="$(KVM_SSH_KEY)" \
+		--from-file=known_hosts="$(KVM_KNOWN_HOSTS)" \
+		--dry-run=client -o yaml | "$(KUBECTL)" apply -f -
+
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG}

PROJECT

@@ -18,4 +18,11 @@ resources:
   kind: Proxy
   path: gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+  domain: example.com
+  group: crawl
+  kind: ProviderHealth
+  path: gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1
+  version: v1alpha1
 version: "3"

README.md

@@ -1,135 +1,308 @@
 # egress-proxies-operator
-// TODO(user): Add simple overview of use/purpose
 
-## Description
-// TODO(user): An in-depth paragraph about your project and overview of use
+A Kubernetes operator that manages a fleet of HTTP egress proxies for
+crawling: each proxy is a `Proxy` custom resource that the operator
+provisions (or merely tracks), actively health-checks **through the proxy
+itself**, and hands out to crawler clients via an HTTP list/lease API.
 
-## Getting Started
+## Architecture in 60 seconds
 
-### Prerequisites
-- go version v1.24.6+
-- docker version 17.03+.
-- kubectl version v1.11.3+.
-- Access to a Kubernetes v1.11.3+ cluster.
+- **`Proxy` CRD** (`crawl.example.com/v1alpha1`, namespaced, `kubectl get px`):
+  `Managed` proxies are provisioned by a configured provider; `External`
+  proxies exist elsewhere and are only tracked and health-checked.
+- **Reconciler** — a crash-safe state machine: every reconcile derives one
+  action from (spec, status, provider Get). Proxies are **immutable
+  cattle**: any meaningful spec change (placement, cloud-init, port)
+  deletes and recreates the VM — never in-place mutation.
+- **Providers** behind one minimal interface: `kubernetes` (a real Squid
+  pod in this cluster — local dev/CI), `gcp` (Compute Engine VMs with
+  ephemeral external IPs — the real egress fleet) and `kvm` (VMs on your
+  own libvirt hypervisors, reached over SSH). Config is a YAML file
+  (`--providers-config`) with named instances (`gcp-eu`, `kvm-home`, ...).
+- **Health engine** probes every proxy by fetching a URL *through* it (a
+  real CONNECT tunnel — a proxy that accepts TCP but can't egress goes
+  Unhealthy), with threshold logic and transition-only status writes.
+- **Discovery API** (`:8090`): list healthy proxies filtered by
+  attributes, lease one (least-loaded, TTL-based), release, and report
+  rate-limiting — reports put the proxy in a per-target cooldown.
+- **Orphan GC** sweeps each provider for tagged instances whose owning CR
+  is gone — the safety net for crashes mid-create.
 
-### To Deploy on the cluster
-**Build and push your image to the location specified by `IMG`:**
+Details, diagrams, and recorded design decisions: [docs/architecture.md](docs/architecture.md).
+
+## Quickstart on kind (~5 minutes)
+
+Requires: kind, kubectl, docker, Go 1.26, jq (optional). The
+kubernetes-pod provider needs no cloud account — proxies are real
+`ubuntu/squid` pods in the kind cluster itself.
+
+The operator runs **in-cluster** for this quickstart. (Running it on your
+laptop with `make run-dev` provisions pods fine, but the health probe then
+originates on your machine, which cannot reach kind's pod IPs — the proxy
+would sit at `Unhealthy` forever. In-cluster, probes run where the pod
+network is routable.)
 
 ```sh
-make docker-build docker-push IMG=<some-registry>/egress-proxies-operator:tag
+kind create cluster --name proxy-operator-demo
+make install                                          # install the CRD
+make docker-build IMG=egress-proxies-operator:dev
+kind load docker-image egress-proxies-operator:dev --name proxy-operator-demo
+make deploy IMG=egress-proxies-operator:dev
+kubectl -n egress-proxies-operator-system rollout status deploy/egress-proxies-operator-controller-manager
 ```
 
-**NOTE:** This image ought to be published in the personal registry you specified.
-And it is required to have access to pull the image from the working environment.
-Make sure you have the proper permission to the registry if the above commands don’t work.
-
-**Install the CRDs into the cluster:**
+Create a proxy and watch it come up:
 
 ```sh
-make install
+kubectl apply -f config/samples/proxy_kubernetes.yaml
+kubectl get px -w
+# NAME                      MODE      PROVIDER     PHASE   IP           HEALTHY
+# proxy-kubernetes-sample   Managed   kubernetes   Ready   10.244.x.x   True
 ```
 
-**Deploy the Manager to the cluster with the image specified by `IMG`:**
+Once it's `Ready`, port-forward the discovery API and use it (full
+reference with schemas and error codes: [docs/api.md](docs/api.md)):
 
 ```sh
-make deploy IMG=<some-registry>/egress-proxies-operator:tag
+kubectl -n egress-proxies-operator-system port-forward \
+  svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090 &
 ```
 
-> **NOTE**: If you encounter RBAC errors, you may need to grant yourself cluster-admin
-privileges or be logged in as admin.
-
-**Create instances of your solution**
-You can apply the samples (examples) from the config/sample:
-
 ```sh
-kubectl apply -k config/samples/
+# List healthy proxies
+curl -s 'localhost:8090/v1/proxies?healthy=true' | jq
+
+# Lease one (5-minute TTL)
+curl -s -XPOST localhost:8090/v1/leases \
+  -d '{"selector":{"geo":"local"},"ttlSeconds":300}' | jq
+# → {"leaseID":"...", "proxy":{"id":"default/proxy-kubernetes-sample", "ip":..., ...}}
+
+# Actually crawl through it (from inside the cluster, or port-forward the pod)
+# curl -x http://<proxy-ip>:3128 https://example.com
+
+# Report the proxy got rate-limited by a site → 15-minute cooldown for that target
+curl -s -XPOST localhost:8090/v1/leases/<leaseID>/report \
+  -d '{"result":"rate_limited","target":"example.com"}'
+
+# Release early (idempotent — 204 both times)
+curl -si -XDELETE localhost:8090/v1/leases/<leaseID>
 ```
 
->**NOTE**: Ensure that the samples has default values to test it out.
-
-### To Uninstall
-**Delete the instances (CRs) from the cluster:**
+Tear down:
 
 ```sh
-kubectl delete -k config/samples/
+kubectl delete -f config/samples/proxy_kubernetes.yaml   # finalizer deletes the pod
+kind delete cluster --name proxy-operator-demo
 ```
 
-**Delete the APIs(CRDs) from the cluster:**
+## Deploying in-cluster
 
 ```sh
-make uninstall
+make docker-build IMG=<registry>/egress-proxies-operator:dev
+make deploy IMG=<registry>/egress-proxies-operator:dev
 ```
 
-**UnDeploy the controller from the cluster:**
+- Provider config comes from the `providers-config` ConfigMap
+  ([config/manager/providers_config.yaml](config/manager/providers_config.yaml));
+  the default ships the kubernetes provider plus a `kvm-home` entry that needs
+  the `kvm-ssh` Secret (without it the manager starts and warns).
+- The discovery API is exposed by the
+  `controller-manager-discovery-service` Service on port 8090.
+- Auth: create the token Secret, or the API serves **unauthenticated**
+  (it warns loudly at startup):
+
+  ```sh
+  kubectl -n egress-proxies-operator-system create secret generic discovery-token \
+    --from-literal=token="$(openssl rand -hex 24)"
+  ```
+
+## GCP setup
+
+1. Add a `gcp` entry to the providers config (see
+   [config/samples/providers-config.yaml](config/samples/providers-config.yaml)) —
+   only `project` is required.
+2. Credentials are **Application Default Credentials**: workload identity
+   in-cluster, `gcloud auth application-default login` locally. No
+   key-file plumbing exists.
+3. The identity needs `roles/compute.instanceAdmin.v1` on the project —
+   plus `roles/iam.serviceAccountUser` if instances attach a service
+   account.
+4. Managed GCP proxies must set all of `placement.zone`,
+   `placement.machineType`, and `placement.image`
+   (see [config/samples/proxy_gcp.yaml](config/samples/proxy_gcp.yaml),
+   which also installs Squid via cloud-init). A missing field fails the
+   Proxy with a message naming it.
+
+## KVM setup
+
+Managed proxies as VMs on libvirt/KVM hosts you own. Full guide:
+[docs/kvm-provider.md](docs/kvm-provider.md).
+
+1. Each host needs `libvirtd`, an SSH user in the `libvirt` group, a
+   bridge to a network with DHCP, and a storage pool.
+2. Create the `kvm-ssh` Secret from the operator's **private** key and a
+   `known_hosts` with each host's key; the manager mounts it at
+   `/etc/proxy-operator/kvm`:
+
+   ```sh
+   ssh-keyscan -H <host> >> ~/.ssh/kvm_known_hosts
+   kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
+     --from-file=id_ed25519=$HOME/.ssh/proxy-operator_ed25519 \
+     --from-file=known_hosts=$HOME/.ssh/kvm_known_hosts
+   # or: make kvm-ssh-secret   (same paths by default; KVM_SSH_KEY / KVM_KNOWN_HOSTS to override)
+   ```
+
+3. Add a `kvm` entry to the providers config with `images`,
+   `machineTypes` and `hosts` (see
+   [config/samples/providers-config.yaml](config/samples/providers-config.yaml)).
+   Base images are downloaded into each host's pool on first use.
+4. A Proxy names a host (`placement.zone`, optional with one host), a
+   size (`placement.machineType`) and an image (`placement.image`) — see
+   [config/samples/proxy_kvm.yaml](config/samples/proxy_kvm.yaml).
+
+Cloud-init from a Secret: the Secret **must** carry the label
+`crawl.example.com/cloud-init: "true"` — the operator's cache only holds
+labelled Secrets, so an unlabelled one is invisible (the Proxy reports
+`CloudInitError`). Rotating the Secret's content triggers VM replacement.
+
+Stuck deletion: a Proxy stays `Terminating` while the operator keeps failing
+to clean its instance up — credentials revoked, a provider or a KVM host
+dropped from the config. Annotate it with
+`crawl.example.com/skip-cleanup: "true"` to abandon the instance and release
+the object. Whatever is still running then has to be removed by hand.
+
+## Providers
+
+`Managed` proxies are provisioned by a provider — a small compute backend
+behind one minimal interface. Providers are configured in the
+`--providers-config` YAML file as **named instances**: `spec.provider` on a
+Proxy refers to an entry's `name`, not its `type`, so `gcp-eu` and `gcp-us`
+can be two differently-configured instances of the same `gcp` type (see
+[config/samples/providers-config.yaml](config/samples/providers-config.yaml)).
+
+### Implemented providers
+
+| Type | Creates | Per-instance config | Notes |
+| --- | --- | --- | --- |
+| `kubernetes` | A real Squid pod (`ubuntu/squid:6.6-24.04_edge` by default) in the same cluster the operator runs in | `image` (optional) | Needs no cloud account — local dev, CI, and the kind quickstart. Pods share the cluster's egress IP, so it exercises the full lifecycle but not distinct egress paths. |
+| `gcp` | A Compute Engine VM with an ephemeral external IP | `project` (required), `network`, `networkTag`, `diskSizeGb` | The real egress fleet. Deliberately uses only four API calls (Insert / Get / Delete / AggregatedList), all fire-and-forget: `Create` returns as soon as the operation is submitted and the reconciler discovers progress by polling `Get`. Auth is Application Default Credentials — workload identity in-cluster, `gcloud` ADC locally; no key-file plumbing. |
+| `kvm` | A libvirt VM on one of your own hypervisors, reached over SSH | `sshKeyFile`, `knownHostsFile` (unless `insecureIgnoreHostKey`), `machineTypes`, `hosts` (required); `images`, `diskSizeGb`, `vendorData`, `installGuestAgent` | Home-lab or on-prem egress. The base image is downloaded into each host's pool on first use; `Create` waits for the VM to start. See [docs/kvm-provider.md](docs/kvm-provider.md). |
+
+### Adding a provider
+
+The generic contract is [docs/provider-contract.md](docs/provider-contract.md),
+the questionnaire a new provider's design fills in is
+[docs/provider-spec-template.md](docs/provider-spec-template.md), and how the
+existing providers meet the contract is
+[docs/providers-existing.md](docs/providers-existing.md);
+[docs/providers.md](docs/providers.md) indexes them. Start there: several rules (a
+self-contained `providerID`, `Get` returning `ErrNotFound` only when the
+instance is really gone, `CreatedAt` read back for orphan GC) are enforced by
+callers, not by the interface.
+
+## Tracing
+
+OpenTelemetry tracing is built in but **off by default** — the operator
+only exports spans when pointed at an OTLP receiver:
+
+```bash
+# In config/manager/manager.yaml (commented-out block is already there):
+#   OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger-collector.observability:4318
+# Local dev: print spans to stdout instead
+OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml
+```
+
+Every reconcile, discovery API request, and GC sweep becomes a trace, with
+child spans for provider and Kubernetes API calls; log lines inside a traced
+operation carry matching `traceID`/`spanID` fields (point Grafana/Loki
+derived fields at `traceID`). Sampling and endpoints follow the standard
+`OTEL_*` env vars; `--trace-health-probes` additionally traces each health
+probe (high volume). Details in `docs/architecture.md` §10.
+
+## Caveats — read these two
+
+**Changing a proxy changes its IP.** Proxies are immutable cattle: editing
+`placement`, `cloudInit` (or rotating its Secret), or `port` deletes the
+VM and creates a replacement with the **same name but a new IP**. Clients
+discover the new address via the discovery API; anything that pinned the
+old IP breaks by design.
+
+**Operator restart drops all leases and cooldowns.** Lease state is
+in-memory (`replicas: 1` accordingly). Clients must tolerate a lease
+vanishing — requests through the proxy keep working; they just re-lease.
+The lease store sits behind an interface so a persistent backend can
+replace it without touching the API handlers.
+
+Smaller notes:
+
+- `status.lastHealthCheckTime` is the time of the last *status-affecting*
+  probe, not the most recent probe — status writes are transition-only by
+  design. True probe recency lives in the metrics
+  (`proxy_operator_healthcheck_*`).
+- The discovery API is served by every replica but is not leader-elected;
+  the operator ships with `replicas: 1` (see the lease caveat above).
+
+## Version pins
+
+Built and verified against the spec's pins with **no substitutions
+needed**: Go 1.26, kubebuilder v4.15.0, controller-runtime v0.24.1,
+k8s.io/* v0.36.3 (Kubernetes 1.36 API level), controller-tools v0.21.0,
+cloud.google.com/go/compute v1.65.0. envtest uses the 1.36.2 binary
+bundle (the latest 1.36 patch with published binaries — do not "fix" the
+Makefile's derived version to 1.36.3, which has none).
+
+## Gitea CI
+
+[.gitea/workflows/build.yaml](.gitea/workflows/build.yaml) builds the
+manager image and pushes it to this Gitea instance's container registry.
+It runs on **any tag push** or manually via **Run workflow** (with a `tag`
+input) — never on branch pushes. A lightweight `check` job (`go vet`,
+`go build`, `go test -short`) gates the build.
+
+Every build pushes two tags to
+`gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator`:
+
+- the human tag (the git tag, or the dispatch input), and
+- an immutable `sha-<12-char-commit>` tag — pin deployments to this one.
+
+`:latest` is additionally updated on real tag pushes only, so a manual
+dispatch of an old ref can never clobber it. The commit is baked into the
+binary (`internal/version.Commit`) via the `GIT_COMMIT` build arg.
+
+### Mandatory Gitea secrets
+
+Set under **Settings → Actions → Secrets** in this repo:
+
+| Secret           | Required by                   | What it is                               |
+| ---------------- | ----------------------------- | ---------------------------------------- |
+| `REGISTRY_TOKEN` | `build.yaml` (registry login) | Gitea PAT with the `write:package` scope |
+
+The token is paired with `${{ github.actor }}` as the username, so it
+must belong to the user triggering the workflow — same convention as the
+other projects on this instance.
+
+Without `REGISTRY_TOKEN` the `check` job still passes but the build job
+fails at the `docker login` step. No other secrets are needed — the
+workflow does not deploy anywhere.
+
+## Development
 
 ```sh
-make undeploy
+make test        # unit + envtest suites, with -race (sets up envtest binaries itself)
+go test -short ./...   # skip the envtest suite
+make run-dev     # run against the current kubeconfig context
 ```
 
-## Project Distribution
+`make run-dev` is for iterating on the operator itself: provisioning,
+replacement, the discovery API, and External proxies all work from your
+laptop. Health checks against in-cluster pods do **not** (see the
+quickstart note) — use the in-cluster deploy to see a kubernetes-provider
+proxy go `Ready`.
 
-Following the options to release and provide this solution to the users.
-
-### By providing a bundle with all YAML files
-
-1. Build the installer for the image built and published in the registry:
-
-```sh
-make build-installer IMG=<some-registry>/egress-proxies-operator:tag
-```
-
-**NOTE:** The makefile target mentioned above generates an 'install.yaml'
-file in the dist directory. This file contains all the resources built
-with Kustomize, which are necessary to install this project without its
-dependencies.
-
-2. Using the installer
-
-Users can just run 'kubectl apply -f <URL for YAML BUNDLE>' to install
-the project, i.e.:
-
-```sh
-kubectl apply -f https://raw.githubusercontent.com/<org>/egress-proxies-operator/<tag or branch>/dist/install.yaml
-```
-
-### By providing a Helm Chart
-
-1. Build the chart using the optional helm plugin
-
-```sh
-kubebuilder edit --plugins=helm/v2-alpha
-```
-
-2. See that a chart was generated under 'dist/chart', and users
-can obtain this solution from there.
-
-**NOTE:** If you change the project, you need to update the Helm Chart
-using the same command above to sync the latest changes. Furthermore,
-if you create webhooks, you need to use the above command with
-the '--force' flag and manually ensure that any custom configuration
-previously added to 'dist/chart/values.yaml' or 'dist/chart/manager/manager.yaml'
-is manually re-applied afterwards.
-
-## Contributing
-// TODO(user): Add detailed information on how you would like others to contribute to this project
-
-**NOTE:** Run `make help` for more information on all potential `make` targets
-
-More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html)
-
-## License
-
-Copyright 2026.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+The full test inventory — what each suite covers, the deliberate gaps,
+and the manual kind verification procedure — is in
+[docs/testing.md](docs/testing.md).
 
+Project layout, reconcile-loop diagrams, and the decision log are in
+[docs/architecture.md](docs/architecture.md); the build history is in
+[docs/plans-executions/](docs/plans-executions/).

api/v1alpha1/providerhealth_types.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ConditionReady summarizes a ProviderHealth's targets: True when every
+// target passed its last check.
+const ConditionReady = "Ready"
+
+// ProviderTarget is the latest check verdict for one thing a provider
+// depends on: a KVM hypervisor, the Kubernetes API server, a GCP project.
+type ProviderTarget struct {
+	// name identifies the target within its provider: the KVM host name,
+	// "apiserver", or the GCP project.
+	// +kubebuilder:validation:MinLength=1
+	// +required
+	Name string `json:"name"`
+
+	// address is where the check connected to.
+	// +optional
+	Address string `json:"address,omitempty"`
+
+	// ready is true when the last check succeeded.
+	// +required
+	Ready bool `json:"ready"`
+
+	// reason is a CamelCase verdict such as Reachable, Unreachable,
+	// Forbidden, SSHHandshakeFailed or PoolNotActive.
+	// +optional
+	Reason string `json:"reason,omitempty"`
+
+	// message is human-readable detail, usually the error of a failed check.
+	// +kubebuilder:validation:MaxLength=1024
+	// +optional
+	Message string `json:"message,omitempty"`
+
+	// latencyMillis is how long the last check took.
+	// +optional
+	LatencyMillis int64 `json:"latencyMillis,omitempty"`
+
+	// lastTransitionTime is when ready last changed.
+	// +required
+	LastTransitionTime metav1.Time `json:"lastTransitionTime"`
+
+	// details are provider-specific facts from the last check, e.g. the
+	// libvirt version and free storage-pool space of a KVM host.
+	// +optional
+	Details map[string]string `json:"details,omitempty"`
+}
+
+// ProviderHealthStatus is the result of the latest check of one provider.
+type ProviderHealthStatus struct {
+	// type is the provider's type from the providers config: kubernetes,
+	// gcp or kvm.
+	// +optional
+	Type string `json:"type,omitempty"`
+
+	// conditions hold Ready, which summarizes the targets.
+	// +listType=map
+	// +listMapKey=type
+	// +optional
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+	// targets hold one verdict per checked target.
+	// +listType=map
+	// +listMapKey=name
+	// +optional
+	Targets []ProviderTarget `json:"targets,omitempty"`
+
+	// targetsReady counts the ready targets as "ready/total".
+	// +optional
+	TargetsReady string `json:"targetsReady,omitempty"`
+
+	// lastCheckTime is when the latest check finished.
+	// +optional
+	LastCheckTime *metav1.Time `json:"lastCheckTime,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster,shortName=ph
+// +kubebuilder:printcolumn:name="Type",type=string,JSONPath=`.status.type`
+// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`
+// +kubebuilder:printcolumn:name="Targets",type=string,JSONPath=`.status.targetsReady`
+// +kubebuilder:printcolumn:name="Last Check",type=date,JSONPath=`.status.lastCheckTime`
+// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
+
+// ProviderHealth reports the periodic health check of one configured
+// provider. It is named after the provider, and the operator creates,
+// updates and deletes it itself; there is no spec to edit.
+type ProviderHealth struct {
+	metav1.TypeMeta `json:",inline"`
+
+	// metadata is a standard object metadata
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitzero"`
+
+	// status is the result of the latest check
+	// +optional
+	Status ProviderHealthStatus `json:"status,omitzero"`
+}
+
+// +kubebuilder:object:root=true
+
+// ProviderHealthList contains a list of ProviderHealth
+type ProviderHealthList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitzero"`
+	Items           []ProviderHealth `json:"items"`
+}

api/v1alpha1/proxy_types.go

@@ -61,6 +61,21 @@ const (
 	// provision. A mismatch against the freshly computed hash means the VM
 	// must be replaced.
 	AnnotationSpecHash = "crawl.example.com/spec-hash"
+
+	// AnnotationSkipCleanup abandons the provider-side instance: set it to
+	// "true" and deletion removes the finalizer without asking the provider
+	// to clean anything up. It is the escape hatch for a cleanup that cannot
+	// succeed — a provider dropped from the config, credentials revoked, a
+	// backend that answers permanently — where the object would otherwise
+	// stay in Terminating forever. The instance is left running and becomes
+	// the administrator's to remove.
+	AnnotationSkipCleanup = "crawl.example.com/skip-cleanup"
+
+	// LabelCloudInit must be set (to "true") on every Secret referenced by
+	// spec.cloudInit.secretRef: the manager's cache only holds Secrets
+	// carrying this label, so an unlabelled Secret is invisible to the
+	// operator — both to the resolve step and to the rotation watch.
+	LabelCloudInit = "crawl.example.com/cloud-init"
 )
 
 // Defaults, applied both by CRD structural defaulting (kubebuilder:default
@@ -346,7 +361,7 @@ type ProxyList struct {
 
 func init() {
 	SchemeBuilder.Register(func(s *runtime.Scheme) error {
-		s.AddKnownTypes(SchemeGroupVersion, &Proxy{}, &ProxyList{})
+		s.AddKnownTypes(SchemeGroupVersion, &Proxy{}, &ProxyList{}, &ProviderHealth{}, &ProviderHealthList{})
 		return nil
 	})
 }

api/v1alpha1/zz_generated.deepcopy.go

@@ -95,6 +95,120 @@ func (in *PlacementSpec) DeepCopy() *PlacementSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderHealth) DeepCopyInto(out *ProviderHealth) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderHealth.
+func (in *ProviderHealth) DeepCopy() *ProviderHealth {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderHealth)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProviderHealth) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderHealthList) DeepCopyInto(out *ProviderHealthList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]ProviderHealth, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderHealthList.
+func (in *ProviderHealthList) DeepCopy() *ProviderHealthList {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderHealthList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProviderHealthList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderHealthStatus) DeepCopyInto(out *ProviderHealthStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]v1.Condition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Targets != nil {
+		in, out := &in.Targets, &out.Targets
+		*out = make([]ProviderTarget, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.LastCheckTime != nil {
+		in, out := &in.LastCheckTime, &out.LastCheckTime
+		*out = (*in).DeepCopy()
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderHealthStatus.
+func (in *ProviderHealthStatus) DeepCopy() *ProviderHealthStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderHealthStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderTarget) DeepCopyInto(out *ProviderTarget) {
+	*out = *in
+	in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
+	if in.Details != nil {
+		in, out := &in.Details, &out.Details
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderTarget.
+func (in *ProviderTarget) DeepCopy() *ProviderTarget {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderTarget)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Proxy) DeepCopyInto(out *Proxy) {
 	*out = *in

cmd/main.go

@@ -14,28 +14,55 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package main is the composition root: it loads the provider config (fail
+// fast), assembles the provider registry, and wires the reconciler, health
+// engine, lease store, discovery API, orphan GC, provider health checks and
+// metrics onto one controller-runtime manager.
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
+	"fmt"
 	"os"
+	goruntime "runtime"
+	"time"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/controller"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/discovery"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/gc"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/metrics"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kvm"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/providerhealth"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -60,6 +87,19 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var tlsOpts []func(*tls.Config)
+
+	var providersConfig string
+	var discoveryAddr string
+	var proxyNamespace string
+	var healthWorkers int
+	var gcInterval, gcMinAge time.Duration
+	var gcAllowNamespaced bool
+	var leaseCooldown, maxLeaseTTL time.Duration
+	var showVersion bool
+	var gcpWireFullPayloads bool
+	var traceHealthProbes bool
+	var providerCheckInterval, providerCheckTimeout time.Duration
+
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -74,13 +114,112 @@ func main() {
 	flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics server")
+
+	flag.StringVar(&providersConfig, "providers-config", "",
+		"Path to the providers config YAML. Required.")
+	flag.StringVar(&discoveryAddr, "discovery-addr", ":8090",
+		"Listen address of the discovery/lease HTTP API.")
+	flag.StringVar(&proxyNamespace, "proxy-namespace", "",
+		"Restrict the manager's cache to one namespace. Empty watches all namespaces. "+
+			"Restricting also disables orphan GC unless --gc-allow-namespaced is set.")
+	flag.IntVar(&healthWorkers, "health-workers", 8,
+		"Number of concurrent health-probe workers.")
+	flag.DurationVar(&gcInterval, "gc-interval", 10*time.Minute,
+		"Interval between orphan GC sweeps.")
+	flag.DurationVar(&gcMinAge, "gc-min-age", 10*time.Minute,
+		"Minimum instance age before orphan GC may delete it.")
+	flag.BoolVar(&gcAllowNamespaced, "gc-allow-namespaced", false,
+		"Allow orphan GC to run although the cache is namespace-restricted. Dangerous: proxies "+
+			"outside the namespace count as orphans and their instances get deleted.")
+	flag.DurationVar(&leaseCooldown, "lease-cooldown", 15*time.Minute,
+		"How long a reported proxy/target pair is excluded from lease selection.")
+	flag.DurationVar(&maxLeaseTTL, "max-lease-ttl", time.Hour,
+		"Maximum lease TTL a client may request.")
+	flag.BoolVar(&showVersion, "version", false,
+		"Print the commit the binary was built from and exit.")
+	flag.BoolVar(&gcpWireFullPayloads, "gcp-wire-log-full-payloads", false,
+		"Log GCP V(5) wire payloads verbatim instead of eliding fields larger than 1KiB.")
+	flag.BoolVar(&traceHealthProbes, "trace-health-probes", false,
+		"Emit one trace span per health probe. Off by default: probes run about "+
+			"once per second per proxy and would dominate trace volume.")
+	flag.DurationVar(&providerCheckInterval, "provider-check-interval", time.Minute,
+		"Interval between provider health checks (can the cluster reach each hypervisor, API "+
+			"and account), published as ProviderHealth objects and metrics. 0 turns them off; "+
+			"otherwise at least 10s.")
+	flag.DurationVar(&providerCheckTimeout, "provider-check-timeout", 0,
+		"Time limit for one round of provider health checks: 15s, or the interval when that is "+
+			"shorter. If set, at most the interval.")
+
 	opts := zap.Options{
 		Development: true,
 	}
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
+	if showVersion {
+		fmt.Println(version.Resolve())
+		os.Exit(0)
+	}
+
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+	setupLog.Info("Starting egress-proxies-operator",
+		"commit", version.Resolve(), "goVersion", goruntime.Version())
+
+	// Off (no-op spans, unchanged logs) unless OTEL_* env opts in; see
+	// internal/tracing. Shutdown is called explicitly after mgr.Start
+	// returns — the os.Exit paths below skip defers.
+	tracingShutdown, err := tracing.Setup(logf.IntoContext(context.Background(), setupLog),
+		"egress-proxies-operator", version.Resolve())
+	if err != nil {
+		setupLog.Error(err, "Failed to set up tracing")
+		os.Exit(1)
+	}
+
+	ctx := ctrl.SetupSignalHandler()
+
+	// Providers load first and fail fast: a manager that comes up without
+	// its backends would just convert every Proxy into an error loop.
+	if providersConfig == "" {
+		setupLog.Error(nil, "--providers-config is required")
+		os.Exit(1)
+	}
+	if err := providerhealth.ValidateSchedule(providerCheckInterval, providerCheckTimeout); err != nil {
+		setupLog.Error(err, "Invalid provider check flags")
+		os.Exit(1)
+	}
+	cfg, err := provider.LoadConfigFile(providersConfig)
+	if err != nil {
+		setupLog.Error(err, "Failed to load providers config", "path", providersConfig)
+		os.Exit(1)
+	}
+	providers, err := registry.Build(ctx, cfg, map[string]registry.Constructor{
+		"kubernetes": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+			// The kubernetes provider builds its own uncached client;
+			// wrap its transport so its API calls join the caller's trace.
+			return kubernetes.NewWithTransportWrapper(ctx, pc, tracing.RestConfigWrapper())
+		},
+		"gcp": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+			return gcp.NewWithWireOptions(ctx, pc, gcp.WireLogOptions{FullPayloads: gcpWireFullPayloads})
+		},
+		"kvm": kvm.New,
+	})
+	if err != nil {
+		setupLog.Error(err, "Failed to build providers")
+		os.Exit(1)
+	}
+	m := metrics.New()
+	// Checkers come from the unwrapped providers, in config order: the
+	// metrics and tracing wrappers implement only Provider, and a check is
+	// not a provider request to be counted anyway.
+	checkTargets := make([]providerhealth.Target, 0, len(cfg.Providers))
+	for _, pc := range cfg.Providers {
+		checker, _ := providers[pc.Name].(provider.Checker)
+		checkTargets = append(checkTargets, providerhealth.Target{Name: pc.Name, Type: pc.Type, Checker: checker})
+	}
+	for name, p := range providers {
+		// Tracing outermost: the span covers the metrics recording too.
+		providers[name] = provider.WithTracing(name, provider.WithMetrics(name, p, m))
+	}
 
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
@@ -128,32 +267,110 @@ func main() {
 		metricsServerOptions.KeyName = metricsCertKey
 	}
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	// The Secret cache is restricted to labelled cloud-init Secrets: the
+	// operator has cluster-wide Secret read RBAC, and without the label
+	// selector it would cache every Secret in scope.
+	cacheOpts := cache.Options{
+		ByObject: map[client.Object]cache.ByObject{
+			&corev1.Secret{}: {
+				Label: labels.SelectorFromSet(labels.Set{crawlv1alpha1.LabelCloudInit: "true"}),
+			},
+		},
+	}
+	if proxyNamespace != "" {
+		cacheOpts.DefaultNamespaces = map[string]cache.Config{proxyNamespace: {}}
+	}
+
+	restCfg := ctrl.GetConfigOrDie()
+	restCfg.Wrap(tracing.RestConfigWrapper())
+	mgr, err := ctrl.NewManager(restCfg, ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
+		Cache:                  cacheOpts,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "b47711d1.example.com",
-		// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
-		// when the Manager ends. This requires the binary to immediately end when the
-		// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
-		// speeds up voluntary leader transitions as the new leader don't have to wait
-		// LeaseDuration time first.
-		//
-		// In the default scaffold provided, the program ends immediately after
-		// the manager stops, so would be fine to enable this option. However,
-		// if you are doing or is intended to do any operation such as perform cleanups
-		// after the manager stops then its usage might be unsafe.
-		// LeaderElectionReleaseOnCancel: true,
 	})
 	if err != nil {
 		setupLog.Error(err, "Failed to start manager")
 		os.Exit(1)
 	}
 
+	store := lease.NewStore(leaseCooldown)
+	if err := mgr.Add(store); err != nil {
+		setupLog.Error(err, "Failed to add lease store")
+		os.Exit(1)
+	}
+
+	engine := health.NewEngine(mgr.GetClient())
+	engine.Workers = healthWorkers
+	engine.Metrics = m
+	engine.TraceProbes = traceHealthProbes
+	if err := mgr.Add(engine); err != nil {
+		setupLog.Error(err, "Failed to add health engine")
+		os.Exit(1)
+	}
+
+	if err := mgr.Add(&discovery.Server{
+		Reader:      mgr.GetClient(),
+		Store:       store,
+		Addr:        discoveryAddr,
+		Token:       os.Getenv("DISCOVERY_TOKEN"),
+		MaxLeaseTTL: maxLeaseTTL,
+		Metrics:     m,
+	}); err != nil {
+		setupLog.Error(err, "Failed to add discovery server")
+		os.Exit(1)
+	}
+
+	if err := mgr.Add(&gc.Sweeper{
+		Reader:              mgr.GetClient(),
+		Providers:           providers,
+		Interval:            gcInterval,
+		MinAge:              gcMinAge,
+		NamespaceRestricted: proxyNamespace != "",
+		AllowNamespaced:     gcAllowNamespaced,
+	}); err != nil {
+		setupLog.Error(err, "Failed to add orphan GC")
+		os.Exit(1)
+	}
+
+	monitor := &providerhealth.Monitor{
+		Client:   mgr.GetClient(),
+		Reader:   mgr.GetAPIReader(),
+		Targets:  checkTargets,
+		Interval: providerCheckInterval,
+		Timeout:  providerCheckTimeout,
+	}
+	if err := mgr.Add(monitor); err != nil {
+		setupLog.Error(err, "Failed to add provider health monitor")
+		os.Exit(1)
+	}
+
+	if err := m.Register(ctrlmetrics.Registry,
+		proxyPhaseCounts(mgr.GetClient()),
+		func() int {
+			total := 0
+			for _, n := range store.Counts() {
+				total += n
+			}
+			return total
+		},
+	); err != nil {
+		setupLog.Error(err, "Failed to register metrics")
+		os.Exit(1)
+	}
+	if err := metrics.RegisterProviderTargets(ctrlmetrics.Registry, providerTargetSamples(monitor)); err != nil {
+		setupLog.Error(err, "Failed to register provider check metrics")
+		os.Exit(1)
+	}
+
 	if err := (&controller.ProxyReconciler{
 		Client:       mgr.GetClient(),
 		Scheme:       mgr.GetScheme(),
+		Providers:    providers,
+		Health:       engine,
+		HealthEvents: engine.Events,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "Failed to create controller", "controller", "proxy")
 		os.Exit(1)
@@ -170,8 +387,62 @@ func main() {
 	}
 
 	setupLog.Info("Starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
-		setupLog.Error(err, "Failed to run manager")
+	startErr := mgr.Start(ctx)
+
+	// Flush pending spans on the way out, error path included. Fresh
+	// context: the signal ctx is already cancelled by the time Start
+	// returns.
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	if err := tracingShutdown(shutdownCtx); err != nil {
+		setupLog.Error(err, "Failed to flush traces on shutdown")
+	}
+	cancel()
+
+	if startErr != nil {
+		setupLog.Error(startErr, "Failed to run manager")
 		os.Exit(1)
 	}
 }
+
+// proxyPhaseCounts reads phase counts from the cache at scrape time. Before
+// the cache has synced (or on any list error) it reports nothing rather
+// than something wrong.
+func proxyPhaseCounts(c client.Reader) func() map[string]int {
+	return func() map[string]int {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		var list crawlv1alpha1.ProxyList
+		if err := c.List(ctx, &list); err != nil {
+			return nil
+		}
+		counts := map[string]int{}
+		for i := range list.Items {
+			phase := string(list.Items[i].Status.Phase)
+			if phase == "" {
+				phase = string(crawlv1alpha1.PhasePending)
+			}
+			counts[phase]++
+		}
+		return counts
+	}
+}
+
+// providerTargetSamples flattens the monitor's latest results for the
+// scrape-time provider-check gauges.
+func providerTargetSamples(mon *providerhealth.Monitor) func() []metrics.ProviderTarget {
+	return func() []metrics.ProviderTarget {
+		var out []metrics.ProviderTarget
+		for _, r := range mon.Snapshot() {
+			for _, ts := range r.Targets {
+				out = append(out, metrics.ProviderTarget{
+					Provider:  r.Provider,
+					Target:    ts.Name,
+					Up:        ts.Ready,
+					Duration:  ts.Latency,
+					CheckedAt: r.CheckedAt,
+				})
+			}
+		}
+		return out
+	}
+}

config/crd/bases/crawl.example.com_providerhealths.yaml

@@ -0,0 +1,193 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.21.0
+  name: providerhealths.crawl.example.com
+spec:
+  group: crawl.example.com
+  names:
+    kind: ProviderHealth
+    listKind: ProviderHealthList
+    plural: providerhealths
+    shortNames:
+    - ph
+    singular: providerhealth
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .status.type
+      name: Type
+      type: string
+    - jsonPath: .status.conditions[?(@.type=="Ready")].status
+      name: Ready
+      type: string
+    - jsonPath: .status.targetsReady
+      name: Targets
+      type: string
+    - jsonPath: .status.lastCheckTime
+      name: Last Check
+      type: date
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: |-
+          ProviderHealth reports the periodic health check of one configured
+          provider. It is named after the provider, and the operator creates,
+          updates and deletes it itself; there is no spec to edit.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          status:
+            description: status is the result of the latest check
+            properties:
+              conditions:
+                description: conditions hold Ready, which summarizes the targets.
+                items:
+                  description: Condition contains details for one aspect of the current
+                    state of this API Resource.
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - type
+                x-kubernetes-list-type: map
+              lastCheckTime:
+                description: lastCheckTime is when the latest check finished.
+                format: date-time
+                type: string
+              targets:
+                description: targets hold one verdict per checked target.
+                items:
+                  description: |-
+                    ProviderTarget is the latest check verdict for one thing a provider
+                    depends on: a KVM hypervisor, the Kubernetes API server, a GCP project.
+                  properties:
+                    address:
+                      description: address is where the check connected to.
+                      type: string
+                    details:
+                      additionalProperties:
+                        type: string
+                      description: |-
+                        details are provider-specific facts from the last check, e.g. the
+                        libvirt version and free storage-pool space of a KVM host.
+                      type: object
+                    lastTransitionTime:
+                      description: lastTransitionTime is when ready last changed.
+                      format: date-time
+                      type: string
+                    latencyMillis:
+                      description: latencyMillis is how long the last check took.
+                      format: int64
+                      type: integer
+                    message:
+                      description: message is human-readable detail, usually the error
+                        of a failed check.
+                      maxLength: 1024
+                      type: string
+                    name:
+                      description: |-
+                        name identifies the target within its provider: the KVM host name,
+                        "apiserver", or the GCP project.
+                      minLength: 1
+                      type: string
+                    ready:
+                      description: ready is true when the last check succeeded.
+                      type: boolean
+                    reason:
+                      description: |-
+                        reason is a CamelCase verdict such as Reachable, Unreachable,
+                        Forbidden, SSHHandshakeFailed or PoolNotActive.
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - name
+                  - ready
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
+              targetsReady:
+                description: targetsReady counts the ready targets as "ready/total".
+                type: string
+              type:
+                description: |-
+                  type is the provider's type from the providers config: kubernetes,
+                  gcp or kvm.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

config/crd/kustomization.yaml

@@ -3,4 +3,5 @@
 # It should be run by config/default
 resources:
 - bases/crawl.example.com_proxies.yaml
+- bases/crawl.example.com_providerhealths.yaml
 # +kubebuilder:scaffold:crdkustomizeresource

config/default/discovery_service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: controller-manager
+    app.kubernetes.io/name: egress-proxies-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: controller-manager-discovery-service
+  namespace: system
+spec:
+  ports:
+  - name: discovery
+    port: 8090
+    protocol: TCP
+    targetPort: discovery
+  selector:
+    control-plane: controller-manager
+    app.kubernetes.io/name: egress-proxies-operator

config/default/kustomization.yaml

@@ -22,6 +22,8 @@ resources:
 #- ../prometheus
 # [METRICS] Expose the controller manager metrics service.
 - metrics_service.yaml
+# Expose the discovery/lease HTTP API inside the cluster.
+- discovery_service.yaml
 
 # Uncomment the patches line if you enable Metrics
 patches:

config/manager/kustomization.yaml

@@ -1,8 +1,9 @@
 resources:
 - manager.yaml
+- providers_config.yaml
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: example.com/egress-proxies-operator
-  newTag: v0.0.1
+  newName: egress-proxies-operator
+  newTag: kvm-dev

config/manager/manager.yaml

@@ -63,12 +63,61 @@ spec:
         args:
           - --leader-elect
           - --health-probe-bind-address=:8081
+          - --providers-config=/etc/proxy-operator/providers.yaml
+          # Add --trace-health-probes to emit one span per health probe
+          # (high volume: ~1/s per proxy).
+          # Provider health checks (kubectl get providerhealth) run every
+          # minute; tune with --provider-check-interval (0 turns them off,
+          # otherwise at least 10s) and --provider-check-timeout (15s, or
+          # the interval if that is shorter).
+        env:
+          # Bearer token for the discovery API. Optional: without the
+          # Secret the API serves unauthenticated (with a loud warning).
+          # Create it with:
+          #   kubectl -n egress-proxies-operator-system create secret \
+          #     generic discovery-token --from-literal=token=<your-token>
+          - name: DISCOVERY_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: discovery-token
+                key: token
+                optional: true
+          # --- OpenTelemetry tracing ---
+          # Tracing is OFF until OTEL_EXPORTER_OTLP_ENDPOINT (or
+          # OTEL_TRACES_EXPORTER) is set; without it the operator runs
+          # exactly as before. The downward-API vars must stay listed
+          # before OTEL_RESOURCE_ATTRIBUTES: $(VAR) expansion only sees
+          # earlier-listed vars.
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: OTEL_SERVICE_NAME
+            value: egress-proxies-operator
+          - name: OTEL_RESOURCE_ATTRIBUTES
+            value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)
+          # Point at an OTLP receiver (Tempo, Jaeger, otel-collector) to
+          # enable tracing; http/protobuf (4318) is the default protocol,
+          # set OTEL_EXPORTER_OTLP_PROTOCOL=grpc for 4317.
+          # - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          #   value: http://jaeger-collector.observability:4318
+          # - name: OTEL_TRACES_SAMPLER
+          #   value: parentbased_traceidratio
+          # - name: OTEL_TRACES_SAMPLER_ARG
+          #   value: "0.1"
         image: controller:latest
         name: manager
         ports:
         - containerPort: 8081
           name: health
           protocol: TCP
+        - containerPort: 8090
+          name: discovery
+          protocol: TCP
         securityContext:
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
@@ -96,7 +145,34 @@ spec:
           requests:
             cpu: 10m
             memory: 64Mi
-        volumeMounts: []
-      volumes: []
+        volumeMounts:
+        - name: providers-config
+          mountPath: /etc/proxy-operator
+          readOnly: true
+        # SSH material for kvm providers (sshKeyFile / knownHostsFile in the
+        # providers config point here). Optional: the mount is empty when the
+        # Secret does not exist, and only kvm entries need it. Create it with:
+        #   kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
+        #     --from-file=id_ed25519=<private key> \
+        #     --from-file=known_hosts=<known_hosts with every host's key>
+        - name: kvm-ssh
+          mountPath: /etc/proxy-operator/kvm
+          readOnly: true
+        # Scratch space: the root filesystem is read-only, and the kvm
+        # provider builds each VM's cloud-init seed ISO through a temp
+        # directory (a few hundred KB, deleted right after upload).
+        - name: tmp
+          mountPath: /tmp
+      volumes:
+      - name: providers-config
+        configMap:
+          name: providers-config
+      - name: kvm-ssh
+        secret:
+          secretName: kvm-ssh
+          optional: true
+      - name: tmp
+        emptyDir:
+          sizeLimit: 64Mi
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10

config/manager/providers_config.yaml

@@ -0,0 +1,54 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: providers-config
+  namespace: system
+  labels:
+    app.kubernetes.io/name: egress-proxies-operator
+    app.kubernetes.io/managed-by: kustomize
+data:
+  # Mounted at /etc/proxy-operator/providers.yaml (--providers-config).
+  # The default ships only the kubernetes-pod provider so the operator runs
+  # out of the box; add gcp entries (type: gcp, gcp.project: ...) or kvm
+  # entries (type: kvm, hosts reached over SSH) for real egress fleets —
+  # see config/samples/providers-config.yaml.
+  providers.yaml: |
+    providers:
+      - name: kubernetes
+        type: kubernetes
+      - name: kvm-home
+        type: kvm
+        kvm:
+          # SSH to each host with the libvirt unix socket forwarded through the
+          # connection; nothing to open on the hypervisors beyond sshd. Both
+          # files come from the optional kvm-ssh Secret mounted by
+          # config/manager/manager.yaml — see docs/kvm-provider.md.
+          sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+          knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+          # insecureIgnoreHostKey: true   # skip host-key checks (not recommended)
+          # diskSizeGb: 10                # per-VM overlay; cloud-init grows the root fs into it
+          # vendorData: |                 # extra vendor-data; the qemu-guest-agent bootstrap is merged
+          #   #cloud-config               # into each Proxy's cloud-config, not configured here
+          #   ...
+          images:
+            # placement.image names a key here (or is an https:// URL directly).
+            # Fetched into each host's pool on first use and cached there as
+            # base-<key>.qcow2; do not change an entry while Proxies use it.
+            debian-13:
+              url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+              # sha256: "<64 hex>"        # optional; verified after download
+              # firmware: uefi            # default; "bios" only for images with no EFI partition
+              #                           # (Debian 13 hangs in GRUB under BIOS headless: Debian bug #1111240)
+            ubuntu-24.04:
+              url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+          machineTypes:                   # placement.machineType names a key here
+            small:  {vcpus: 1, memoryMiB: 1024}
+            medium: {vcpus: 2, memoryMiB: 2048}
+          hosts:                          # placement.zone names a host; optional with one host
+            - name: beelink
+              address: beelink.tail8c69c.ts.net:22
+              user: novakj                  # must be in the libvirt group
+              bridge: br0                 # to a LAN with DHCP; the VM's IP comes from there
+              # socket: /var/run/libvirt/libvirt-sock
+              # pool: default
+

config/rbac/kustomization.yaml

@@ -25,4 +25,7 @@ resources:
 - proxy_admin_role.yaml
 - proxy_editor_role.yaml
 - proxy_viewer_role.yaml
+# ProviderHealth objects are written only by the operator, so they get a
+# viewer role and nothing else.
+- providerhealth_viewer_role.yaml
 

config/rbac/providerhealth_viewer_role.yaml

@@ -0,0 +1,29 @@
+# This rule is not used by the project egress-proxies-operator itself.
+# It is provided to allow the cluster admin to help manage permissions for users.
+#
+# Grants read-only access to ProviderHealth objects, the results of the
+# operator's periodic provider checks. The operator creates, updates and
+# deletes them itself, so no admin or editor role is provided.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: egress-proxies-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: providerhealth-viewer-role
+rules:
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths/status
+  verbs:
+  - get

config/rbac/role.yaml

@@ -4,6 +4,16 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -12,6 +22,22 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths/status
+  verbs:
+  - get
+  - patch
 - apiGroups:
   - crawl.example.com
   resources:

config/samples/crawl_v1alpha1_proxy.yaml

@@ -1,9 +0,0 @@
-apiVersion: crawl.example.com/v1alpha1
-kind: Proxy
-metadata:
-  labels:
-    app.kubernetes.io/name: egress-proxies-operator
-    app.kubernetes.io/managed-by: kustomize
-  name: proxy-sample
-spec:
-  # TODO(user): Add fields here

config/samples/kustomization.yaml

@@ -1,4 +1,9 @@
 ## Append samples of your project ##
+# providers-config.yaml is deliberately absent: it is a sample
+# --providers-config file, not a Kubernetes manifest.
 resources:
-- crawl_v1alpha1_proxy.yaml
+- proxy_kubernetes.yaml
+- proxy_gcp.yaml
+- proxy_kvm.yaml
+- proxy_external.yaml
 # +kubebuilder:scaffold:manifestskustomizesamples

config/samples/providers-config.yaml

@@ -0,0 +1,57 @@
+# Sample --providers-config file (not a Kubernetes manifest). In-cluster
+# this content lives in the providers-config ConfigMap
+# (config/manager/providers_config.yaml); for `make run-dev` a
+# kubernetes-only variant is at hack/providers-dev.yaml.
+#
+# Named provider instances: "gcp-eu" and "gcp-us" are two configs of the
+# same type. spec.provider on a Proxy refers to the name, not the type.
+providers:
+  - name: kubernetes
+    type: kubernetes
+    # kubernetes:
+    #   image: ubuntu/squid:6.6-24.04_edge   # the default
+  - name: gcp-eu
+    type: gcp
+    gcp:
+      project: my-project
+      # network: default            # VPC network name
+      # networkTag: proxy-operator  # firewall tag on created instances
+      # diskSizeGb: 10
+      # auth: Application Default Credentials (workload identity
+      # in-cluster, gcloud ADC locally). No key-file plumbing.
+  - name: kvm-home
+    type: kvm
+    kvm:
+      # SSH to each host with the libvirt unix socket forwarded through the
+      # connection; nothing to open on the hypervisors beyond sshd. Both
+      # files come from the optional kvm-ssh Secret mounted by
+      # config/manager/manager.yaml — see docs/kvm-provider.md.
+      sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+      knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+      # insecureIgnoreHostKey: true   # skip host-key checks (not recommended)
+      # diskSizeGb: 10                # per-VM overlay; cloud-init grows the root fs into it
+      # installGuestAgent: false      # skip the vendor-data script that installs qemu-guest-agent
+      # vendorData: |                 # extra vendor-data, carried beside the agent script as a
+      #   #cloud-config               # MIME multipart. The Proxy's own cloud-init is never rewritten.
+      #   ...
+      images:
+        # placement.image names a key here (or is an https:// URL directly).
+        # Fetched into each host's pool on first use and cached there as
+        # base-<key>.qcow2; do not change an entry while Proxies use it.
+        debian-13:
+          url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+          # sha256: "<64 hex>"        # optional; verified after download
+          # firmware: uefi            # default; "bios" only for images with no EFI partition
+          #                           # (Debian 13 hangs in GRUB under BIOS headless: Debian bug #1111240)
+        ubuntu-24.04:
+          url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+      machineTypes:                   # placement.machineType names a key here
+        small:  {vcpus: 1, memoryMiB: 1024}
+        medium: {vcpus: 2, memoryMiB: 2048}
+      hosts:                          # placement.zone names a host; optional with one host
+        - name: beelink
+          address: beelink:22
+          user: novakj                  # must be in the libvirt group
+          bridge: br0                 # to a LAN with DHCP; the VM's IP comes from there
+          # socket: /var/run/libvirt/libvirt-sock
+          # pool: default

config/samples/proxy_external.yaml

@@ -0,0 +1,15 @@
+# An External proxy: the VM exists outside the operator's control; the
+# operator only tracks and health-checks it through the endpoint. No
+# finalizer, no provider calls, and deleting the CR touches nothing.
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-external-sample
+spec:
+  mode: External
+  endpoint:
+    host: 203.0.113.7
+    port: 3128
+  attributes:
+    geo: eu
+    purpose: crawl

config/samples/proxy_gcp.yaml

@@ -0,0 +1,36 @@
+# A Managed proxy backed by GCP: the operator creates a VM with an
+# ephemeral external IP and installs Squid via cloud-init. Requires a
+# providers-config entry named "gcp-eu" (see providers-config.yaml) and
+# Application Default Credentials with compute.instanceAdmin.v1.
+#
+# All three placement fields are required for GCP; the operator sets the
+# Proxy to Failed with a message naming any missing one.
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-gcp-sample
+spec:
+  mode: Managed
+  provider: gcp-eu
+  placement:
+    zone: europe-west1-b
+    machineType: e2-micro
+    image: projects/debian-cloud/global/images/family/debian-12
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_port 3128
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+  attributes:
+    geo: eu
+    purpose: crawl

config/samples/proxy_kubernetes.yaml

@@ -0,0 +1,14 @@
+# A Managed proxy backed by the kubernetes-pod provider: the operator runs
+# a real Squid pod in this cluster. This is the local-dev/CI sample — pods
+# share the cluster's egress IP, so it exercises the full lifecycle but
+# does not provide a distinct egress path (use the gcp provider for that).
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-kubernetes-sample
+spec:
+  mode: Managed
+  provider: kubernetes
+  attributes:
+    geo: local
+    purpose: crawl

config/samples/proxy_kvm.yaml

@@ -0,0 +1,43 @@
+# A Managed proxy backed by a libvirt/KVM hypervisor on the home network:
+# the operator boots a stock Debian cloud image as a VM on the named host,
+# with a NIC on the host's bridge (IP from the LAN's DHCP), and installs
+# Squid via cloud-init. Requires a providers-config entry named "kvm-home"
+# (see providers-config.yaml) and the kvm-ssh Secret (docs/kvm-provider.md).
+#
+# placement.zone is the host name from kvm.hosts (optional when only one
+# host is configured); machineType and image are keys in the provider's
+# machineTypes and images maps. The first Proxy for an image on a host
+# waits for the image download; later ones reuse the cached copy.
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-kvm-sample
+spec:
+  mode: Managed
+  provider: kvm-home
+  placement:
+    zone: nas1
+    machineType: small
+    image: debian-13
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_port 3128
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+      # Optional: log in with `ssh debian@<ip>` (the Debian image's default
+      # user) to look around; `virsh console <name>` on the host works too.
+      # ssh_authorized_keys:
+      #   - ssh-ed25519 AAAA... you@laptop
+  attributes:
+    geo: home
+    purpose: crawl

docs/api.md

@@ -0,0 +1,323 @@
+# Discovery API reference
+
+The operator serves an HTTP API (the *discovery API*) that crawler clients
+use to find and lease egress proxies: list healthy proxies filtered by
+attributes, acquire a TTL-based lease on one, release it early, and report
+how a target site treated the proxy. It is implemented in
+[`internal/discovery`](../internal/discovery/) with lease state in
+[`internal/lease`](../internal/lease/); the only Kubernetes interaction is
+reading `Proxy` resources from the manager's cache.
+
+## Base URL
+
+The API listens on `:8090` (`--discovery-addr`) inside the manager pod and
+is exposed by a Service
+([config/default/discovery_service.yaml](../config/default/discovery_service.yaml)).
+
+In-cluster:
+
+```text
+http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090
+```
+
+From a workstation, port-forward:
+
+```sh
+kubectl -n egress-proxies-operator-system port-forward \
+  svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090 &
+```
+
+Set `BASE_URL` to wherever you reach the API; all examples below use it:
+
+```sh
+export BASE_URL=localhost:8090   # via the port-forward above
+# or, from inside the cluster:
+# export BASE_URL=http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090
+```
+
+## Authentication
+
+A single static bearer token, read from the `DISCOVERY_TOKEN` environment
+variable at startup. The shipped Deployment populates it from the
+`discovery-token` Secret (key `token`), which is **optional** — if the
+Secret is absent or the token is empty, the API serves **unauthenticated**
+(the manager logs a loud warning at startup). Create the Secret:
+
+```sh
+kubectl -n egress-proxies-operator-system create secret generic discovery-token \
+  --from-literal=token="$(openssl rand -hex 24)"
+```
+
+Send the token on every request:
+
+```sh
+export TOKEN=<the token>
+curl -s -H "Authorization: Bearer $TOKEN" "$BASE_URL/v1/proxies" | jq
+```
+
+A missing or wrong token gets `401 {"error":"unauthorized",...}`.
+`GET /healthz` is always exempt.
+
+The curl examples below omit the `-H "Authorization: Bearer $TOKEN"` flag
+for brevity — add it to every call when auth is enabled.
+
+## Conventions
+
+- Requests and responses are JSON. Errors share one envelope:
+
+  ```json
+  {"error": "<machine_code>", "message": "<human-readable text>"}
+  ```
+
+- Request bodies are capped at **64 KiB** (larger bodies fail the JSON
+  decode with `400 invalid_body`).
+- Proxies with a deletion timestamp (being finalized) are excluded from
+  every response and never offered for lease.
+
+## Configuration
+
+| Setting | Default | Meaning |
+|---|---|---|
+| `--discovery-addr` | `:8090` | Listen address of the API |
+| `--max-lease-ttl` | `1h` | Maximum `ttlSeconds` a client may request |
+| `--lease-cooldown` | `15m` | Cooldown window applied on `rate_limited`/`banned` reports |
+| `DISCOVERY_TOKEN` (env) | empty | Bearer token; empty disables auth |
+
+The shipped Deployment passes none of these flags, so the defaults apply.
+
+## Endpoints
+
+### `GET /healthz`
+
+Liveness check. Unauthenticated, always `200` with body `ok`.
+
+```sh
+curl -s "$BASE_URL/healthz"
+```
+
+### `GET /v1/proxies` — list proxies
+
+Query parameters (all optional):
+
+| Parameter | Values | Effect |
+|---|---|---|
+| `healthy` | `true` \| `false` | Keep only proxies whose `Healthy` condition matches. Any other value → `400 invalid_query`. |
+| `attr.<key>` | any string | Exact match on `spec.attributes[<key>]`. Repeatable; **all** given pairs must match. |
+
+List everything:
+
+```sh
+curl -s "$BASE_URL/v1/proxies" | jq
+```
+
+List healthy proxies in a given geo:
+
+```sh
+curl -s "$BASE_URL/v1/proxies?healthy=true&attr.geo=eu" | jq
+```
+
+Response — `200`, proxies sorted by `id`, an empty match is `200` with
+`"count": 0` (never `404`):
+
+```json
+{
+  "proxies": [
+    {
+      "id": "default/proxy-kubernetes-sample",
+      "ip": "10.244.1.7",
+      "port": 3128,
+      "attributes": {"geo": "local"},
+      "phase": "Ready",
+      "healthy": true,
+      "latencyMillis": 42,
+      "activeLeases": 1,
+      "maxLeases": 5
+    }
+  ],
+  "count": 1
+}
+```
+
+Proxy object fields (the same shape appears inside lease responses):
+
+| Field | Meaning |
+|---|---|
+| `id` | `namespace/name` of the `Proxy` resource; used as the stable key everywhere |
+| `ip` | Effective host — `spec.endpoint.host` for `External` proxies, `status.ip` for `Managed` (empty until the backing VM/pod is up) |
+| `port` | Effective port (default `3128`) |
+| `attributes` | `spec.attributes` — free-form selection labels (`geo`, `asn`, `purpose`, …); omitted when empty |
+| `phase` | `Pending` \| `Provisioning` \| `Ready` \| `Unhealthy` \| `Deleting` \| `Failed` |
+| `healthy` | `true` iff the `Healthy` condition is `True` (the through-the-proxy health probe passes) |
+| `latencyMillis` | Latency of the last status-affecting health probe |
+| `activeLeases` | Currently active leases on this proxy |
+| `maxLeases` | Lease capacity (default `5`; an explicit `0` means unleasable) |
+
+### `POST /v1/leases` — acquire a lease
+
+Picks a healthy proxy with free capacity matching the selector and grants
+an exclusive-slot, TTL-based lease on it.
+
+Request body (every field optional; `{}` is valid):
+
+```json
+{
+  "selector": {"geo": "eu"},
+  "ttlSeconds": 300,
+  "target": "example.com"
+}
+```
+
+| Field | Default | Meaning |
+|---|---|---|
+| `selector` | none | Attribute equality filter, same semantics as `attr.<key>` above |
+| `ttlSeconds` | `300` (5 min) | Lease lifetime; must be ≤ `--max-lease-ttl` (default 1 h), else `400 invalid_ttl` |
+| `target` | none | The site you intend to crawl; enables per-target cooldowns (see below) |
+
+```sh
+curl -s -XPOST "$BASE_URL/v1/leases" \
+  -d '{"selector":{"geo":"eu"},"ttlSeconds":300,"target":"example.com"}' | jq
+```
+
+Success — `201`:
+
+```json
+{
+  "leaseID": "P3X6HHQTPCM5UTGVGE3B5UPS3A",
+  "proxy": {
+    "id": "default/proxy-eu-1",
+    "ip": "34.88.10.20",
+    "port": 3128,
+    "attributes": {"geo": "eu"},
+    "phase": "Ready",
+    "healthy": true,
+    "latencyMillis": 42,
+    "activeLeases": 1,
+    "maxLeases": 5
+  },
+  "expiresAt": "2026-08-11T22:05:00Z",
+  "ttlSeconds": 300
+}
+```
+
+Use `proxy.ip` and `proxy.port` as an HTTP proxy for the lease's lifetime:
+
+```sh
+curl -x http://34.88.10.20:3128 https://example.com
+```
+
+No match — `409` with diagnostic counts explaining why nothing qualified:
+
+```json
+{
+  "error": "no_match",
+  "message": "no healthy proxy with free capacity matched the selector",
+  "considered": 3,
+  "atCapacity": 1,
+  "inCooldown": 1,
+  "unhealthy": 1
+}
+```
+
+| Count | Meaning |
+|---|---|
+| `considered` | Proxies that matched the selector (before health/capacity checks) |
+| `atCapacity` | Skipped because `activeLeases >= maxLeases` |
+| `inCooldown` | Skipped because of an active cooldown for this target (or a global one) |
+| `unhealthy` | Skipped because the `Healthy` condition is not `True` |
+
+Leases expire on their own — releasing is only needed to free the slot
+early. There is no renew/extend endpoint; acquire a new lease instead.
+
+### `DELETE /v1/leases/{id}` — release early
+
+Frees the lease's capacity slot immediately. Idempotent: always `204`,
+including for unknown or already-expired lease IDs.
+
+```sh
+curl -si -XDELETE "$BASE_URL/v1/leases/P3X6HHQTPCM5UTGVGE3B5UPS3A"
+```
+
+### `POST /v1/leases/{id}/report` — report an outcome
+
+Tell the operator how the target site treated the proxy. This is the
+feedback signal that drives cooldowns.
+
+Request body:
+
+```json
+{"result": "rate_limited", "target": "example.com"}
+```
+
+| Field | Values | Meaning |
+|---|---|---|
+| `result` | `ok` \| `rate_limited` \| `banned` | Anything else → `400 invalid_result` |
+| `target` | optional | Which site produced the result; falls back to the lease's `target`, then to global |
+
+```sh
+curl -si -XPOST "$BASE_URL/v1/leases/P3X6HHQTPCM5UTGVGE3B5UPS3A/report" \
+  -d '{"result":"rate_limited","target":"example.com"}'
+```
+
+Responses: `204` on success, `404 unknown_lease` if the lease ID was never
+issued or has aged out.
+
+Semantics:
+
+- `ok` is a pure acknowledgement — nothing is recorded.
+- `rate_limited` and `banned` currently behave **identically**: both put
+  the proxy in one cooldown window (default 15 min, `--lease-cooldown`)
+  for the resolved target.
+- An expired lease remains reportable for one cooldown window past its
+  TTL, so a late "we got rate-limited" still lands.
+
+## Proxy selection and cooldowns
+
+How `POST /v1/leases` picks among eligible proxies (healthy, matching the
+selector, not being deleted, not at capacity, not in cooldown), in order:
+
+1. fewest `activeLeases` (least-loaded),
+2. lowest `latencyMillis`,
+3. lexicographic `id` (deterministic tie-break).
+
+Cooldowns are keyed by **(proxy, target)**:
+
+- A report **with a target** blocks that proxy only for lease requests
+  naming the **same target**. Other targets — and requests with no
+  target — still get the proxy.
+- A report **without a target**, on a lease that also had no target,
+  creates a **global** cooldown: the proxy is blocked for *all* lease
+  requests until the window passes. Always pass `target` on leases and
+  reports unless you really mean "this proxy is bad for everyone".
+
+## End-to-end example
+
+```sh
+# 1. Acquire a lease for crawling example.com through an EU proxy
+LEASE=$(curl -s -XPOST "$BASE_URL/v1/leases" \
+  -H "Authorization: Bearer $TOKEN" \
+  -d '{"selector":{"geo":"eu"},"ttlSeconds":600,"target":"example.com"}')
+LEASE_ID=$(echo "$LEASE" | jq -r .leaseID)
+PROXY=$(echo "$LEASE" | jq -r '"\(.proxy.ip):\(.proxy.port)"')
+
+# 2. Crawl through the leased proxy
+curl -x "http://$PROXY" https://example.com/some/page
+
+# 3. Got a 429? Report it — example.com-bound leases will avoid this
+#    proxy for the next 15 minutes
+curl -s -XPOST "$BASE_URL/v1/leases/$LEASE_ID/report" \
+  -H "Authorization: Bearer $TOKEN" \
+  -d '{"result":"rate_limited","target":"example.com"}'
+
+# 4. Done early? Release the slot (otherwise the TTL frees it)
+curl -s -XDELETE "$BASE_URL/v1/leases/$LEASE_ID" \
+  -H "Authorization: Bearer $TOKEN"
+```
+
+## Caveats
+
+- **Lease and cooldown state is in-memory and per-process.** An operator
+  restart drops all active leases and cooldowns. Clients must tolerate a
+  granted lease disappearing (a subsequent report returns `404`).
+- **Run a single replica.** The API is served by every manager replica but
+  is not leader-elected, and lease state is not shared between replicas;
+  the shipped Deployment pins `replicas: 1`.

docs/architecture.md

@@ -1,10 +1,22 @@
 # Architecture
 
-> **Status:** the operator is built through Step 5 (health engine) of
-> [docs/plans/2026-08-07-1747-proxy-operator.md](plans/2026-08-07-1747-proxy-operator.md).
-> This document currently covers the event/reconcile flow; the components
-> table and the Decisions section arrive with Step 10, and the diagrams
-> below grow as the lease store, discovery API, and orphan GC land.
+## Components
+
+| Component | Package | Runs as | Leader-elected | Role |
+|---|---|---|---|---|
+| Proxy + ProviderHealth CRDs | `api/v1alpha1` | types | — | `Proxy` spec/status, CEL validation, defaulting, pure helpers; the status-only, cluster-scoped `ProviderHealth` |
+| Reconciler | `internal/controller` | controller | yes (with the manager) | the state machine: provision, replace, delete, represent health |
+| Provider contract | `internal/provider` | library | — | `Provider` interface, optional `Checker`, error taxonomy, deterministic naming, config, metrics decorator; rules in [provider-contract.md](provider-contract.md), per-provider profile table in [providers-existing.md](providers-existing.md) |
+| kubernetes provider | `internal/provider/kubernetes` | library | — | real Squid pods in this cluster (local dev/CI) |
+| gcp provider | `internal/provider/gcp` | library | — | Compute Engine VMs, four API calls, fire-and-forget ops |
+| kvm provider | `internal/provider/kvm` | library | — | VMs on libvirt hosts over SSH-forwarded sockets; image cache, overlay disks, NoCloud seeds |
+| Health engine | `internal/health` | Runnable | yes | through-the-proxy probes, thresholds, transition events |
+| Lease store | `internal/lease` | Runnable (expiry sweep) | no | in-memory leases + cooldowns, single mutex |
+| Discovery API | `internal/discovery` | Runnable | no | HTTP list/lease/release/report on `:8090` |
+| Orphan GC | `internal/gc` | Runnable | yes | deletes tagged instances whose CR is gone |
+| Provider health monitor | `internal/providerhealth` | Runnable | yes | periodic provider checks → one `ProviderHealth` per provider, target gauges, transition logs |
+| Metrics | `internal/metrics` | library | — | explicit registration, scrape-time collectors |
+| Composition root | `cmd/main.go` | binary | — | flags, provider registry, wires everything onto one manager |
 
 ## Event flow: cluster events → reconciler functions
 
@@ -117,20 +129,51 @@ reconcileDelete(ctx, p)                          reconcileExternal(ctx, p)
         ──► RequeueAfter: DeletionPoll (poll until gone)
 ```
 
-### 5. What provider calls do back in the cluster (kubernetes pod provider)
+### 5. What provider calls do in the outside world
 
 ```text
+kubernetes pod provider (internal/provider/kubernetes/)
 prov.Create ──► buildPod (pure) ──► client.Create(corev1.Pod)   ─┐ these cause Pod events,
 prov.Get    ──► client.Get(Pod) → phase/IP → InstanceState       │ but the operator does NOT
 prov.Delete ──► client.Delete(Pod, tolerate NotFound)            │ watch Pods — it observes
 prov.ListByTag ─► client.List(Pods by labels, all namespaces)   ─┘ them by polling prov.Get
                                                                     on each RequeueAfter tick
+
+gcp provider (internal/provider/gcp/) — instances.{Insert,Get,Delete,AggregatedList}, nothing else
+prov.Create ──► buildInsertRequest (pure) ──► instances.Insert   ─┐ fire-and-forget:
+                  409 alreadyExists = success (idempotent retry)  │ Operation.Wait is never
+prov.Get    ──► instances.Get → status/NatIP → InstanceState      │ called; readiness is
+                  RUNNING without NatIP = still Provisioning      │ discovered by Get polls,
+prov.Delete ──► instances.Delete (404 = success)                  │ exactly like the pod
+prov.ListByTag ─► AggregatedList(label filter,                   ─┘ provider
+                    ReturnPartialSuccess: true)
+providerID = zones/<zone>/instances/<name> — zone-qualified, so Get/Delete
+stay correct even mid-replacement after a zone edit
+
+kvm provider (internal/provider/kvm/) — libvirt RPC over an SSH-forwarded unix socket, per host
+prov.Create ──► DomainLookupByName: exists → start if needed, done      ─┐ every step tolerates
+                else: ensure base image in pool (download on first use) │ the previous attempt
+                      overlay volume (exists = reuse)                    │ having got that far;
+                      seed ISO (always rebuilt — a cut-off upload must   │ the seed is the one
+                        never be trusted)                                │ thing redone
+                      DomainDefineXML + DomainCreate                    ─┘
+prov.Get    ──► DomainGetState + DomainGetMetadata + DomainInterfaceAddresses
+                  (ARP, then guest agent; first routable IPv4)
+                  running without IP = Provisioning; nostate = Provisioning
+prov.Delete ──► destroy (not running = fine) → undefine → delete overlay + seed by name
+prov.ListByTag ─► ConnectListAllDomains on every host, keep domains whose
+                    <metadata> carries managed=true; one host down fails the sweep,
+                    one unreadable domain is skipped
+providerID = hosts/<host>/domains/<name>; GC identity lives in the domain's
+<metadata> element (libvirt has no labels). All RPCs run under a context-
+bounded call: go-libvirt never times out on its own, so a cancelled reconcile
+closes the SSH connection to unblock it.
 ```
 
-The reconciler never watches provider-side resources (Pods now, GCP VMs
-later). All instance-state observation is poll-based through the
-`Provider` interface, so the same flow works identically for a cloud API
-that has no watch mechanism at all.
+The reconciler never watches provider-side resources (Pods or GCP VMs).
+All instance-state observation is poll-based through the `Provider`
+interface, so the same flow works identically for a cloud API that has no
+watch mechanism at all.
 
 ### 6. Health engine (`internal/health/`) — probes and transitions
 
@@ -183,3 +226,390 @@ Consequence worth knowing: `status.lastHealthCheckTime` is the time of the
 last *status-affecting* probe, not the most recent probe — suppressed
 probes deliberately never write status. True probe recency will live in
 metrics (Step 9).
+
+### 7. Discovery + lease API (`internal/discovery/`, `internal/lease/`)
+
+HTTP-driven, not cluster-event-driven: crawler clients call in; the only
+Kubernetes interaction is reading Proxies from the manager's cache. The
+server is a non-leader-elected Runnable (all replicas would serve, but the
+deployment ships `replicas: 1` because lease state is per-process — an
+operator restart drops all leases and cooldowns, a documented caveat).
+Client-facing reference with request/response schemas and curl examples:
+[api.md](api.md).
+
+```text
+crawler client
+  │  Authorization: Bearer $DISCOVERY_TOKEN   (empty token = auth disabled, loud startup warning)
+  ▼
+Server.handler()      middleware, outermost first        (server.go)
+  recover → request-log → MaxBytesReader(64KiB) → bearer auth (constant-time; /healthz exempt)
+  │
+  ├─ GET  /healthz            ──► 200 ok (unauthenticated)
+  │
+  ├─ GET  /v1/proxies?attr.k=v&healthy=true            (handlers.go)
+  │     Reader.List(Proxies) ── manager cache
+  │     filter: attributes equality + Healthy condition
+  │     + Store.Counts() for activeLeases
+  │     ──► 200 {"proxies":[...], "count":N}   (empty list is 200, not 404)
+  │
+  ├─ POST /v1/leases  {"selector":{...},"ttlSeconds":300,"target":"..."}
+  │     Reader.List → filter selector; unhealthy matches counted, not offered
+  │     Store.Acquire(healthy candidates, target, ttl)   ── one lock: select+insert
+  │     │    selection: fewest active leases, then latency, then name
+  │     ├─ granted ──► 201 {leaseID, proxy:{...}, expiresAt, ttlSeconds}
+  │     └─ ErrNoMatch ──► 409 {"error":"no_match", considered, atCapacity,
+  │                            inCooldown, unhealthy}
+  │
+  ├─ DELETE /v1/leases/{id} ──► Store.Release ──► always 204 (idempotent)
+  │
+  └─ POST /v1/leases/{id}/report {"result":"ok|rate_limited|banned","target":"..."}
+        Store.Report ── rate_limited/banned ⇒ cooldown[{proxy,target}] for
+        │               CooldownWindow (target falls back: report → lease → global)
+        ├─ 204 │ 400 invalid_result │ 404 unknown_lease
+        └─ an expired lease still resolves for CooldownWindow past its TTL —
+           a late report lands exactly when the proxy is being rate-limited
+
+Store.Start(ctx)  ── manager Runnable, NOT leader-elected: sweeps expired
+                     leases + cooldowns; correctness never depends on the
+                     sweep (every read checks ExpiresAt against the clock)
+```
+
+### 8. Orphan GC (`internal/gc/`) — the crash-safety net
+
+Timer-driven, leader-elected (destructive ⇒ single writer). Exists for the
+one gap the reconciler cannot close alone: a crash after a provider Create
+but before the status write that records the instance.
+
+```text
+Sweeper.Start(ctx)   ── refuses to run when the cache is namespace-
+  │                     restricted unless --gc-allow-namespaced is explicit
+  │                     (an incomplete live set would "orphan" live VMs)
+  └─ every Interval (10m; first sweep a full interval after start):
+       sweep(ctx)
+         │ Reader.List(Proxies) → live UID set
+         │   List fails → skip the whole sweep (never guess)
+         │   a CR with deletionTimestamp still counts as LIVE — its
+         │   finalizer owns that deletion; GC racing it double-deletes
+         └ per provider: ListByTag
+             │ error → log, continue with the next provider
+             └ delete only when ALL hold:
+                 has the proxy-operator-uid label   (ownership proof)
+                 older than MinAge (10m)            (not mid-create)
+                 UID matches no existing CR         (truly orphaned)
+               each kill logged loudly with provider, providerID, UID
+```
+
+### 9. Metrics (`internal/metrics/`)
+
+Registered explicitly from `cmd/main.go` (no `init()`; tests use fresh
+registries). Two kinds:
+
+- **Scrape-time collectors** — `proxy_operator_proxies{phase}` and
+  `proxy_operator_leases_active` read the cache / lease store at every
+  scrape; reconcile-incremented gauges inevitably drift and leak series.
+  The provider-check gauges (§11) read the health monitor's snapshot the
+  same way.
+- **Fed vectors** — `healthcheck_duration_seconds{proxy}` and
+  `healthcheck_failures_total{proxy}` observe EVERY probe (status writes
+  are transition-only; metrics carry the high-frequency signal), and the
+  health engine deletes a proxy's series when it prunes its state;
+  `lease_requests_total{outcome}` from the discovery handlers;
+  `provider_requests_total{provider,op,result}` from the
+  `provider.WithMetrics` decorator — the one place `Class()` is called
+  purely for observability.
+
+Each consuming package defines its own small recorder interface
+(`health.ProbeMetrics`, `discovery.LeaseMetrics`, `provider.RequestRecorder`);
+`metrics.Metrics` satisfies all of them structurally, so no package other
+than `cmd/main.go` imports the metrics package.
+
+### 10. Tracing (`internal/tracing/`)
+
+OpenTelemetry tracing, integrated with — not replacing — the logr/zap
+logging. Everything hangs off standard `OTEL_*` env vars: with no
+`OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` set, no SDK is
+installed, spans are no-ops, and logs are byte-for-byte what they were.
+`OTEL_SDK_DISABLED=true` and `OTEL_TRACES_EXPORTER=none` force it off;
+`console` prints spans to stdout for local dev; sampling follows
+`OTEL_TRACES_SAMPLER(_ARG)`.
+
+Span topology — each unit of work is a new trace (watch events carry no
+incoming trace context):
+
+- `Reconcile Proxy` (root, via a `reconcile.Reconciler` decorator) →
+  `reconcile.managed` / `reconcile.replaceInstance` / `reconcile.delete`,
+  plus `status.patch` from the deferred flush. `reconcileExternal` does no
+  I/O and is unspanned.
+- `provider.create|get|delete|list` (client spans, `provider.WithTracing`
+  decorator wired outermost around `WithMetrics`); the GCP SDK's own
+  otelhttp transport contributes HTTP child spans automatically.
+- Kubernetes API calls become client child spans via a wrapped
+  `rest.Config` transport — gated on an existing parent span, so informer
+  list/watch long-polls and leader-election renewals never create root
+  spans. The kube-apiserver ignores incoming `traceparent` by design;
+  these spans are leaves.
+- Discovery API: one server span per request (named from the route
+  pattern), incoming W3C `traceparent` honored so clients' traces continue
+  into the operator; `/healthz` excluded.
+- `gc.sweep` per GC pass; `providerhealth.check` per round of provider
+  checks; `health.probe` per probe **only** with
+  `--trace-health-probes` (default off — probes run ~1/s per proxy).
+
+Log correlation: logr sinks never see a context, so trace IDs ride on the
+logger — `tracing.Start` re-derives the ctx logger from a captured base
+with `traceID`/`spanID` values (lowerCamel, matching `reconcileID`;
+Grafana/Loki derived-field regexes must match `traceID`, not `trace_id`).
+Re-deriving from the base rather than layering keeps zap from emitting
+duplicate keys on nested spans. GCP V(5) wire logs get the same keys from
+the slog handler's ctx.
+
+### 11. Provider health checks (`internal/providerhealth/`)
+
+Answers "can this cluster reach each provider's backend right now?"
+without waiting for a Proxy to fail. A provider opts in by implementing
+`provider.Checker`: `Check(ctx) []TargetStatus` returns one verdict per
+target and never fails as a whole. All three providers do:
+
+| Provider | Target | What one check does |
+|---|---|---|
+| kvm | each `kvm.hosts` entry | over the pooled SSH+libvirt connection: libvirt version and hostname, then the configured pool must exist and be running; reports pool capacity and free space ([details](kvm-provider.md#health-check)) |
+| kubernetes | `apiserver` | lists at most one managed Pod cluster-wide: the verb and scope `ListByTag` and GC rely on |
+| gcp | the project | one page of an aggregated instance list with `maxResults=1`: credentials, Compute API enabled, `compute.instances.list` |
+
+```text
+Monitor.Start(ctx)   ── leader-only; --provider-check-interval (1m; 0 = off,
+  │                     otherwise ≥ 10s), --provider-check-timeout (15s, or
+  │                     the interval if that is shorter)
+  ├─ once: delete managed-by-labelled ProviderHealth objects whose provider
+  │   is not configured (all of them when checks are off) — the providers
+  │   config is read once per process
+  └─ at start, then every Interval: checkOnce
+       │ every provider's Check concurrently, under one context.WithTimeout
+       │ snapshot → the scrape-time gauges
+       │ log transitions: down or reason changed → "WARNING: provider target
+       │   not ready"; back up → "Provider target ready"; steady → silent
+       └ per provider: APIReader.Get → Create if missing → status merge patch
+```
+
+- **One status write per provider per round**, since `lastCheckTime`
+  moves every round. At the 10s minimum interval that is still far below
+  leader-election renewals. Proxy status avoids writes like this only
+  because its probes run once a second per proxy.
+- **Reads go through the API reader**, not the cache. The objects are
+  cluster-scoped; a cached Get would start a cluster-wide informer (and
+  need `watch`) even with `--proxy-namespace`.
+- **Checks bypass the provider decorators.** `cmd/main.go` collects the
+  Checkers before wrapping the providers in `WithTracing(WithMetrics(...))`,
+  so a check never counts as a provider request. KVM RPCs still appear in
+  the V(1) `libvirt call` trace, and each round is one
+  `providerhealth.check` span.
+- **Not wired to `/readyz`.** A dead hypervisor must not make the manager
+  unready while the other providers keep working.
+- **Gauges** `proxy_operator_provider_target_up`,
+  `proxy_operator_provider_check_duration_seconds` and
+  `proxy_operator_provider_check_timestamp_seconds`, each labelled
+  `{provider,target}`. Only the leader exports them.
+
+Illustrative output:
+
+```sh
+$ kubectl get providerhealth          # short name: ph
+NAME         TYPE         READY   TARGETS   LAST CHECK   AGE
+kubernetes   kubernetes   True    1/1       12s          3d
+kvm-home     kvm          False   0/1       12s          3d
+$ kubectl get ph kvm-home -o yaml     # per target: reason, message, details
+```
+
+## Decisions
+
+Judgment calls the spec left open, and deliberate deviations — recorded so
+they read as choices, not accidents. Chronological by build step.
+
+- **Registry takes its constructor map as a parameter** instead of holding
+  a package-level map: avoids the provider⇄registry import cycle and puts
+  the wiring at the composition root, where it is visible.
+- **Mock provider replaced by the kubernetes-pod provider** (user
+  decision, mid-build): a simulated in-memory provider was too far from
+  the real system to build confidence in. Local dev/CI now runs real
+  `ubuntu/squid` pods (Canonical's actively maintained image, verified
+  50M+ pulls, pinned tag) in the operator's own cluster. Trade-off
+  accepted: envtest has no kubelet, so end-to-end proof lives in the kind
+  quickstart, and cluster pods share one egress IP — distinct egress
+  paths remain the GCP provider's job.
+- **`RequeueAfter: RequeueNow` instead of the plan's `Requeue: true`:**
+  `ctrl.Result.Requeue` is deprecated in controller-runtime v0.24; a fifth
+  configurable interval (default 1s) keeps identical semantics and stays
+  shrinkable in tests.
+- **Quota exhaustion is a wait, not a failure:** `ErrQuotaExceeded` sets a
+  condition and requeues slowly (5m) with a nil error — off the backoff
+  curve, out of the error log, and never `phase: Failed`. Only
+  `ErrPermanent` latches Failed, keyed to the generation so a spec edit
+  auto-recovers.
+- **The finalizer path never latches permanent failures:** a permanent
+  error during deletion keeps retrying visibly instead — latching there
+  would wedge the object forever with no path out but manual finalizer
+  surgery.
+- **Health transitions travel reconciler-ward over a channel**
+  (`source.Channel`), not direct status patches: `phase` derives from both
+  provisioning and health, so two status writers would race and flap. One
+  writer of status; the engine owns health *state*, the reconciler its
+  *representation*; write-only-on-transition falls out for free.
+- **Health state seeds from the existing Healthy condition on leader
+  handover** (verdict kept, counters zeroed, first probe jittered), so a
+  healthy fleet doesn't flap to Unknown on restart — but a real
+  transition still needs a full threshold run. A never-probed proxy skips
+  the jitter and probes on the next tick: startup spread matters for
+  restarts, not for a single new proxy.
+- **Latency suppression is `max(20ms, 50%)` + a 60s rate limit, and only
+  while the verdict is healthy.** The spec's bare ">50% change" is
+  undefined at 0 and lets a proxy jittering 40↔61ms write status forever;
+  the healthy-only guard (found by test) stops a below-threshold success
+  streak from emitting latency updates for a proxy still reported
+  unhealthy. Consequence: `status.lastHealthCheckTime` means "last
+  status-affecting probe" — true probe recency is in the metrics.
+- **Deterministic instance names** are `proxy-` + 16 chars of
+  base32(SHA-256(CR UID)): legal for both GCP (`[a-z2-7]` ⊂ `[-a-z0-9]`,
+  22 ≤ 63 chars) and Pod names, 80 bits against birthday collisions at a
+  fleet of tens. The replacement VM therefore has the *same name* as the
+  one being deleted — which is why replacement polls to NotFound before
+  recreating instead of racing a 409.
+- **`banned` and `rate_limited` share one cooldown window:** a second
+  duration knob the spec doesn't ask for; the report's semantic
+  difference is preserved in the API but not the store.
+- **Report targets fall back report → lease → global**, so a client that
+  leased with a target can't accidentally poison the proxy's global pool
+  by omitting the target in its report.
+- **The 409 body's `considered` counts unhealthy matches too** (the store
+  only ever sees healthy candidates): `considered = atCapacity +
+  inCooldown + unhealthy + eligible-but-outranked`, keeping the numbers
+  additive for a human debugging "why no proxy?".
+- **TTLs above `--max-lease-ttl` are a 400, not a silent clamp** — a
+  client asking for a week should find out.
+- **Discovery is not leader-elected and ships `replicas: 1`:** caches
+  start before non-leader-election runnables (verified in
+  controller-runtime's ordering), and a leader-elected server would leave
+  non-leader replicas as broken Service endpoints. One replica because
+  lease state is per-process.
+- **GCP `Create` requires zone, machineType, and image** and fails
+  `ErrPermanent` naming the missing field — inventing machine-type
+  defaults would silently create billable VMs of arbitrary shape.
+- **Unknown GCP instance statuses map to `Stopped`:** the reconciler's
+  answer to Stopped is delete-and-recreate, the always-safe move for
+  cattle when the API grows a new state.
+- **KVM hosts are reached over SSH with the libvirt socket forwarded,**
+  not libvirtd's TCP/TLS listener: nothing to configure or expose on the
+  hypervisors beyond sshd, pure Go on the operator side (no cgo, no
+  libvirt libraries in the image), and the hosts list can grow by a
+  config edit because the providerID carries the host name.
+- **KVM base images are fetched by the operator into the host's pool**
+  through libvirt's upload stream, rather than expected to be pre-placed:
+  the operator then needs no filesystem access on the host and a new
+  host is usable with zero manual staging. The cost is a synchronous
+  first-time download inside Create.
+- **The KVM seed ISO is rebuilt on every Create attempt** before the
+  domain exists, instead of "exists → skip": a reconcile cancelled mid
+  upload leaves a truncated seed, and the compensating delete cannot run
+  because the connection was already closed to unblock the RPC. Skipping
+  it would boot a VM with no cloud-init and no retry would ever repair it.
+- **KVM `nostate` maps to `Provisioning`, not `Stopped`:** unlike GCP's
+  unknown statuses, `nostate` means libvirt could not determine the
+  state at all; answering that with delete-and-recreate is too drastic,
+  so the next poll decides.
+- **KVM guests boot UEFI by default, secure boot off.** Found on the
+  first real host: Debian 13's generic image hangs in GRUB under SeaBIOS
+  on a headless VM (Debian bug #1111240: `terminal_output gfxterm
+  serial` with a serial console and no video device) regardless of CPU
+  model or seed, and boots under OVMF with the serial console working.
+  The alternatives were a virtio console (loses the kernel's serial
+  output) or a video device the VMs have no use for. Every
+  current distro cloud image supports UEFI, so it is the default and
+  `firmware: bios` is the per-image escape; libvirt's firmware
+  autoselection (`<os firmware="efi">`) keeps the OVMF path out of the
+  operator's config. Secure boot is off so unsigned images boot too.
+- **A KVM domain GC cannot read is skipped, a host it cannot reach fails
+  the sweep.** GC never deletes what it does not see, so leaving one
+  domain out is safe; one corrupt domain blocking GC for every other one
+  is not. A whole host missing from the view would hide many, so that
+  still aborts.
+- **Kubernetes 403s classify as `ErrPermanent`** even though quota
+  exhaustion also surfaces as 403 (indistinguishable from RBAC denial in
+  `apierrors`): not hammering an API server that may never allow the
+  request is the safer default; a real ResourceQuota 403 forgoes the
+  gentler quota backoff. Documented at the classification site.
+- **GC kills log at Info with a `WARNING:` prefix** — logr has no Warn
+  level; the plan's "log at Warn" is met in spirit with provider,
+  providerID, and UID always attached. Same convention as the
+  discovery server's empty-token warning.
+- **GC trusts only provable orphans:** instances without the UID label
+  are never deleted, a CR with a deletionTimestamp still counts as live
+  (its finalizer owns that deletion), and an unreadable Proxy list skips
+  the whole sweep. The namespace guard refuses to sweep a
+  namespace-restricted cache without `--gc-allow-namespaced`.
+- **Cloud-init Secrets must carry `crawl.example.com/cloud-init: "true"`:**
+  the manager caches only labelled Secrets (the operator holds
+  cluster-wide Secret read RBAC — an unrestricted cache would hold every
+  Secret in scope). Unlabelled referenced Secrets are invisible by
+  construction, surfacing as `CloudInitError`.
+- **Deletion never gives up on its own, but can be released by hand:** a
+  permanent cleanup failure keeps retrying with the finalizer on, because
+  dropping it automatically would leak a running instance on any provider
+  that answers "permission denied". Where `Get` fails permanently the
+  reconciler still calls `Delete`, since it is idempotent and may finish
+  where `Get` cannot converge. `crawl.example.com/skip-cleanup: "true"`
+  abandons the instance and releases the object; it is checked before the
+  provider lookup, so it also frees a Proxy whose provider is gone from the
+  config.
+- **Events RBAC from the plan is omitted:** nothing wires an
+  EventRecorder in the prototype, and granting verbs nothing uses would
+  be RBAC lint noise. Add the marker together with the recorder if events
+  land later.
+- **logr, not slog, inside controller paths:** the repo convention says
+  `slog`, but `log.FromContext(ctx)` hands controller-runtime's logr
+  logger to everything running under the manager — fighting that would
+  mean two logging systems in one process. Noted as a deviation rather
+  than silently ignored.
+- **Tracing is env-gated, not flag-gated:** it activates only when
+  `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` is set (user
+  decision). No collector configured means no SDK installed, no-op spans,
+  unchanged logs — the safe default for every existing deployment. The
+  one flag is `--trace-health-probes`, off by default, because probes at
+  ~1/s per proxy would dominate trace volume.
+- **Trace keys are `traceID`/`spanID`**, lowerCamel like `reconcileID` and
+  `providerID`, deliberately not the `trace_id` many Grafana derived-field
+  examples assume — configure the derived-field regex accordingly. logr
+  sinks can't read ctx, so the IDs ride on the ctx logger, re-derived from
+  a captured base per span so zap never emits duplicate keys.
+- **Probe→reconcile trace links are not attempted:** the health engine's
+  `GenericEvent` carries only namespace/name (no ctx), and the workqueue
+  coalesces events, so any link would be a guess. A health-triggered
+  reconcile starts a fresh trace; the probe that caused it is findable via
+  its own (opt-in) span and shared proxy attributes.
+- **No `traceparent` toward probe targets:** probe transports stay
+  uninstrumented so trace headers can never leak through a proxy to
+  external sites. The kube-apiserver ignores incoming `traceparent` by
+  design (public endpoint), so k8s client spans are leaves — in-process
+  traces, not cross-process ones.
+- **Provider health is a status-only, cluster-scoped CRD** (user
+  decision). `kubectl get providerhealth` works without Prometheus, which
+  the deployment does not ship (its ServiceMonitor is commented out). There
+  is no spec: the providers config file stays the one place providers are
+  configured, and the object only reports on it. The gauges carry the same
+  verdicts for alerting.
+- **KVM checks reuse the pooled connection instead of dialing fresh.** A
+  fresh dial would not pick up a rotated Secret anyway, since the SSH
+  material is cached. It would also open an sshd session per host per
+  minute. And a working cached connection is exactly what reconciles use.
+  Sharing must not cost the reconciles anything, so the check calls
+  through `hostPool.probe`. A check that runs out of time abandons its
+  RPCs instead of closing the connection, which would kill an upload or a
+  Create running on it. At most one abandoned check per host is left
+  waiting, and until it returns, the next check reports `Timeout` at once.
+  Waiting for a host's connection slot also ends with the caller's
+  deadline, so a reconcile redialing a dead host cannot stall the round.
+- **Provider transitions are logs, not Events**, in line with the
+  EventRecorder decision above. They log at Info with a `WARNING:` prefix,
+  and each target's `lastTransitionTime` in the object records when it
+  flipped.
+- **A provider name that is not a valid object name** gets metrics but no
+  `ProviderHealth` object, with a warning at start. Tightening config
+  validation instead would crash-loop existing deployments on upgrade.

docs/demo/create-gcp-proxies.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+# Demo: create N Managed proxies backed by the gcp provider, named
+# proxy-gcp-demo-1 .. proxy-gcp-demo-N, each in a randomly picked EU zone
+# so the fleet gets egress IPs from different locations.
+#
+# Usage:
+#   ./create-gcp-proxies.sh <count>
+#
+# Optional environment:
+#   NAMESPACE     namespace to create the proxies in (default: current context)
+#   GCP_PROVIDER  provider NAME from providers.yaml (default: gcp-eu)
+#   ZONES         space-separated zone list to pick from (default: EU zones below)
+set -euo pipefail
+
+if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
+    echo "usage: $(basename "$0") <count>  (positive integer)" >&2
+    exit 1
+fi
+count="$1"
+provider="${GCP_PROVIDER:-gcp-eu}"
+
+# GCP zones in the EU where e2-micro is generally available. Override with
+# ZONES="zone1 zone2 ..." if your project has quota only in some of them.
+default_zones=(
+    europe-west1-b europe-west1-c europe-west1-d       # Belgium
+    europe-west2-a europe-west2-b europe-west2-c       # London
+    europe-west3-a europe-west3-b europe-west3-c       # Frankfurt
+    europe-west4-a europe-west4-b europe-west4-c       # Netherlands
+    europe-west6-a europe-west6-b europe-west6-c       # Zurich
+    europe-west8-a europe-west8-b europe-west8-c       # Milan
+    europe-west9-a europe-west9-b europe-west9-c       # Paris
+    europe-central2-a europe-central2-b europe-central2-c   # Warsaw
+    europe-north1-a europe-north1-b europe-north1-c    # Finland
+    europe-southwest1-a europe-southwest1-b europe-southwest1-c   # Madrid
+)
+if [ -n "${ZONES:-}" ]; then
+    read -r -a zones <<< "${ZONES}"
+else
+    zones=("${default_zones[@]}")
+fi
+
+if ! command -v kubectl &> /dev/null; then
+    echo "ERROR: kubectl is required but not installed" >&2
+    exit 1
+fi
+
+ns_args=()
+if [ -n "${NAMESPACE:-}" ]; then
+    ns_args=(-n "${NAMESPACE}")
+fi
+
+for i in $(seq 1 "${count}"); do
+    zone="${zones[RANDOM % ${#zones[@]}]}"
+    echo "Creating proxy-gcp-demo-${i} in ${zone} ..."
+    kubectl apply "${ns_args[@]}" -f - << EOF
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-gcp-demo-${i}
+spec:
+  mode: Managed
+  provider: ${provider}              # must match a provider NAME in providers.yaml
+  placement:
+    zone: ${zone}
+    machineType: e2-micro
+    # debian-cloud images have no cloud-init, so spec.cloudInit (passed as
+    # user-data metadata) would be silently ignored there. Ubuntu images do.
+    image: projects/ubuntu-os-cloud/global/images/family/ubuntu-2404-lts-amd64
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      package_update: true
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+  attributes:
+    geo: eu
+    zone: ${zone}
+    purpose: crawl
+EOF
+done
+
+echo ""
+echo "Created ${count} proxies. VMs take a few minutes to provision and pass"
+echo "the health check. Watch them come up with:"
+echo "  kubectl get px ${ns_args[*]:-} -w"

docs/demo/create-kubernetes-proxies.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Demo: create N Managed proxies backed by the kubernetes-pod provider,
+# named proxy-kubernetes-demo-1 .. proxy-kubernetes-demo-N. Pods share the
+# cluster's egress IP — this exercises the full lifecycle, not distinct
+# egress paths (use create-gcp-proxies.sh for that).
+#
+# Usage:
+#   ./create-kubernetes-proxies.sh <count>
+#
+# Optional environment:
+#   NAMESPACE  namespace to create the proxies in (default: current context)
+set -euo pipefail
+
+if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
+    echo "usage: $(basename "$0") <count>  (positive integer)" >&2
+    exit 1
+fi
+count="$1"
+
+if ! command -v kubectl &> /dev/null; then
+    echo "ERROR: kubectl is required but not installed" >&2
+    exit 1
+fi
+
+ns_args=()
+if [ -n "${NAMESPACE:-}" ]; then
+    ns_args=(-n "${NAMESPACE}")
+fi
+
+for i in $(seq 1 "${count}"); do
+    echo "Creating proxy-kubernetes-demo-${i} ..."
+    kubectl apply "${ns_args[@]}" -f - << EOF
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-kubernetes-demo-${i}
+spec:
+  mode: Managed
+  provider: kubernetes
+  attributes:
+    geo: local
+    purpose: crawl
+EOF
+done
+
+echo ""
+echo "Created ${count} proxies. Watch them come up with:"
+echo "  kubectl get px ${ns_args[*]:-} -w"

docs/demo/run-demo.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Demo driver: opens a tmux session with a 2x2 pane grid:
+#
+#   top-left:     show-egress-ips-table.sh in a 10s loop, run inside the
+#                 netshoot pod against the in-cluster discovery Service
+#   top-right:    watch -n3 kubectl get px
+#   bottom-left:  create-kubernetes-proxies.sh <count>
+#   bottom-right: create-gcp-proxies.sh <count>
+#
+# Usage:
+#   ./run-demo.sh
+#
+# Optional environment:
+#   COUNT         proxies each create script makes (default: 4)
+#   SESSION       tmux session name (default: proxy-demo; an existing
+#                 session with this name is killed and recreated)
+#   NETSHOOT_POD  pod to exec into for the egress-IP loop (default: netshoot)
+#   DEMO_DIR      where the demo scripts live (default: this script's dir)
+set -euo pipefail
+
+COUNT="${COUNT:-4}"
+SESSION="${SESSION:-proxy-demo}"
+NETSHOOT_POD="${NETSHOOT_POD:-netshoot}"
+DEMO_DIR="${DEMO_DIR:-$(cd "$(dirname "$0")" && pwd)}"
+BASE_URL="http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090"
+
+for tool in tmux kubectl; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+if ! kubectl get pod "${NETSHOOT_POD}" &> /dev/null; then
+    echo "ERROR: pod ${NETSHOOT_POD} not found — start one with:" >&2
+    echo "  kubectl run netshoot --image=nicolaka/netshoot -- sleep infinity" >&2
+    exit 1
+fi
+
+echo "Copying show-egress-ips-table.sh into pod ${NETSHOOT_POD} ..."
+kubectl cp "${DEMO_DIR}/show-egress-ips-table.sh" "${NETSHOOT_POD}:/tmp/show-egress-ips-table.sh"
+
+if tmux has-session -t "${SESSION}" 2> /dev/null; then
+    echo "Killing existing tmux session ${SESSION}"
+    tmux kill-session -t "${SESSION}"
+fi
+
+# 2x2 grid: after these splits pane indexes are 0 top-left, 1 top-right,
+# 2 bottom-left, 3 bottom-right; tiled layout evens them into quarters.
+tmux new-session -d -s "${SESSION}"
+tmux split-window -h -t "${SESSION}:0"
+tmux split-window -v -t "${SESSION}:0.0"
+tmux split-window -v -t "${SESSION}:0.1"
+tmux select-layout -t "${SESSION}:0" tiled
+
+loop_cmd="BASE_URL=${BASE_URL}; while true; do bash /tmp/show-egress-ips-table.sh \"\$BASE_URL\"; echo; sleep 10; done"
+tmux send-keys -t "${SESSION}:0.0" "kubectl exec -it ${NETSHOOT_POD} -- bash -c '${loop_cmd}'" C-m
+tmux send-keys -t "${SESSION}:0.1" "watch -n3 kubectl get px" C-m
+tmux send-keys -t "${SESSION}:0.2" "bash ${DEMO_DIR}/create-kubernetes-proxies.sh ${COUNT}" C-m
+tmux send-keys -t "${SESSION}:0.3" "bash ${DEMO_DIR}/create-gcp-proxies.sh ${COUNT}" C-m
+
+tmux select-pane -t "${SESSION}:0.2"
+if [ -n "${TMUX:-}" ]; then
+    tmux switch-client -t "${SESSION}"
+else
+    tmux attach-session -t "${SESSION}"
+fi

docs/demo/show-egress-ips-table.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+# Demo: condensed-table variant of show-egress-ips.sh, made to fit a small
+# tmux pane. One line per proxy: which proxy the request goes through, its
+# endpoint, location (zone/geo attribute), and the egress IP the IP-echo
+# site saw — or unhealthy/FAILED.
+#
+# Usage:
+#   ./show-egress-ips-table.sh <BASE_URL>    e.g. ./show-egress-ips-table.sh localhost:8090
+#
+# Optional environment:
+#   TOKEN        bearer token for the discovery API (see docs/api.md)
+#   IP_ECHO_URL  site that returns the caller's IP as JSON with an "ip" field
+#                (default: https://api.ipify.org?format=json)
+set -euo pipefail
+
+if [ $# -ne 1 ]; then
+    echo "usage: $(basename "$0") <BASE_URL>  (e.g. localhost:8090)" >&2
+    exit 1
+fi
+BASE_URL="$1"
+IP_ECHO_URL="${IP_ECHO_URL:-https://api.ipify.org?format=json}"
+
+for tool in curl jq; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+auth_args=()
+if [ -n "${TOKEN:-}" ]; then
+    auth_args=(-H "Authorization: Bearer ${TOKEN}")
+fi
+
+if ! proxies_json=$(curl -sS --fail "${auth_args[@]}" "${BASE_URL}/v1/proxies") \
+    || ! echo "${proxies_json}" | jq -e . > /dev/null 2>&1; then
+    echo "ERROR: could not fetch proxy list from ${BASE_URL}" >&2
+    exit 1
+fi
+
+total=$(echo "${proxies_json}" | jq -r '.count')
+fmt="%-31s %-21s %-21s %s\n"
+
+echo "${total} proxies @ $(date +%H:%M:%S)"
+# shellcheck disable=SC2059
+printf "${fmt}" "PROXY" "ENDPOINT" "LOCATION" "EGRESS-IP"
+
+probed=0
+skipped=0
+failed=0
+while IFS= read -r proxy; do
+    id=$(echo "${proxy}" | jq -r '.id')
+    endpoint=$(echo "${proxy}" | jq -r '"\(.ip):\(.port)"')
+    location=$(echo "${proxy}" | jq -r '.attributes.zone // .attributes.geo // "-"')
+    healthy=$(echo "${proxy}" | jq -r '.healthy')
+
+    if [ "${healthy}" != "true" ]; then
+        # shellcheck disable=SC2059
+        printf "${fmt}" "${id}" "${endpoint}" "${location}" "(unhealthy)"
+        skipped=$((skipped + 1))
+        continue
+    fi
+
+    if response=$(curl -sS --max-time 10 -x "http://${endpoint}" "${IP_ECHO_URL}" 2> /dev/null); then
+        egress=$(echo "${response}" | jq -r '.ip // "?"' 2> /dev/null || echo "?")
+        probed=$((probed + 1))
+    else
+        egress="FAILED"
+        failed=$((failed + 1))
+    fi
+    # shellcheck disable=SC2059
+    printf "${fmt}" "${id}" "${endpoint}" "${location}" "${egress}"
+done < <(echo "${proxies_json}" | jq -c '.proxies[]')
+
+echo "-- ${probed} probed, ${skipped} unhealthy, ${failed} failed --"

docs/demo/show-egress-ips.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Demo: list proxies from the discovery API and show the egress IP each
+# healthy one provides, by calling an IP-echo site through it.
+#
+# Usage:
+#   ./show-egress-ips.sh <BASE_URL>          e.g. ./show-egress-ips.sh localhost:8090
+#
+# Optional environment:
+#   TOKEN        bearer token for the discovery API (see docs/api.md)
+#   IP_ECHO_URL  site that returns the caller's IP as JSON
+#                (default: https://api.ipify.org?format=json)
+set -euo pipefail
+
+if [ $# -ne 1 ]; then
+    echo "usage: $(basename "$0") <BASE_URL>  (e.g. localhost:8090)" >&2
+    exit 1
+fi
+BASE_URL="$1"
+IP_ECHO_URL="${IP_ECHO_URL:-https://api.ipify.org?format=json}"
+
+for tool in curl jq; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+auth_args=()
+if [ -n "${TOKEN:-}" ]; then
+    auth_args=(-H "Authorization: Bearer ${TOKEN}")
+fi
+
+echo "Fetching proxies from ${BASE_URL}/v1/proxies ..."
+if ! proxies_json=$(curl -sS --fail "${auth_args[@]}" "${BASE_URL}/v1/proxies"); then
+    echo "ERROR: could not fetch proxy list from ${BASE_URL}" >&2
+    exit 1
+fi
+if ! echo "${proxies_json}" | jq -e . > /dev/null; then
+    echo "ERROR: response from ${BASE_URL}/v1/proxies is not valid JSON" >&2
+    exit 1
+fi
+
+total=$(echo "${proxies_json}" | jq -r '.count')
+echo "Found ${total} proxies"
+echo ""
+
+probed=0
+skipped=0
+failed=0
+while IFS= read -r proxy; do
+    id=$(echo "${proxy}" | jq -r '.id')
+    ip=$(echo "${proxy}" | jq -r '.ip')
+    port=$(echo "${proxy}" | jq -r '.port')
+    healthy=$(echo "${proxy}" | jq -r '.healthy')
+
+    if [ "${healthy}" != "true" ]; then
+        echo "--- skipping ${id} (unhealthy) ---"
+        echo ""
+        skipped=$((skipped + 1))
+        continue
+    fi
+
+    echo "=== via ${id} — http://${ip}:${port} ==="
+    if response=$(curl -sS --max-time 10 -x "http://${ip}:${port}" "${IP_ECHO_URL}"); then
+        echo "${response}" | jq . 2> /dev/null || echo "${response}"
+        probed=$((probed + 1))
+    else
+        echo "WARNING: request through ${id} failed" >&2
+        failed=$((failed + 1))
+    fi
+    echo ""
+done < <(echo "${proxies_json}" | jq -c '.proxies[]')
+
+echo "Done: ${total} proxies — ${probed} probed, ${skipped} skipped (unhealthy), ${failed} failed"

docs/gcp-in-specific-project.md

@@ -0,0 +1,255 @@
+## Project egress-proxy
+
+```bash
+PROJECT_ID=egress-proxy
+
+# 1. Create the service account
+gcloud iam service-accounts create proxy-operator \
+  --project ${PROJECT_ID} \
+  --display-name "egress-proxies-operator"
+
+# Output: 
+# Created service account [proxy-operator].
+# Service account email: proxy-operator@egress-proxy.iam.gserviceaccount.com
+
+# 2. Grant compute.instanceAdmin.v1 on the project
+gcloud projects add-iam-policy-binding ${PROJECT_ID} \
+  --member "serviceAccount:proxy-operator@${PROJECT_ID}.iam.gserviceaccount.com" \
+  --role roles/compute.instanceAdmin.v1
+
+# Output:
+# ---------
+# Updated IAM policy for project [egress-proxy].
+# bindings:
+# - members:
+#   - serviceAccount:proxy-operator@egress-proxy.iam.gserviceaccount.com
+#   role: roles/compute.instanceAdmin.v1
+# - members:
+#   - serviceAccount:541231138892@cloudservices.gserviceaccount.com
+#   role: roles/compute.instanceGroupManagerServiceAgent
+# - members:
+#   - serviceAccount:service-541231138892@compute-system.iam.gserviceaccount.com
+#   role: roles/compute.serviceAgent
+# - members:
+#   - user:admin@fujultimate.cz
+#   role: roles/owner
+# etag: BwZYuFXko24=
+# version: 1
+
+# 3. Create the JSON key (this is what goes into the Secret)
+SA_KEY_PATH=sa_key.json
+gcloud iam service-accounts keys create $SA_KEY_PATH \
+  --iam-account proxy-operator@${PROJECT_ID}.iam.gserviceaccount.com
+
+# output:
+# created key [fdff85174a8e80bbd684e76c4d9fe28e2f4b2ddf] of type [json] as [sa_key.json] for [proxy-operator@egress-proxy.iam.gserviceaccount.com]
+
+# 4. A **firewall rule**: created VMs get network tag `proxy-operator` (the
+#    default; configurable as `gcp.networkTag`), an ephemeral external IP,
+#    and Squid listening on 3128.
+
+gcloud compute firewall-rules create allow-proxy-operator \
+    --project $PROJECT_ID \
+    --network default \
+    --allow tcp:3128 \
+    --target-tags proxy-operator \
+    --source-ranges 94.230.145.216/32
+```
+
+## Phase 2 - resources in kube
+
+```bash
+SA_KEY_PATH=sa_key.json
+kubectl -n egress-proxies-operator-system create secret generic gcp-credentials \
+  --from-file=key.json=$SA_KEY_PATH
+```
+
+
+## Appendix - full manifests
+
+```bash
+# crawl CR
+kubectl apply -f - <<'EOF'
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-gcp-sample
+spec:
+  mode: Managed
+  provider: gcp-eu                   # must match a provider NAME in providers.yaml
+  placement:
+    zone: europe-west1-b
+    machineType: e2-micro
+    # debian-cloud images have no cloud-init, so spec.cloudInit (passed as
+    # user-data metadata) would be silently ignored there. Ubuntu images do.
+    image: projects/ubuntu-os-cloud/global/images/family/ubuntu-2404-lts-amd64
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      package_update: true
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+  attributes:
+    geo: eu
+    purpose: crawl
+EOF
+
+
+
+# configmap
+kubectl apply -f - <<'EOF'
+apiVersion: v1
+data:
+  providers.yaml: |
+    providers:
+      - name: kubernetes
+        type: kubernetes
+      - name: gcp-eu          # spec.provider on a Proxy refers to this NAME, not the type
+        type: gcp
+        gcp:
+            project: egress-proxy
+            # network: default            # these three default as shown
+            # networkTag: proxy-operator
+            # diskSizeGb: 10
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: egress-proxies-operator
+  name: egress-proxies-operator-providers-config
+  namespace: egress-proxies-operator-system
+EOF
+
+# operator deployment
+kubectl apply -f - <<'EOF'
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    deployment.kubernetes.io/revision: "2"
+  labels:
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: egress-proxies-operator
+    control-plane: controller-manager
+  name: egress-proxies-operator-controller-manager
+  namespace: egress-proxies-operator-system
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: egress-proxies-operator
+      control-plane: controller-manager
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/default-container: manager
+      labels:
+        app.kubernetes.io/name: egress-proxies-operator
+        control-plane: controller-manager
+    spec:
+      containers:
+        - args:
+            - --metrics-bind-address=:8443
+            - --leader-elect
+            - --health-probe-bind-address=:8081
+            - --providers-config=/etc/proxy-operator/providers.yaml
+          command:
+            - /manager
+          env:
+            - name: DISCOVERY_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  key: token
+                  name: discovery-token
+                  optional: true
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: /var/secrets/gcp/key.json
+          image: egress-proxies-operator:dev
+          imagePullPolicy: IfNotPresent
+          livenessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthz
+              port: 8081
+              scheme: HTTP
+            initialDelaySeconds: 15
+            periodSeconds: 20
+            successThreshold: 1
+            timeoutSeconds: 1
+          name: manager
+          ports:
+            - containerPort: 8081
+              name: health
+              protocol: TCP
+            - containerPort: 8090
+              name: discovery
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /readyz
+              port: 8081
+              scheme: HTTP
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          resources:
+            limits:
+              cpu: 500m
+              memory: 128Mi
+            requests:
+              cpu: 10m
+              memory: 64Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: true
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /etc/proxy-operator
+              name: providers-config
+              readOnly: true
+            - mountPath: /var/secrets/gcp
+              name: gcp-credentials
+              readOnly: true
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      serviceAccount: egress-proxies-operator-controller-manager
+      serviceAccountName: egress-proxies-operator-controller-manager
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - configMap:
+            defaultMode: 420
+            name: egress-proxies-operator-providers-config
+          name: providers-config
+        - name: gcp-credentials
+          secret:
+            defaultMode: 420
+            secretName: gcp-credentials
+EOF
+```

docs/gcp-vm-validation.md

@@ -0,0 +1,158 @@
+# Validating real VM creation on GCP
+
+Recipe for wiring the GCP provider into a live cluster and watching a
+`Proxy` CR create a real Compute Engine VM. Angle brackets mark values you
+supply: `<PROJECT_ID>`, `<SA_KEY_PATH>`, `<ZONE>`, `<CLUSTER_EGRESS_IP>`,
+`<REGISTRY_IMAGE>`.
+
+The one important fact up front: **the operator takes no GCP credentials
+through its own config.** The client is built with Application Default
+Credentials (`internal/provider/gcp/gcp.go`, `New()`); there is no
+key-file field in the providers config. The only secret to prepare is a
+service-account JSON key, injected via the standard
+`GOOGLE_APPLICATION_CREDENTIALS` mechanism. On GKE you would use workload
+identity instead and skip the key entirely.
+
+## 1. GCP-side prerequisites (prepared outside the cluster)
+
+1. A project — `<PROJECT_ID>` — with the **Compute Engine API enabled**.
+2. A **service account** with `roles/compute.instanceAdmin.v1` on the
+   project. The operator only calls instances
+   `Insert`/`Get`/`Delete`/`AggregatedList` and does not attach a service
+   account to the VMs it creates, so no `iam.serviceAccountUser` is
+   needed.
+3. A **JSON key** for that service account, saved at `<SA_KEY_PATH>`.
+4. A **firewall rule**: created VMs get network tag `proxy-operator` (the
+   default; configurable as `gcp.networkTag`), an ephemeral external IP,
+   and Squid listening on 3128.
+
+   ```sh
+   gcloud compute firewall-rules create allow-proxy-operator \
+     --project <PROJECT_ID> \
+     --network default \
+     --allow tcp:3128 \
+     --target-tags proxy-operator \
+     --source-ranges <CLUSTER_EGRESS_IP>/32
+   ```
+
+   The source range must cover the cluster's egress IP — the operator's
+   CONNECT health probes originate there, and without the rule the Proxy
+   hangs at `Running`/unhealthy instead of reaching `Ready`. ⚠️ The
+   sample cloud-init configures `http_access allow all`, so on a public
+   IP this is an open proxy — keep the source ranges tight.
+
+## 2. Create the credentials Secret
+
+Namespace is `egress-proxies-operator-system` after kustomize prefixing:
+
+```sh
+kubectl -n egress-proxies-operator-system create secret generic gcp-credentials \
+  --from-file=key.json=<SA_KEY_PATH>
+```
+
+## 3. Add a GCP entry to the providers ConfigMap
+
+Edit `config/manager/providers_config.yaml` (mounted at
+`/etc/proxy-operator/providers.yaml`):
+
+```yaml
+providers:
+  - name: kubernetes
+    type: kubernetes
+  - name: gcp-eu          # spec.provider on a Proxy refers to this NAME, not the type
+    type: gcp
+    gcp:
+      project: <PROJECT_ID>
+      # network: default            # these three default as shown
+      # networkTag: proxy-operator
+      # diskSizeGb: 10
+```
+
+The config is validated fail-fast at startup — a typo shows up
+immediately in the manager log, not on first use.
+
+## 4. Mount the Secret and point ADC at it
+
+In `config/manager/manager.yaml`, add to the manager container:
+
+```yaml
+env:
+  - name: GOOGLE_APPLICATION_CREDENTIALS
+    value: /var/secrets/gcp/key.json
+volumeMounts:
+  - name: gcp-credentials
+    mountPath: /var/secrets/gcp
+    readOnly: true
+volumes:
+  - name: gcp-credentials
+    secret:
+      secretName: gcp-credentials
+```
+
+(`volumeMounts` merges into the existing container list; `volumes` into
+the existing pod-level list.)
+
+## 5. Deploy and create the Proxy
+
+```sh
+make deploy IMG=<REGISTRY_IMAGE>
+```
+
+`config/samples/proxy_gcp.yaml` is usable as-is once `spec.provider`
+matches the name from step 3. All three placement fields are mandatory
+for GCP — a missing one sets the Proxy to `Failed` with a message naming
+it:
+
+```yaml
+spec:
+  mode: Managed
+  provider: gcp-eu
+  placement:
+    zone: <ZONE>                     # e.g. europe-west1-b
+    machineType: e2-micro
+    image: projects/debian-cloud/global/images/family/debian-12
+```
+
+```sh
+kubectl apply -f config/samples/proxy_gcp.yaml
+```
+
+## 6. What you should see
+
+```sh
+kubectl get proxy -w
+```
+
+`Provisioning` → `Running` (VM's external IP published in status) →
+`Ready` (CONNECT health probe succeeded through the public IP). Then:
+
+```sh
+# the VM exists and carries the GC labels
+gcloud compute instances list --project <PROJECT_ID> \
+  --filter 'labels.proxy-operator-managed=yes'
+
+# the proxy actually tunnels — should print the VM's external IP
+curl -x http://<EXTERNAL_IP>:3128 https://ifconfig.me
+```
+
+Cleanup — the finalizer deletes the VM:
+
+```sh
+kubectl delete proxy proxy-gcp-sample
+gcloud compute instances list --project <PROJECT_ID>   # should be empty again
+```
+
+## Gotchas
+
+- **The orphan GC sweeps the whole project**: any VM labeled
+  `proxy-operator-managed=yes` whose UID does not match a live Proxy CR
+  in *this* cluster is deleted once past the age threshold. Do not point
+  two operator installs at the same project, and do not hand-create VMs
+  with that label.
+- **VM creation is fire-and-forget** — the provider never waits on the
+  insert operation; progress is discovered by polling `Get`. A quota
+  error or bad image name surfaces on the Proxy's status/conditions a
+  reconcile later, not synchronously. `kubectl describe proxy` is the
+  place to look when something stalls.
+- **e2-micro costs pennies but is not free everywhere** — remember to
+  delete the CR (or check `gcloud compute instances list`) when done.

docs/kvm-provider.md

@@ -0,0 +1,373 @@
+# The KVM provider
+
+Runs Managed proxies as VMs on your own libvirt/KVM hypervisors. The
+operator reaches each host over SSH, forwards the libvirt unix socket
+through that connection, and talks libvirt's own RPC over it — so the
+hosts need nothing beyond `sshd` and `libvirtd`, and the operator image
+carries no libvirt libraries. VMs boot a stock distro cloud image that the
+operator downloads into the host's storage pool on first use, with the
+Proxy's cloud-init delivered on a NoCloud seed ISO built in-process.
+
+## Hypervisor prerequisites
+
+Per host, once:
+
+1. **libvirt + KVM** (`libvirtd` running, `/dev/kvm` present).
+2. **An SSH user in the `libvirt` group**, so it may open
+   `/var/run/libvirt/libvirt-sock`. The operator authenticates with a
+   private key; nothing else on the host is touched.
+3. **A bridge to a network with DHCP** (e.g. `br0` bridged onto the LAN).
+   Each VM gets one virtio NIC on it and takes its IP from that network's
+   DHCP; the cluster must be able to reach those IPs for health probes
+   and for crawlers to use the proxy.
+4. **A storage pool** (default `default`, usually `/var/lib/libvirt/images`)
+   with room for one base image per distro (a few hundred MB) plus the
+   per-VM overlays (`diskSizeGb` each, thin-provisioned).
+5. **UEFI firmware for guests**: the `ovmf` package on Debian/Ubuntu,
+   `edk2-ovmf` on Fedora/RHEL. VMs boot UEFI by default (libvirt picks
+   the OVMF binary itself); a domain start fails with "no firmware
+   found" without it. Only `firmware: bios` images skip this.
+6. Optionally, **outbound internet from the VMs**: a vendor-data script
+   installs `qemu-guest-agent` from the distro mirrors on first boot.
+   Without it the VM still works — the operator falls back to the host's
+   ARP table to find the IP, which fills in once the guest sends traffic —
+   but the ARP table only sees traffic to the host itself, so on a plain
+   bridge the agent is in practice required.
+
+Check from the operator's point of view before wiring it in:
+
+```sh
+ssh virt@nas1.home.lan virsh -c qemu:///system list --all   # user can drive libvirt
+ssh virt@nas1.home.lan virsh -c qemu:///system pool-info default
+ssh virt@nas1.home.lan ip link show br0
+```
+
+## Cluster-side setup
+
+1. Create the SSH Secret from a key pair dedicated to the operator and a
+   `known_hosts` file holding every host's key. The worked example below
+   uses `~/.ssh/proxy-operator_ed25519` and `~/.ssh/kvm_known_hosts`,
+   the same paths the integration test and `make kvm-ssh-secret` default
+   to.
+
+   ```sh
+   # once: a key pair for the operator; the .pub goes on every host
+   ssh-keygen -t ed25519 -f ~/.ssh/proxy-operator_ed25519 -N '' -C proxy-operator
+   ssh-copy-id -i ~/.ssh/proxy-operator_ed25519.pub virt@nas1.home.lan
+
+   # once per host: its host key, under the address the *operator* will
+   # dial (hosts[].address), which may differ from what your laptop uses
+   ssh-keyscan -H nas1.home.lan >> ~/.ssh/kvm_known_hosts
+
+   # the Secret: the PRIVATE key (not the .pub) plus known_hosts
+   kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
+     --from-file=id_ed25519=$HOME/.ssh/proxy-operator_ed25519 \
+     --from-file=known_hosts=$HOME/.ssh/kvm_known_hosts
+   ```
+
+   `make kvm-ssh-secret` runs that last command as an apply, so it also
+   updates an existing Secret; override the paths with
+   `KVM_SSH_KEY=... KVM_KNOWN_HOSTS=...`. Putting the `.pub` in the
+   Secret is the classic mistake: the operator then fails at startup
+   with a "not a private key" error. Without `known_hosts`, set
+   `insecureIgnoreHostKey: true` and accept that a MITM could hand you a
+   fake hypervisor.
+
+   `config/manager/manager.yaml` mounts the Secret (optional) at
+   `/etc/proxy-operator/kvm` as two files, `id_ed25519` and
+   `known_hosts`, which is what `sshKeyFile` and `knownHostsFile` in the
+   provider config point at. The deployment starts fine without the
+   Secret, even with a `kvm` provider configured: only that provider's
+   proxies fail, and they recover once the Secret exists. Rotating the
+   key: re-run the command (or the make target) and restart the manager;
+   both files are read once, on the first connection, and cached.
+
+2. Add a provider entry to the providers config
+   (`config/manager/providers_config.yaml` in-cluster; full sample in
+   `config/samples/providers-config.yaml`):
+
+   ```yaml
+   - name: kvm-home
+     type: kvm
+     kvm:
+       sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+       knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+       images:
+         debian-13:
+           url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+       machineTypes:
+         small: {vcpus: 1, memoryMiB: 1024}
+       hosts:
+         - name: nas1
+           address: nas1.home.lan:22
+           user: virt
+           bridge: br0
+   ```
+
+   The operator checks both files at startup and logs a `WARNING:` line
+   when either is unreadable, but it does not refuse to start: the
+   provider set is built all-or-nothing, so exiting here would crash-loop
+   the manager and take the `kubernetes` and `gcp` providers with it.
+   The files are read for real on the first connection. A missing file is
+   a transient failure, so proxies heal by themselves once the Secret
+   lands; a key that does not parse is permanent and needs a fix.
+
+3. Apply a Proxy (`config/samples/proxy_kvm.yaml`) and watch it:
+
+   ```sh
+   kubectl get px -w
+   # NAME               MODE      PROVIDER   PHASE          IP             HEALTHY
+   # proxy-kvm-sample   Managed   kvm-home   Provisioning
+   # proxy-kvm-sample   Managed   kvm-home   Running        192.168.1.73
+   # proxy-kvm-sample   Managed   kvm-home   Ready          192.168.1.73   True
+   ```
+
+   The first Proxy for a given image on a given host sits in
+   `Provisioning` for the length of the image download; the operator log
+   shows `fetching base image into host pool` and `base image cached`
+   with the size and duration.
+
+## Config reference
+
+| Field | Required | Default | Meaning |
+|---|---|---|---|
+| `sshKeyFile` | yes | | private key used for every host |
+| `knownHostsFile` | unless `insecureIgnoreHostKey` | | OpenSSH `known_hosts` to verify host keys against |
+| `insecureIgnoreHostKey` | | `false` | skip host-key verification entirely |
+| `diskSizeGb` | | `10` | capacity of each VM's overlay disk; cloud-init's growpart expands the root fs into it |
+| `vendorData` | | | extra cloud-init vendor-data placed on every seed next to the Proxy's user-data. Beware that cloud-init drops any top-level key of a vendor-data **cloud-config** that the user-data also defines. Shipped as one part of a MIME multipart alongside the agent script when `installGuestAgent` is on |
+| `installGuestAgent` | | `true` | add the vendor-data script that installs and starts `qemu-guest-agent`. Turn off for images that ship it already or cannot reach a package mirror; IP discovery then depends on the host's ARP table |
+| `images.<key>` | | | the key names the image in `placement.image`. Letters, digits, dots, dashes and underscores only — it becomes the cached volume name `base-<key>.<format>` |
+| `images.<key>.url` | yes per entry | | http(s) URL of a cloud image (qcow2 or raw) |
+| `images.<key>.sha256` | | | expected digest; verified after download, a change triggers a re-download |
+| `images.<key>.format` | | `qcow2` | `qcow2` or `raw` |
+| `images.<key>.firmware` | | `uefi` | `uefi` (OVMF, secure boot off) or `bios` (SeaBIOS) for images without an EFI system partition; Debian 13 hangs in GRUB under BIOS on a headless VM, see *Known gaps* |
+| `machineTypes.<key>` | ≥ 1 | | `{vcpus, memoryMiB}` size profile |
+| `hosts[].name` | yes | | the `placement.zone` value that selects this host; unique. Letters, digits, dots, dashes and underscores only — it becomes part of the providerID `hosts/<name>/domains/<vm>` |
+| `hosts[].address` | yes | | SSH endpoint, `host:port` |
+| `hosts[].user` | yes | | SSH user, member of the `libvirt` group |
+| `hosts[].socket` | | `/var/run/libvirt/libvirt-sock` | libvirt socket path on the host |
+| `hosts[].pool` | | `default` | storage pool for base images, overlays and seeds |
+| `hosts[].bridge` | yes | | host bridge for the VM's NIC |
+
+## Placement
+
+| Proxy field | Meaning for kvm |
+|---|---|
+| `placement.zone` | host name from `hosts`. May be omitted when exactly one host is configured; otherwise required. Unknown → `Failed` |
+| `placement.machineType` | key in `machineTypes`. Unknown → `Failed` |
+| `placement.image` | key in `images`, or an `http(s)://` URL used directly. Anything else → `Failed` |
+| `placement.region` | ignored |
+
+`Failed` here means the reconciler stops retrying and the message names
+the offending value; fix the spec (which replaces the VM) or the config.
+
+## What lands on the host
+
+For a Proxy whose deterministic name is `proxy-abc…`, on the chosen
+host's pool:
+
+| Object | Name | Lifetime |
+|---|---|---|
+| base image | `base-<key>.qcow2` (or `base-<url-hash>.qcow2` for a URL) | cached; survives Proxy deletion |
+| image marker | `base-<key>.qcow2.sha256` | with the base image |
+| overlay disk | `proxy-abc….qcow2`, qcow2 backed by the base | with the Proxy |
+| seed ISO | `proxy-abc…-seed.iso` | with the Proxy |
+| domain | `proxy-abc…` | with the Proxy |
+
+The domain is a q35 machine with host-passthrough CPU, the overlay on
+virtio, the seed on a SATA cdrom, one virtio NIC on the bridge, a serial
+console, the guest-agent channel, a virtio RNG and no graphics. Its
+`<metadata>` carries the operator's element (managed flag, owning UID,
+creation time) — that is how orphan GC recognises its own VMs, since
+libvirt has no labels.
+
+The seed's `user-data` is the Proxy's `cloudInit`, **byte for byte**. Any
+format cloud-init accepts works, including a Jinja template, a MIME
+archive, a shell script or a gzipped payload; the operator never parses or
+re-renders it.
+
+The guest agent rides along in `vendor-data`, as a **shell script** rather
+than a cloud-config. That is the only shape cloud-init cannot drop:
+vendor-data cloud-config is merged under the user's own document with
+mergers that keep the user-data's value for every key both define and never
+append lists, so a Proxy carrying its own `packages:` silently lost the
+agent and sat in `Provisioning` forever. The strategy cannot be overridden
+from either document — `merge_how` in user-data governs only the first
+source, and `merge_how` in vendor-data is consumed and discarded before the
+document is stored. A vendor-data script skips all of it: cloud-init gives
+vendor-data its own handler chain, writes scripts to a separate directory
+and runs them from `cc_scripts_vendor`, after the user's packages are
+installed and before the user's `runcmd`. The script is idempotent, picks
+whichever of `apt-get`, `dnf`, `yum`, `zypper` or `apk` the image has, and
+logs into `/var/log/cloud-init-output.log`. Set `installGuestAgent: false`
+to leave it out. When `vendorData` is also configured, the two travel as
+separate parts of a MIME multipart vendor-data.
+
+## The image cache
+
+The base image is downloaded **by the operator, through libvirt**, into
+the host's pool the first time a Proxy asks for it on that host — nothing
+is pre-placed on the hypervisor, and the operator needs no filesystem
+access there. The bytes stream from the URL through the operator into
+the pool with a running sha256; nothing is buffered. The marker volume is
+written last and holds the digest, so a crash mid-download leaves a base
+volume without a marker, which the next attempt deletes and redoes.
+
+A re-download happens when: the marker is missing, or `sha256` is
+configured and differs from the marker's content. Otherwise the cached
+copy is used for as long as it exists, even if the URL's content changed
+upstream (a `latest` URL is pinned at the moment of first fetch, per
+host).
+
+To evict manually:
+
+```sh
+ssh virt@nas1 virsh -c qemu:///system vol-delete --pool default base-debian-13.qcow2.sha256
+ssh virt@nas1 virsh -c qemu:///system vol-delete --pool default base-debian-13.qcow2
+```
+
+⚠️ **Do not change an image entry's `url` or `sha256`, and do not evict
+its base volume, while Proxies using that image exist on the host.** The
+overlays back onto the base file; deleting or replacing it corrupts every
+VM that uses it. Delete those Proxies first, or add the new image under a
+new key and move Proxies over by changing their `placement.image` (which
+replaces the VM).
+
+## Logging into a VM
+
+- Put your key in the Proxy's cloud-init (`ssh_authorized_keys`) and
+  `ssh debian@<ip>` (Debian) / `ssh ubuntu@<ip>` (Ubuntu).
+- Or on the host: `virsh console proxy-abc…` attaches to the serial
+  console; cloud-init's output scrolls by there on first boot.
+- `virsh domifaddr proxy-abc… --source arp` / `--source agent` shows what
+  the operator sees when it looks for the IP.
+
+## Logging
+
+The provider logs through controller-runtime's logger, named `kvm`, with
+`provider`, `host` and `name` (the domain) on every line.
+
+- `--zap-log-level=debug` (logr V(1)): one line per provider operation
+  (`domain defined`, `KVM domain ensured running`, ...) **and one
+  `libvirt call` line per libvirt RPC**, named `kvm.libvirt`, with
+  `rpc` (the libvirt API function, e.g. `virDomainLookupByName`), the
+  identifiers it was given (`domain`, `pool`, `volume`), curated results
+  (`state`, `path`, `interfaces`), `duration`, and on failure `error`
+  plus `libvirtError` (the code name, e.g. `ErrNoDomain`) and
+  `libvirtCode`. Connection events (`libvirt host connected`, dial
+  failures, `libvirt connection dropped` with a reason) log at the same
+  level. A Create is about ten RPCs; a Get is four.
+- `--zap-log-level=2`: the same lines also carry the domain and volume
+  `xml`, the `metadata` element, the interface `addresses` and the
+  domain `names` from a listing.
+- Cloud-init is never logged at any level: the seed ISO upload reports
+  only `length` and `duration`. There is no raw wire tier like the GCP
+  provider's V(5); go-libvirt exposes no hook for it.
+
+## Health check
+
+The operator checks every configured host every `--provider-check-interval`
+(default 1m). It publishes the result in two places: the `ProviderHealth`
+object named after the provider (`kubectl get ph kvm-home -o yaml`), and the
+`proxy_operator_provider_target_up{provider,target}` gauge.
+
+All hosts are checked concurrently, each over the same pooled connection
+that Create and Get use:
+
+1. The SSH and libvirt handshakes. These only happen on the first check, or
+   after the link dropped.
+2. `virConnectGetLibVersion` and `virConnectGetHostname`.
+3. `virStoragePoolLookupByName` and `virStoragePoolGetInfo` on the host's
+   `pool`.
+
+A passing target's `details` carry `hostname`, `libvirtVersion`, `pool`,
+`poolState`, `poolCapacity` and `poolAvailable` (in GiB). The free space is
+the closest thing this provider has to a capacity signal. Every target also
+shows its SSH `user`. A failing target says where the check broke:
+
+| Reason | Meaning | Look at |
+|---|---|---|
+| `SSHMaterialUnavailable` | the key or known_hosts file is missing or does not parse | the `kvm-ssh` Secret (see *Cluster-side setup*) |
+| `Unreachable` | the TCP connect failed, or the link died mid-check | DNS, Tailscale or a firewall between the cluster and `address` |
+| `SSHHandshakeFailed` | the host key is missing from known_hosts or has changed, or `user` rejected the key | the known_hosts entry; `user`'s `authorized_keys` |
+| `LibvirtConnectFailed` | SSH worked, the libvirt socket did not | `user` in the `libvirt` group, `libvirtd` running, the `socket` path |
+| `Timeout` | no answer within `--provider-check-timeout` (default 15s), or the previous check of the host is still waiting | a hung `libvirtd`, a saturated link, or a reconcile stuck redialing the host |
+| `PoolNotFound` | there is no pool named `pool` | `virsh pool-list --all` |
+| `PoolNotActive` | the pool exists but is not running | `virsh pool-start <pool>` and `virsh pool-autostart <pool>` |
+| `LibvirtError` | any other libvirt error | the message carries libvirt's own text |
+
+Sharing the connection has two consequences:
+
+- The leader now keeps one SSH connection per host open from startup, with a
+  keepalive every 15s. Before, the first connection opened with the first KVM
+  reconcile.
+- A check that runs out of time gives up on its RPCs but leaves the
+  connection alone, so an image upload or a Create running on it is not cut
+  off. At most one abandoned check per host keeps waiting; until it
+  returns, the next check reports `Timeout` straight away. A link that is
+  really dead is closed by the keepalive within about two keepalive
+  intervals.
+- A check never waits longer than its timeout for the host's connection,
+  even while a reconcile holds it to redial a host that is down.
+
+## Known gaps
+
+- **No quota or capacity detection.** libvirt reports an out-of-space
+  pool or an over-committed host as an ordinary error; the reconciler
+  treats it as transient and retries with backoff rather than the slower
+  quota backoff. The health check does report each pool's free space
+  (`poolAvailable`), so a pool that is filling up shows before Creates
+  start failing.
+- **A rotated `kvm-ssh` Secret needs a manager restart.** The key and
+  known_hosts files are read on first use and cached for the life of the
+  process. Replacing them in the Secret therefore changes nothing for
+  later connections, health checks included. After a host key change, the
+  check keeps reporting `SSHHandshakeFailed` until the manager restarts.
+- **Orphan volumes without a domain are invisible to GC.** A crash
+  between creating the volumes and defining the domain leaves an overlay
+  and seed that `ListByTag` cannot see (it lists domains). They are
+  cleaned when the same Proxy reconciles again (Delete removes volumes by
+  name even with no domain), but a Proxy CR that is gone for good leaves
+  them for `virsh vol-list` to find.
+- **A host removed from the config takes its VMs with it.** `Delete` for a
+  providerID naming an unconfigured host is a no-op success, so the Proxy
+  finishes deleting instead of pinning its finalizer, but the domain and its
+  volumes stay on the hypervisor and orphan GC cannot see them either (it
+  only sweeps configured hosts). Delete the Proxies before retiring a host,
+  or clean up afterwards with `virsh undefine --nvram` plus `virsh vol-delete`.
+  `Get` still fails permanently for such a providerID on purpose: a typo in
+  `hosts` must not look like a missing VM and trigger a rebuild elsewhere.
+- **The first image fetch is synchronous.** It holds a reconcile worker
+  for the download (LAN speed, a few hundred MB, once per image per
+  host). Other Proxies keep reconciling on the remaining workers.
+- **Replacing an image in use breaks running VMs** — see the warning
+  under *The image cache*.
+- **IP discovery needs either guest traffic or the agent.** A VM on a
+  bridge with no DHCP never gets an address and stays `Provisioning`
+  forever; check `virsh console`. The same symptom with a booted, addressed
+  VM means the guest agent never came up: check `systemctl status
+  qemu-guest-agent`, `cloud-init status --long` and the vendor script's
+  output in `/var/log/cloud-init-output.log` (a VM without outbound
+  internet cannot install it). A user-data that disables vendor-data
+  (`vendor_data: {enabled: false}`) also removes the agent, by design.
+- **Debian 13 cloud images hang in GRUB under BIOS on a headless VM**
+  ([Debian bug #1111240](https://bugs.debian.org/1111240), open, in
+  `grub2`): trixie's image ships `terminal_output gfxterm serial` in its
+  GRUB config, and with a serial console but no video device (exactly
+  the domain this provider defines) GRUB prints "Booting" and never
+  hands over to the kernel. Not host-specific; reproduced by the Debian
+  cloud team. The same image boots under UEFI with the serial console
+  intact, which is why `firmware` defaults to `uefi`. Debian 12
+  (bookworm) images are unaffected and boot either way. Images that
+  need `firmware: bios` are the exception, not the rule.
+
+## Testing
+
+Unit tests run over an in-memory fake of the libvirt API. The optional
+integration test against a real host is described in
+[testing.md](testing.md#the-kvm-integration-test-real-hypervisor); its
+`-v` output includes the `libvirt call` trace above, at the verbosity
+`KVM_TEST_LOG_LEVEL` selects (default 1). `TestKVMCheck_realHost` runs the
+health check against the same host and creates nothing:
+`go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/`.

docs/plans-executions/2026-08-07-1747-proxy-operator.md

@@ -10,13 +10,13 @@ Pairs with [docs/plans/2026-08-07-1747-proxy-operator.md](../plans/2026-08-07-17
 - [x] Step 3 — Kubernetes pod provider (`internal/provider/kubernetes/`; first built as an in-memory mock, then replaced — see the two Step 3 sections below)
 - [x] Step 4 — Reconciler (`internal/controller/`)
 - [x] Step 5 — Health engine (`internal/health/`)
-- [ ] Step 6 — Lease store (`internal/lease/`)
-- [ ] Step 7 — Discovery API (`internal/discovery/`)
-- [ ] Step 8 — GCP provider (`internal/provider/gcp/`)
-- [ ] Step 9 — Orphan GC + metrics
-- [ ] Step 10 — Wiring, config, docs
-- [ ] Step 11 — Tests
-- [ ] Verification (vet/test/kind e2e) + commit, push, open MR
+- [x] Step 6 — Lease store (`internal/lease/`)
+- [x] Step 7 — Discovery API (`internal/discovery/`)
+- [x] Step 8 — GCP provider (`internal/provider/gcp/`)
+- [x] Step 9 — Orphan GC + metrics
+- [x] Step 10 — Wiring, config, docs
+- [x] Step 11 — Tests
+- [x] Verification (vet/test/kind e2e) + commit, push, open MR
 
 ## Step 0 — Branch and scaffold
 
@@ -694,3 +694,438 @@ engine deliberately knows nothing about conditions except reading one at
 seed time, keeping the state/representation split honest. The
 `hint`-driven `wg.Go` idiom (Go 1.25+) replaced the classic
 `wg.Add/defer wg.Done` in the worker pool.
+
+## Step 6 — Lease store (`internal/lease/`)
+
+Implemented `store.go` per the plan: `Acquire` takes the whole candidate
+set so selection and insertion happen under the one store mutex (no
+overcommit between concurrent requests), selection is a linear scan +
+`slices.SortFunc` on `(activeLeases asc, latency asc, name asc)`,
+`AcquireStats{Considered, AtCapacity, InCooldown}` feeds Step 7's 409
+body, cooldowns live in a `map[{proxy, target}]time.Time` (empty target =
+global pool), and expired leases are retained for `CooldownWindow` past
+their TTL so a late `Report` — arriving exactly when a proxy is being
+rate-limited — still resolves and records its cooldown.
+
+Semantics pinned against the spec (§8) rather than guessed:
+
+- Report results are exactly `ok | rate_limited | banned` (`ParseResult`
+  gives the API layer its 400 check). `rate_limited` and `banned` both
+  record a cooldown for the same window; `ok` records nothing.
+  Distinguishing ban duration from rate-limit duration would be a second
+  knob the spec doesn't ask for — noted for the Decisions section.
+- Cooldown scoping: the global cooldown (empty target) always applies; a
+  target-scoped cooldown additionally blocks acquisitions for that target;
+  acquisitions without a target see only the global pool ("a proxy
+  rate-limited by one site is still fine for everyone else").
+- A `Report` without a target falls back to the lease's own target before
+  falling back to global — so a client that leased with a target doesn't
+  accidentally poison the whole proxy by omitting it in the report.
+
+Design notes:
+
+- **Correctness never depends on the sweep.** Every read path
+  (`Acquire`/`ActiveCount`/`Counts`) compares `ExpiresAt` against the
+  injected clock, so TTL expiry frees capacity immediately even if the
+  background loop hasn't run; the sweep is purely garbage collection. The
+  plan's `ExpireLoop` became `Start(ctx)` + `NeedLeaderElection() false`
+  so the store satisfies `manager.Runnable` directly — Step 10 just
+  `mgr.Add(store)`s it. Not leader-elected because lease state is
+  per-process and must expire wherever the discovery API is serving.
+- The store knows nothing about Proxy objects — `Candidate` carries the
+  opaque key, `MaxLeases`, and latency; the discovery layer does the
+  health/attribute filtering. The spec's `LeaseStore` interface will be
+  defined consumer-side in `internal/discovery` (Step 7), per Go idiom;
+  this package exports only the concrete in-memory `*Store`.
+- Lease IDs come from `crypto/rand.Text()` (Go 1.24+); returned `Lease`
+  values are copies so callers can't mutate store internals.
+
+Tests (94.8% coverage, `-race -count=2` clean): capacity + release
+freeing slots, `MaxLeases=0` unleasable, least-loaded/latency/name
+selection order, target-scoped vs global cooldown scoping, cooldown
+expiry via the injected fake clock, TTL freeing capacity with no sweep,
+report-on-expired-but-retained lease (then `ErrUnknownLease` after
+retention), `ok` recording nothing, idempotent release, `ParseResult`,
+40 concurrent acquires against `MaxLeases=5` granting exactly 5, and the
+`Start` loop sweeping then stopping cleanly on cancel.
+
+```bash
+go test -race -count=2 ./internal/lease/
+make test    # whole repo green, other packages' coverage unchanged
+```
+
+Worth noting: `docs/architecture.md` was not extended this step — the
+lease store is HTTP-driven, not cluster-event-driven, so its diagram
+belongs with the discovery API and lands in Step 7 (banner updated to say
+so).
+
+## Step 7 — Discovery API (`internal/discovery/`)
+
+Implemented `server.go` (Runnable + middleware chain) and `handlers.go`
+(the four endpoints + `proxyView` wire shape) per the plan: stdlib
+`http.ServeMux` method+wildcard routing (no third-party router — see the
+plan clarification commit: this is a stdlib feature since Go 1.22, the
+project stays on the pinned Go 1.26), middleware outermost-first recover →
+request-log → `MaxBytesReader(64KiB)` → constant-time bearer auth with
+`/healthz` exempt, empty `DISCOVERY_TOKEN` serving unauthenticated with a
+loud startup warning, `NeedLeaderElection() = false` with the plan's
+runnable-ordering rationale in the doc comment, and graceful `Shutdown`
+with a 10 s grace on ctx cancel.
+
+The `LeaseStore` interface landed consumer-side in this package (spec §8
+wants handlers swappable to a CRD/Redis store); `internal/lease.*Store`
+satisfies it without modification.
+
+Judgment calls the plan/spec left open:
+
+- **409 arithmetic:** the store only ever sees healthy candidates, so its
+  `Considered` excludes unhealthy matches. The handler counts unhealthy
+  selector-matches itself and reports `considered = healthy + unhealthy`,
+  keeping the plan's example arithmetic (7 = 2+2+3) consistent.
+- **TTL handling:** omitted/zero `ttlSeconds` → 300 s default; negative or
+  above `MaxLeaseTTL` (default 1h, flag in Step 10) → 400 `invalid_ttl`
+  rather than silent clamping — a client asking for a week-long lease
+  should find out, not get an hour quietly.
+- **Grant response includes the fresh `activeLeases`** (the just-granted
+  lease counted), read back via `Store.Counts()` after the acquire.
+- Proxies with a deletionTimestamp are filtered out of both list and
+  candidate selection — a proxy mid-teardown shouldn't be advertised.
+
+Tests (87.3% coverage, `-race -count=2` clean, green on first run):
+httptest over the real handler chain with a fake cache reader and a real
+`lease.Store` — auth on/off/wrong-token/healthz-exempt, list filtering
+(attributes, healthy, combined, empty-is-200), grant shape (201, default
+TTL, lowest-latency pick, RFC3339 expiresAt, activeLeases=1), the full
+409 body arithmetic, invalid TTL/body/result, idempotent 204 release,
+report→cooldown→409 round-trip, 404 on unknown lease, and a real
+`Start` on `127.0.0.1:0` (via the new `BoundAddr()` accessor) serving
+healthz then shutting down cleanly on cancel.
+
+```bash
+go test -race -count=2 ./internal/discovery/
+make test    # whole repo green
+```
+
+Worth noting: `go mod tidy` promoted `github.com/go-logr/logr` from
+indirect to direct (the server holds a `logr.Logger` field). The
+`--discovery-addr`, `--max-lease-ttl` flags and the `DISCOVERY_TOKEN`
+Secret mount arrive with `cmd/main.go` in Step 10. `docs/architecture.md`
+gained §7 covering the whole HTTP path and the store's sweep Runnable.
+
+## Step 8 — GCP provider (`internal/provider/gcp/`)
+
+Implemented per the plan: `gcp.go` (Provider + the flattened `instancesAPI`
+test seam + providerID handling + state mapping), `insert.go` (pure
+`buildInsertRequest` + config defaults), `errors.go` (HTTP-code → taxonomy
+classification). Only the four calls the spec allows — instances Insert /
+Get / Delete / AggregatedList — and `Operation.Wait` is never called:
+Create/Delete return once the operation is submitted, `409 alreadyExists`
+on Insert and `404` on Delete both count as success, which is what makes
+repeat calls after a crash correct.
+
+Dependency added (the plan's environment check pinned it):
+
+```bash
+go get cloud.google.com/go/compute@v1.65.0 google.golang.org/api@latest
+# resolved google.golang.org/api v0.292.0; go mod tidy pulled the auth/gax chain
+```
+
+Key shapes, all straight from the plan:
+
+- **providerID `zones/<zone>/instances/<name>`** — Get/Delete parse the
+  zone out of the ID instead of re-reading `spec.placement.zone`, which is
+  wrong exactly when a zone edit is the replacement being processed.
+- **The seam is not an SDK mirror** — verified the plan's premise against
+  the vendored source before designing around it:
+  `InstancesScopedListPairIterator` has an unexported `nextFunc`, so a
+  fake cannot construct one. The seam flattens `AggregatedList` to
+  `[]*computepb.Instance` and returns operations as just their name.
+- `AggregatedList` sets `ReturnPartialSuccess: true` (one unreachable
+  zone must not fail a GC sweep) and filters by
+  `labels.proxy-operator-managed = true`.
+- `RUNNING` without a `NatIP` maps to `Provisioning` — never publish an
+  empty IP. Unknown/new GCP statuses map to `Stopped`: the reconciler's
+  response is delete-and-recreate, always safe for cattle.
+- Classification: 404→NotFound; 429 and 403-with-
+  `quotaExceeded`/`rateLimitExceeded`→Quota; 400/401/403-other→Permanent;
+  everything else (408, 5xx, network, unknown)→Transient.
+
+Judgment call: `placement.zone`/`machineType`/`image` are all required at
+`Create` — missing values fail as `ErrPermanent` with a message naming
+the empty fields, rather than inventing defaults the spec doesn't define.
+A wrong guess here would silently create billable VMs of an arbitrary
+shape; a Failed condition telling the user what to set is strictly better.
+
+Tests (75.2%, `-race -count=2` clean, green on first run): the plan's
+primary field-by-field `buildInsertRequest` assertion (machine-type URL,
+boot disk, the exact `{External NAT, ONE_TO_ONE_NAT}` access config,
+user-data metadata, GC labels, network tag) plus config overrides and
+no-metadata-without-cloud-init; the full classification table including
+`errors.Is` AND `errors.As` through the multi-unwrap; and fake-seam tests
+for zone-qualified IDs, 409-is-success, permanent-on-bad-placement (no
+API call made), the nine-row state-mapping table, 404 paths, malformed
+providerIDs, and the ListByTag filter/partial-success assertions. The
+uncovered remainder is `New()` (dials real Google with ADC) and the
+`realInstances` adapter — the same deliberately-untested posture as the
+kubernetes provider's `New()`.
+
+Worth noting: gopls suggested replacing `proto.String(x)` with Go 1.26's
+`new(x)` expression; left as `proto.String` — it is the universal
+protobuf-construction idiom and matches every example in the SDK docs.
+Registry wiring (`"gcp": gcp.New`) happens at the composition root in
+Step 10, as designed in Step 2. `docs/architecture.md` §5 now shows both
+providers' call mappings.
+
+## Step 9 — Orphan GC + metrics
+
+Implemented `internal/gc/gc.go` (the `Sweeper` Runnable) and
+`internal/metrics/metrics.go` (explicit-registration metric set), plus the
+`provider.WithMetrics` decorator deferred from Step 2 into
+`internal/provider/metrics.go`, and the observation hooks in the health
+engine and discovery server.
+
+**GC**, per the plan: `NeedLeaderElection() = true` (destructive ⇒ single
+writer), 10 min ticker with the first sweep one full interval after start,
+per-provider `ListByTag` with log-and-continue on provider errors, and a
+kill requires all three of: our UID label present, older than `MinAge`
+(10 min), and the UID matching no existing CR — where a CR with a
+`deletionTimestamp` still counts as live (its finalizer owns that
+deletion; GC racing it would double-delete). Two safety behaviors worth
+naming: a failed Proxy `List` skips the whole sweep (an unreadable live
+set proves nothing orphaned), and the namespace-scope guard makes `Start`
+refuse to run against a namespace-restricted cache unless
+`--gc-allow-namespaced` is explicit, with the flag named in the error.
+One deviation of record: the plan says kills log "at Warn", but logr has
+no Warn level — kills log at Info with a `WARNING:` prefix carrying
+provider, providerID, and UID, same convention as the discovery server's
+empty-token warning.
+
+**Metrics**, per the plan: no `init()` — `Metrics.Register(reg, phases,
+leases)` is called explicitly by the composition root (Step 10), which is
+also what lets every test use a fresh registry (asserted by a test that
+registers two sets on two registries). `proxy_operator_proxies{phase}`
+and `proxy_operator_leases_active` are scrape-time collectors fed by
+closures — a reconcile-incremented gauge drifts and leaks series on
+delete; reading the source of truth at scrape time cannot. The
+probe vectors observe **every** probe (status stays transition-only;
+metrics carry the high-frequency signal), and the health engine calls
+`ForgetProxy` when it prunes a state entry so per-proxy series don't leak.
+`provider_requests_total{provider,op,result}` comes from the
+`WithMetrics` decorator — the one place `Class()` is called purely for
+observability, with results labelled ok / not_found / quota_exceeded /
+permanent / transient.
+
+**Decoupling shape:** each consuming package defines its own small
+recorder interface (`health.ProbeMetrics`, `discovery.LeaseMetrics`,
+`provider.RequestRecorder`); `metrics.Metrics` satisfies all of them
+structurally. Only `cmd/main.go` will import `internal/metrics`.
+
+Tests (`-race -count=2` clean): GC's true-orphan matrix in one sweep
+(live kept, deleting-CR kept, young kept, unlabelled kept, orphan
+deleted), broken-provider isolation, list-failure skips sweep, the
+namespace guard both ways, and the Start loop sweeping then stopping;
+metrics via `prometheus/testutil` — `GatherAndCompare` on the scrape-time
+collectors, ForgetProxy dropping series, outcome/result label counts;
+the decorator's five-way classification table with error passthrough
+asserted. Coverage: gc 86.1%, metrics 95.0%, provider up to 97.1%;
+health/discovery re-ran green with the hooks in place.
+
+```bash
+go test -race -count=2 ./internal/gc/ ./internal/metrics/ ./internal/provider/ ./internal/health/ ./internal/discovery/
+make test    # whole repo green
+```
+
+Worth noting: `prometheus/client_golang` was already in the module via
+controller-runtime's metrics server, so no new dependency — `go mod tidy`
+just promoted it to direct. `docs/architecture.md` gained §8 (GC sweep)
+and §9 (metrics shape).
+
+## Step 10 — Wiring, config, docs
+
+The composition root and everything around it. `cmd/main.go` now: parses
+the plan's flag set (plus `--gc-allow-namespaced` from Step 9), loads the
+provider config first and fails fast, builds the registry with
+`{"kubernetes": kubernetes.New, "gcp": gcp.New}`, wraps every provider in
+`provider.WithMetrics`, then adds the lease store, health engine,
+discovery server, and GC sweeper to one manager and hands the reconciler
+its providers + health snapshotter + events channel. Metrics register on
+controller-runtime's global registry with scrape-time closures (phase
+counts from the cache, active leases summed from `store.Counts()`).
+
+Two cache decisions became concrete here:
+
+- The Secret cache is restricted to Secrets labelled
+  `crawl.example.com/cloud-init=true` (new constant
+  `v1alpha1.LabelCloudInit`) — the operator holds cluster-wide Secret
+  read RBAC, and an unrestricted cache would hold every Secret in scope.
+  Consequence documented in the README: an unlabelled referenced Secret
+  is invisible → `CloudInitError`.
+- `--proxy-namespace` restricts the whole cache via `DefaultNamespaces`
+  and flips the GC sweeper's `NamespaceRestricted` guard.
+
+Manifests: `config/manager/manager.yaml` gained the
+`--providers-config` arg, the optional `DISCOVERY_TOKEN` secretKeyRef
+(`optional: true` — without the Secret the API runs unauthenticated with
+its loud warning), the ConfigMap volume mount, and containerPort 8090;
+new `config/manager/providers_config.yaml` (kubernetes-only default) and
+`config/default/discovery_service.yaml`. RBAC: the pods marker landed in
+the controller RBAC block (cluster-scoped role — the kubernetes
+provider's ListByTag spans namespaces). The plan's events RBAC was
+deliberately omitted: nothing wires an EventRecorder, and unused verbs
+are lint noise — recorded in Decisions.
+
+Samples: `proxy_kubernetes.yaml` / `proxy_gcp.yaml` (with a working
+Squid-installing cloud-init) / `proxy_external.yaml` replace the scaffold
+placeholder; `providers-config.yaml` documents both provider blocks;
+`hack/providers-dev.yaml` + a new `run-dev` Makefile target run locally
+with plain-HTTP metrics:
+
+```bash
+make run-dev
+# go run ./cmd/main.go --providers-config hack/providers-dev.yaml \
+#   --metrics-bind-address :8080 --metrics-secure=false
+```
+
+Docs: README rewritten per the plan (60-second architecture, kind
+quickstart, in-cluster deploy incl. token Secret creation, GCP setup with
+the IAM roles, the two prominent caveats, the no-substitutions version
+pins note). `docs/architecture.md` gained the components table and the
+full Decisions section — the plan's listed decisions plus everything
+accumulated in this log (RequeueNow, quota≠Failed, 409 arithmetic,
+TTL-400-not-clamp, GCP required placement, unknown-status→Stopped,
+banned==rate_limited window, GC logging convention, the Secret label
+contract, events-RBAC omission, logr-not-slog). `CHANGELOG.md` got its
+first entry with a real timestamp.
+
+Verified: `make test` green across the repo (coverage unchanged),
+`bin/kustomize build config/default` and `config/samples` render clean,
+`make build` produces the binary, e2e-tagged build + vet clean. The kind
+end-to-end run is deliberately still ahead — it is the Verification
+step's job, after Step 11 closes the remaining test gaps.
+
+Worth noting: `make run-dev` passes `--metrics-secure=false` because the
+scaffold's secure-serving default requires authn/authz reachability that
+a local process doesn't have; in-cluster deployments keep the secure
+default from the kustomize patch. The `providers` map wrapping happens
+*before* any consumer sees it, so the reconciler and GC only ever hold
+instrumented providers.
+
+## Step 11 — Tests
+
+Most of the plan's Step 11 list was deliberately front-loaded into the
+step that built each component (the action-table suite, computePhase and
+SpecHash tables, lease-store matrix incl. the concurrent `-race` case,
+discovery httptest suite, health threshold/CONNECT tests, GCP
+`buildInsertRequest` + classification, name-derivation tests from
+Step 2). This step closed what remained — the envtest-only coverage —
+and audited the list item by item.
+
+Added to `internal/controller/proxy_controller_test.go`:
+
+- **The CEL cases only a real API server can test** (fake clients run
+  neither CEL nor structural defaulting): six invalid-create rejections
+  (Managed-without-provider, External-with-provider,
+  External-without-endpoint, Managed-with-endpoint, cloudInit
+  both/neither), mode-mutation rejection, provider mutation *and removal*
+  rejection (the `has(self.x)==has(oldSelf.x)` form exists exactly for
+  the removal case), and the `+kubebuilder:default={}` assertion — a
+  Proxy created with no `healthCheck` comes back with every nested
+  default materialized, plus port and maxLeases defaults.
+- **Ready-through-health**: Managed proxy walks to Running (phase still
+  Provisioning — "no health verdict yet must not be Ready"), then a fake
+  `HealthSnapshotter` supplies a healthy snapshot and the phase flips to
+  Ready with latency in status.
+- **Quota + permanent, envtest edition**: quota → condition
+  QuotaExceeded, `RequeueAfter = QuotaRetry`, nil error, phase *not*
+  Failed; then permanent → Failed and the generation latch provably stops
+  further provider calls.
+- **Adopt**: strip the spec-hash annotation off a Running proxy (as an
+  operator upgrade with a changed hash-input struct would), reconcile,
+  and assert the annotation is restored byte-identical, the providerID
+  unchanged, and zero provider deletes.
+
+One repo-wide change: `make test` now runs with `-race` (the plan's
+"everything runs with -race" was previously only true of the manual
+verification runs, not the canonical target):
+
+```make
+go test -race $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+```
+
+Everything green on the first run of the new specs; full suite ~9 s for
+the controller package under race, `-short` still skips envtest in 0.6 s.
+
+Worth noting: the provider-removal CEL test has a subtlety worth keeping —
+removing `provider` alone would also trip the required-iff rule, so the
+test flips mode and adds an endpoint in the same update to isolate the
+immutability rules as the thing that rejects. The plan's remaining
+checklist item is Verification: the throwaway-kind-cluster run of the
+README quickstart, then push + MR.
+
+## Verification — kind end-to-end
+
+Static checks first (`go vet ./...`, `make build`, full `make test` with
+`-race`): all green. Then the real thing, per the spec's §13 "run the kind
+quickstart yourself and fix what breaks" — and two things broke, both now
+fixed.
+
+**Finding 1 — `make run-dev` cannot produce a Ready proxy on kind.** The
+operator on the host provisions the pod fine (Provisioned=True, IP
+published), but the health probe originates on the host, and kind pod IPs
+(10.244.x.x) are not host-routable — every probe fails by construction
+and the proxy latches `Unhealthy`:
+
+```text
+Healthy=False: Get "https://www.gstatic.com/generate_204":
+  proxyconnect tcp: dial tcp 10.244.0.5:3128: connect: connection refused
+```
+
+Everything around the failure worked exactly as designed (thresholds,
+condition, phase, and the finalizer delete ran clean from the host). Fix:
+the README quickstart now deploys the operator **in-cluster**
+(docker-build → kind load → deploy → port-forward 8090), with the
+run-dev limitation documented in both the quickstart and the Development
+section.
+
+**Finding 2 — Squid was OOM-killed at startup in-cluster.** With the
+operator deployed in-cluster the pod crash-looped (`OOMKilled`, empty
+logs). Root cause: squid sizes its file-descriptor tables from
+`RLIMIT_NOFILE`, and containerd under kind sets that effectively
+unlimited (~10^9) — squid allocates gigabytes before it ever listens.
+Fix in the generated config (`internal/provider/kubernetes/pod.go`):
+`max_filedescriptors 1024` (the load-bearing line) plus `cache_mem 16 MB`
+(a crawling forward proxy gains nothing from squid's 256 MB default),
+with a regression assertion added to `pod_test.go`.
+
+**With both fixes, the full pass:**
+
+```bash
+kind create cluster --name proxy-operator-demo
+make install
+make docker-build IMG=egress-proxies-operator:dev
+kind load docker-image egress-proxies-operator:dev --name proxy-operator-demo
+make deploy IMG=egress-proxies-operator:dev
+kubectl apply -f config/samples/proxy_kubernetes.yaml
+# → Ready 10.244.0.9 lat=89ms Provisioned=True Healthy=True   (~30 s)
+kubectl -n egress-proxies-operator-system port-forward svc/...-discovery-service 8090:8090 &
+curl -s 'localhost:8090/v1/proxies?healthy=true'          # count:1, latencyMillis:89
+curl -s -XPOST localhost:8090/v1/leases -d '{"selector":{"geo":"local"},"ttlSeconds":300}'
+#   → 201 {leaseID, proxy(activeLeases:1), expiresAt, ttlSeconds:300}
+curl -XPOST .../report -d '{"result":"rate_limited","target":"example.com"}'   # 204
+curl -XPOST /v1/leases -d '{...,"target":"example.com"}'  # 409 {inCooldown:1} ✓
+curl -XDELETE /v1/leases/<id>   # 204, and 204 again ✓
+kubectl delete -f config/samples/proxy_kubernetes.yaml    # finalizer: pod Terminating, CR gone
+kind delete cluster --name proxy-operator-demo
+```
+
+A real Squid pod went Ready through a real CONNECT probe, a lease was
+held on it, the cooldown machinery answered a 409 with correct
+arithmetic, and the finalizer cleaned up — the plan's success bar, met
+with the actual product.
+
+Worth noting: `make deploy` runs `kustomize edit set image` and mutates
+`config/manager/kustomization.yaml` in the working tree — reverted before
+committing (the repo keeps the pinned stanza). The health probe's ~89 ms
+latency is gstatic-through-squid from a kind pod on this machine;
+metrics-side observations were not separately checked in-cluster (covered
+by unit tests).

docs/plans-executions/2026-08-11-1742-gcp-provider-verbose-logging.md

@@ -0,0 +1,42 @@
+# Execution: Verbose (V-level) logging in the GCP provider
+
+Plan: `docs/plans/2026-08-11-1742-gcp-provider-verbose-logging.md`
+
+- [x] Step 0 — Save and commit the plan
+- [x] Step 1 — V(1)/V(2) logging in `internal/provider/gcp` (gcp.go, errors.go)
+- [x] Step 2 — Tests (verbosity tiers, error detail, cloud-init leak guard)
+- [ ] Step 3 — CHANGELOG entry (after the user confirms it works live)
+
+## Step 1 — logging in the provider
+
+Went as planned: context-carried logger (`logf.FromContext(ctx).WithName("gcp")`),
+V(1) one line per API call, V(2) request/list detail, opNames captured from the
+`instancesAPI` seam instead of being discarded. `logAPIError` lives in
+`errors.go` (next to `classify`, whose imports it shares) rather than `gcp.go`
+as loosely implied by the plan — same package, so no behavioural difference.
+These are the first `.V(n)` calls and the first logging import anywhere under
+`internal/provider/`.
+
+Worth noting: the gopls `errorsastype` suggestion fired on the new
+`errors.As` in `logAPIError` (Go's newer `errors.AsType`); kept `errors.As`
+for consistency with the three existing uses in the same file. Same for the
+`newexpr` (`proto.String` → `new`) suggestions — the codebase consistently
+uses `proto.String`.
+
+## Step 2 — tests
+
+`funcr.New` as the capturing sink, injected via `logr.NewContext`, exactly the
+seam the plan predicted. One deviation: instead of a single
+`TestLogging_verbosity` table, it split into three tests — `_verbosityTiers`
+(table over V=0/1/2, incl. the cloud-init sentinel leak assertion),
+`_apiErrorKeepsHTTPDetail` (403 quotaExceeded keeps `httpStatus`/reason at
+V(1)), and `_treatedAsSuccessPathsAreExplicit` (409-on-create /
+404-on-delete each log their "treated as success" line) — the last two
+exercise fake error wiring that didn't fit the tier table cleanly.
+
+Verified with:
+
+```bash
+go test -race ./internal/provider/gcp/
+go test ./...
+```

docs/plans-executions/2026-08-11-1802-bake-commit-version.md

@@ -0,0 +1,61 @@
+# Execution: Bake the git commit into the operator binary and log it at startup
+
+Plan: `docs/plans/2026-08-11-1802-bake-commit-version.md`
+
+- [x] Step 0 — Save and commit the plan
+- [x] Step 1 — `internal/version` package + tests
+- [x] Step 2 — `cmd/main.go`: `--version` flag + startup log line
+- [x] Step 3 — Makefile `GIT_COMMIT` + `--build-arg` wiring
+- [x] Step 4 — Dockerfile `-ldflags` stamp + OCI revision label
+- [ ] Step 5 — CHANGELOG entry (after the user confirms it works live)
+
+## Steps 1–4
+
+Mostly as planned. One deviation worth recording: the plan claimed
+`build`/`run` targets need no changes because Go's automatic VCS stamp covers
+host builds — that turned out to be only half true. Go skips VCS stamping
+when the build target is a *file argument* rather than a package pattern, and
+the Makefile's `build` target used `go build -o bin/manager cmd/main.go`.
+Verified empirically:
+
+```bash
+go build -o bin/manager cmd/main.go && ./bin/manager --version   # unknown (no vcs settings)
+go build -o bin/manager ./cmd       && ./bin/manager --version   # e4d2a191d0c2-dirty
+```
+
+So `build:` now uses `go build -o bin/manager ./cmd`. `go run` never stamps
+VCS info regardless of invocation form — `make run`/`run-dev` print
+`commit=unknown`, which is acceptable for dev loops (the Dockerfile path uses
+the explicit ldflags stamp and is unaffected; it kept `cmd/main.go`).
+
+The ldflags path was verified independently:
+
+```bash
+go build -ldflags "-X gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version.Commit=deadbeef1234" \
+  -o bin/manager-stamped cmd/main.go
+./bin/manager-stamped --version   # deadbeef1234
+```
+
+Worth noting: `cmd/main.go` imports k8s apimachinery as `runtime`, so the
+stdlib runtime needed an alias (`goruntime "runtime"`) for
+`goruntime.Version()` in the startup line. The `--version` check happens
+right after `flag.Parse()`, before logger and manager setup, so it works
+without a kubeconfig.
+
+## Verification
+
+```bash
+go vet ./... && go test ./...           # all green, incl. new resolve() table tests
+go build -o bin/manager ./cmd && ./bin/manager --version   # e4d2a191d0c2-dirty
+```
+
+The image-level check initially failed (Docker daemon not running); after
+the daemon was started it passed in full:
+
+```bash
+make docker-build IMG=egress-proxies-operator:dev
+docker run --rm egress-proxies-operator:dev --version
+# ae434a7167ec-dirty  (matches git rev-parse --short=12 HEAD + untracked files)
+docker inspect egress-proxies-operator:dev --format '{{index .Config.Labels "org.opencontainers.image.revision"}}'
+# ae434a7167ec-dirty
+```

docs/plans-executions/2026-08-11-1838-gcp-http-wire-logging-v5.md

@@ -0,0 +1,63 @@
+# Execution: GCP HTTP wire logging at V(5)
+
+Plan: `docs/plans/2026-08-11-1838-gcp-http-wire-logging-v5.md`
+
+- [x] Step 1 — `wireLogger` + `option.WithLogger` wiring in `internal/provider/gcp/gcp.go`
+- [x] Step 2 — Tests (`TestWireLogger_gatesAtV5`, `TestWireLogger_infoLandsAtV1`)
+- [x] Step 3 — Live verification at `--zap-log-level=5` (user, on cluster)
+- [x] Step 3b — Post-verification fix: drop auth records, elide huge fields
+- [ ] Step 4 — CHANGELOG entry (after live confirmation of 3b; batch with the
+      two earlier pending entries: GCP V-logging, version stamp)
+
+## Steps 1–2
+
+Went exactly as planned — the whole feature is ~10 lines of production code
+because both halves already existed: the compute SDK logs full HTTP
+request/response records at slog Debug to an injectable logger, and
+`logr.ToSlogHandler` does the slog→logr bridging. The only real design
+content is the level shift (`base.V(1)` + slog-Debug's +4 = V(5)) and the
+startup warning line when V(5) is active (raw payloads include cloud-init
+user-data, which the curated V(2) logging deliberately hides).
+
+Worth noting for future readers:
+
+- `option.WithLogger` **disables** `GOOGLE_SDK_GO_LOGGING_LEVEL` for this
+  client (documented SDK precedence) — `--zap-log-level` is now the only knob
+  for GCP wire logs.
+- The V(5) check in `New` runs once at startup; that is sound because the zap
+  level is fixed by flags at process start.
+- Added `TestWireLogger_infoLandsAtV1` beyond the plan's table — it pins the
+  shift arithmetic from the other side (slog Info → V(1)), so a future logr
+  mapping change would fail loudly.
+
+Verified with:
+
+```bash
+go test -race ./internal/provider/gcp/
+go build ./... && go test ./...
+```
+
+## Step 3b — what live verification exposed, and the fix
+
+Live V(5) output revealed two problems the plan missed:
+
+1. **Security: the injected logger propagates into `cloud.google.com/go/auth`**,
+   which logs its own token exchange (`auth.go:571/576`) — signed JWT
+   assertion in the request, full bearer access token in the response. The
+   plan's "auth token is safe" analysis only covered the compute client's
+   request headers, not the auth library's own records. Fix: `wireLogger`
+   now wraps the handler in a filter that drops every Debug record except
+   the compute client's `"api request"`/`"api response"` (allowlist, so
+   future SDK additions fail closed); Warn/Error still pass through.
+2. **Readability: GCP responses embed multi-KB blobs** (Shielded-VM UEFI
+   dbx databases) that swamp the line. Fix: string fields >1KiB are elided
+   to `[elided N bytes]` by default, recursively through payload
+   maps/arrays. Opt-out via new manager flag
+   `--gcp-wire-log-full-payloads` (threaded through a constructor closure
+   in `cmd/main.go` → `gcp.NewWithWireOptions`; the `registry.Constructor`
+   signature stays unchanged). Chosen by the user: elision on by default,
+   verbatim available on demand. Auth records are dropped in both modes.
+
+The filter/elision logic lives in `internal/provider/gcp/wirelog.go` with
+tests covering: auth-record drop (both modes), elision marker + small-field
+preservation, verbatim mode, and the original V(5) gating.

docs/plans-executions/2026-08-11-1935-gitea-build-workflow.md

@@ -0,0 +1,53 @@
+# Execution: Distilled Gitea Actions image-build workflow
+
+Plan: `docs/plans/2026-08-11-1935-gitea-build-workflow.md`
+
+- [x] Step 1 — Create `.gitea/workflows/build.yaml`
+- [x] Step 2 — Replace CLAUDE.md CI TODO with a CI/CD subsection
+- [x] Step 3 — Push branch + open MR
+- [ ] Step 4 — CHANGELOG entry (after the first successful run is confirmed)
+
+## Steps 1–2 — workflow + CLAUDE.md
+
+The workflow distills the house pattern from 9 sibling projects (survey in the plan)
+plus improvements none of them combine: an immutable `sha-<12>` tag, a lightweight
+test gate, `:latest` moving only on real tag pushes, and a `concurrency` group.
+
+Work happened in a git worktree off `origin/main` so the main checkout (which had
+unrelated uncommitted changes) stayed untouched:
+
+```bash
+git worktree add -b feat/gitea-build-workflow \
+  "$SCRATCH/wt-build" origin/main
+```
+
+Deviation from the plan's assumptions: while planning, `feat/proxy-operator` was
+still unmerged and the plan noted `main` lacked `internal/`/`test/`. By execution
+time `origin/main` had moved (`076bc66..f7000f7` — the proxy-operator MR merged),
+so the branch and its CI gate cover the full operator code.
+
+Verified the workflow parses and the check-gate commands pass on this exact tree
+(ruby stands in for a YAML linter because the system python3 has no `yaml` module):
+
+```bash
+ruby -ryaml -e "YAML.load_file('.gitea/workflows/build.yaml'); puts 'YAML OK'"
+go vet ./... && go build ./... && go test -short ./...   # all packages ok
+```
+
+## Step 3 — push + MR
+
+Branch pushed and MR opened with `tea` (the worktree was then removed and the main
+checkout switched onto the branch so the files are visible locally):
+
+```bash
+tea pr create --title "Add Gitea Actions image-build workflow" \
+  --description "..." --base main --head feat/gitea-build-workflow
+# → https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/pulls/2
+```
+
+Worth noting: the workflow itself cannot run end-to-end until (a) the MR merges
+(it only triggers on tags / manual dispatch, not branch pushes) and (b) the
+`REGISTRY_TOKEN` secret is created in this repo's Gitea settings (PAT with
+`write:package`). First real verification = manual dispatch with tag
+`manual-test`, expecting `manual-test` + `sha-…` in Packages and `:latest`
+untouched.

docs/plans-executions/2026-08-11-2152-discovery-api-docs.md

@@ -0,0 +1,38 @@
+# Execution: Discovery API documentation
+
+Plan: [2026-08-11-2152-discovery-api-docs.md](../plans/2026-08-11-2152-discovery-api-docs.md)
+
+- [x] Step 1 — Write `docs/api.md` full API reference
+- [x] Step 2 — Add pointers in README and architecture.md
+
+## Step 1 — docs/api.md
+
+Wrote the full reference: base URL (in-cluster FQDN + port-forward), bearer
+auth, error envelope, configuration table, all five routes with schemas,
+status codes and curl examples, the selection/cooldown semantics section,
+an end-to-end curl walkthrough, and the in-memory/single-replica caveats.
+All facts were taken from the code, not from memory of prior docs.
+
+Worth noting: the doc explicitly calls out two things no earlier doc
+stated for clients — that a report **without** a target on a targetless
+lease creates a *global* cooldown (blocking the proxy for everyone), and
+that `rate_limited` and `banned` currently behave identically. Both came
+straight from `internal/lease/store.go` and are easy to trip over.
+
+## Step 2 — Pointers + verification
+
+Added one-line links to the new doc in README's quickstart (above the curl
+block) and in `docs/architecture.md` §7. Verified the documented behavior
+against the tree rather than trusting the write-up:
+
+```sh
+go build ./... && go test -short ./internal/discovery/ ./internal/lease/
+```
+
+Both pass; a grep of `server_test.go` confirmed every documented status
+code and error code (`invalid_ttl`, `invalid_query`, `invalid_result`,
+`no_match`, `unknown_lease`, 201/204/401/404/409) is asserted by tests.
+
+Worth noting: CHANGELOG entry deliberately deferred until the user
+confirms the docs read well, per the CHANGELOG convention's
+"once the user confirms it works" clause.

docs/plans-executions/2026-08-11-2220-demo-scripts.md

@@ -0,0 +1,55 @@
+# Execution: Demo scripts
+
+Plan: [2026-08-11-2220-demo-scripts.md](../plans/2026-08-11-2220-demo-scripts.md)
+
+- [x] Step 0 — Branch `feat/demo-scripts` + plan commit
+- [x] Step 1 — `docs/demo/show-egress-ips.sh`
+- [x] Extra (added iteratively, not in the original plan) — proxy-creation scripts
+
+## Step 0 + Step 1
+
+Branched off `main`, committed the plan alone, then wrote
+`docs/demo/show-egress-ips.sh`: takes `BASE_URL` as its argument, lists
+`/v1/proxies` (bearer auth via optional `TOKEN` env), probes each healthy
+proxy with `curl -x http://ip:port` against an IP-echo site
+(`IP_ECHO_URL`, default ipify JSON), banners which proxy each request goes
+through, skips unhealthy ones, and ends with a probed/skipped/failed
+summary. Per user request the script was left uncommitted for iteration
+and no verification beyond `bash -n` was run.
+
+## Extra — create-kubernetes-proxies.sh, create-gcp-proxies.sh
+
+Added on the same branch before the first commit:
+
+- `create-kubernetes-proxies.sh <count>` — creates
+  `proxy-kubernetes-demo-1..N` with the kubernetes provider, spec taken
+  from `config/samples/proxy_kubernetes.yaml`, applied via
+  `kubectl apply -f -` heredocs. Optional `NAMESPACE` env.
+- `create-gcp-proxies.sh <count>` — creates `proxy-gcp-demo-1..N` from the
+  user-supplied gcp-eu manifest (e2-micro, Ubuntu 24.04, Squid
+  cloud-init), each with a zone picked randomly from a hardcoded list of
+  27 EU zones so the fleet gets egress IPs from different locations.
+  Env overrides: `ZONES`, `GCP_PROVIDER` (default `gcp-eu`), `NAMESPACE`.
+
+## Extra — run-demo.sh, show-egress-ips-table.sh
+
+Second round of iterative additions:
+
+- `run-demo.sh` — tmux demo driver: 2x2 tiled grid with the egress-IP
+  table looping every 10s inside a netshoot pod (script `kubectl cp`'d
+  into the pod, `BASE_URL` set to the in-cluster Service FQDN),
+  `watch -n3 kubectl get px`, and both create scripts auto-running with
+  `COUNT` proxies each (default 4). Env knobs: `COUNT`, `SESSION`,
+  `NETSHOOT_POD`, `DEMO_DIR` (defaults to the script's own dir).
+- `show-egress-ips-table.sh` — condensed one-line-per-proxy variant of
+  `show-egress-ips.sh` sized for a tmux pane: PROXY / ENDPOINT /
+  LOCATION (zone→geo attribute fallback) / EGRESS-IP columns, with
+  `(unhealthy)` and `FAILED` inline instead of verbose output. The
+  verbose script stays for standalone use; the demo driver uses the
+  table variant.
+
+Worth noting: beyond the user's sample manifest, the gcp script also
+writes the picked zone into `attributes.zone`, so the discovery API
+exposes each proxy's location and leases can select on it. The zone list
+is static — if a project lacks quota in some region, `ZONES` narrows the
+pool; nothing validates zones against the live project.

docs/plans-executions/2026-08-24-1025-otel-tracing.md

@@ -0,0 +1,183 @@
+# Execution log: OpenTelemetry tracing integrated with logr/zap logging
+
+Plan: `docs/plans/2026-08-24-1025-otel-tracing.md`
+
+- [x] Step 1 — Dependencies
+- [x] Step 2 — New package `internal/tracing`
+- [x] Step 3 — `provider.WithTracing` decorator
+- [x] Step 4 — `cmd/main.go` wiring
+- [x] Step 5 — Reconciler spans
+- [x] Step 6 — Discovery server
+- [x] Step 7 — GC + health
+- [x] Step 8 — GCP wire-log enrichment
+- [x] Step 9 — Manifests + docs
+
+## Step 1 — Dependencies
+
+Aligned the pre-existing indirect skew (otel core v1.44.0 vs otlptrace exporters
+v1.40.0) and added the new direct deps in one shot:
+
+```bash
+go get go.opentelemetry.io/otel@v1.45.0 \
+  go.opentelemetry.io/otel/sdk@v1.45.0 \
+  go.opentelemetry.io/otel/trace@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/stdout/stdouttrace@v1.45.0 \
+  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0
+go mod tidy
+```
+
+MVS side-effects worth recording: `logr v1.4.3→v1.4.4`, `httpsnoop
+v1.0.4→v1.1.0`, `grpc-gateway/v2 v2.27.7→v2.29.0`, `proto/otlp v1.9.0→v1.11.0`,
+plus a `genproto/googleapis/api` pseudo-version bump. Full `make test` passed
+against the bumped graph (`google.golang.org/api v0.292.0` and k8s v0.36
+tolerate otelhttp v0.70.0).
+
+Worth noting: the first `go mod tidy` ran *before* any first-party code
+imported `otlptracehttp`/`stdouttrace`, so it silently dropped those two
+modules again; the Step 2 tidy re-added them. The plan's semconv question
+resolved to `semconv/v1.43.0` — that's what `sdk@v1.45.0/resource/builtin.go`
+imports, so first-party code uses the same version to avoid
+`ErrSchemaURLConflict` in the common path.
+
+## Step 2 — `internal/tracing` package
+
+Landed as planned: `tracing.go` (env-gated `Setup`, hand-rolled exporter
+selection), `logger.go` (`Start`/`StartSpan`/`ContextWithLogger` with the
+base-logger ctx key that prevents duplicate `traceID` zap fields on nested
+spans), `reconciler.go`, `transport.go`, `http.go`, `options.go`; tests for
+all of it (first use of `sdk/trace/tracetest` in the repo).
+
+Two deviations from the plan's letter:
+
+- **k8s transport gating is a custom RoundTripper, not `otelhttp.WithFilter`**
+  (`parentGatedTransport`): requests without a parent span bypass the otel
+  transport entirely, so the no-root-spans-from-informers guarantee doesn't
+  depend on otelhttp filter semantics for transports.
+- **`HTTPMiddleware` must copy the route pattern back.** The logger-injecting
+  inner handler wraps the request via `WithContext` (a shallow copy), so the
+  mux records the matched pattern on the copy while otelhttp's post-routing
+  span rename reads the original. Found by the middleware test (span named
+  `"GET"` instead of `"GET /v1/things/{id}"`); fixed with `r.Pattern =
+  r2.Pattern` after `next.ServeHTTP`.
+
+Also: both the transport and the middleware pass explicit W3C propagators
+instead of relying on the global, so behavior is deterministic under tests
+and when tracing is disabled. `Setup` tests reset the global provider to a
+fresh noop per case — restoring otel's own default delegate triggers a
+"Setting tracer provider to its current value" warning from the SDK.
+
+## Step 3 — `provider.WithTracing`
+
+`internal/provider/tracing.go` mirrors `metrics.go` exactly (same wrap shape,
+same `resultLabel` classification reused as the `provider.result` span
+attribute); tests reuse `staticProvider` from `metrics_test.go`. `ErrNotFound`
+maps to span status Ok as planned — the reconciler polls `Get` to NotFound
+during replacement/deletion, so it's an answer, not a failure.
+
+One small API addition to `internal/tracing` for this: exported
+`tracing.Tracer(opts...)`, since the decorator lives in package `provider`
+and couldn't reach the unexported option resolver. The `cmd/main.go` wiring
+(`WithTracing` outermost around `WithMetrics`) lands with Step 4's commit —
+same file, one commit.
+
+## Step 4 — `cmd/main.go` wiring
+
+As planned: `tracing.Setup` right after `ctrl.SetLogger`; manager rest.Config
+wrapped via `restCfg := ctrl.GetConfigOrDie(); restCfg.Wrap(...)` (the
+previous inline call left nowhere to wrap); providers decorated tracing-
+outermost; explicit trace flush after `mgr.Start` returns on both the error
+and clean paths, with a fresh 10s context since the signal ctx is already
+cancelled by then.
+
+The kubernetes provider grew `NewWithTransportWrapper(ctx, cfg, wrap)` (its
+client is built from its own `ctrl.GetConfig()`, invisible to the manager's
+wrapped config); `New` now delegates with a nil wrapper, so `newWithClient`
+tests stayed untouched, and main registers a closure — the same pattern the
+gcp constructor already used for wire-log options.
+
+Deviation: the plan put `--trace-health-probes` here, but the flag needs the
+`health.Engine.TraceProbes` field that Step 7 introduces — moved there to
+keep every commit compiling.
+
+## Step 5 — Reconciler spans
+
+`SetupWithManager` completes with `tracing.NewReconciler("Proxy", r)`;
+sub-spans `reconcile.managed` / `reconcile.replaceInstance` /
+`reconcile.delete` open at the top of each state machine, and `status.patch`
+opens inside the deferred flush closure so it stays within the root span
+while its error still folds into the recorded result. `reconcileExternal`
+left unspanned as planned (no I/O).
+
+Two small judgment calls: the sub-spans carry no extra attributes — the
+provider decorator already records `provider.id`, and the root span carries
+the object identity, so duplicating them was noise; and `status.patch` is
+emitted every reconcile even when nothing changed (the no-op compare is the
+span's content — a real PATCH shows up as its k8s HTTP child). Tests that
+call `r.Reconcile` directly bypass the wrapper; with no global tracer set
+they see no-op spans, so the existing fake-client and envtest suites run
+unchanged.
+
+## Step 6 — Discovery server
+
+Middleware chain is now recover → tracing (server span + request logger) →
+request-log → body cap → auth; the request-log middleware and the three
+handler error sites log via `logf.FromContext(r.Context())` — which is
+`s.log` enriched with the request's traceID/spanID by the tracing
+middleware, not a different logger (logr sinks can't read ctx at log time,
+so per-request values must ride on the logger instance in the ctx; user
+asked, answered in-session).
+
+Deviation from the plan's letter: `recoverMiddleware` keeps `s.log`. It sits
+*outside* the tracing layer, so its request ctx never has the enriched
+logger — switching it to `FromContext` would silently drop the `"discovery"`
+name and gain nothing.
+
+## Step 7 — GC + health
+
+`gc.sweep` runs in a root span with `gc.providers` / `gc.deleted` counters
+(a failed orphan delete no longer counts as deleted — the loop grew an
+explicit `continue`). The health engine got the deferred `--trace-health-probes`
+flag and `TraceProbes` field; the probe call moved into `runProbe`, which
+wraps it in a `health.probe` client span only when enabled, ending before
+`record` (which stays ctx-free by design). Probe transports remain
+uninstrumented so no traceparent can leak through a proxy to external
+targets. Probe→reconcile span links stay out of scope (GenericEvent carries
+no ctx; the workqueue coalesces events) — recorded in the architecture
+Decisions in Step 9.
+
+## Step 8 — GCP wire-log enrichment
+
+`wireFilterHandler.Handle` clones the record and appends `traceID`/`spanID`
+when the ctx carries a valid span — same keys as the logr enrichment, added
+before the elision pass (both values are well under the 1KiB threshold).
+The new `wirelog_test.go` covers the enrichment, that the JWT/token security
+filter still drops non-wire Debug records even with a span present, and that
+spanless records stay unchanged.
+
+## Step 9 — Manifests, docs, lint
+
+`config/manager/manager.yaml` gained the OTel env block (downward-API
+`POD_NAME`/`POD_NAMESPACE` deliberately listed *before*
+`OTEL_RESOURCE_ATTRIBUTES` — `$(VAR)` expansion only sees earlier vars),
+with the OTLP endpoint and sampler left as commented examples so deployments
+stay tracing-off by default. `docs/architecture.md` got "### 10. Tracing"
+plus five Decisions entries; README got a Tracing section.
+
+`make lint` surfaced 16 new-on-branch issues (checked with
+`golangci-lint run --new-from-rev main`; the other ~43 pre-date this work):
+
+- one gofmt slip, 13 goconst repeats — fixed with `env*`/`exporter*`
+  constants in `internal/tracing/tracing.go` shared by the tests;
+- two logcheck hits ("function takes both a context and a logger"):
+  `Setup` now reads its logger from ctx (`logf.FromContext`, caller seeds
+  via `logf.IntoContext` — the linter's sanctioned pattern), and
+  `ContextWithLogger` carries a justified `//nolint:logcheck`, since being
+  the IntoContext-analogue is its purpose.
+
+Worth noting: pre-existing lint debt (50 goconst etc. on `main`) was left
+untouched — `make lint` still fails overall; only branch-introduced issues
+were cleaned. The lint fixes changed `Setup`'s signature after Step 2/4 had
+landed, so those commits show the two-arg version; this commit is where it
+settles.

docs/plans-executions/2026-08-24-1224-tracing-e2e.md

@@ -0,0 +1,68 @@
+# Execution log: e2e test — OTel tracing against real Tempo
+
+Plan: `docs/plans/2026-08-24-1224-tracing-e2e.md`
+
+- [x] Step 1 — `test/e2e/tracing_test.go`
+- [x] Step 2 — Makefile
+- [x] Step 3 — Docs
+
+## Steps 1–3 — spec, Makefile, docs (one commit)
+
+The three steps landed together — the spec is one new file and the other
+two are its wiring. `Describe("OTel tracing", Ordered)` is fully
+self-contained (own ns create → `make install`/`make deploy` → teardown)
+because Ginkgo randomizes top-level container order, so it cannot share the
+Manager Describe's deployment. It reuses the package-level `namespace` /
+`managerImage` and the suite's idioms (`utils.Run`, curl-pod with the
+restricted-PSS overrides JSON, log-substring `Eventually`s).
+
+Judgment calls beyond the plan's letter:
+
+- The squid pre-pull (`docker pull` + `kind load`) is **best effort** — a
+  missing docker binary logs a note and continues rather than failing the
+  spec; the 5m Ready timeout still covers an in-cluster pull.
+- The OTLP preflight pod prints per-attempt HTTP codes and a final
+  `OTLP_OK`/`OTLP_UNREACHABLE` marker; the assertion quotes the pod's
+  output, so an unreachable endpoint names itself in the failure.
+- `kubectl set env` is passed the literal
+  `OTEL_RESOURCE_ATTRIBUTES=...$(POD_NAME)...` string via `exec.Command` —
+  no shell involved, kubectl stores `$()` verbatim, and the in-place update
+  keeps the var after the downward-API vars it references.
+- Tempo helpers are stdlib-only; `/api/traces/<id>` is decoded as
+  OTLP-JSON (`batches[].scopeSpans[].spans[].name`), which is Tempo's
+  actual shape (not Jaeger's).
+
+Verified so far: `go vet -tags=e2e ./...` clean.
+
+## Live run against homelab Tempo
+
+First attempt failed before the suite started — Docker Desktop wasn't
+running (`kind` could not create the cluster), and the failure was masked
+to exit 0 by a `| tail` pipe on the make invocation (no pipefail in that
+shell). Rerun with Docker started first and no pipe:
+
+```bash
+TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e
+```
+
+Result: **SUCCESS — 4/4 specs (Manager smoke + both tracing Its), 219 s**,
+kind cluster auto-deleted. Run id `e2e-1787567624494810000`; the traces are
+findable in Grafana with TraceQL
+`{resource.test.run.id="e2e-1787567624494810000"}`. Both proxies reached
+`Ready` on real squid pods; the create trace carried
+`Reconcile Proxy` / `reconcile.managed` / `provider.create` /
+`status.patch` and the expected resource attrs; the deletion trace carried
+`reconcile.delete` / `provider.delete`.
+
+Worth noting: the OTLP preflight and both Tempo polls succeeded on the
+in-cluster → LAN path (kind on macOS reaches 192.168.0.30 through Docker
+Desktop's NAT), so no extra networking setup is needed on this machine.
+
+The plan's negative verification also ran: with a deliberately wrong
+endpoint (`OTLP_ENDPOINT=http://192.168.0.30:9999`) the tracing spec
+failed in the BeforeAll preflight after ~52 s with
+`OTLP endpoint http://192.168.0.30:9999 is not reachable from inside the
+kind cluster; curl output: OTLP_UNREACHABLE` — a named, fast failure
+instead of a 2-minute opaque search timeout — and teardown still deleted
+the kind cluster (`make cleanup-test-e2e` run explicitly, since a failing
+`go test` skips the Makefile's cleanup step).

docs/plans-executions/2026-09-04-1404-kvm-provider.md

@@ -0,0 +1,673 @@
+# Execution: KVM/libvirt provider
+
+Plan: [2026-09-04-1404-kvm-provider.md](../plans/2026-09-04-1404-kvm-provider.md)
+
+- [x] Step 0 — Save plan + branch `feat/kvm-provider`
+- [x] Step 1 — Config (`KVMConfig` & friends, `validate()` case, tests)
+- [x] Step 2 — Pure builders (`domainxml.go`, `volume.go`, `seed.go`, `id.go`, `metadata.go`)
+- [x] Step 3 — Transport + errors (`conn.go`, `errors.go`)
+- [x] Step 3b — Image cache (`image.go`)
+- [x] Step 4 — Provider methods (`kvm.go`) over a fake API
+- [x] Step 5 — Wiring, manifests, samples, docs, CHANGELOG
+- [x] Step 6 — Integration test against a real hypervisor (written and compiled; first real run is part of Step 7)
+- [x] Step 7 — Manual validation + MR
+
+## Step 0 — Save plan + branch
+
+Plan committed alone on `feat/proxy-operator`, then branched:
+
+```bash
+git checkout -b feat/kvm-provider
+```
+
+## Step 1 — Config
+
+The new structs (`KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage`) and
+`validateKVM` went into their own file, `internal/provider/config_kvm.go`,
+rather than being appended to `config.go` as the plan said: the KVM block has
+far more fields and rules than the other two providers, and keeping it
+separate leaves `config.go` readable. Tests likewise live in
+`config_kvm_test.go` instead of extending the table in `config_test.go`; the
+invalid-config table covers every required field, duplicate host names, image
+URL scheme, sha256 shape, and the disk format enum.
+
+While adding the `kvm` case to `validate()`, the pairwise "type is X but a Y
+block is set" checks (which would have needed six branches with three types)
+were replaced by a `setBlocks()` helper that lists the non-nil blocks and
+rejects any that don't match `type`. The error strings are unchanged, so the
+existing GCP/kubernetes tests still pass.
+
+Dependencies were fetched up front so the API signatures in the plan could be
+checked with `go doc`; they stay `// indirect` in `go.mod` until Step 2 imports
+them:
+
+```bash
+go get github.com/digitalocean/go-libvirt@latest github.com/kdomanski/iso9660@v0.4.0
+# go-libvirt resolved to v0.0.0-20260814190004-1a83157e1858 (no tagged release)
+# side effect: golang.org/x/crypto 0.54.0 → 0.56.0, x/mod, x/text, x/tools bumped
+```
+
+Worth noting: `go doc` showed that go-libvirt's `New(net.Conn)` is deprecated
+in favour of `NewWithDialer(socket.Dialer)`, where `Dialer` is a one-method
+`Dial() (net.Conn, error)` interface. Step 3 should wrap the SSH client's unix
+forward in that interface instead of using `New`; the plan's transport design
+is otherwise unaffected.
+
+## Step 2 — Pure builders
+
+`internal/provider/kvm/` now has the five builders from the plan plus
+`kvm.go` holding the package doc and `withDefaults` (disk size 10 GiB, the
+default vendor-data, `/var/run/libvirt/libvirt-sock`, pool `default`, image
+format `qcow2`). Everything is a pure function over `encoding/xml` structs or
+strings, unit-tested by unmarshalling the output back and checking fields.
+
+Two design choices differ from the plan text:
+
+- **Domain disks reference pool volumes, not host paths.** The plan had
+  `buildDomainXML` take the overlay and seed paths (needing two
+  `StorageVolGetPath` round-trips per Create). The domain XML now uses
+  `<disk type="volume"><source pool=... volume=.../>`, so only the *base*
+  image path is ever looked up (the overlay's `<backingStore>` needs a real
+  path). Fewer RPCs and the builder's inputs are all names.
+  **Reverted in Step 7**: volume-referenced disks cannot start on an
+  AppArmor host, see there. The domain XML is back to `type="file"` with
+  the two `StorageVolGetPath` calls the plan had.
+- **Capacities are always `unit="bytes"`.** libvirt's `G` means GiB but
+  `GB` means 10^9; spelling the byte count out removes the ambiguity from
+  both the overlay and the seed/base volumes.
+
+The metadata element is written with a default namespace
+(`<instance xmlns="…/kvm">`) because `encoding/xml` cannot emit prefixes on
+marshal. libvirt may hand it back prefixed; `parseInstanceMetadata` is
+tested against both spellings since `encoding/xml` matches on the namespace
+URI. Whether libvirt accepts the default-namespace form on `DomainDefineXML`
+is confirmed in Step 6 against a real host, not here. (Step 7: it does,
+but `DomainGetMetadata` returns the element with no namespace at all; the
+parser now matches on local name only.)
+
+Checked before relying on `github.com/kdomanski/iso9660` for the seed: its
+filename mangling lowercases and keeps `-`, so `user-data`/`meta-data`/
+`vendor-data` come out as `user-data;1` etc. in the directory record, which
+the Linux isofs driver (and therefore cloud-init's NoCloud mount) presents as
+`user-data`. The seed test opens the ISO with the same library's reader and
+asserts label `cidata`, all three files, and byte-identical output for
+identical inputs. `NewWriter()` stages files in a temp directory, so
+`Cleanup()` is deferred.
+
+`go mod tidy` ran as part of this step; it promoted `kdomanski/iso9660` to a
+direct dependency and dropped `go-libvirt` again because nothing imports it
+yet — Step 3 brings it back.
+
+Worth noting: `withDefaults` copies the `Images` map and `Hosts` slice
+instead of mutating the config in place, so the registry's shared
+`ProviderConfig` stays untouched. The default vendor-data installs
+`qemu-guest-agent` via `packages:` + `runcmd`, which means the first boot
+needs network access to the distro mirrors; a Proxy on an isolated bridge
+would still work (ARP discovery), just without the agent.
+
+## Step 3 — Transport + errors
+
+`conn.go` holds the `libvirtAPI` seam (17 methods, signatures as in the
+plan), a per-host connection pool, and the SSH dialer; `errors.go` holds
+`classify`, `isLibvirtError`, `hasCode` and `logAPIError`. Both are
+unit-tested; the whole package still runs in about a second under `-race`.
+
+The one thing the plan did not anticipate: **go-libvirt RPCs take no
+context and never time out**. A hung hypervisor would pin a reconcile worker
+forever. So the pool exposes `call(ctx, host, fn)` rather than handing out
+raw clients: `fn` runs in a goroutine, and if `ctx` ends first the pool
+closes the SSH connection out from under it (which makes the blocked RPC
+return) and reports `ctx.Err()`. The same helper (`awaitOrCancel`) bounds
+`lv.Connect()` during dial. Two more liveness measures for the same reason:
+the SSH handshake gets a deadline (ctx's, or 30 s), and each connection runs
+an OpenSSH `keepalive@openssh.com` loop every 15 s that closes the client
+when a request errors or goes unanswered for one interval.
+
+Connection-drop policy in `call`: any error that is *not* a `libvirt.Error`
+(so io.EOF, "connection closed", SSH failures) drops the cached connection
+and the next call redials; a daemon-reported `libvirt.Error` keeps it. Dials
+are serialised per host (concurrent Creates on one host share a dial) but
+hosts don't block each other.
+
+As Step 1 noted, the dialer uses `libvirt.NewWithDialer` with a
+`socket.Dialer` that wraps `ssh.Client.Dial("unix", socket)` rather than the
+deprecated `New(net.Conn)`.
+
+Tests: the pool is exercised through a fake dialer (reuse, redial after
+`Disconnected`, drop on transport error, keep on libvirt error, cancel
+releases a blocked call, concurrent callers share one dial, hosts
+independent). The SSH layer is tested against an in-process
+`x/crypto/ssh` server that serves `direct-streamlocal@openssh.com` channels
+by dialing the requested unix path (an echo socket standing in for
+`libvirt-sock`), with keys generated per test and a real `known_hosts` file
+written via `knownhosts.Line`. Covered: forward round-trip and the socket
+path the server saw, unknown host key rejected with `knownhosts.KeyError`,
+`insecureIgnoreHostKey`, wrong client key, cancelled ctx, missing key or
+known_hosts file, and keepalive both closing a stalled client and leaving a
+healthy one alone. The libvirt handshake (`Connect()`) itself is *not*
+covered here — faking libvirt's XDR RPC just for that is not worth it; the
+Step 6 integration test covers it against a real host.
+
+`go mod tidy` promoted `go-libvirt` and `golang.org/x/crypto` to direct
+dependencies.
+
+Worth noting: `classify` maps `ErrOperationInvalid` (e.g. destroying a
+domain that is already off) to Transient; Delete in Step 4 must check that
+code itself via `hasCode` and treat it as success rather than relying on
+the classifier. Nothing maps to `ErrQuotaExceeded` — libvirt has no quota
+concept; an out-of-space pool surfaces as a Transient `ErrInternalError`.
+
+## Step 3b — Image cache
+
+`image.go` follows the plan closely: `resolveImage` maps `placement.image`
+to a configured key or a direct http(s) URL (else Permanent);
+`imageCache.ensure` checks base volume + marker (+ marker content when a
+checksum is configured), tears down and re-fetches when anything is off,
+streams the HTTP body through `StorageVolUpload` with a `sha256` tee,
+refreshes the pool, then writes the marker last. A per-`(host, volume)`
+mutex serialises concurrent callers; the pool itself is the only cache
+state.
+
+Errors from this file are pre-tagged with the taxonomy sentinel via
+`fmt.Errorf("%w: …", provider.ErrPermanent, …)` and `classify` now honours
+such tags before looking at libvirt codes. That keeps one `wrapErr` path in
+the provider instead of special-casing image errors.
+
+`fake_test.go` landed with this step rather than Step 4: an in-memory
+`libvirtAPI` with pools, volumes (content, format, capacity, backing path),
+domains (state, metadata, per-source interface addresses), per-method error
+injection (`fail` always / `failOnce`) and an `uploadHook` to break a stream
+mid-way. It models the libvirt error codes the provider branches on
+(`ErrStorageVolExist`, `ErrNoStorageVol`, `ErrNoDomain`,
+`ErrOperationInvalid`, `ErrNoDomainMetadata`, `ErrAgentUnresponsive`).
+
+Tests cover every case in the plan plus: no checksum configured (marker is
+still written with the computed digest; second call needs no marker read),
+403/500 alongside 404/503, missing pool → Permanent, cancelled ctx →
+Transient with nothing created, and different hosts fetching independently.
+
+One test-harness lesson worth keeping: `httptest`'s server drops
+`Content-Length` and switches to chunked encoding for bodies larger than
+its 2 KiB write buffer, so the fake mirror sets the header explicitly.
+Real distro mirrors send it; a server that doesn't is reported as
+Permanent with a message naming the header.
+
+Worth noting: the base volume is created with `capacity = Content-Length`
+and the *image's* format (`qcow2`), and the raw bytes are uploaded over it.
+That relies on libvirt's upload writing the file verbatim (which is how
+`virsh vol-upload` populates volumes); the following `StoragePoolRefresh`
+makes libvirt re-read the real virtual size from the qcow2 header. Step 6
+confirms this on a real host. Also, the whole download runs inside one
+`pool.call`, so cancellation aborts the HTTP body (ctx-bound) and the
+upload stream fails; the half-written base volume is deleted on the spot
+and, failing that, on the next attempt via the missing marker.
+
+## Step 4 — Provider methods
+
+`kvm.go` now holds `Provider`, `New` (loads the SSH key and known_hosts
+up front via `newSSHAuth`, so a bad path fails at startup), the
+`newWithDialer` seam, and `Create`/`Get`/`Delete`/`ListByTag`. Each
+method resolves its host from config, then runs the whole libvirt
+conversation inside one `hostPool.call`, so a cancelled reconcile drops
+the connection and the blocked RPC returns. Errors pass through `classify`
+in one `wrapErr`, as the Step 3b note intended.
+
+Behaviour that goes beyond the plan text, or that a future reader should
+know before touching this file:
+
+- **Create looks up the domain first.** If it exists, only the running
+  check runs and the volumes are never touched. If not, nothing can be
+  using the volumes yet: the overlay is reused when it exists (its
+  creation is atomic; `ErrStorageVolExist` → reuse) but the **seed is
+  always deleted and rebuilt**. The plan said "exists → skip upload"; the
+  reviewer pointed out that a reconcile cancelled mid-upload leaves a
+  truncated seed that the compensating delete cannot remove, because
+  `hostPool.call` has already closed the connection the delete would go
+  over. Skipping such a seed would boot the VM without cloud-init and no
+  retry would ever repair it. Rebuilding costs one ~40 KiB upload per
+  attempt before the domain exists, nothing afterwards. The table test
+  breaks Create after each step and re-runs it, and a separate test
+  starts from a leftover truncated seed.
+- **Get swallows address-source errors.** `DomainInterfaceAddresses` from
+  ARP returns nothing until the guest has sent traffic and the agent
+  source errors with `ErrAgentUnresponsive` until `qemu-guest-agent` is
+  up, so both are logged at V(2) and mean "no IP yet" → `Provisioning`.
+  Loopback, link-local and unspecified addresses are skipped when
+  choosing the first IPv4 (the agent reports `lo` too).
+- **State mapping:** `running`/`blocked` are active; `nostate` →
+  Provisioning (the reconciler answers Stopped with delete-and-recreate,
+  too drastic for a state libvirt could not determine); everything else
+  (paused, pmsuspended, shutdown, shutoff, crashed) → Stopped.
+- **Delete treats a missing pool as "nothing to clean"** rather than
+  Permanent: failing there would pin the Proxy's finalizer on a config
+  edit that renamed the pool. Domain removal is destroy (`ErrOperationInvalid`
+  on a stopped domain is fine) → undefine with managed-save, snapshot
+  metadata and nvram flags → overlay and seed deleted by name.
+- **ListByTag** reads each domain's metadata once (passed into the same
+  `inspect` that Get uses, so no second `DomainGetMetadata` round-trip)
+  and skips domains without the operator's element. A domain that cannot
+  be read at all — vanished mid-sweep, corrupt metadata, state call
+  failing — is *skipped*, not fatal: GC never deletes what it does not
+  see, so leaving one out is safe, whereas one bad domain blocking GC for
+  every other one (the first version's behaviour, caught in review) is
+  not. An unreachable host still fails the whole sweep.
+  `ConnectListAllDomains` is called with flags 0 (active and inactive).
+- A host named in a providerID but no longer in `kvm.hosts` is Permanent
+  for Get and Delete: the operator cannot reach it, and retrying forever
+  would hide the config problem.
+
+Tests run the provider over a `hostDialer` that maps host names to
+`fakeAPI` instances (an unmapped host is "unreachable"). The image is
+pre-seeded in the pool for most tests; one test runs the full first-use
+download through `httptest` and confirms a second Proxy on the same host
+does not download again.
+
+Found while running the package: `TestBuildSeedISO_deterministic` from
+Step 2 was wrong. `iso9660.ImageWriter` stamps `time.Now()` into the
+volume descriptor, so two builds in different seconds differ by six
+timestamp bytes; the test had only passed because both builds landed in
+the same second. Byte identity is nothing the provider depends on (an
+existing seed is never re-uploaded), so the test now compares label,
+files and size, and the comment on `buildSeedISO` no longer claims
+identical output.
+
+The `go-operator-reviewer` agent ran on the first version of this step
+and found the two bugs above (truncated seed, wedged GC sweep) plus the
+double metadata read and the `nostate` mapping; all were fixed before
+committing. One finding it raised is deliberately **not** fixed here
+because it lives in Step 3b's `image.go` and changes the volume naming
+the plan chose: re-fetching a base image (checksum edited in config, or
+upstream content changed) deletes `base-<key>.<fmt>` while existing
+overlays still back onto it, breaking those VMs' disks. Content-addressed
+base names would avoid it. Flagged for the user; until decided, Step 5's
+docs must warn that an image's `sha256`/`url` must not change while
+Proxies using it exist.
+
+Worth noting: `go test -cover` reports 88.2 % for the package; the
+uncovered lines are the real SSH/libvirt dial path and log-only
+branches. `go vet` and `-race` (three runs) are clean. Wiring into
+`cmd/main.go` is Step 5, so `New` is not yet reachable from a running
+operator.
+
+## Step 6 — Integration test
+
+Done before Step 5, at the user's request, after a discussion of what the
+unit tests can and cannot prove: they run over an in-memory fake of the
+libvirt calls, so every assumption about libvirt's real behaviour (domain
+XML accepted, raw qcow2 upload + pool refresh usable as a backing image,
+guest boots and reports an address) needed a test against a real host.
+The wiring in Step 5 is not a prerequisite — the test drives the provider
+directly.
+
+`internal/provider/kvm/kvm_integration_test.go` is behind
+`//go:build integration` and skips itself under `-short` or when any of
+`KVM_TEST_HOST`, `KVM_TEST_USER`, `KVM_TEST_SSH_KEY`, `KVM_TEST_BRIDGE`,
+`KVM_TEST_IMAGE_URL` is unset. Optional knobs beyond the plan's list:
+`KVM_TEST_KNOWN_HOSTS` (without it the run disables host-key checking
+and logs that), `KVM_TEST_IMAGE_SHA256`, `KVM_TEST_POOL`,
+`KVM_TEST_SOCKET`, `KVM_TEST_TIMEOUT` (default 20 m, must cover a cold
+image download) and `KVM_TEST_KEEP` to leave a failed run's VM behind for
+`virsh console`.
+
+One sequential test, `TestKVM_lifecycle`, not subtests: each stage
+depends on hypervisor state the previous one left. Create (timed, so the
+log shows the image fetch), re-Create returns the same ID, poll Get to
+Running with an IPv4 (Stopped fails fast with a hint to use the console),
+UID and CreatedAt survive the metadata round-trip, ListByTag includes it,
+Delete, Get → NotFound, second Delete is a no-op, ListByTag excludes it,
+and the pool is inspected through the provider's own connection pool
+(`p.hosts.call`, reachable because the test is in package `kvm`): overlay
+and seed gone, base image and marker kept. A TCP connect to the guest's
+port 22 is attempted but only logged, since the test machine may not
+route to the bridge network. Cleanup deletes the VM on failure with its
+own two-minute budget, because the test context may already be spent.
+
+Makefile target, so the invocation is not something to remember:
+
+```bash
+make test-integration-kvm
+# = go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/
+```
+
+`docs/testing.md` gained the kvm package rows in the suite table, a
+"live libvirt" bullet under the deliberately-not-covered list, and a
+section on running this test, with the deviation from the testcontainers
+convention called out as the plan asked.
+
+Worth noting: **the test has not been executed yet.** This session had
+no hypervisor credentials; it was verified with
+`go vet -tags=integration ./...` and by running it without the
+environment (self-skip). Its first real run is the opening act of
+Step 7's manual validation, and it is the point at which the Step 2 and
+Step 3b "confirm on a real host" notes get answered. The test's config
+is built directly rather than through the provider package's validation
+(unexported), so a typo in an env var surfaces from `New` or from
+libvirt, not from a validation message.
+
+## Step 5 — Wiring, manifests, samples, docs, CHANGELOG
+
+Done after Step 6 (see that entry for why the order flipped). Everything
+the plan listed, plus a few things it did not:
+
+- `cmd/main.go`: `"kvm": kvm.New` in the builtins map. No RBAC change —
+  the provider talks to hypervisors, not the API server.
+- `config/manager/manager.yaml`: `kvm-ssh` Secret mounted read-only at
+  `/etc/proxy-operator/kvm` with `optional: true`, so deployments without
+  a kvm provider are unaffected. It nests under the providers-config
+  mount at `/etc/proxy-operator`; kustomize renders it (`kubectl
+  kustomize config/default` checked). The container runs as uid 65532
+  and Secret files default to 0644, so the key is readable without an
+  `fsGroup`; the `kubectl create secret` recipe is in a comment next to
+  the mount, in `docs/kvm-provider.md` and in the README.
+- `config/samples/providers-config.yaml`: `kvm-home` block with Debian 13
+  and Ubuntu 24.04 official image URLs, two machine types, one host, and
+  every optional field shown commented out. Parsed through
+  `provider.LoadConfigFile` in a throwaway test to make sure the sample
+  validates. `config/samples/proxy_kvm.yaml` uses the GCP sample's squid
+  cloud-init and shows `ssh_authorized_keys` commented out rather than
+  set — a sample with a placeholder key would either fail cloud-init or
+  invite pasting a real key into a committed file. Added to the samples
+  kustomization; `config/manager/providers_config.yaml`'s comment now
+  mentions kvm.
+- `docs/kvm-provider.md`: prerequisites with a three-line preflight
+  (`virsh list`, `pool-info`, `ip link` over ssh), cluster setup, the full
+  config reference table, placement table, a table of what lands on the
+  host and its lifetime, how the image cache works and how to evict, the
+  **warning not to change or evict an image while Proxies use it** (the
+  reviewer's finding from Step 4, carried as a documented rule since the
+  content-addressed-naming alternative was not chosen), how to log in,
+  and the known gaps.
+- README: providers bullet and a "KVM setup" section mirroring the GCP
+  one. `docs/architecture.md`: components row, a §5 block for the kvm
+  call flow in the same style as the other two, and five decision-log
+  entries (SSH transport, operator-fetched images, seed always rebuilt,
+  `nostate` → Provisioning, skip-vs-fail in the GC sweep).
+- `CHANGELOG.md` entry at the top, stating explicitly that the provider
+  is not yet validated on a real host.
+
+Verification: `go build ./...`, `go vet ./...`, `go test -short` on
+`cmd` and `internal/provider/...`, and both kustomizations rendered.
+`hack/providers-dev.yaml` untouched, as planned.
+
+Worth noting: the CRD has no rule requiring all three placement fields
+(that is GCP's own check), so the single-host "zone may be omitted"
+behaviour works without an API change. And with this step the branch is
+feature-complete but **unproven**: Step 7 runs the integration test and
+the end-to-end checklist against a real hypervisor before the MR.
+
+## Step 7 — Manual validation + MR (in progress)
+
+Host: `beelink` (Ubuntu, libvirt 11.6.0, QEMU 10.1.0, pool `default` at
+`/srv/vms/images` on local ext4, bridge `br0`), reached over Tailscale as
+`beelink.tail8c69c.ts.net`, SSH user `novakj`. Not the `nas1.home.lan`
+the plan and samples name; that host does not resolve from the dev box.
+
+```bash
+export KVM_TEST_HOST=beelink.tail8c69c.ts.net:22
+export KVM_TEST_USER=novakj
+export KVM_TEST_SSH_KEY=$HOME/.ssh/proxy-operator_ed25519
+export KVM_TEST_KNOWN_HOSTS=$HOME/.ssh/kvm_known_hosts
+export KVM_TEST_BRIDGE=br0
+export KVM_TEST_IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+make test-integration-kvm
+```
+
+### Run 1 and 2: `Could not open '/srv/vms/images/base-test-image.qcow2': Permission denied`
+
+Everything up to `DomainCreate` worked on the first try against a real
+host, which answers the open notes from Steps 2 and 3b: the SSH-forwarded
+libvirt handshake, the raw qcow2 upload plus pool refresh as a backing
+image, the default-namespace metadata element, and `DomainDefineXML` for
+the generated XML were all accepted by libvirt 11.6. QEMU then exited at
+start with `Permission denied` on the base image.
+
+Ownership was ruled out the slow way: the file was `root:root 0600` (the
+dir-pool default; libvirt's DAC driver chowns the chain at start and
+restores it after a failed start, so that observation meant nothing),
+`qemu.conf` was all defaults (`libvirt-qemu`, `dynamic_ownership=1`), and
+after a manual `chown libvirt-qemu:kvm` the second run failed identically.
+
+The actual cause is the Step 2 choice of `<disk type="volume">`.
+libvirt's AppArmor helper builds each domain's profile from the domain XML
+without consulting the storage driver, and its `add_file_path` only emits a
+rule when the source has a `path`; a volume-referenced disk has pool and
+volume names instead, so neither the overlay nor its backing image gets a
+rule and QEMU is denied on the first one it opens. Every stock Debian and
+Ubuntu hypervisor confines QEMU this way. The unit tests could not have
+caught it: the fake accepts either form.
+
+Fix, in this step: `buildDomainXML` takes `DiskPath`/`SeedPath` and emits
+`type="file"` disks; `ensureOverlay` and `writeSeed` return the volume
+handles (the exists-already branch now does a `StorageVolLookupByName`),
+and Create resolves both paths with `StorageVolGetPath` before defining
+the domain. Two extra cheap RPCs per Create. `TestBuildDomainXML_fieldByField`
+now fails on any `type="volume"` or `pool=` attribute so the design cannot
+drift back, and `TestCreate_freshHost` asserts the disk sources equal the
+paths the fake reports.
+
+Worth noting: the first two runs left the base image and its marker in
+the pool as designed, so later runs skip the 435 MB download.
+
+### Run 3: `parsing domain metadata: expected element <instance> in name space … but have no name space`
+
+With file-backed disks, Create returned in 1 s (image cached) and the
+domain started. Get then failed to parse the metadata: libvirt 11.6 finds
+the element by our URI but returns it as a bare `<instance>` with no
+namespace declaration at all. Step 2 had tested the default-namespace and
+prefixed spellings and left "does libvirt accept the default-namespace
+form" for the real host; it does, it just does not give it back that way.
+
+Fix: `parseInstanceMetadata` matches on the local name only, since the
+lookup already selected the element by URI. The fake now strips the
+declaration when storing metadata, so the provider's Get and ListByTag
+tests parse the shape libvirt really returns, and a new
+`TestParseInstanceMetadata_libvirtBareForm` pins it.
+
+### Run 4: guest hangs at GRUB's "Booting" line
+
+Create and start passed; Get polled `Provisioning` with no IP while the
+VM sat at 100 % CPU after GRUB printed "Booting `Debian GNU/Linux`" and
+before the kernel wrote anything to the serial console. The domain XML
+matches a working VM on the same host (q35, BIOS, host-passthrough,
+virtio disk, SATA cdrom, br0), so the difference is the disk content:
+the cached image was downloaded without a checksum and never verified.
+Diagnosis was handed to the user (sha512 against Debian's published
+sums, `qemu-img check`, and a `virt-install --import` of the same image
+outside the operator). Because the provider's own logs were invisible in
+the test output, this run also produced a side plan:
+[2026-09-10-0805-kvm-libvirt-tracing](2026-09-10-0805-kvm-libvirt-tracing.md),
+which traces every libvirt RPC at V(1) and routes it into `-v` output.
+
+### Run 5: the guest hang is Debian 13's BIOS boot path; UEFI fixes it
+
+The cached image's sha512 matched Debian's published sum, and
+`qemu-img info` showed a sound 3 GiB qcow2, which also confirms that
+the raw upload over a libvirt-created volume plus pool refresh yields a
+byte-identical file (the Step 3b note is answered). Bisected by hand on
+beelink with `virt-install --import` on a fresh overlay of the same
+image:
+
+| variant | seed | CPU | firmware | result |
+|---|---|---|---|---|
+| operator's domain | ours | host-passthrough | SeaBIOS | hangs after GRUB "Booting", 100 % CPU |
+| bare | none | host-passthrough | SeaBIOS | same hang |
+| bare | none | host-model | SeaBIOS | same hang |
+| bare | none | host-passthrough | **OVMF (`--boot uefi`)** | **boots to login** |
+
+So neither the seed ISO, the domain XML nor the CPU model was the
+problem: the image's legacy GRUB → kernel handoff does not work on this
+host, while its EFI stub path does. Not diagnosed further; it is
+Debian's chain, not ours, and every current cloud image boots UEFI.
+
+Change: `images.<key>.firmware` (`uefi` default, `bios` escape),
+threaded through `withDefaults` → `imageSource` → `domainParams`, and
+`buildDomainXML` emits `<os firmware="efi"><firmware><feature
+enabled="no" name="secure-boot"/></firmware>` so libvirt selects the
+OVMF binary and creates the NVRAM file; the `nvram` undefine flag Delete
+already passes removes it (seen on the host: `virsh undefine` without
+`--nvram` refuses such a domain). Direct-URL images default to UEFI
+too. Docs: OVMF is a new hypervisor prerequisite, config table row,
+sample comment, known-gaps entry, decision-log entry.
+
+Worth noting: the first image download went through with no checksum
+configured, and the cache reuses a base image by key, so switching
+`KVM_TEST_IMAGE_URL` requires deleting `base-test-image.qcow2` and its
+marker by hand first. That is the reviewer's Step 4 finding showing up
+in practice; still documented rather than fixed.
+
+Follow-up, same day: the cause is known upstream. [Debian bug
+#1111240](https://bugs.debian.org/1111240) (filed 2025-08-16 against
+the cloud image, reassigned to `grub2`, still open as of 2026-09-10;
+trixie ships grub2 2.12-9+deb13u2) reports exactly this with a domain
+XML almost identical to ours: BIOS, virtio disk, SATA cdrom seed, bridge
+NIC, `<console><target type='serial'/>`, no `<graphics>`/`<video>`.
+The cloud team reproduced it: trixie's grub.cfg has `terminal_output
+gfxterm serial`, and with a serial port present but no video adapter
+GRUB never hands over to the kernel. Their fix in the image
+(`terminal_output serial`) is not something an operator can apply;
+the reporter's workaround (console `target type='virtio'`) boots but
+takes the kernel's `console=ttyS0` output with it. UEFI sidesteps the
+gfxterm path and keeps the serial console, so the decision stands, but
+the docs now name the bug rather than blaming the host, and note that
+bookworm images are unaffected. `latest` on cloud.debian.org was the
+2026-08-31 build when this was hit, so every trixie point release to
+date carries it.
+
+### Run 6: pass
+
+With UEFI the full lifecycle passed in 23 s on a warm cache: Create 2 s
+(domain start 0.9 s), guest agent answering with `192.168.0.89` after
+20 s (ARP never did; the host had not talked to the guest), TCP 22
+reachable from the dev machine, ListByTag seeing it, Delete leaving only
+the base image and marker in the pool, second Delete a no-op. Trace
+excerpt worth keeping for the record:
+
+```
+09:44:28.603 libvirt virDomainDefineXML xmlBytes=1509 domain=proxy-hk5oavoqk45n3twq duration=13.51ms
+09:44:28.603 kvm domain defined uid=9bbceb41-… firmware=uefi
+09:44:29.542 libvirt virDomainCreate domain=proxy-hk5oavoqk45n3twq duration=938.705ms
+09:44:49.573 libvirt virDomainInterfaceAddresses domain=proxy-hk5oavoqk45n3twq source=DomainInterfaceAddressesSrcAgent interfaces=2 duration=1.742ms
+09:44:49.573 kvm KVM domain fetched state=Running ip=192.168.0.89 uid=9bbceb41-…
+```
+
+Worth noting: the sweep listed a second managed domain,
+`proxy-qxge3dbm2mnpc3ih`, shut off, left behind by an earlier failed
+run whose cleanup never ran. That is exactly the orphan GC exists for;
+on the host it needs `virsh undefine --nvram` plus its two volumes by
+hand. The `ListByTag` sweep costs one `virDomainGetMetadata` per
+domain on the host (21 here, ~0.4 ms each over the tunnel), which is
+fine at this scale but is the call to watch on a busy hypervisor.
+
+### Run 7: in-cluster, two defects the integration test could not see
+
+The e2e pass ran the operator inside the kind cluster
+(`egress-proxies-operator:kvm-dev`, built with `make docker-build`,
+loaded with `kind load docker-image`, deployed via the kustomize
+`config/manager` with the `kvm-home` provider pointing at beelink) and
+two Proxies with real cloud-configs: one plain squid, one with a debug
+login user. Both defects come from the difference between the test
+harness and the real thing, not from libvirt.
+
+**Read-only root filesystem.** The manager runs with
+`readOnlyRootFilesystem: true`, and `iso9660.NewWriter` stages the seed
+in a temp directory, so the first Create failed with `creating iso
+writer: read-only file system`. Fix: a 64 MiB `emptyDir` on `/tmp` in
+`config/manager/manager.yaml`. Side finding: the error is classified
+`ErrPermanent` by `writeSeed`, yet `hostPool.call` treated any
+non-libvirt error as a transport failure and dropped the SSH connection,
+so every retry redialled. `call` now leaves the connection alone for
+errors already carrying a provider sentinel (`isClassified`), with a
+test.
+
+**The guest agent never got installed.** Both VMs booted (confirmed on
+the console by hand) but sat in `Provisioning` for 25 minutes: the
+trace showed `virDomainInterfaceAddresses` from the agent source failing
+with `ErrAgentUnresponsive` on every poll and the ARP source empty. The
+integration test had passed with `CloudInit: "#cloud-config\n"`; the
+real Proxies have `packages: [squid]` and `runcmd:`. cloud-init merges
+vendor-data under user-data with its default mergers, which keep the
+user-data's value for any key both define and never append lists, so the
+vendor-data's `packages`/`runcmd` were dropped wholesale. Confirmed
+locally with the dev box's cloud-init 26.1 before changing anything:
+
+```sh
+python3 - <<'EOF'
+from cloudinit import mergers
+user = {"packages": ["squid"], "runcmd": ["systemctl restart squid"]}
+vendor = {"packages": ["qemu-guest-agent"], "runcmd": [["systemctl", "enable", "--now", "qemu-guest-agent"]]}
+print(mergers.construct(mergers.default_mergers()).merge(dict(user), dict(vendor)))
+EOF
+# {'packages': ['squid'], 'runcmd': ['systemctl restart squid']}
+```
+
+Fix: `withGuestAgent` in `seed.go` merges the bootstrap into the Proxy's
+cloud-config itself (append to `packages` and `runcmd`, skip if present,
+re-render with `sigs.k8s.io/yaml`), so vendor-data is no longer defaulted
+and `vendorData` is plain extra data. User-data that is not a cloud-config
+document (script, `#include`, MIME) still gets the bootstrap as
+vendor-data since there is nothing to collide with; a cloud-config that
+does not parse as a mapping fails the Proxy as `ErrPermanent`. The
+rendered document for the squid Proxy passed `cloud-init schema
+--config-file`. The logging test's fixture (`#cloud-config` + a bare
+sentinel line) is not a mapping and had to become `hostname: <sentinel>`.
+
+Rebuilt, reloaded, restarted, deleted and recreated both Proxies: both
+`Ready`/`HEALTHY True` in 2 min 1 s from `kubectl apply`, IP via the
+agent 110 s after `virDomainCreate` (the guest installs squid and the
+agent from the mirrors first), and `curl -x http://<ip>:3128
+https://www.gstatic.com/generate_204` from the dev box returns 204
+through both.
+
+Worth noting: the `#cloud-config` round-trip drops comments and reorders
+keys alphabetically; semantics are preserved (numbers, booleans and
+block scalars survive, checked in tests) but a user diffing the seed
+against their CR will see a different document. Vendor-data supplied via
+`vendorData` is subject to the same cloud-init merge rule and is now
+documented as such. The kind pod also restarted twice during the session
+from "Pod sandbox changed" (the node briefly lost the API server), which
+is unrelated to the operator. Still uncommitted on purpose: the
+`kustomization.yaml` image override (`egress-proxies-operator:kvm-dev`)
+and the `beelink` host entries in `config/manager/providers_config.yaml`
+and `config/samples/providers-config.yaml`, which are this machine's dev
+setup rather than the repo's defaults.
+
+### Run 8: review fixes
+
+A `/code-review high` pass over the branch found five defects, fixed in
+[plan 2026-09-10-2222](../plans/2026-09-10-2222-kvm-review-fixes.md) with
+its own execution log. Two of them land on Run 7's conclusions above and
+supersede them:
+
+- The guest-agent **merge described in Run 7 is gone**. It could not be made
+  safe for user-data that is not a plain cloud-config, and its YAML
+  round-trip re-typed unquoted scalars. The bootstrap now ships as a
+  vendor-data shell script, which cloud-init never merges, and the Proxy's
+  cloud-init reaches the seed byte for byte.
+- The read-only-filesystem fix from Run 7 still stands, but the eager SSH
+  key load it sat next to does not: a missing `kvm-ssh` Secret aborted the
+  whole provider set and crash-looped the manager. The material is read on
+  first use now.
+
+Re-validated the same way: integration test against beelink, both Proxies
+recreated in kind, 204 through squid.
+
+### MR opened
+
+[PR #5](https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/pulls/5),
+`feat/kvm-provider` → `main`, 19 commits. `tea` is not installed on the
+dev box and the HTTPS remote has no credential helper, so the push used a
+one-off helper reading a Gitea access token from `~/.config/gitea-token`,
+and the MR was created with the API instead of `tea pr create`:
+
+```sh
+git -c credential.helper='!f() { echo username=kacerr; echo "password=$(tr -d "[:space:]" < ~/.config/gitea-token)"; }; f' \
+  push -u origin feat/kvm-provider
+curl -s -H "Authorization: token $(tr -d '[:space:]' < ~/.config/gitea-token)" \
+  -H 'Content-Type: application/json' -d @pr.json \
+  https://gitea.home.hrajfrisbee.cz/api/v1/repos/kacerr/egress-proxies-operator/pulls
+# pr.json: {"title": ..., "head": "feat/kvm-provider", "base": "main", "body": ...}
+```
+
+The push printed a libsecret "store failed" warning from the system
+credential store; harmless, the token was not persisted anywhere but the
+file.

docs/plans-executions/2026-09-10-0805-kvm-libvirt-tracing.md

@@ -0,0 +1,109 @@
+# Execution: Trace every libvirt call in the KVM provider
+
+Plan: [2026-09-10-0805-kvm-libvirt-tracing.md](../plans/2026-09-10-0805-kvm-libvirt-tracing.md)
+
+- [x] Step 0 — Save plan
+- [x] Step 1 — `errors.go`: shared error fields
+- [x] Step 2 — `trace.go`: the `tracingAPI` decorator
+- [x] Step 3 — `conn.go`: pool logging and wrapping
+- [x] Step 4 — Call sites
+- [x] Step 5 — Integration test output
+- [x] Step 6 — Unit tests
+- [x] Step 7 — Docs, execution log, CHANGELOG
+
+## Steps 1–6 — Tracer, pool logging, tests, integration wiring
+
+Done as one commit on `feat/kvm-provider`; the plan's assumption that
+this belongs on the KVM branch rather than its own feature branch was
+accepted at plan approval.
+
+The code follows the plan closely. Things that came out differently
+or are worth knowing:
+
+- **Logger names join with `/` in funcr, `.` in zap.** The trace lines
+  are `kvm/libvirt` in the unit-test capture and in `testr` output, and
+  `kvm.libvirt` under the operator's zap logger. The docs use the zap
+  spelling; the unit test asserts the funcr one. First run of the tier
+  test failed on exactly this.
+- **Cancel is not reported as a transport error.** The original `call`
+  dropped the connection twice on cancellation (once from the abort
+  closure, once from the "not a libvirt error" branch, harmlessly, since
+  drop is idempotent). With a log line on each drop that would have
+  printed both `context cancelled` and `transport error` for one event,
+  so the transport branch now skips `ctx.Err()`. Covered by
+  `TestHostPool_call_logsDropReason`.
+- **`ConnectListAllDomains` in the fake includes every defined domain**,
+  so the trace test's fixture (one running managed domain plus one
+  stopped domain for `DomainCreate` to start) lists two; the V(2)
+  assertion checks for the `names` key rather than an exact list.
+- **`libvirtErrorFields` also feeds `logAPIError`**, which therefore
+  gains `libvirtCodeName` (e.g. `ErrNoDomain`) next to the numeric code.
+- The seam-coverage guard (`reflect` count of `libvirtAPI` methods vs
+  the test table) is the only thing standing between a future 18th RPC
+  and a silently untraced call; keep it.
+
+Verification run before committing:
+
+```bash
+go build ./... && go vet ./... && go vet -tags=integration ./internal/provider/kvm/
+go test -short ./...
+go test -race -count=1 -cover ./internal/provider/kvm/   # 90.2 %, up from 88.2
+go test -tags=integration -count=1 -run TestKVM ./internal/provider/kvm/   # self-skips without env
+```
+
+Worth noting: the live run (`make test-integration-kvm` against beelink)
+had not happened when this was written; it is what Step 7 of the KVM
+provider plan is waiting on, and its output is the acceptance test for
+this feature. Expected shape is in the plan's Verification section.
+
+## Revision after the first live run — less noise, libvirt names
+
+The first live run on beelink showed the trace working but hard to read:
+`testr` printed `"ts"`, `"level"`, `"msg"` and the three per-run
+constants on every line, the error text appeared twice (`error` and
+`libvirtMessage`), `host` appeared twice on image-cache lines (the
+logger already carried it), and `method=DomainLookupByName` was
+go-libvirt's name rather than something to look up in libvirt's docs.
+The user asked for "less verbose but more informative". Changes:
+
+- The tracer's headline field is now `rpc=virDomainLookupByName`: the
+  public libvirt API function, which go-libvirt's method names are
+  exactly minus the `vir` prefix (they come from the same
+  `remote_protocol.x`). Not gRPC, for the record: libvirt's own
+  XDR-based RPC over the forwarded unix socket.
+- Errors are one `error` field plus `libvirtError=ErrNoDomain` and
+  `libvirtCode=42`; `libvirtMessage` is gone from both the tracer and
+  `logAPIError`, since it is the error text.
+- `image.go` no longer adds `host` to lines whose logger has it.
+- The integration test replaces `testr` with a small `compactSink`
+  (`logr.LogSink` plus `GetCallStackHelper`, so `t.Log` still
+  attributes lines to `trace.go`): `HH:MM:SS.mmm libvirt
+  virDomainCreate domain=… duration=… error=… libvirtError=…`. The
+  `rpc` value becomes the headline, `provider`/`host`/`name` are
+  dropped (the run's first `t.Log` line states them), and values with
+  spaces are quoted. Production zap output is unchanged apart from the
+  field renames; structured logs are meant to repeat context.
+
+Previewed by driving the sink through the fake provider with an
+injected `DomainCreate` failure (a throwaway integration-tagged test,
+not committed):
+
+```
+trace.go:55: 09:04:33.165 libvirt virDomainLookupByName domain=proxy-abc123def456ghij duration=4µs error="Domain not found: …" libvirtError=ErrNoDomain libvirtCode=42
+trace.go:55: 09:04:33.165 libvirt virStorageVolUpload pool=default volume=proxy-abc123def456ghij-seed.iso offset=0 length=0 duration=71µs
+kvm.go:262:  09:04:33.165 kvm seed volume uploaded volume=proxy-abc123def456ghij-seed.iso bytes=45056
+trace.go:55: 09:04:33.165 libvirt virDomainCreate domain=proxy-abc123def456ghij duration=0s error="internal error: process exited…" libvirtError=ErrResourceBusy libvirtCode=87
+```
+
+Worth noting: `length=0` on `virStorageVolUpload` is what the provider
+passes (libvirt reads to EOF); the following `seed volume uploaded`
+line has the byte count.
+
+## Step 7 — Live run confirmed; CHANGELOG
+
+The compact trace was accepted on the first passing live run
+(2026-09-10, beelink): one line per RPC with the libvirt function name,
+the provider's own lines between them, `Provisioning` polls readable at
+a glance (`SrcArp interfaces=0`, `SrcAgent … ErrAgentUnresponsive`
+until the agent came up at +20 s), and a full `ListByTag` sweep over 21
+domains in 15 ms. CHANGELOG entry added under the KVM validation entry.

docs/plans-executions/2026-09-10-2222-kvm-review-fixes.md

@@ -0,0 +1,229 @@
+# Execution: fix the five code-review findings on `feat/kvm-provider`
+Plan: [docs/plans/2026-09-10-2222-kvm-review-fixes.md](../plans/2026-09-10-2222-kvm-review-fixes.md)
+
+- [x] Step 0 — Save plan into the repo
+- [x] Step 1 — Unwedge deletion (finding 1)
+- [x] Step 2 — Validate names that become identifiers (finding 2)
+- [x] Step 3 — Read the SSH material lazily (finding 3)
+- [x] Step 4 — Guest agent via vendor-data script (findings 4 and 5)
+- [x] Step 5 — Validate, record, push
+
+## Step 0 — plan committed
+
+Copied out of `~/.claude/plans/` and committed alone before touching code, as
+the repo's plan rules require.
+
+## Step 1 — Unwedge deletion
+
+Three changes, in the order the bug unfolds.
+
+`reconcileDelete` calls `Get` first and only drops the finalizer when it
+answers not-found, so any permanent error there retried forever. It now
+switches on the class: a permanent `Get` still calls `Delete`, which is
+idempotent, and releases the object when that succeeds. Polling a `Get`
+that cannot converge was the wedge; `Delete` is the only call that can
+still make progress.
+
+`crawl.example.com/skip-cleanup: "true"` abandons the instance and removes
+the finalizer. It is checked **before** the provider lookup, which also
+frees a Proxy whose provider was deleted from the config — a second wedge
+sitting three lines above the reported one, with the same shape. No
+`EventRecorder` exists on the reconciler and none was added, so the
+abandonment is an Info log; `docs/architecture.md` already records that
+decision for the whole controller.
+
+In the KVM provider, `Delete` for a providerID naming an unconfigured host
+now logs and returns success. The precedent is two functions further down:
+`removeVolumes` treats a missing pool the same way, with a comment saying
+outright that refusing "would pin the Proxy's finalizer".
+
+**`Get` deliberately still fails permanently for that case.** Making it
+answer not-found would have been a one-line fix for the wedge, but the live
+reconcile path reads not-found as "the VM is gone" and rebuilds it
+elsewhere. A typo in `hosts` would then quietly duplicate every Proxy on
+that host, with the originals still running where GC cannot see them, since
+`ListByTag` only sweeps configured hosts. The asymmetry is now pinned by a
+test that asserts both halves at once.
+
+## Step 2 — Validate names that become identifiers
+
+One regexp, `^[A-Za-z0-9][A-Za-z0-9_.-]*$`, applied to `hosts[].name` and to
+the `images` map keys. Dots stay legal because a host name like
+`nas1.home.lan` is ordinary and harmless; the character that actually
+breaks things is `/`.
+
+The review only reported the host name. The image key has the same defect
+and is arguably worse: it is not merely embedded in an ID but pasted into a
+file name, `base-<key>.<format>`, so `../etc/x` would be a path traversal
+into the storage pool rather than a parse failure. Both are validated in the
+same pass. Machine-type keys are left alone deliberately — they are only
+ever compared against `placement.machineType`, never embedded anywhere.
+
+Worth noting: this is validation of the operator's own config file, which
+only a cluster admin can write, so the traversal was never a privilege
+boundary. It is still cheaper to reject at load than to explain a stuck
+Proxy later. Both new table cases assert the message text, so the wording
+that names the consequence is now pinned by tests.
+
+## Step 3 — Read the SSH material lazily
+
+`kvm.New` no longer loads the key; `lazyAuth` does, on the first connection,
+caching the result once it works. `registry.Build` is all-or-nothing by
+design, so the old constructor error did not merely fail the KVM provider,
+it exited the manager before any controller started.
+
+The plan said "lazy dialer", and the first attempt was exactly that — one
+closure wrapping both the loading and the dial. It could not be tested: the
+fixture's SSH server is real, but the socket behind it is an echo server, so
+every successful dial dies in the libvirt handshake and the assertion could
+not tell "the key worked" from "the key did not". Splitting the seam in two,
+`lazyAuth` for the material and `lazyDialer` for the wiring, made both
+testable: the healing path against `lazyAuth`, and a separate test that the
+dialer really authenticates, asserted by the error being past SSH and by the
+server having seen the forward request.
+
+Error classes carry the intent. A missing file stays unclassified, so
+`provider.Class` defaults it to transient and a Proxy heals on its own when
+the Secret appears — the common case, since the Secret is usually created
+after the manager. A key that does not parse wraps `ErrPermanent`, because
+retrying cannot fix it. `newSSHAuth` distinguishes the two with
+`errors.Is(err, fs.ErrNotExist)`.
+
+Worth noting: the startup check is kept, just non-fatally, logged with the
+`WARNING:` prefix the repo already uses where logr has no warn level. That
+preserves the diagnostic the eager version was written for. The caching test
+deliberately deletes the key file after a successful load and asserts the
+cached value is reused, which pins the rotation behaviour the docs promise:
+a rotated Secret needs a manager restart. `TestNew_badKeyPathFailsFast` was
+inverted rather than deleted, so the old contract stays visible in the diff
+next to the new one.
+
+## Step 4 — Guest agent via vendor-data script
+
+The review filed two findings against `withGuestAgent` — detection too
+narrow, and YAML re-typing on the round-trip. Reading cloud-init 26.1's
+sources while planning turned both into one finding: the operator should
+not be editing the user's document at all.
+
+Vendor-data gets its own handler chain (`stages.py:_default_vendordata_handlers`),
+with separate script and cloud-config paths. A vendor-data **script** is
+therefore written to the vendor script directory and run by
+`cc_scripts_vendor`, never touching the config merger that drops colliding
+keys. On the final-stage module list it lands after
+`package_update_upgrade_install` and before the user's `runcmd`, so the
+agent now comes up *earlier* than the old `runcmd` append did.
+
+That also closed the door the previous approach had left ajar. The merge
+cannot be rescued from inside either document: `merge_how` in user-data
+governs only the first source, and `merge_how` in vendor-data is popped by
+`dict_extract_mergers` and discarded before the document is written. There
+was no fix that kept the merge.
+
+`withGuestAgent`, `isCloudConfig` and `appendUnique` are gone, along with
+seed.go's `sigs.k8s.io/yaml` dependency. `installGuestAgent` (default true)
+turns the script off. When `vendorData` is also configured the two ship as
+a MIME multipart, with the user's document declared `text/plain` — a type
+cloud-init re-sniffs from the content's own header line, so this code never
+classifies what the user wrote.
+
+Before deleting the old path, its damage was captured rather than taken on
+trust. Round-tripping a plausible cloud-config through it:
+
+```sh
+# temporary test in the package, deleted afterwards
+go test -run TestZZProbeRetyping -v ./internal/provider/kvm/
+# in:  version: 1.10
+# out: version: 1.1
+```
+
+`1.10` arriving as `1.1` is worse than the reviewer's `passwd` example,
+since nothing about it looks wrong in the seed. Comments were dropped and
+keys reordered on the same pass.
+
+The generated vendor-data was then checked against the cloud-init actually
+installed on this machine, rather than against the docs:
+
+```sh
+python3 -c 'from cloudinit import user_data, handlers; ...'
+# vendor-script:    multipart=False  sniffed=text/x-shellscript
+# vendor-multipart: multipart=True   part 1 text/plain -> text/cloud-config
+#                                    part 2 text/x-shellscript
+```
+
+and the script itself through `sh -n`.
+
+Worth noting: the script has to choose a package manager where cloud-init's
+`packages:` key was distro-agnostic. It covers apt, dnf, yum, zypper and
+apk, and both systemd and OpenRC for the enable step, but an image outside
+that set now needs `installGuestAgent: false` and the agent baked in. The
+seed's user-data is byte-identical to the Proxy's `cloudInit` from here on,
+which is what the new `TestBuildSeedISO_userDataIsNeverRewritten` pins
+across seven formats. One consequence worth remembering: a user-data that
+sets `vendor_data: {enabled: false}` disables the agent, and cloud-init
+documents that as intended behaviour.
+
+## Step 5 — Validation
+
+Unit suite, `go vet` and `gofmt` clean across the repo. Then the two real
+runs.
+
+**Integration against beelink**, unchanged environment from the KVM plan's
+log, passed in 22 s on a warm cache, with
+`virDomainInterfaceAddresses source=DomainInterfaceAddressesSrcAgent
+interfaces=2` — the agent installed by the new vendor script answering, on
+a stock Debian 13 image.
+
+**In-cluster**, both Proxies deleted and recreated against the rebuilt
+image: Ready with an address 2 min 3 s after `kubectl apply`, 204 through
+squid from the dev box for both. Inside the debug VM:
+
+```sh
+ssh novakj@192.168.0.94 'systemctl is-active qemu-guest-agent; cloud-init status; \
+  sudo head -4 /var/lib/cloud/instance/user-data.txt; sudo ls /var/lib/cloud/instance/scripts/vendor/'
+# active
+# status: done
+# #cloud-config
+# # Login for debugging: user "novakj", password "kvm-dev", passwordless sudo.
+# ...
+# part-001
+```
+
+Two things worth having seen with your own eyes there: the user's **comment
+lines survived** into the guest, which the old merge stripped, and the
+script really did land in the vendor directory rather than being merged
+into the user's config.
+
+**Negative check for the Secret**, done by pointing the deployment's volume
+at a name that does not exist rather than deleting the real Secret:
+
+```sh
+kubectl -n egress-proxies-operator-system patch deploy egress-proxies-operator-controller-manager \
+  --type=json -p '[{"op":"replace","path":"/spec/template/spec/volumes/1/secret/secretName","value":"kvm-ssh-absent"}]'
+```
+
+The manager came up and stayed up, 0 restarts, logging
+`WARNING: kvm ssh material is not usable ... open /etc/proxy-operator/kvm/id_ed25519:
+no such file or directory`. The Kubernetes Proxies kept reconciling; the
+KVM ones failed per reconcile and retried. Before this branch the same
+config exited the process before any controller started. Patched back
+afterwards, everything Ready.
+
+Worth noting: the KVM Proxies kept reporting `Ready` during that window
+rather than degrading visibly. That is existing behaviour, not a
+regression — a transient provider error rides the workqueue and does not
+clear a healthy status — but it does mean the only signal of a missing
+Secret is the log and the reconcile errors. Also, the ListByTag sweep on
+beelink still reports `managed=2` where one domain is an orphan from an
+older failed run; unrelated to this work, and the same leftover noted in
+the KVM plan's Run 6.
+
+Worth noting: the review filed this as a KVM finding, but the fix is mostly
+in the controller, because KVM is the only provider that resolves a
+providerID segment against operator config at all. GCP and Kubernetes hand
+the lookup to the backend, which answers 404 for a retired zone or a deleted
+namespace, so they reach not-found and release on their own. Anything else
+that classifies permanently on the delete path — revoked RBAC on the
+Kubernetes provider is the realistic one — was equally stuck before this
+step and is what the annotation is really for. The KVM table case
+`"host not configured"` moved out of `TestDelete_errors` into its own named
+test, since it is no longer an error.

docs/plans-executions/2026-09-11-1526-provider-health-checks.md

@@ -0,0 +1,378 @@
+# Execution: provider health checks
+Plan: [docs/plans/2026-09-11-1526-provider-health-checks.md](../plans/2026-09-11-1526-provider-health-checks.md)
+
+- [x] Step 0 — Save the plan
+- [x] Step 1 — ProviderHealth API and CRD
+- [x] Step 2 — Contract plus the kubernetes and GCP checks
+- [x] Step 3 — KVM check
+- [x] Step 4 — Monitor
+- [x] Step 5 — Metrics and wiring
+- [x] Step 6 — Docs
+- [x] Step 7 — Validate, record, push, open the PR
+
+## Step 0 — Save the plan
+
+Branched off `main` (at the PR #5 merge, `eb8f075`). The plan was copied out of
+`~/.claude/plans/` and committed on its own, before any code, as the repo's plan rules
+require:
+
+```bash
+git checkout -b feat/provider-health main
+cp ~/.claude/plans/now-i-would-like-flickering-wall.md docs/plans/2026-09-11-1526-provider-health-checks.md
+```
+
+The uncommitted `.claude/settings.json` change was left unstaged and stays out of this
+branch.
+
+## Step 1 — ProviderHealth API and CRD
+
+`kubebuilder` isn't installed here (only `bin/controller-gen`, `bin/kustomize` and
+`bin/setup-envtest` are), so `api/v1alpha1/providerhealth_types.go` and the `PROJECT`
+entry were written by hand instead of with `kubebuilder create api`. For a cluster-scoped
+resource `PROJECT` simply omits `namespaced`, which is what kubebuilder itself writes.
+
+The new types are registered in the existing `AddKnownTypes` call in `proxy_types.go`,
+not in a second `init()`.
+
+The deepcopy code, the CRD and the RBAC come from the usual generators:
+
+```bash
+make generate manifests
+go build ./... && go vet ./api/... && go test -short ./api/...
+```
+
+To confirm the new CRD and the viewer role actually reach the rendered install, I built
+the kustomization and grepped for them:
+
+```bash
+bin/kustomize build config/default | grep -n providerhealth
+# providerhealths.crawl.example.com (scope: Cluster, shortName ph)
+# egress-proxies-operator-providerhealth-viewer-role
+```
+
+Worth noting:
+- The only role added is the viewer role. The operator is the object's only writer, so
+  the admin and editor roles kubebuilder scaffolds for Proxy were not added.
+- The type file has no unit test, since it has no logic. The schema, including the
+  required `ready`/`lastTransitionTime` and the list-map keys, is proven by the envtest
+  round-trip in Step 4.
+- `config/rbac/role.yaml` is unchanged so far. The manager's own permissions arrive in
+  Step 4 with the Monitor's RBAC markers.
+
+## Step 2 — Contract plus the kubernetes and GCP checks
+
+`provider.Checker` is an optional interface in `internal/provider/check.go`, kept out of
+`Provider`. Each implementation has a compile-time assertion, because the monitor's
+type assertion would otherwise fail silently.
+
+- **kubernetes:** lists at most one managed Pod cluster-wide, using the same client, verb
+  and scope as `ListByTag`. The target is `apiserver`, at the `restCfg.Host` now stored
+  in the new `apiHost` field.
+- **GCP:** calls a new seam method, `Probe`. `realInstances.Probe` lives in `check.go`,
+  next to its only caller. `ListByTag`'s `Sprintf` filter became the shared
+  `managedFilter` constant, so the check and the GC sweep can't drift apart.
+
+Before relying on "MaxResults is only a page size", I checked the SDK's REST
+`AggregatedList` in the module cache:
+
+```bash
+grep -n "func (c \*instancesRESTClient) AggregatedList" -A80 \
+  "$(go list -m -f '{{.Dir}}' cloud.google.com/go/compute)"/apiv1/instances_client.go
+# InternalFetch: req.MaxResults = pageSize ... return elems, resp.GetNextPageToken(), nil
+```
+
+```bash
+go build ./... && go vet ./internal/provider/... && go test -race -count=1 ./internal/provider/...
+```
+
+Worth noting:
+- The planned `TestRealInstances_Probe_fetchesOnePage` became a three-case
+  `TestRealInstances_Probe` table.
+  - It keeps the first page's `nextPageToken`, and asserts exactly one request with
+    `maxResults=1` for each case.
+  - "empty project" covers `iterator.Done`.
+  - "api error keeps its status" proves the real client returns a `*googleapi.Error`, so
+    a disabled Compute API is reported as `Forbidden` rather than `Unreachable`.
+- Once a context has ended, both checks report `Timeout` whatever the error says. The
+  monitor's timeout is the usual cause, and the underlying error only repeats it.
+
+## Step 3 — KVM check
+
+`conn.go` now wraps each of the four dial stages in a `dialError`: ssh-material, tcp,
+ssh-handshake and libvirt-connect. `Error()` returns the inner message unchanged and
+`Unwrap()` returns the cause, so `classify`/`isClassified` and every existing
+message-matching test behave as before. `TestDialError_keepsClassification` pins this
+down.
+
+The three new RPCs went into `libvirtAPI`, `tracingAPI` and `fakeAPI`. The method-count
+guard in `TestTracingAPI_logsEveryMethod` fails until a traced row exists for each new
+method, which is what forced the three new rows.
+
+`Check` runs one goroutine per host, over the shared connection, through
+`hostPool.call`. The pool verdict is decided only after `call` returns, because
+returning it as an error from inside the callback would make `call` drop a perfectly
+good connection.
+
+Build, vet (also with the integration tag, since that file changed) and race tests:
+
+```bash
+go build ./internal/provider/... && go vet ./internal/provider/kvm/ \
+  && go vet -tags=integration ./internal/provider/kvm/ \
+  && go test -race -count=1 ./internal/provider/kvm/
+```
+
+Worth noting:
+- **Pool state wording:** `poolState` uses virsh's words (`running`, `inactive`, …)
+  rather than go-libvirt's constant names, so it reads like `virsh pool-list` output.
+- **Details:** each host also reports its SSH `user`. That is the first thing to check
+  when the reason is `SSHHandshakeFailed`.
+- **Tests renamed or merged:**
+  - The planned `TestSSHAuth_dial_stages` is `TestLazySSH_dial_tagsTheFailingStage`. It
+    dials through `lazySSH`, so the ssh-material stage is covered as well as the three
+    stages the in-process SSH server can trigger.
+  - The pool-inactive and pool-missing cases are merged into `TestCheck_libvirtVerdicts`,
+    which also covers a daemon error. Each case runs two checks and asserts a single
+    dial.
+- **Tests added beyond the plan:**
+  - `TestCheck_timeout`: a daemon that stops answering gives `Timeout`.
+  - `TestDialError_keepsClassification`.
+- **Not run yet:** `TestKVMCheck_realHost` in the integration file only compiles here.
+  It needs beelink, so it goes on the Step 7 checklist for the user to run.
+
+## Step 4 — Monitor
+
+`internal/providerhealth` has the `Monitor` runnable plus the pure `buildStatus`.
+
+- **Scheduling:** the Monitor is leader-only. It cleans up once at start, checks
+  immediately, then checks on a ticker.
+- **Each round:** providers are checked concurrently under one timeout, then logged and
+  published one after another. A failed write is logged, and the next round writes
+  again.
+- **Reads:** objects are read through the reader it is given. `main.go` passes the
+  manager's API reader (Step 5).
+
+The first `make manifests` left `config/rbac/role.yaml` unchanged, with no error.
+I had put the `+kubebuilder:rbac` markers inside `Monitor`'s doc comment, and
+controller-gen only reads RBAC markers from package-level comment groups. So they were
+silently attached to the type and ignored. An empty `git diff config/rbac/role.yaml` was
+the only sign. Moving them into their own comment block fixed it:
+
+```bash
+make manifests && git diff config/rbac/role.yaml
+# + providerhealths: create, delete, get, list
+# + providerhealths/status: get, patch
+```
+
+Vet, race tests and the envtest round-trip (using the `bin/k8s/1.36.2-linux-amd64`
+binaries `make setup-envtest` had already fetched):
+
+```bash
+go vet ./internal/providerhealth/ && go test -race -count=1 -v ./internal/providerhealth/
+# --- PASS: TestMonitor_envtestRoundTrip (6.45s) ... ok
+```
+
+Worth noting:
+- **Fake client scope:** the fake client needs a RESTMapper that registers
+  ProviderHealth as `RESTScopeRoot`. Its default mapper treats every kind as namespaced.
+- **Why the envtest calls `publish` directly:** `checkOnce` only logs write errors, so a
+  schema rejection would look like a missing object. Calling `publish` makes the API
+  server's answer the test failure.
+  - Its first round writes a 2000-character message; the stored message is truncated to
+    1024.
+  - Its second round proves that only the target that flipped gets a new
+    `lastTransitionTime`.
+- **No guard against a hung checker:** the Monitor trusts `Checker`'s contract to return
+  once the context ends, and doesn't abandon a checker that ignores it. All three
+  checkers stop when the context ends: KVM through `awaitOrCancel`, and kubernetes and
+  GCP through their clients.
+- **Test renamed:** the planned `TestMonitor_timeoutBoundsBlockingChecker` is
+  `TestMonitor_checkOnce_timeoutBoundsEachRound`. It also asserts that a fast provider
+  still publishes next to a hung one.
+
+## Step 5 — Metrics and wiring
+
+`metrics.RegisterProviderTargets` registers a scrape-time collector for three gauges,
+each labelled `{provider,target}`: `up`, check duration and check timestamp.
+`Register`'s signature is unchanged.
+
+`cmd/main.go`:
+- gains `--provider-check-interval` (1m) and `--provider-check-timeout` (15s);
+- validates them before even loading the providers config;
+- collects the `Checker`s from the unwrapped providers in config order, just before the
+  metrics and tracing wrap loop;
+- adds the Monitor with the manager's API reader;
+- registers the collector through a small `providerTargetSamples` closure next to
+  `proxyPhaseCounts`.
+
+Build, vet and metrics tests:
+
+```bash
+go build ./... && go vet ./cmd/... ./internal/metrics/ && go test -race -count=1 ./internal/metrics/
+```
+
+To confirm bad flags fail fast, before anything touches the cluster, I ran the binary
+with a 5s interval. zap's development mode prints a stack trace after the error, hence
+the grep:
+
+```bash
+go run ./cmd --providers-config=/dev/null --provider-check-interval=5s 2>&1 | grep -m1 "Invalid provider check flags"
+# ERROR setup Invalid provider check flags {"error": "--provider-check-interval must be 0 (off) or at least 10s, got 5s"}
+go vet -tags=e2e ./test/e2e/
+```
+
+The e2e suite gets a new ordered spec, "should publish provider health". It waits for
+`providerhealth kubernetes` to report `Ready=True`. It runs before the existing metrics
+spec, which now also asserts
+`proxy_operator_provider_target_up{provider="kubernetes",target="apiserver"} 1`. By then
+the first round is known to be done, so the series is certain to exist.
+
+Worth noting:
+- **Main-only wiring:** `providerTargetSamples` is an 8-line loop in `package main`,
+  which has no test file. The e2e metrics assertion is its only coverage.
+- **The KVM entry isn't asserted in e2e:** the default providers config also ships
+  `kvm-home` (beelink), which would report `SSHMaterialUnavailable` in kind. That entry
+  is the user's own host, and asserting it would break the suite as soon as the default
+  config changes.
+- **Manifest:** `config/manager/manager.yaml` only gained a comment about the two flags.
+  The defaults need no args.
+
+## Step 6 — Docs
+
+- **`docs/architecture.md`:**
+  - component rows for the CRD, the `Checker` contract and the monitor;
+  - a new §11 covering the provider table, the loop diagram, the write, read and
+    decorator choices, the gauges, and sample `kubectl get ph` output;
+  - cross-references from the metrics (§9) and tracing (§10) sections;
+  - four Decisions entries.
+- **`docs/kvm-provider.md`:**
+  - a "Health check" section with the reason → meaning → where-to-look table and the
+    two consequences of sharing the connection;
+  - under Known gaps, the capacity bullet now points at `poolAvailable`, and a new
+    bullet says a rotated `kvm-ssh` Secret needs a restart;
+  - the Testing section names `TestKVMCheck_realHost`.
+- **`docs/testing.md`:**
+  - suite rows for the three `check_test.go` files, `internal/providerhealth` (unit and
+    envtest) and the metrics gauges;
+  - "17 libvirt calls" is now 20;
+  - the integration and e2e sections describe the new tests.
+- **`CLAUDE.md`:**
+  - the Architecture TODO now points at `docs/architecture.md`;
+  - the stale "only `go.mod` exists" gotcha is replaced by a note that the "What this
+    is" and "Commands" sections predate the code, plus Step 4's RBAC-marker gotcha.
+
+Worth noting:
+- `docs/api.md` is untouched. It documents the discovery HTTP API, which this change
+  doesn't alter.
+- `CLAUDE.md`'s "What this is" and "Commands" sections were left stale on purpose.
+  Rewriting them is outside this plan, so the new gotcha flags them instead of leaving
+  them silently wrong.
+
+## Step 7 — Validate, record, push, open the PR
+
+The full suite runs codegen, `go fmt` and `go vet`, then `go test -race` on every
+package, with both envtest suites included:
+
+```bash
+make test
+# every package ok; internal/providerhealth 92.5 %, internal/provider/kvm 92.0 % coverage
+```
+
+`go fmt` rewrote nothing, and `git status` showed only the Step 6 docs.
+
+`make lint` could not run, for a reason unrelated to this branch. The Makefile rebuilt
+the custom golangci-lint, which reports "built with go1.27.0", and its bundled
+staticcheck (honnef.co/go/tools v0.7.0) panics on the standard library before it
+reaches any repo package:
+
+```bash
+make lint
+# buildir: package "poll" (isInitialPkg: false, needAnalyzeSource: true): unexpected expr: *ast.KeyValueExpr
+# make: *** [Makefile:107: lint] Error 3
+```
+
+The local toolchain is go1.27.0 while `go.mod` says 1.26.0; the otel-tracing plan's
+log shows `make lint` running normally before the toolchain moved. Retrying with
+`GOTOOLCHAIN=go1.26.0` would have meant downloading a toolchain, and the user declined
+that. `go vet` is clean in all three configurations: plain inside `make test`, plus
+`-tags=integration` and `-tags=e2e`.
+
+The kind e2e suite creates and deletes its own `egress-proxies-operator-test-e2e`
+cluster. I checked `KIND_CLUSTER` first, because the two clusters already on the
+machine (`kind`, `proxy-operator-demo`) must not be torn down by `cleanup-test-e2e`:
+
+```bash
+make test-e2e > <scratch>/e2e.log 2>&1
+# Ran 3 of 5 Specs in 86.210 seconds
+# SUCCESS! -- 3 Passed | 0 Failed | 0 Pending | 2 Skipped
+```
+
+In the deployed manager, the `kubernetes` ProviderHealth reported `Ready=True` about a
+second after the pod was up (20:30:05.6 → 20:30:06.7). The metrics scrape that followed
+contained `proxy_operator_provider_target_up{provider="kubernetes",target="apiserver"} 1`.
+That exercises the generated RBAC, the API-reader path and the `main.go` gauge wiring
+end to end. The two skipped specs are the Tempo-gated tracing specs; `TEMPO_URL` was not
+set.
+
+### Follow-up — the check must not cost the reconciles their connection
+
+Reviewing the shared-connection decision after the automated validation turned up two
+ways the health check could hurt the work it shares the link with, both of which the
+plan's "a check against a hung host drops the shared connection" line accepted too
+easily:
+
+- A check that timed out went through `hostPool.call`, whose cancel path drops the
+  connection. A `libvirtd` slow to answer one `ConnectGetLibVersion` would therefore
+  cut off an image upload or a `Create` running on the same link.
+- `hostEntry` was guarded by a `sync.Mutex`, so waiting for a host's slot could not be
+  interrupted. A reconcile redialing a host that is down holds the slot for as long as
+  the SSH dial takes — minutes on a blackholed address — and the whole check round
+  would wait behind it, past its own timeout.
+
+So the check now calls `hostPool.probe` instead of `call`: on `ctx` ending it abandons
+the RPC and leaves the connection alone (a link that is really dead is closed by the
+keepalive anyway, which ends the abandoned `fn` too). One abandoned probe per host is
+allowed to stay waiting — `hostEntry.probing` is an `atomic.Bool`, not a counter, so a
+daemon that never answers cannot pile up goroutines; further probes get the new
+`errProbeBusy`, which `checkReason` maps to `Timeout`. `hostEntry.mu` became a one-slot
+channel with an `acquire(ctx)` that only bounds *waiting*: an uncontended caller takes a
+free slot even with an expired context, so `call`'s behaviour is unchanged where nothing
+is contended.
+
+The second change came out of the same pass: `--provider-check-timeout` defaulted to a
+flat 15s while the flag validation rejected a timeout longer than the interval, so
+`--provider-check-interval=12s` alone failed to start. The flag now defaults to 0 and
+`Monitor.setDefaults` derives `min(15s, Interval)`; `ValidateSchedule` accepts 0 and
+rejects only negatives and timeouts past the interval.
+
+Re-verified with the same commands as above:
+
+```bash
+go vet ./...
+go test -race -count=1 ./internal/provider/kvm/ ./internal/providerhealth/
+make test
+# every package ok; internal/provider/kvm 92.2 %, internal/providerhealth 93.7 % coverage
+```
+
+Worth noting:
+- `make lint` still cannot run, for the toolchain reason recorded above; unchanged by
+  this follow-up.
+- The two new `kvm` tests are timing-shaped by necessity — they assert that a caller
+  with a 50ms deadline does not wait out a dial that is blocked on a channel. The
+  assertions are one-sided (`> 1s` fails) so a slow machine does not make them flaky.
+- `docs/architecture.md`, `docs/kvm-provider.md` and `docs/testing.md` were corrected in
+  the same commit: the old text promised the drop-on-timeout behaviour that is now gone.
+- The real-hypervisor commands from the plan's Verification section are still the user's
+  to run; nothing here replaces them.
+
+### Real-hypervisor validation and PR
+
+The user ran the plan's real-hypervisor checks against beelink (the `TestKVMCheck`
+integration test plus the manual `ProviderHealth` and metrics checks) and confirmed
+they passed. After that the CHANGELOG entry went in, and the branch was pushed with
+the token-file credential helper. The PR against `main` was opened through the Gitea
+API because `tea` isn't installed.
+
+Worth noting:
+- The branch also carries `docs/plans/2026-09-16-0841-provider-blueprint.md`, on
+  purpose: that plan reaches `main` with this merge, and the blueprint work starts on a
+  new branch afterwards.

docs/plans-executions/2026-09-16-0841-provider-blueprint.md

@@ -0,0 +1,177 @@
+# Execution: provider blueprint
+Plan: [docs/plans/2026-09-16-0841-provider-blueprint.md](../plans/2026-09-16-0841-provider-blueprint.md)
+
+- [x] Step 0 — Save the plan; wait for the merge
+- [x] Step 1 — Collect evidence (read-only, in parallel)
+- [x] Step 2 — Write `docs/providers.md`
+- [x] Step 3 — Verify, then open the MR
+
+## Step 0 — Save the plan; wait for the merge
+
+The plan was committed on `feat/provider-health` (`80b2e9c`) and reached `main`
+through PR #6. After the user confirmed the merge, the branch was cut from the
+updated `main`:
+
+```bash
+git switch main && git pull --ff-only
+git switch -c feat/provider-blueprint
+```
+
+Worth noting: `.claude/settings.json` has local, uncommitted permission edits
+that carried over through the switch. They are deliberately left out of this
+branch.
+
+## Step 1 — Collect evidence (read-only, in parallel)
+
+Three Explore agents answered the plan's 12-item questionnaire, one each for
+kubernetes, gcp and kvm. Meanwhile I read the shared layer:
+`internal/provider/*.go`, `registry.go`, the provider wiring in `cmd/main.go`,
+`reconcileManaged`/`replaceInstance`/`reconcileDelete` in the controller,
+`internal/gc/gc.go`, and how the monitor uses `Checker`.
+
+Before relying on the agents' claims, I spot-checked them against the code. One
+was wrong and did not go into the doc: the kvm agent said `Delete` skips volume
+removal when `DomainDestroy` returns `ErrNoDomain`. In fact `removeDomain`
+returns nil there, and `Delete` still calls `removeVolumes` afterwards
+(`kvm.go:477-482`).
+
+Rules that exist only in the callers, and not in any comment, were found by
+reading the controller and GC. They went into the doc as MUST rules:
+- A zero `CreatedAt` makes an instance immediately collectable (`gc.go:120`).
+- `Get` returning `ErrPermanent` on the deletion path makes the reconciler call
+  `Delete` anyway.
+- Replacement waits for `Get` to return NotFound before creating an instance
+  with the same name.
+- `Stopped` and `Terminated` both lead to delete-and-recreate.
+
+Worth noting, found outside this plan's scope and not changed here:
+- `docs/gcp-vm-validation.md` filters on `labels.proxy-operator-managed=yes`,
+  but the value is `true`.
+- According to the gcp agent, `docs/gcp-in-specific-project.md` holds a real
+  project ID, IAM output and an egress IP. The user was told.
+
+## Step 2 — Write `docs/providers.md`
+
+Wrote the eight sections the plan lists. Linked the new doc from the Provider
+contract row in `docs/architecture.md` and from the CLAUDE.md "Architecture"
+section.
+
+Judgment calls:
+- The file:line evidence sits in per-provider notes under the matrix, as
+  planned. The contract rules in §2 cite the reconciler and GC lines each rule
+  protects.
+- The gap list includes documentation drift (kvm-provider.md's ARP order and
+  log keys, gcp-vm-validation.md's label value and its claim that async errors
+  reach status), as well as code gaps. Both would mislead the next
+  provider author in the same way.
+- "Proposed features" stays a flat list with no design, as the user decided.
+
+Worth noting: the line numbers in the doc are a snapshot. The doc says so and
+asks readers to look for the named function when a line has moved.
+
+## Step 3 — Verify, then open the MR
+
+An Explore agent re-checked the doc against the code: about 150 claims,
+including every matrix cell. It found four mismatches, and all four were fixed:
+- **Startup dial:** I had listed kvm-provider.md's "connection kept open from
+  startup" as doc drift. It isn't: the pool dials lazily, but the provider
+  health monitor runs a check as soon as the leader starts
+  (`monitor.go:147`, default interval 1m), and that check opens the connection.
+  The bullet was removed and the kvm notes now say this.
+- **Checker wording:** §5 said the monitor type-asserts `Checker`. It is
+  `cmd/main.go:216`; the monitor only sees nil or not.
+- **Reasons, not conditions:** §3 called `QuotaExceeded` and `PermanentError`
+  conditions. They are reasons on `Provisioned`.
+- **Latch lines:** the line range is `:133-139`, not `:130-135`.
+
+It also noted that none of the providers assert `var _ provider.Provider`, and
+that the kubernetes provider's RBAC markers live in the controller package. The
+§7 checklist now says both.
+
+Link check (every relative `](…)` target and anchor in the new and edited docs
+resolves), then the sanity build:
+
+```bash
+go build ./... && go test -short ./...
+# all packages ok
+```
+
+Worth noting: no CHANGELOG entry yet. Per CLAUDE.md it goes in after the user
+confirms the doc.
+
+### Follow-up — tracing was missing from the blueprint
+
+The user pointed out, after the PR was opened, that the blueprint said almost
+nothing about tracing. The plan's questionnaire item 10 had collected the
+evidence, but the doc reduced it to a matrix row about logging and a checklist
+line telling authors not to open spans. That line even suggests tracing needs
+no work. A provider author needs the opposite message: the operation span comes
+free, and child spans for backend calls are theirs to add.
+
+Added a new §6, "Observability: tracing, logs and metrics", and renumbered the
+matrix, checklist and gaps sections to §7–§9. It covers:
+- **Three layers:** decorator operation spans; child spans per backend call,
+  with parent-gated HTTP transports or a span decorator over the SDK seam; and
+  logs.
+- **The ctx and logger rules** that keep `traceID` on log lines.
+- **A per-provider status table:**
+  - kubernetes: parent-gated `RestConfigWrapper` spans.
+  - gcp: the Google SDK's own `otelhttp`, never disabled by the provider.
+  - kvm: no spans. `trace.go` is a log decorator.
+
+Also added two tracing rows to the §7 matrix, rewrote checklist item 8 as
+"Tracing and logging" with an in-memory span-recorder test, and added gaps (kvm
+has no child spans; only kubernetes is checked against Tempo, and only at
+operation level). A proposal notes that the kvm `libvirtAPI` seam takes no
+`ctx`, so per-RPC spans need that plumbed in.
+
+The links were re-checked, including the new `architecture.md#10-tracing-internaltracing` anchor.
+
+### Follow-up — a completeness sweep
+
+The user asked for one more pass over everything provider-related. After the
+tracing miss, this pass read beyond the provider packages:
+- the Proxy API types and the spec hash;
+- the health engine and discovery call sites that consume `Instance.IP`;
+- the manager Deployment, the providers ConfigMap and the samples;
+- README, testing.md, kvm-provider.md and the GCP docs;
+- the initial prompt, the proxy-operator and kvm plans, and the 2026-08-10 PR
+  review;
+- the CHANGELOG.
+
+What the blueprint was missing, now added:
+- **What the instance is for.** The IP and port must be reachable from the
+  operator's health probes and from discovery clients. Providers don't know
+  which proxy runs; the proxy comes from cloud-init, except for kubernetes.
+- **Concurrency.** Three reconciles, GC and checks can call a provider at once,
+  so it MUST be safe for concurrent use.
+- **The runtime envelope:** read-only root filesystem, a 64Mi `/tmp`, a 128Mi
+  memory limit, credentials as optional mounts.
+- **Create:** `placement` can't be validated by the CRD, so error messages must
+  name the field. Catalogue edits in config aren't in the spec hash.
+- **Delete** is re-called every `DeletionPoll` while a delete is in progress.
+- **Provider names** must be DNS-1123 for `ProviderHealth`, and `spec.provider`
+  is immutable.
+- **Metric names** in §6.
+- **Three matrix rows:** proxy software, IP reachability, distinct egress.
+- **A deployment checklist item:** credentials mount, config entries, sample
+  Proxy, Makefile helper.
+- **Gaps:**
+  - still open from the PR review: GC has no installation identity; an
+    unconfigured provider latches Proxies; kubernetes Pod RBAC always installed
+    and no `OwnerReference` on Pods;
+  - the duplicated type list;
+  - samples that can't work: `proxy_gcp.yaml` uses a debian-cloud image without
+    cloud-init, and `proxy_kvm.yaml` places on `nas1` where configs define
+    `beelink`;
+  - the `"mock"` example in the `spec.provider` doc.
+
+Outside the plan's "leave other docs alone" line, README.md changed:
+- Its "Adding a provider" section restated the contract, with the old "Create
+  never blocks" rule that kvm contradicts. It now points to `docs/providers.md`
+  instead of keeping a second copy that disagrees.
+- The "Implemented providers" table gained the missing kvm row.
+- The deploy note no longer claims the default ConfigMap ships only the
+  kubernetes provider.
+
+The links were re-checked for providers.md and README.md.

docs/plans-executions/2026-09-16-1108-provider-docs-split.md

@@ -0,0 +1,228 @@
+# Execution: Split the provider blueprint into contract, appendix, template and index
+
+Plan: [docs/plans/2026-09-16-1108-provider-docs-split.md](../plans/2026-09-16-1108-provider-docs-split.md)
+
+- [x] Step 0 — Save and commit the plan
+- [x] Step 1 — Write `docs/provider-contract.md`
+- [x] Step 2 — Write `docs/provider-spec-template.md`
+- [x] Step 3 — Write `docs/providers-existing.md`
+- [x] Step 4 — Rewrite `docs/providers.md` as the index; link edits
+- [x] Step 5 — Verification sweep
+- [x] Step 6 — Final summary, CHANGELOG after confirmation
+
+## Step 0 — Save and commit the plan
+
+Committed as `cf89ac4` on the existing `feat/provider-blueprint` branch, on
+its own, before any document work. The pre-existing local modification of
+`.claude/settings.json` was left out of the commit.
+
+## Step 1 — Write `docs/provider-contract.md`
+
+2026-09-16 11:17. Written from scratch rather than edited down from the old
+blueprint, so that nothing provider-specific could survive by accident. Every
+MUST names its caller by function (`reconcileManaged`, `replaceInstance`,
+`reconcileDelete`, `providerFailure`, `deletionFailure`, `Sweeper.sweep`,
+`Monitor.checkOnce`, `Monitor.run`, `registry.Build`) and states what breaks.
+
+**Axis codes frozen** (19, §4.1 to §4.19): `readiness`, `create-return`,
+`delete-return`, `ip-source`, `gc-mark`, `created-at`, `list-partial`,
+`quota`, `not-found`, `primary-key`, `provisioning-state`, `async-ops`,
+`secondary-resources`, `credentials`, `client-kind`, `timeouts`, `logging`,
+`child-spans`, `backend-test`. The appendix table and the template are keyed
+on these strings; the §4.20 summary table's last column gives the template
+entry number each axis maps to.
+
+**Two rules that were not in the old blueprint**, both found by reading the
+callers while verifying:
+
+- `Sweeper.sweep` loops providers sequentially in one goroutine, and no
+  caller puts a deadline on the context (only `Monitor.checkOnce` uses
+  `WithTimeout`). So a hung `ListByTag` stalls orphan GC for every provider.
+  Recorded in §1.5 and the `timeouts` axis.
+- The Stopped branch of `reconcileManaged` calls `Delete` and requeues
+  without clearing `ProviderID`, then deletes again on every poll until
+  NotFound. A backend that reports a freshly created, never started instance
+  as stopped would have it deleted right after creation. Recorded as a MUST
+  in §2.2 and as the `provisioning-state` axis with three allowed profiles.
+
+Checks run on the draft (all clean):
+
+```sh
+grep -niE "kubernetes|gcp|kvm|libvirt|squid|google|compute engine" docs/provider-contract.md
+grep -n "—" docs/provider-contract.md
+grep -nE '\.go:[0-9]+|:[0-9]+-[0-9]+' docs/provider-contract.md
+grep -ohE "(internal|cmd|config|docs|api|hack|test)/[A-Za-z0-9_./-]+" docs/provider-contract.md | sort -u | while read p; do test -e "$p" || echo "MISSING $p"; done
+```
+
+The path check reports three expected misses: `config/samples/proxy_<type>.yaml`
+is a pattern, `docs/provider-spec-template.md` is written in Step 2, and
+`internal/provider/providertest` is a proposal named in §13.
+
+Worth noting: the contract refers to "the operator pod" and "the cluster's
+own API" rather than naming the platform, so that the abstractness grep stays
+meaningful. The `hack/providers-dev.yaml` rule is phrased as "only if the type
+needs no credentials" because that file exists for `make run-dev` without any
+Secret.
+
+## Step 2 — Write `docs/provider-spec-template.md`
+
+2026-09-16 11:28. Twenty-four entries in a fixed shape (Question, Allowed
+answers, Obligations, Answer, Evidence). Thirteen of them carry an axis code
+and their numbers are the ones the contract's §4.20 table points at; the
+other eleven (identity, placement, image, cloud-init, machine type, config
+block, Checker, concurrency, deployment, docs, accepted limits) are decisions
+the contract leaves open without being a profile axis. Entries 9, 13 and 15
+carry an empty table to fill in (state mapping, error classification, config
+fields); entry 21 carries the contract §11 test list as checkboxes so a
+filled spec doubles as the test plan.
+
+Check that every axis code appears in the template:
+
+```sh
+for c in $(grep -oE '^### 4\.[0-9]+ `[a-z-]+`' docs/provider-contract.md | grep -oE '`[a-z-]+`' | tr -d '`'); do
+  grep -q "\`$c\`" docs/provider-spec-template.md || echo "template missing $c"
+done
+```
+
+Clean. Worth noting: the preamble allows "verify against the target version"
+only on an Evidence line, never as an Answer, because the whole point of the
+file is that the implementer does not guess.
+
+## Step 3 — Write `docs/providers-existing.md`
+
+2026-09-16 11:35. The profile table has exactly one row per contract axis
+code, checked by the same loop as Step 2 with a row count. Everything else
+comes from the old blueprint's §6, §7 and §9 plus a fresh read of the three
+packages, with every `file:line` replaced by a symbol name.
+
+Corrections to earlier notes applied while writing: kvm's Checker `Details`
+also carry `user` and `pool`; the kubernetes provider has no seam interface
+(`newWithClient` injects a `client.Client`, the fake client stands in); kvm's
+`classify` has an `isClassified` pass-through. Timeouts were checked per
+provider for the `timeouts` row: kubernetes sets no `rest.Config.Timeout`,
+gcp relies on SDK defaults, kvm bounds only the SSH handshake
+(`defaultHandshakeTimeout` in `dialSSH`), so all three are weaker than the
+contract's SHOULD and the table says so.
+
+Symbol check on the appendix (every backticked identifier grepped for in
+Go, Makefile and YAML sources):
+
+```sh
+grep -ohE '`[A-Za-z_][A-Za-z0-9_.]*`' docs/providers-existing.md | tr -d '`' | sort -u | while read s; do
+  grep -rqwF --include='*.go' --include='Makefile' --include='*.yaml' -- "$s" . || echo "$s"
+done
+```
+
+The residue is file names, dotted `Type.method` forms, the `timeouts` axis
+code, libvirt's `nostate`, and the API names `OwnerReference` and
+`rest.Config.Timeout`; nothing unexpected.
+
+Worth noting: matrix rows that the profile table now carries (Create
+non-blocking, GC marks, IP source, quota detection, Checker presence,
+credentials, logging, tracing, tests) were dropped from "Other differences"
+rather than kept twice.
+
+## Step 4 — Rewrite `docs/providers.md` as the index; link edits
+
+2026-09-16 11:40. `docs/providers.md` was rewritten in place (same path, so
+the inbound links from `README.md`, `CLAUDE.md`, `docs/architecture.md` and
+the historical plan files still resolve, and the pre-split blueprint stays
+reachable in the path's git history). The three inbound sentences were
+updated to name the new files:
+
+- `README.md` "### Adding a provider": the first sentence now names the
+  contract, the template and the appendix, with `docs/providers.md` as the
+  index; the "Start there" sentence is unchanged.
+- `CLAUDE.md` "## Architecture": the providers.md sentence now names all
+  three files and the index.
+- `docs/architecture.md` components row "Provider contract": "rules in
+  provider-contract.md, per-provider profile table in providers-existing.md".
+
+The historical files under `docs/plans/` and `docs/plans-executions/` keep
+their `providers.md` links on purpose.
+
+## Step 5 — Verification sweep
+
+2026-09-16 11:45. All ten checks from the plan, run against the working tree
+after Step 4; nothing needed fixing.
+
+Links (file targets and anchors) across the four new files plus `README.md`,
+`CLAUDE.md` and `docs/architecture.md`, with anchors compared against
+slugified headings of the target file:
+
+```sh
+python3 - <<'PY'
+import re, pathlib
+files = ["docs/provider-contract.md","docs/provider-spec-template.md","docs/providers-existing.md","docs/providers.md","README.md","CLAUDE.md","docs/architecture.md"]
+def slug(h):
+    h = re.sub(r'`','',h).strip().lower(); h = re.sub(r'[^\w\s-]','',h)
+    return re.sub(r'\s+','-',h).strip('-')
+def headings(p):
+    return {slug(m.group(2)) for m in (re.match(r'^(#{1,6})\s+(.*)$', l) for l in open(p)) if m}
+bad = 0
+for f in files:
+    for m in re.finditer(r'\]\(([^)\s]+)\)', open(f).read()):
+        t = m.group(1)
+        if t.startswith(('http://','https://','mailto:')): continue
+        path, _, anchor = t.partition('#')
+        target = pathlib.Path(f) if path == '' else pathlib.Path(f).parent/path
+        if not target.exists(): print(f, "missing", t); bad += 1; continue
+        if anchor and target.suffix == '.md' and anchor not in headings(target): print(f, "bad anchor", t); bad += 1
+print("link problems:", bad)
+PY
+```
+
+Output: `link problems: 0`.
+
+Em-dashes, line references and the contract's abstractness:
+
+```sh
+grep -n "—" docs/provider-contract.md docs/provider-spec-template.md docs/providers-existing.md docs/providers.md
+git diff 2d959af -- README.md CLAUDE.md docs/architecture.md | grep '^+' | grep "—"
+grep -nE '\.go:[0-9]+|:[0-9]+-[0-9]+' docs/provider-contract.md docs/provider-spec-template.md docs/providers-existing.md docs/providers.md
+grep -niE "kubernetes|gcp|kvm|libvirt|squid|google|compute engine" docs/provider-contract.md
+```
+
+The only em-dash hit is the pre-existing "Leader-elected" cell of the
+`architecture.md` components row, which shows as an added line because the
+row's last cell was edited; no new em-dash was written. The other three
+greps are empty.
+
+Nothing lost from the old blueprint's §9: the bold leads of every gap and
+proposal bullet in `git show 0bb699b:docs/providers.md` were looked up in
+the new files. The literal-prefix match reported 22 misses, all caused by
+rewording ("isn't" to "is not", bold removed), so each topic was re-checked
+by a distinctive phrase and all 22 are present in `providers-existing.md`
+or contract §13.
+
+Paths and symbols: every `path/...` token in the four files exists except
+the `config/samples/proxy_<type>.yaml` pattern and the
+`internal/provider/providertest` proposal; every backticked identifier in
+the contract and template resolves in Go, Makefile or YAML sources except
+file names, dotted `Type.method` forms, profile names and the `NewWith...`
+placeholder.
+
+Axis alignment: the 19 axis codes from contract §4 each appear once as a
+row in the appendix table (six cells per row) and at least once in the
+template.
+
+`go build ./... && go test -short ./...`: all packages `ok`.
+
+## Step 6 — Final summary
+
+The split is complete on `feat/provider-blueprint`: five commits after the
+plan commit, one per step. The CHANGELOG entry is deliberately not written
+yet; per `CLAUDE.md` it is added once the user confirms the documents read
+the way they want. Nothing has been pushed.
+
+Worth noting for whoever picks this up:
+
+- The contract's §4.20 table maps axis codes to template entry numbers; if
+  entries are renumbered, that column must follow.
+- The `CHANGELOG.md` heading format in `CLAUDE.md` uses an em-dash; the
+  "no em-dashes" rule applied to the four new documents only.
+- The stale `mock` comments in `internal/provider/provider.go`,
+  `internal/provider/registry/registry.go` and `api/v1alpha1/proxy_types.go`
+  are still listed as gaps, not fixed here.
+- The next planned use of these files is a Proxmox provider: copy the
+  template to `docs/proxmox-provider-spec.md` and fill it in before any code.

docs/plans/2026-08-07-1747-proxy-operator.md

@@ -391,7 +391,10 @@ pods would refuse connections while still being Service endpoints. The 1-replica
 constraint comes from lease state being per-process, which the spec already accepts —
 both facts go in the README caveats.
 
-stdlib `http.ServeMux` with Go 1.22 method+wildcard patterns:
+stdlib `http.ServeMux` using its method+wildcard patterns (`"GET /path"`,
+`"/{id}"` + `r.PathValue`) — a stdlib feature available since Go 1.22, used
+here so no third-party router is needed; the project itself stays on the
+pinned Go 1.26:
 `GET /v1/proxies`, `POST /v1/leases`, `DELETE /v1/leases/{id}`,
 `POST /v1/leases/{id}/report`, plus unauthenticated `GET /healthz`.
 Middleware outermost-first: recover → request-log → `MaxBytesReader(64KiB)` → bearer

docs/plans/2026-08-11-1742-gcp-provider-verbose-logging.md

@@ -0,0 +1,87 @@
+# Plan: Verbose (V-level) logging in the GCP provider
+
+**Created:** 2026-08-11 17:42
+
+## Context
+
+Debugging GCP provisioning is currently blind: the operator has zero `.V(n)` calls
+anywhere, so `--zap-log-level=debug` (or any numeric level) reveals nothing about
+what the GCP provider is doing — which API calls it makes, with what parameters,
+and what came back. The goal: with debug/V-level logging enabled, see the details
+of every GCP Compute API call (Insert/Get/Delete/AggregatedList) including a
+summary of the response; with default `info` level, the provider stays as quiet
+as today.
+
+## Approach (the "how")
+
+**Logger source — context-carried, not injected.** Provider methods all take
+`ctx`, and the reconciler already builds a per-request logger
+(`logf.FromContext(ctx)` in `internal/controller/proxy_controller.go:117`) that
+carries the proxy's name/namespace. The GCP provider will do
+`log := logf.FromContext(ctx).WithName("gcp").WithValues("provider", p.name)` at
+the top of each public method. Zero wiring changes (no registry/constructor/struct
+changes), and every provider log line automatically inherits the reconcile
+context (which Proxy triggered it). Calls from the GC sweeper inherit its
+`orphan-gc` logger name the same way.
+
+**Verbosity scheme** (logr convention: `.Info()` = V(0), `debug` flag = V(1)):
+
+- **V(1)** — one line per GCP API call, after it returns: operation, identifying
+  params, outcome. Examples:
+  - `Create`: `"GCP insert instance"` with `zone`, `name`, `machineType`,
+    `image`, `opName` (currently discarded at gcp.go:117 — capture it, it's the
+    only handle for correlating with GCP's operation log), plus a line for the
+    409-already-exists path.
+  - `Get`: `"GCP get instance"` with `zone`, `name`, `status`, mapped `state`, `ip`.
+  - `Delete`: `"GCP delete instance"` with `zone`, `name`, `opName`, and the
+    404-treated-as-success path.
+  - `ListByTag`: `"GCP aggregated list"` with `filter`, `count`.
+  - Error paths at V(1) too: log the raw classification (HTTP status / reason
+    from `googleapi.Error`) before it's wrapped, since the wrapped error the
+    reconciler sees is coarser.
+- **V(2)** — request/response detail: full curated insert-request summary
+  (network, networkTag, diskSizeGB, port, labels, `cloudInitBytes` = `len`),
+  per-instance lines in `ListByTag` (id, state, uid, age).
+
+**Curated fields, never raw proto dumps.** `CreateRequest.CloudInit` is resolved
+user-data possibly sourced from a Secret, and it lands in the insert request's
+metadata — so logging the request proto wholesale would leak it. Log named safe
+fields only; for cloud-init, log only its byte length. This is a hard rule, and
+a test asserts it.
+
+**Where the calls live: the `Provider` methods in
+`internal/provider/gcp/gcp.go`** (Create/Get/Delete/ListByTag), not in
+`realInstances` (deliberately untested by design, gcp.go:86) and not an HTTP
+round-tripper (would log auth headers/user-data, unredactable). The
+`instancesAPI` fake seam (`newWithAPI`, gcp.go:96) keeps everything testable.
+To surface `opName`, change `Provider.Create`/`Delete` to capture the string
+their `instancesAPI` calls already return instead of discarding it.
+
+## Files to change
+
+- `internal/provider/gcp/gcp.go` — add `logf` import; V(1)/V(2) logging in
+  `Create`, `Get`, `Delete`, `ListByTag`; capture opNames. Only file with
+  production changes.
+- `internal/provider/gcp/gcp_test.go` — new table-driven test
+  `TestLogging_verbosity` (name TBD per house `Test<Function>_<scenario>`
+  style): inject a capturing logger via `logf.IntoContext(ctx, funcr.New(...))`
+  (`github.com/go-logr/logr/funcr`, logr already a direct dep), assert:
+  - at V(1): expected message + keys per operation (incl. opName),
+  - at V(0): nothing logged,
+  - **cloud-init content never appears in any log output** (grep the captured
+    lines for a sentinel string placed in `CloudInit`).
+- No changes to `provider.Provider` interface, registry, `cmd/main.go`,
+  manifests, or the kubernetes provider (it can copy this pattern later).
+
+## Verification
+
+```bash
+go test -race ./internal/provider/gcp/...
+go build ./...
+```
+
+Optional live check: run the manager with `--zap-log-level=2` against the GCP
+project and confirm insert/get lines appear during a Proxy reconcile, and that
+`--zap-log-level=info` stays quiet.
+
+Also append a CHANGELOG.md entry per house convention once confirmed working.

docs/plans/2026-08-11-1802-bake-commit-version.md

@@ -0,0 +1,75 @@
+# Plan: Bake the git commit into the operator binary and log it at startup
+
+**Created:** 2026-08-11 18:02
+
+## Context
+
+There is no versioning yet, and the image tag (`egress-proxies-operator:dev`,
+`imagePullPolicy: IfNotPresent`) says nothing about what code is actually
+running. The user wants the commit hash baked into the image at build time,
+and — the key requirement — the binary itself must know it and print it into
+the log stream during initialization, so `kubectl logs | head` answers "which
+version is running". Docker builds cannot use Go's automatic VCS stamp because
+`.dockerignore` excludes `.git` (correctly — re-including it would bust layer
+caching), so the hash must travel git → Makefile → `--build-arg` → `-ldflags -X`.
+
+## Implementation
+
+**1. New package `internal/version`** (`version.go` + `version_test.go`):
+
+- `var Commit string` — stamped at link time via
+  `-ldflags "-X gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version.Commit=<hash>"`.
+- `func Resolve() string` — returns `Commit` if non-empty; otherwise falls
+  back to `debug.ReadBuildInfo()` VCS settings (`vcs.revision` truncated to
+  12 chars, `-dirty` suffix when `vcs.modified=true`), so plain host builds
+  (`make build`, `make run`, `go run`) are stamped for free since `.git` is
+  present there; `"unknown"` when neither source is available (e.g. `go test`).
+- Internal `resolve(ldflagsCommit string, readBuildInfo func() (*debug.BuildInfo, bool)) string`
+  so the fallback logic is table-testable with a fake build-info func
+  (house style: stdlib testing, `t.Parallel()`, subtests,
+  `Test<Function>_<scenario>` names).
+
+**2. `cmd/main.go`**:
+
+- Add `--version` bool flag; immediately after the existing `flag.Parse()`
+  (main.go:132), if set: print `version.Resolve()` to stdout and exit 0
+  (before logger/manager setup).
+- Right after `ctrl.SetLogger(...)` (main.go:134):
+  `setupLog.Info("Starting egress-proxies-operator", "commit", version.Resolve(), "goVersion", runtime.Version())`
+  — first line of every run, plain V(0) so it appears at any log level.
+
+**3. `Makefile`**:
+
+- Near the other variables:
+  `GIT_COMMIT ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)$(shell test -z "$$(git status --porcelain 2>/dev/null)" || echo -dirty)`
+  (`git status --porcelain` catches staged and untracked changes, which
+  `git diff --quiet` misses).
+- `docker-build`: add `--build-arg GIT_COMMIT=$(GIT_COMMIT)`.
+- `docker-buildx`: add the same `--build-arg` to the `buildx build` line.
+- `build`/`run`/`run-dev` stay untouched — the ReadBuildInfo fallback covers them.
+
+**4. `Dockerfile`**:
+
+- `ARG GIT_COMMIT=unknown` in the builder stage; extend the existing
+  `go build` with `-ldflags "-X <module>/internal/version.Commit=${GIT_COMMIT}"`.
+- Re-declare `ARG GIT_COMMIT` in the distroless stage and add
+  `LABEL org.opencontainers.image.revision="${GIT_COMMIT}"` so the hash is
+  also visible via `docker inspect` without running the binary.
+
+No changes to deploy manifests, providers, or the reconciler. CHANGELOG entry
+after the user confirms it works (house convention).
+
+## Verification
+
+```bash
+go test ./internal/version/            # resolve() table tests
+go build ./... && go test ./...        # nothing else broke
+go run ./cmd/main.go --version         # host build: VCS-stamped hash (+ -dirty), exit 0
+make docker-build IMG=egress-proxies-operator:dev
+docker run --rm egress-proxies-operator:dev --version          # prints the baked commit
+docker inspect egress-proxies-operator:dev \
+  --format '{{index .Config.Labels "org.opencontainers.image.revision"}}'
+```
+
+Live check: redeploy on the cluster and confirm the first log line carries
+`"commit"`.

docs/plans/2026-08-11-1838-gcp-http-wire-logging-v5.md

@@ -0,0 +1,85 @@
+# Plan: GCP HTTP wire logging at V(5)
+
+**Created:** 2026-08-11 18:38
+
+## Context
+
+The GCP provider logs curated call summaries at V(1)/V(2), but when debugging
+against the real API the user wants ground truth: the actual HTTP requests and
+responses ("gory details") — visible at `--zap-log-level=5`, in the same log
+stream as everything else. The compute SDK already produces exactly this:
+`cloud.google.com/go/compute@v1.65.0/apiv1/helpers.go:60,70` logs
+`"api request"`/`"api response"` (method, URL, headers, full JSON payloads,
+lazily via `internallog.HTTPRequest/HTTPResponse`) to an injectable
+`*slog.Logger` at slog Debug level. We inject one bridged to the operator's
+zap sink, level-shifted so those Debug records surface only at V(5).
+
+Level scheme after this change: V(1) call outcomes, V(2) curated detail,
+V(5) raw HTTP traffic. V(3)/V(4) reserved.
+
+## Mechanism (verified in module sources)
+
+- `option.WithLogger(*slog.Logger)` exists in `google.golang.org/api@v0.292.0`
+  (option.go:529) and **takes precedence over `GOOGLE_SDK_GO_LOGGING_LEVEL`**
+  — after this change, V(5) is the single knob for this client; document that.
+- `logr.ToSlogHandler` (go-logr/logr v1.4.3, already a direct dep) maps
+  slog Debug → logr V(4), plus the base logger's V-bias. logr's own docs
+  (sloghandler.go:180-184): `slog.New(ToSlogHandler(logrV2)).Debug()` ≈ V(6).
+  So a base of `.V(1)` lands Debug at exactly V(5).
+- Gating is cheap: the slog handler's `Enabled()` consults the zap sink, so
+  below level 5 the SDK's lazy `LogValuer`s are never evaluated.
+
+## Implementation
+
+**`internal/provider/gcp/gcp.go`** (only production file):
+
+1. New pure function:
+   ```go
+   // wireLogger returns the slog logger handed to the SDK: its Debug-level
+   // "api request"/"api response" records (slog Debug = +4 on the logr
+   // scale) land at V(5) on top of the base's V(1) shift.
+   func wireLogger(base logr.Logger) *slog.Logger {
+       return slog.New(logr.ToSlogHandler(base.V(1)))
+   }
+   ```
+2. In `New` (gcp.go:96): pass it to the client —
+   `compute.NewInstancesRESTClient(ctx, option.WithLogger(wireLogger(logf.Log.WithName("gcp").WithName("http"))))`.
+   Base is the process-root `logf.Log` (client is built once at startup;
+   `ctrl.SetLogger` runs before `registry.Build` in cmd/main.go, so it
+   resolves to the real zap logger).
+3. One-time notice in `New`: if `logf.Log.V(5).Enabled()`, log at Info:
+   `"GCP HTTP wire logging active — request payloads include cloud-init user-data"`
+   (the secret-leak warning our curated V(2) logging exists to avoid; at V(5)
+   the user has explicitly opted into raw payloads).
+4. New imports: `log/slog`, `google.golang.org/api/option` (module already in
+   go.mod as a direct dep; `option` package is a first-time import in the repo).
+
+**`internal/provider/gcp/gcp_test.go`**:
+
+- `TestWireLogger_gatesAtV5`: table over funcr sink verbosities
+  (`funcr.Options{Verbosity: N}`, pattern already used by `captureContext`):
+  at 5 a `Debug("api request", ...)` through `wireLogger` emits (message and
+  attrs present); at 4 it emits nothing; an `Info` record through the same
+  logger lands at V(1) (sanity-check of the shift).
+- `New` itself stays untested by design (dials real Google endpoints —
+  existing convention, gcp.go:86-87).
+
+No changes to manifests, Makefile, other providers, or the reconciler.
+CHANGELOG entry after the user confirms it works (house convention) — this
+plus the two earlier pending entries (GCP V-logging, version stamp).
+
+## Verification
+
+```bash
+go test -race ./internal/provider/gcp/
+go build ./... && go test ./...
+```
+
+Live (the real proof, needs the cluster):
+
+```bash
+# rebuild + load image, set --zap-log-level=5, restart, then:
+kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager -f \
+  | grep -m2 'api request\|api response'   # full URL/headers/payload visible
+# and at --zap-log-level=2: the same grep stays silent while V(2) lines still appear
+```

docs/plans/2026-08-11-1935-gitea-build-workflow.md

@@ -0,0 +1,135 @@
+# Plan: Distilled Gitea Actions image-build workflow
+**Created:** 2026-08-11 19:35
+
+## Context
+
+This repo (`egress-proxies-operator`) has a Dockerfile, a Makefile with `docker-build`/`docker-push` targets, and a Gitea remote — but no CI workflow (CLAUDE.md flags this as a TODO). A survey of all projects under `/Users/jan.novak/srv` found 9 image-build workflows, all variations of one lineage: trigger on `workflow_dispatch` + tag push, `docker login` to `gitea.home.hrajfrisbee.cz` with `secrets.REGISTRY_TOKEN`, raw `docker build`/`docker push`, `runs-on: ubuntu-latest`, `permissions: {contents: read, packages: write}`.
+
+The best individual ideas are scattered:
+- **aviso_v2**: quality-gate job before build; computes `sha-<short>` as a second immutable tag via `$GITHUB_OUTPUT`.
+- **gateway-helper-operator** (closest sibling — same kubebuilder shape): passes build args (`GIT_COMMIT` etc.), tags `:latest` alongside the version tag.
+- **psmf-data-sync test.yaml**: `actions/setup-go@v5` with `go-version-file: go.mod` + module cache (proven to work on the act_runner).
+
+Goal: distill these into one `build.yaml` for this repo. User decisions: triggers = **tags + manual dispatch only** (house convention, no builds from main); build tool = **raw docker CLI** (the runner bind-mounts docker.sock, so this just works); **lightweight test gate** (`go vet` + `go build` + `go test -short`, no envtest download); **amd64 only**.
+
+## The workflow
+
+Create `.gitea/workflows/build.yaml`:
+
+```yaml
+name: Build and Push
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Image tag'
+        required: true
+        default: 'latest'
+  push:
+    tags:
+      - '*'
+
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Vet
+        run: go vet ./...
+      - name: Build
+        run: go build ./...
+      - name: Test (short)
+        run: go test -short ./...
+
+  build:
+    needs: check
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Compute image tags
+        id: meta
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            TAG="${{ inputs.tag }}"
+          else
+            TAG="${{ github.ref_name }}"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "sha=sha-$(echo '${{ github.sha }}' | cut -c1-12)" >> "$GITHUB_OUTPUT"
+
+      - name: Login to Gitea registry
+        run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login -u ${{ github.actor }} --password-stdin gitea.home.hrajfrisbee.cz
+
+      - name: Build and push
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker build \
+            --build-arg GIT_COMMIT=$(echo '${{ github.sha }}' | cut -c1-12) \
+            --label org.opencontainers.image.source=https://gitea.home.hrajfrisbee.cz/${{ github.repository }} \
+            --label org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
+            -t "$IMAGE:${{ steps.meta.outputs.tag }}" \
+            -t "$IMAGE:${{ steps.meta.outputs.sha }}" \
+            .
+          docker push "$IMAGE:${{ steps.meta.outputs.tag }}"
+          docker push "$IMAGE:${{ steps.meta.outputs.sha }}"
+
+      - name: Push latest (tag builds only)
+        if: github.event_name == 'push'
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker tag "$IMAGE:${{ steps.meta.outputs.tag }}" "$IMAGE:latest"
+          docker push "$IMAGE:latest"
+```
+
+### What's distilled vs. improved over the existing workflows
+
+Distilled (house patterns kept as-is): triggers, `REGISTRY_TOKEN` + `github.actor` login, image name `gitea.home.hrajfrisbee.cz/${{ github.repository }}`, `ubuntu-latest`, raw docker CLI, `permissions` block.
+
+Improvements none of the existing workflows have all of:
+1. **`sha-<12>` immutable tag** alongside the human tag (aviso_v2 had this; nobody else) — lets deployments pin exactly what was built.
+2. **Test gate** (aviso_v2 had one; the Go projects don't) — lightweight variant per user choice; uses `go-version-file: go.mod` so the Go version never drifts from the module.
+3. **`GIT_COMMIT` build arg** matches the Makefile/Dockerfile contract — the binary's `internal/version.Commit` and the `org.opencontainers.image.revision` label get the real commit (12-char, same width as the Makefile's `git rev-parse --short=12`; no `-dirty` needed since CI checkouts are clean).
+4. **`:latest` only on real tag pushes**, not manual dispatch — gateway-helper pushed `latest` unconditionally, which lets an ad-hoc dispatch of an old ref clobber `latest`.
+5. **`concurrency` group** — cancels a superseded run of the same ref (none of the 20 surveyed workflows have this).
+6. **OCI `source`/`created` labels** added at build time (revision label already comes from the Dockerfile).
+
+## Files
+
+- **Create** `.gitea/workflows/build.yaml` — content above.
+- **Update** `CLAUDE.md` — replace the `TODO: no .gitea/workflows/ CI pipeline exists yet` note in the Git Commits section with a short CI/CD subsection describing the workflow (triggers, secret, tags produced).
+- **Update** `CHANGELOG.md` — new top entry (after user confirms it works, per convention; timestamp via `date "+%Y-%m-%d %H:%M %Z"`).
+- Copy this plan to `docs/plans/YYYY-MM-DD-HHMM-gitea-build-workflow.md` (timestamp via `date "+%Y-%m-%d-%H%M"`) and commit it first, per CLAUDE.md ordering rule.
+
+## Branch & MR
+
+House convention: feature → own branch + MR. Dockerfile and `cmd/` already exist on `main`, so:
+
+1. `git checkout -b feat/gitea-build-workflow origin/main` (do not touch the current `feat/proxy-operator` branch's uncommitted `.claude/settings.json` change — leave it be).
+2. Commit plan file, then the workflow + CLAUDE.md update (with `Co-Authored-By: Claude <noreply@anthropic.com>`).
+3. `git push -u origin feat/gitea-build-workflow`, open MR with `tea pr create --base main --head feat/gitea-build-workflow`. Do not merge.
+
+Note: `main` has no `internal/`/`test/` dirs yet (those are on `feat/proxy-operator`), which is fine — the workflow only fires on tags/dispatch, and by then the operator branch will be merged. `go build ./...` / `go test -short ./...` work on both branch states.
+
+## Prerequisite (user action)
+
+`REGISTRY_TOKEN` secret must exist in this repo's Gitea settings (Settings → Actions → Secrets): a personal access token with `write:package` scope — same as every other project uses. Flag this in the MR description.
+
+## Verification
+
+The workflow doesn't trigger on branch pushes, so end-to-end verification happens after merge:
+1. Local sanity: `docker build --build-arg GIT_COMMIT=test -t scratch-check .` (confirms the build args/labels line is valid) — or at minimum a YAML parse check.
+2. After the MR merges: run the workflow manually via Gitea UI (Actions → Build and Push → Run workflow, tag `manual-test`), confirm both `manual-test` and `sha-…` tags appear under Packages, and that `:latest` was NOT updated.
+3. Then push a real version tag (e.g. `v0.1.0`) and confirm `v0.1.0`, `sha-…`, and `latest` all appear.

docs/plans/2026-08-11-2152-discovery-api-docs.md

@@ -0,0 +1,58 @@
+# Plan: Discovery API documentation (docs/api.md)
+**Created:** 2026-08-11 21:52
+
+
+## Context
+
+The operator serves an HTTP discovery/lease API on `:8090` ([internal/discovery/](internal/discovery/)) that crawler clients use to list proxies, acquire TTL leases, release them, and report rate-limiting. There is no dedicated API reference today: README has four quickstart curls (no auth header, no schemas), and `docs/architecture.md` §7 has an ASCII route map. The user wants full documentation with curl examples for every feature.
+
+**Decisions made with user:** doc lives in a new `docs/api.md`; commit straight to `main` (no MR).
+
+## Deliverable
+
+### 1. New file `docs/api.md` — full API reference
+
+Content (all facts verified against code during planning):
+
+- **Overview & base URL** — what the API is; in-cluster FQDN `http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090` (Service: `config/default/discovery_service.yaml`); local access via `kubectl port-forward svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090`.
+- **Authentication** — static bearer token from `DISCOVERY_TOKEN` env var (populated from the optional `discovery-token` Secret, key `token`; `config/manager/manager.yaml`). Empty token ⇒ auth disabled with startup warning. Curl: `-H "Authorization: Bearer $TOKEN"` on every example. `/healthz` always exempt.
+- **Conventions** — JSON everywhere; error envelope `{"error":"<code>","message":"<text>"}`; request bodies capped at 64 KiB; proxies with a deletion timestamp are excluded from all responses.
+- **Configuration table** — `--discovery-addr` (default `:8090`), `--max-lease-ttl` (default 1h), `--lease-cooldown` (default 15m), `DISCOVERY_TOKEN`. Note the shipped Deployment passes none of these flags, so defaults apply.
+- **Endpoints**, each with request/response schema, status codes, and a copy-pasteable curl example:
+  - `GET /healthz` — liveness, unauthenticated.
+  - `GET /v1/proxies` — filters `healthy=true|false` (else 400 `invalid_query`) and repeatable `attr.<key>=<value>` (verbatim equality on `spec.attributes`, all pairs must match). Response `{"proxies":[proxyView...],"count":N}` sorted by id. Full `proxyView` field table: `id` (ns/name), `ip`, `port`, `attributes`, `phase` (Pending/Provisioning/Ready/Unhealthy/Deleting/Failed), `healthy` (condition `Healthy` == True), `latencyMillis`, `activeLeases`, `maxLeases` (default 5; explicit 0 = unleasable).
+  - `POST /v1/leases` — body `{selector, ttlSeconds, target}` all optional; TTL defaults 5m, capped at max-lease-ttl (else 400 `invalid_ttl`). 201 `{leaseID, proxy, expiresAt, ttlSeconds}`; 409 `no_match` with `considered/atCapacity/inCooldown/unhealthy` counts (documented meanings).
+  - `DELETE /v1/leases/{id}` — early release; always 204, idempotent.
+  - `POST /v1/leases/{id}/report` — body `{result: ok|rate_limited|banned, target}`; 204, 404 `unknown_lease`, 400 `invalid_result`. `ok` is a pure ack; `rate_limited` and `banned` behave identically today (both start one cooldown window).
+- **Selection & cooldown semantics** (short section — this is the non-obvious part clients need):
+  - Selection order: fewest active leases → lowest latency → lexicographic id; deterministic; only healthy, non-deleting proxies with free capacity are candidates.
+  - Cooldown: 15m default (`--lease-cooldown`), keyed `{proxy, target}`. Report with a target blocks only leases requesting that target; report without a target (and lease without one) creates a **global** cooldown blocking all acquisitions of that proxy — call this footgun out explicitly.
+  - Expired leases stay reportable for one cooldown window past TTL.
+- **End-to-end workflow example** — numbered curl walkthrough: acquire → use `proxy.ip:port` as HTTP proxy (`curl -x`) → report `rate_limited` on 429 → release. Using a `jq`-extracted `leaseID`.
+- **Caveats** — lease/cooldown state is in-memory and per-process: single replica only, operator restart drops all leases and cooldowns.
+
+### 2. Pointers to the new doc (small edits)
+
+- `README.md`: one-line link near the quickstart curl section ("full reference: docs/api.md").
+- `docs/architecture.md` §7: one-line link to `docs/api.md` as the detailed reference.
+
+### 3. Housekeeping per CLAUDE.md
+
+- First action post-approval: copy this plan to `docs/plans/<timestamp>-discovery-api-docs.md` (timestamp from `date "+%Y-%m-%d-%H%M"`), commit it alone.
+- Then write the docs, commit to `main` with `Co-Authored-By: Claude <noreply@anthropic.com>` trailer, push.
+- Append execution summary + status checklist to `docs/plans-executions/<same-timestamp>-discovery-api-docs.md` in the docs commit.
+- Add `CHANGELOG.md` entry (timestamp via `date "+%Y-%m-%d %H:%M %Z"`) once the user confirms.
+
+## Key source files (facts source of truth)
+
+- [internal/discovery/handlers.go](internal/discovery/handlers.go), [internal/discovery/server.go](internal/discovery/server.go) — routes, schemas, status codes, auth, limits.
+- [internal/lease/store.go](internal/lease/store.go) — selection order, cooldown/retention, stats.
+- [api/v1alpha1/proxy_types.go](api/v1alpha1/proxy_types.go), [api/v1alpha1/helpers.go](api/v1alpha1/helpers.go) — defaults (port 3128, maxLeases 5), phases, conditions.
+- [cmd/main.go](cmd/main.go) — flags/env defaults.
+- [config/default/discovery_service.yaml](config/default/discovery_service.yaml), [config/manager/manager.yaml](config/manager/manager.yaml) — service DNS, token secret.
+
+## Verification
+
+- Cross-check every documented status code / field name against `internal/discovery/server_test.go` expectations.
+- Sanity-run `go build ./... && go test -short ./internal/discovery/ ./internal/lease/` (no code changes expected — confirms docs match current behavior, not a stale tree).
+- Optionally lint the curl JSON bodies by piping each through `jq .` locally.

docs/plans/2026-08-11-2220-demo-scripts.md

@@ -0,0 +1,33 @@
+# Plan: Demo script — egress IP check through each healthy proxy
+**Created:** 2026-08-11 22:20
+
+
+## Context
+
+First of a planned series of demo scripts under `docs/demo/`. This one showcases the discovery API end to end without leases: list proxies from `$BASE_URL/v1/proxies`, and for each **healthy** one, call an IP-echo site through it (`curl -x`) to show the egress IP that proxy provides — clearly labeling which proxy each request goes through. The user explicitly asked to start by switching to a new branch. The script will be iterated on: **write it but do not commit it** — the user wants to add things to it before anything is committed.
+
+## Steps
+
+### Step 0 — Branch
+
+Create `feat/demo-scripts` off `main`. Commit only the plan copy (`docs/plans/<timestamp>-demo-scripts.md`, timestamp via `date "+%Y-%m-%d-%H%M"`) per CLAUDE.md — nothing else gets committed this round.
+
+### Step 1 — `docs/demo/show-egress-ips.sh` (new file, executable, left uncommitted)
+
+Style: match [.devcontainer/post-install.sh](.devcontainer/post-install.sh) — `#!/bin/bash`, `set -euo pipefail`, `ERROR:`/`WARNING:` messages, `${VAR}` braces.
+
+Behavior:
+
+1. **Args/env:** `BASE_URL` is `$1` (required; missing → usage text + exit 1, e.g. `usage: show-egress-ips.sh <BASE_URL>  (e.g. localhost:8090)`). Optional env: `TOKEN` (bearer token, same name docs/api.md uses; when set, send `Authorization: Bearer $TOKEN`), `IP_ECHO_URL` (default `https://api.ipify.org?format=json` — returns `{"ip":"..."}`).
+2. **Dependency check:** `command -v curl`, `command -v jq` → `ERROR` + exit 1 if missing.
+3. **Fetch** `"$BASE_URL/v1/proxies"` once (no server-side `healthy` filter — fetch all so unhealthy ones can be shown as skipped, which makes the demo more informative). Fail with a clear error if curl or JSON parsing fails.
+4. **Iterate** proxies with `jq -c '.proxies[]'`; for each, extract `id`, `ip`, `port`, `healthy`:
+   - unhealthy → print `--- skipping <id> (unhealthy) ---`
+   - healthy → print a clear banner naming the proxy before the request, e.g. `=== via <id> — http://<ip>:<port> ===`, then `curl -sS --max-time 10 -x "http://${ip}:${port}" "$IP_ECHO_URL"`; print the JSON response. A failed probe prints `WARNING: request through <id> failed` and continues (guard so `set -e` doesn't kill the loop).
+5. Finish with a one-line summary: N proxies, M probed, K skipped/failed.
+
+Reference for API shapes: [docs/api.md](docs/api.md) (`proxies[].id/ip/port/healthy`; `curl -x http://ip:port` usage is already documented there and in README).
+
+### Deliberately deferred (user will iterate on the script first)
+
+- No commit of the script, no push beyond the plan commit, no MR, no execution summary, no CHANGELOG — all wait until the user says the script (or script set) is ready.

docs/plans/2026-08-24-1025-otel-tracing.md

@@ -0,0 +1,107 @@
+# Plan: OpenTelemetry tracing integrated with the existing logr/zap logging
+**Created:** 2026-08-24 10:25
+
+## Context
+
+The operator has structured logging (controller-runtime zap → logr, `logf.FromContext(ctx)` everywhere) and Prometheus metrics, but no tracing. A single reconcile fans out into k8s API calls, provider calls (GCP VMs / squid Pods), and status patches; the discovery API serves lease requests; GC and health engines run on tickers. Today correlating one operation across those hops means grepping for `reconcileID` or `providerID`. Goal: proper OTel traces — a new trace per unit of work (reconcile, HTTP request, GC sweep, optionally health probe), child spans for k8s/provider/HTTP calls — **and every log line inside a traced operation enriched with the trace context**, without disturbing the existing logr/zap system.
+
+User decisions (confirmed):
+- **Enablement:** auto from standard `OTEL_*` env vars — no collector configured ⇒ tracing fully off, zero new flags for the common case.
+- **Health probes:** not traced by default; new `--trace-health-probes` flag enables one root span per probe.
+- **Backend:** none exists yet — verification uses the console exporter and a throwaway Jaeger all-in-one in kind; manifests ship commented-out OTLP env examples.
+
+## Key design facts (verified, non-obvious)
+
+- **logr can't read ctx:** `logr.LogSink` never sees `context.Context`, so trace IDs cannot be added inside the zap sink. They must be attached as `WithValues` at span-creation points via `logf.IntoContext`. zap does **not** dedupe repeated keys, so enrichment must happen exactly once per ctx chain (see `tracing.Start` below).
+- Log keys: **`traceID` / `spanID`** (lowerCamel matches `reconcileID`/`providerID` house style; note in docs that Grafana/Loki derived-field regexes must match `traceID`, not the `trace_id` default).
+- **GCP calls trace themselves:** `google.golang.org/api/transport/http` already wraps its transport in `otelhttp.NewTransport` — once a global TracerProvider is set, every Compute call emits an HTTP client span. Do not add HTTP-level spans in the gcp provider; the `provider.*` decorator span becomes their parent.
+- **kube-apiserver ignores incoming `traceparent`** by design; our k8s-API client spans are leaves. Don't promise cross-process traces into the apiserver.
+- The Go SDK does **not** implement `OTEL_SDK_DISABLED` — handle it ourselves.
+- `contrib/exporters/autoexport` drags in metric/log/prometheus exporters (binary + deps on a 128Mi pod) — hand-roll trace-exporter selection (~40 lines) instead.
+- Span events are deprecated (OTel spec, Mar 2026): use `span.SetStatus` + `error.type`-style attributes, not `RecordError`/`AddEvent`.
+- `rest.Config.Wrap` puts our transport innermost (after auth), so spans see the final request — correct; but informer list/watch + leader election run outside any span ⇒ the transport must **filter out requests with no parent span** or every watch becomes a long-lived root span.
+- Manager runnables receive no logger in ctx (only HTTP servers do) — `logf.FromContext` falls back to the global logger there, which is fine; the discovery server must keep seeding from `s.log`.
+
+## Steps
+
+### Step 1 — Dependencies
+
+`go get` (aligning the existing indirect skew: otel core v1.44 / otlptrace v1.40):
+`go.opentelemetry.io/otel@v1.45.0`, `otel/sdk@v1.45.0`, `otel/trace@v1.45.0`,
+`otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0`, `.../otlptracehttp@v1.45.0`,
+`otel/exporters/stdout/stdouttrace@v1.45.0`,
+`go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0`, plus the
+`semconv` version matching the SDK's `resource.Default()` (treat `ErrSchemaURLConflict` from `resource.New` as non-fatal if versions drift).
+`go mod tidy`; full build + `make test` to confirm `google.golang.org/api v0.292.0` / k8s v0.36 tolerate the bump (MVS picks the higher otelhttp).
+
+### Step 2 — New package `internal/tracing`
+
+- **`tracing.go` — `Setup(ctx, log logr.Logger, service, version string) (shutdown func(context.Context) error, err error)`**
+  - Disabled (returns no-op shutdown, installs nothing) when `OTEL_SDK_DISABLED=true`, or when neither `OTEL_TRACES_EXPORTER` nor `OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is set, or `OTEL_TRACES_EXPORTER=none`. One startup Info line either way stating the effective state/exporter.
+  - Exporter selection (hand-rolled, no autoexport): `OTEL_TRACES_EXPORTER` ∈ `otlp` (default) | `console`; for otlp, `OTEL_EXPORTER_OTLP_(TRACES_)PROTOCOL` ∈ `http/protobuf` (default) | `grpc` → `otlptracehttp` / `otlptracegrpc` (endpoint/TLS/headers via their native env handling).
+  - `sdktrace.NewTracerProvider` with batch processor (`WithMaxQueueSize(512)` — bounded on the 128Mi pod; sampler stays the SDK default so `OTEL_TRACES_SAMPLER(_ARG)` works), resource = `resource.New(WithAttributes(service.name, service.version=commit), WithFromEnv())` with env last so it wins.
+  - Globals: `otel.SetTracerProvider`, `otel.SetTextMapPropagator(TraceContext+Baggage)`, `otel.SetLogger(log.WithName("otel"))`, `otel.SetErrorHandler` → logr at `V(1)` (an unreachable collector errors every few seconds; Error level would spam).
+- **`logger.go` — the one logger-enrichment mechanism** (used by everything; prevents duplicate `traceID` keys):
+  - private ctx key holding the *base* (pre-enrichment) logr.Logger.
+  - `Start(ctx, spanName, opts...) (context.Context, trace.Span)`: starts a span from the global tracer; if the resulting span context is valid, capture base = stored base or `logf.FromContext(ctx)` on first call, then `logf.IntoContext(ctx, base.WithValues("traceID", ..., "spanID", ...))`. Nested `Start` re-derives from base — never stacks. Documented trade-off: `WithValues` pushed via `logf.IntoContext` *between* two `Start` calls is dropped (nothing first-party does that; controller-runtime's `reconcileID` logger is the base and survives).
+  - `ContextWithLogger(ctx, base)` seeds the base explicitly (discovery middleware).
+- **`reconciler.go` — `NewReconciler(kind string, inner reconcile.Reconciler, opts...) reconcile.Reconciler`**: root span `Reconcile <kind>` (SpanKind Internal) via `Start`; attrs `k8s.namespace.name`, `k8s.object.name`, `reconcile.id` (`controller.ReconcileIDFromContext`); on return records `reconcile.requeue_after`, sets Error status from err. `WithTracerProvider` option for tests.
+- **`transport.go` — `RestConfigWrapper() transport.WrapperFunc`**: `otelhttp.NewTransport` with `WithFilter` requiring `trace.SpanContextFromContext(r.Context()).IsValid()` — k8s API spans only under an existing trace (excludes informers, leader election, metrics authn).
+- **`http.go` — `HTTPMiddleware(operation string, base logr.Logger) func(http.Handler) http.Handler`**: `otelhttp.NewHandler` (filter `/healthz`; otelhttp renames the span from `r.Pattern` after routing — no custom naming needed) + inner layer calling `ContextWithLogger`-then-enrich so handler logs carry `traceID`. Accepts incoming `traceparent` (clients continue their traces).
+
+Tests (all with `sdktrace` + `tracetest.NewSpanRecorder` via the `WithTracerProvider` options, parallel; `Setup` env tests use `t.Setenv`, not parallel):
+- `Start`: nested spans → child span recorded; **funcr-captured log line contains `traceID` exactly once** (pattern: `internal/provider/gcp/gcp_test.go:277`); disabled tracer → no `traceID` in logs.
+- `Setup`: table over env combinations (unset→disabled, `none`→disabled, endpoint set→otlp, `console`, `OTEL_SDK_DISABLED`).
+- Reconciler decorator: fake inner returns result/err → span name/attrs/status.
+- Transport filter: request without parent span produces no span.
+- Middleware: `httptest` request → server span named from route pattern; `/healthz` unspanned; handler logger carries `traceID`.
+
+### Step 3 — `provider.WithTracing` decorator
+
+`internal/provider/tracing.go`, mirroring `metrics.go`: `WithTracing(name string, p Provider, opts...) Provider` — one Client span per call (`provider.create|get|delete|list`), attrs `provider.name`, `provider.id` (where present), result class attr from `resultLabel(err)`; status Error only for quota/permanent/transient — `ErrNotFound` stays Ok (expected during deletion polling). Returns the inner error **unmodified**. Uses `tracing.Start` so provider-internal logs (gcp `p.logger(ctx)`) inherit `traceID`. Tests in `internal/provider/tracing_test.go` reusing `staticProvider` from `metrics_test.go:19`.
+
+Wiring in `cmd/main.go:174-177` — tracing outermost: `provider.WithTracing(name, provider.WithMetrics(name, p, m))`.
+
+### Step 4 — `cmd/main.go` wiring
+
+- `tracing.Setup(...)` immediately after `ctrl.SetLogger` (cmd/main.go:148); on error: log + `os.Exit(1)`.
+- Explicit `shutdown(context.WithTimeout(context.Background(), 10*time.Second))` (fresh ctx — signal ctx is already cancelled) after `mgr.Start` returns, on **both** the error path before `os.Exit(1)` and the clean path. Earlier `os.Exit` sites have nothing to flush.
+- `cfg := ctrl.GetConfigOrDie(); cfg.Wrap(tracing.RestConfigWrapper())` before `ctrl.NewManager` (cmd/main.go:239).
+- kubernetes provider: add `kubernetes.NewWithTransportWrapper(ctx, cfg, wrap transport.WrapperFunc)` beside `New` (internal/provider/kubernetes/kubernetes.go:42) and register a closure in main's constructor map (same pattern as gcp at cmd/main.go:166) passing `tracing.RestConfigWrapper()` — keeps `internal/tracing` out of provider packages and `newWithClient` tests untouched.
+- New flag `--trace-health-probes` (default false) → `engine.TraceProbes`.
+
+### Step 5 — Reconciler spans
+
+- `SetupWithManager` (proxy_controller.go:412): `b.Complete(tracing.NewReconciler("Proxy", r))`.
+- Sub-spans via `tracing.Start`: `reconcile.managed` (:116), `reconcile.replaceInstance` (:226), `reconcile.delete` (:254), and `status.patch` opened **inside the deferred flush closure** (:98-104, runs within the root span; its error already folds into `err`). `reconcileExternal` stays unspanned — no I/O, pure noise.
+- Useful attrs where cheap: phase transitions, `providerID`.
+
+### Step 6 — Discovery server
+
+- `handler()` (internal/discovery/server.go:151-156): chain becomes recover → `tracing.HTTPMiddleware("discovery", s.log)` → log → maxBytes → auth (otelhttp must stay outside `maxBytesMiddleware`).
+- Handlers and `logMiddleware`/`recoverMiddleware` switch `s.log` → `logf.FromContext(r.Context())` (handlers.go:85, :141, :234; server.go:163, :191) so request logs carry `traceID`. `s.log` remains for startup lines.
+- k8s `List` calls in handlers already use `r.Context()` → child spans appear via the wrapped rest transport.
+
+### Step 7 — GC + health
+
+- `gc.Sweeper.sweep` (internal/gc/gc.go:83): wrap in root span `gc.sweep` via `tracing.Start` (before the `.WithName("orphan-gc")` derivation so it inherits), attrs: providers swept, orphans deleted. Provider/k8s child spans come free from Steps 3/4.
+- `health.Engine`: new field `TraceProbes bool`. In the worker loop (engine.go:158) when enabled: root span `health.probe` (SpanKind Client) around `probeFn`, attrs proxy key/ok/latency/error class, ended before `record` (which stays ctx-free). No transport wrap and no propagation — never inject `traceparent` toward external targets through the proxies. Probe→reconcile span links are **out of scope**: `GenericEvent` (engine.go:330) carries no ctx and the workqueue coalesces events; documented in architecture Decisions.
+- Lease store sweep: not traced (in-memory only).
+
+### Step 8 — GCP wire-log enrichment
+
+`wireFilterHandler.Handle` (internal/provider/gcp/wirelog.go:46) receives the real request ctx from the SDK's slog calls: append `traceID`/`spanID` attrs when `trace.SpanContextFromContext(ctx)` is valid — V(5) wire logs become correlatable with the `provider.*` span. Add `wirelog_test.go` case.
+
+### Step 9 — Manifests + docs
+
+- `config/manager/manager.yaml` env (after `DISCOVERY_TOKEN`): `POD_NAME`/`POD_NAMESPACE` via downward API **first**, then `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)` ($(VAR) expansion only sees earlier vars); commented-out `OTEL_EXPORTER_OTLP_ENDPOINT` + `OTEL_TRACES_SAMPLER` examples with a "tracing is off until an endpoint is set" comment. Note `--trace-health-probes` beside the args list.
+- `docs/architecture.md`: new "### 10. Tracing" section after Metrics (:281) — span topology, enablement, `traceID` log-key choice; Decisions entries (:303): env-gated enablement, probes off by default, no probe→reconcile links (why), `traceID` naming vs Grafana defaults, no traceparent into apiserver.
+- README ops note; CHANGELOG entry (dated via `date`) once the user confirms it works.
+- Run `make lint` (CI has no lint step; `lll`/`revive` will hit new files).
+
+## Verification
+
+1. `make test` (unit, includes all new `tracetest` assertions) and `make lint`.
+2. Console smoke: `OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml ...` against kind — create a Proxy, confirm (a) `Reconcile Proxy` span JSON on stdout with nested `provider.create` + k8s PATCH spans, (b) the reconcile log lines carry the same `traceID` as the span, (c) no spans from informer watches or leader election.
+3. End-to-end in kind: deploy Jaeger all-in-one (`jaegertracing/all-in-one`, OTLP 4318), set `OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318` on the manager Deployment; exercise a reconcile + a `POST /v1/leases` (with and without a client `traceparent`) + wait one GC interval; in the Jaeger UI: reconcile trace shows provider+k8s children; lease trace shows route-named server span with k8s child; grep manager logs for a `traceID` and resolve it in Jaeger.
+4. Negative: run with no `OTEL_*` env — startup line says tracing disabled, no export-error spam, logs unchanged (no `traceID` keys).

docs/plans/2026-08-24-1224-tracing-e2e.md

@@ -0,0 +1,150 @@
+# Plan: e2e test — OTel tracing against real Tempo
+**Created:** 2026-08-24 12:24
+
+## Context
+
+The tracing feature (MR #4, branch `feat/otel-tracing`) is covered by unit
+tests with an in-memory span recorder, but nothing proves the full pipeline:
+operator in a real cluster → OTLP export → Tempo ingest → queryable traces
+with the documented span topology. The homelab Tempo lives at
+`http://192.168.0.30:3200` (query API — verified live from this host:
+`/api/echo`, TraceQL `GET /api/search?q=…`, `GET /api/traces/<id>`), with
+OTLP ingest on `:4318` (HTTP, verified 200) and `:4317` (gRPC, open). User
+decisions: **Go Ginkgo e2e test** (not a shell script), **structural
+assertions** (span tree, not just trace-exists), flow per the user's sketch:
+deploy operator in kind with tracing on → create kubernetes-provider proxies
+→ delete them → shut down — with traces tagged for easy discovery per test
+run. This work continues on `feat/otel-tracing`; MR #4 stays the vehicle.
+
+## Design decisions (from review, with evidence)
+
+- **Proxy CRs go in the `default` namespace, not the operator namespace.**
+  The squid pods the kubernetes provider creates carry no securityContext
+  (internal/provider/kubernetes/pod.go:33-49) and are created in the CR's
+  namespace; the operator ns is labeled `pod-security enforce=restricted`
+  (test/e2e/e2e_test.go:61-62) and would reject them. Pod RBAC is
+  cluster-scoped (config/rbac/role.yaml:10), so `default` works. All
+  kubectl calls use explicit `-n`.
+- **Anchor assertions on `provider.create`, not the root-span search.** A
+  `name="Reconcile Proxy"` hit may be the finalizer-add or a drift
+  reconcile (no provider call inside). TraceQL matches span names anywhere
+  in a trace, so search `{resource.test.run.id="<runID>" && name="provider.create"}`,
+  fetch *that* traceID, then assert companion span names.
+- **Run identification with zero code changes:** OTel resource attribute
+  `test.run.id=<runID>` via `OTEL_RESOURCE_ATTRIBUTES` (env wins —
+  `resource.WithFromEnv()` merges last, internal/tracing/tracing.go). Tempo
+  vParquet searches arbitrary resource attrs without config; fallback query
+  documented: `{resource.service.name="egress-proxies-operator" && name="provider.create"}`
+  + start/end window (Unix **seconds**).
+- **OTLP preflight from inside the cluster:** host-reachability of
+  192.168.0.30 doesn't prove pod-reachability from kind on macOS, and
+  export failures log only at V(1) (invisible) — without a preflight, a
+  broken path is a 2-minute opaque timeout. Reuse the suite's curl-pod
+  pattern (e2e_test.go:218-248, incl. the restricted-PSS overrides JSON)
+  to POST to `<OTLP_ENDPOINT>/v1/traces` and fail fast.
+- **No homelab defaults baked into the Makefile.** The spec Skips unless
+  `TEMPO_URL`/`OTLP_ENDPOINT` are set (otherwise everyone without that LAN
+  host inherits a 2-minute failure). The copy-pasteable invocation lives in
+  docs/testing.md.
+- **Ginkgo ordering:** top-level container order is randomized, so the
+  tracing Describe is fully self-contained (own ns create → make install →
+  make deploy → teardown), same shape as the Manager Describe. `kubectl
+  delete ns` blocks until termination, so no create/delete race between
+  the two Describes.
+
+## Steps
+
+### Step 1 — `test/e2e/tracing_test.go` (new; `//go:build e2e`, package e2e)
+
+`Describe("OTel tracing", Ordered)`:
+
+- **BeforeAll:**
+  1. Read `TEMPO_URL` + `OTLP_ENDPOINT`; `Skip("TEMPO_URL/OTLP_ENDPOINT not set")` when empty.
+  2. `runID := "e2e-" + strconv.FormatInt(time.Now().UnixNano(), 10)`;
+     print it to `GinkgoWriter` so the run is findable in Grafana by hand.
+  3. Create operator ns + restricted-PSS label; `make install`;
+     `make deploy IMG=<managerImage>` (same commands as e2e_test.go:54-75,
+     via `utils.Run`).
+  4. Pre-pull the proxy image to kill the biggest flake source:
+     `docker pull ubuntu/squid:6.6-24.04_edge` + `kind load docker-image`
+     (via `utils.Run`, honoring `KIND`/`KIND_CLUSTER` env like
+     utils.LoadImageToKindClusterWithName).
+  5. OTLP preflight: curl pod POSTing `{}` to `<OTLP_ENDPOINT>/v1/traces`,
+     assert HTTP 200 in its logs (fail message names the endpoint).
+  6. `kubectl set env deployment/egress-proxies-operator-controller-manager
+     -n <ns> OTEL_EXPORTER_OTLP_ENDPOINT=<OTLP_ENDPOINT>
+     "OTEL_RESOURCE_ATTRIBUTES=k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE),test.run.id=<runID>"`
+     — in-place update keeps the var after the downward-API vars ($(VAR)
+     expansion needs that ordering); `exec.Command` means no shell mangling
+     of `$()`. Then `kubectl rollout status`.
+  7. `Eventually`: newest controller pod (sorted by creationTimestamp — a
+     terminating old pod may otherwise be picked) logs contain
+     `"tracing enabled"`.
+- **It "creates proxies and reports reconcile traces":**
+  - Write 2 Proxy manifests (names `proxy-tracing-e2e-1/2`,
+    `provider: kubernetes`, modeled on config/samples/proxy_kubernetes.yaml)
+    to `GinkgoT().TempDir()` (absolute paths — `utils.Run` chdirs the
+    process); `kubectl apply -n default`.
+  - `Eventually` (5m — first squid start) `.status.phase == "Ready"`
+    (confirmed field, api/v1alpha1/proxy_types.go:41).
+  - `Eventually`: Tempo search
+    `{resource.test.run.id="<runID>" && name="provider.create"}` ≥1 hit
+    (BSP flushes every ~5s; 2m default timeout is ample).
+  - Fetch that traceID via `/api/traces/<id>`; assert span names include
+    `Reconcile Proxy`, `reconcile.managed`, `provider.create`,
+    `status.patch`, and resource attr `service.name=egress-proxies-operator`.
+- **It "traces deletion":**
+  - `kubectl delete proxy -n default …`; `Eventually` CRs gone (finalizer
+    → `provider.delete`, DeletionPoll 10s).
+  - `Eventually`: Tempo search finds `provider.delete` for the runID; fetch
+    and assert `reconcile.delete` in the same trace.
+- **AfterAll** (= the sketch's "shut down operator"): delete leftover
+  proxies in `default`; `make undeploy` (SIGTERM → `tracingShutdown`
+  flushes); `make uninstall`; delete operator ns — errors discarded
+  (`_, _ = utils.Run(...)`), mirroring e2e_test.go:79-95.
+- **Tempo client helpers** (same file, stdlib only — the test process runs
+  on the host, which reaches Tempo directly):
+  - `tempoSearch(traceql string)` → `GET {TEMPO_URL}/api/search?q=…&start=…&end=…`
+    (Unix seconds, window = suite start − 5m → now); minimal struct
+    `{Traces []struct{ TraceID string }}`.
+  - `tempoTrace(id string)` → `/api/traces/<id>`, decoded as **OTLP-JSON**
+    (`batches[].scopeSpans[].spans[].name`, resource attrs as
+    `{key, value:{stringValue}}`) — not Jaeger's shape. Helper flattens to
+    a span-name set + resource attr map.
+
+### Step 2 — Makefile
+
+- `test-e2e`: pass `TEMPO_URL`/`OTLP_ENDPOINT` through to `go test` env
+  (no defaults) and add `-timeout 30m` — the suite already runs
+  docker-build + kind-load in BeforeSuite plus 3–5m Eventuallys, and this
+  adds a second full deploy cycle + image pulls; the 10m default will be
+  exceeded on cold caches.
+
+### Step 3 — Docs
+
+- `docs/testing.md` (§ e2e, currently "scaffold… only asserts manager runs
+  and serves metrics"): document the tracing spec, its Skip gate, and the
+  invocation:
+  `TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`.
+- Execution-log entries per step in `docs/plans-executions/2026-08-24-1224-tracing-e2e.md`.
+- CHANGELOG entry only after the user confirms a green run (house rule).
+
+## Not doing (and why)
+
+- No dedicated flush assertion on shutdown — teardown exercises the flush
+  path, but attributing a specific late span to it is guesswork.
+- No gRPC (4317) variant — http/protobuf is the operator default and the
+  verified path; a variant run is a one-env-var change if ever wanted.
+- No changes to `internal/` — the whole test works through public surface
+  (env vars, kubectl, Tempo API), which is the point of an e2e test.
+
+## Verification
+
+1. `TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`
+   → tracing spec passes (both Its); without the env vars → spec reports
+   Skipped, remainder of suite unaffected.
+2. In Grafana: TraceQL `{resource.test.run.id="<runID printed in test log>"}`
+   shows the run's traces with the expected tree.
+3. `go vet -tags=e2e ./...` clean (documented routine).
+4. A deliberately wrong `OTLP_ENDPOINT` (e.g. port 9) fails fast in the
+   preflight step with a clear message, not a 2-minute search timeout.

docs/plans/2026-09-04-1404-kvm-provider.md

@@ -0,0 +1,313 @@
+# Plan: KVM/libvirt provider
+
+**Created:** 2026-09-04 14:04
+
+## Context
+
+The operator provisions egress proxies through named provider instances (today
+`gcp` and `kubernetes`). The user has several KVM hypervisors on the home
+network and wants proxy VMs created there, using stock Debian/Ubuntu cloud
+images so the VM stays a normal machine they can customise through their own
+cloud-init user-data. Longer term the operator may become a generic "run a VM
+or container with a set of scripts" tool; the KVM provider should not
+foreclose that.
+
+The provider contract (`internal/provider/provider.go`: `Create`/`Get`/`Delete`/
+`ListByTag`, keyed by `NameFromUID`, cloud-init passed verbatim, GC-by-tag) fits
+libvirt with **no CRD or reconciler changes**. All work is a new provider
+package plus config/wiring/docs, following the GCP provider's shape.
+
+Decisions taken with the user (2026-09-04):
+
+- **Transport:** SSH to each hypervisor, forwarding the libvirt unix socket
+  (`ssh.Client.Dial("unix", "/var/run/libvirt/libvirt-sock")` → `libvirt.New(conn)`).
+  Pure Go, no cgo, no libvirt libs in the image. Verified: `x/crypto/ssh`
+  `Client.Dial` has a `"unix"` case (`dialStreamLocal`), go-libvirt `New(net.Conn)`.
+- **Topology:** one provider instance can span several hosts from day one
+  (config has a `hosts` list; providerID encodes the host), so adding hosts
+  later is a config edit with no providerID migration. With one host configured
+  `placement.zone` may be omitted.
+- **Networking:** VM NIC on a host bridge to the LAN, IP from the router's
+  DHCP. IP discovery via `DomainInterfaceAddresses` (ARP source first, then
+  guest agent). Provider-injected `vendor-data` installs `qemu-guest-agent`
+  so agent discovery works on stock images without touching user-data.
+- **Base image:** a stock Debian/Ubuntu cloud image (qcow2, cloud-init
+  preinstalled) that **the operator fetches itself** on first use per host:
+  `placement.image` names an entry in the provider's `images:` map (url +
+  optional sha256) or is an `https://` URL directly. The provider streams the
+  download through libvirt's `StorageVolUpload` into the host's pool, so
+  nothing is pre-placed and the operator needs no filesystem access on the
+  host. Per-VM disk is a qcow2 overlay with the cached base as backing store.
+- **Seed:** NoCloud ISO (`user-data`, `meta-data`, `vendor-data`) built in
+  pure Go with `github.com/kdomanski/iso9660` (`WriteTo(w, "cidata")`) and
+  streamed to the pool with `StorageVolUpload(vol, io.Reader, ...)`. Nothing
+  to install on the hypervisors.
+
+## Design
+
+### Config (`internal/provider/config.go`)
+
+```yaml
+- name: kvm-home
+  type: kvm
+  kvm:
+    sshKeyFile: /etc/proxy-operator/kvm/id_ed25519        # required
+    knownHostsFile: /etc/proxy-operator/kvm/known_hosts   # required unless insecureIgnoreHostKey
+    insecureIgnoreHostKey: false
+    diskSizeGb: 10                    # overlay capacity; cloud-init growpart expands root fs
+    vendorData: |                     # optional override; default installs qemu-guest-agent
+      #cloud-config
+      ...
+    images:                           # optional; placement.image names a key (or is an https URL)
+      debian-13:
+        url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+        sha256: "…"                   # optional; verified after download, re-fetch when it changes
+        format: qcow2                 # default qcow2
+      ubuntu-24.04:
+        url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+    machineTypes:                     # required, ≥1; placement.machineType names a key
+      small:  {vcpus: 1, memoryMiB: 1024}
+      medium: {vcpus: 2, memoryMiB: 2048}
+    hosts:                            # required, ≥1, unique names
+      - name: nas1                    # placement.zone value
+        address: nas1.home.lan:22     # required
+        user: virt                    # required; must be in the libvirt group
+        socket: /var/run/libvirt/libvirt-sock   # default
+        pool: default                 # default "default"
+        bridge: br0                   # required
+```
+
+Add `KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage` structs; a `"kvm"`
+case in `validate()` (cross-block checks like the others, required fields
+above, unique host names, image URLs must be `http(s)`, sha256 if set must be
+64 hex chars). Tests in `config_test.go` extend the existing table.
+
+### Placement mapping
+
+| Placement field | KVM meaning |
+|---|---|
+| `zone` | host name from `hosts`; empty → the single configured host, else `ErrPermanent` |
+| `machineType` | key in `machineTypes`; unknown → `ErrPermanent` |
+| `image` | key in `images:`, or an `https://` URL; anything else → `ErrPermanent`. Fetched into the host's pool on first use (see *Image cache*) |
+| `region` | ignored |
+
+### providerID
+
+`hosts/<host>/domains/<name>` with `formatProviderID`/`parseProviderID`
+(same self-contained style as GCP's `zones/<zone>/instances/<name>`).
+
+### GC contract without labels
+
+Domain XML gets a `<metadata>` child in a private namespace, e.g.
+`xmlns="https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm"`,
+carrying `<managed>true</managed><uid>…</uid><createdAt>RFC3339</createdAt>`.
+`Get` reads it via `DomainGetMetadata(dom, DomainMetadataElement, uri)`;
+`ListByTag` does `ConnectListAllDomains` on every host and keeps domains where
+that call succeeds and `managed == "true"`. `CreatedAt` comes from the metadata
+(libvirt has no creation time). A host that is unreachable makes `ListByTag`
+return an error (GC skips the sweep that round rather than acting on a partial view).
+
+### Connection management (`internal/provider/kvm/conn.go`)
+
+Per-host lazily dialed `*ssh.Client` + `*libvirt.Libvirt`, cached under a
+mutex, re-dialed when `Disconnected()` is closed or a call fails with a
+transport error. SSH auth: private key from `sshKeyFile`; host key check via
+`knownhosts.New(knownHostsFile)` unless `insecureIgnoreHostKey`. Dial timeout
+from `ctx`. The provider calls libvirt through a narrow interface:
+
+```go
+type libvirtAPI interface {
+    DomainLookupByName(name string) (libvirt.Domain, error)
+    DomainDefineXML(xml string) (libvirt.Domain, error)
+    DomainCreate(d libvirt.Domain) error
+    DomainDestroy(d libvirt.Domain) error
+    DomainUndefineFlags(d libvirt.Domain, f libvirt.DomainUndefineFlagsValues) error
+    DomainGetState(d libvirt.Domain, flags uint32) (int32, int32, error)
+    DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, f libvirt.DomainModificationImpact) (string, error)
+    DomainInterfaceAddresses(d libvirt.Domain, src uint32, flags uint32) ([]libvirt.DomainInterface, error)
+    ConnectListAllDomains(need int32, f libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error)
+    StoragePoolLookupByName(name string) (libvirt.StoragePool, error)
+    StorageVolLookupByName(p libvirt.StoragePool, name string) (libvirt.StorageVol, error)
+    StorageVolGetPath(v libvirt.StorageVol) (string, error)
+    StorageVolCreateXML(p libvirt.StoragePool, xml string, f libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error)
+    StorageVolUpload(v libvirt.StorageVol, r io.Reader, off, n uint64, f libvirt.StorageVolUploadFlags) error
+    StorageVolDownload(v libvirt.StorageVol, w io.Writer, off, n uint64, f libvirt.StorageVolDownloadFlags) error
+    StorageVolDelete(v libvirt.StorageVol, f libvirt.StorageVolDeleteFlags) error
+    StoragePoolRefresh(p libvirt.StoragePool, flags uint32) error
+}
+```
+
+(All signatures verified against go-libvirt `main` on 2026-09-04.) A
+`hostDialer func(ctx, KVMHost) (libvirtAPI, io.Closer, error)` is the test seam;
+`newWithDialer(cfg, dialer)` mirrors GCP's `newWithAPI`.
+
+### Image cache (`internal/provider/kvm/image.go`)
+
+`ensureImage(ctx, host, api, img) (basePath string, err error)` makes the base
+image present in the host's pool, once per host per image:
+
+- Volume names: `base-<key>.<format>` for an `images:` key, or
+  `base-<sha256(url)[:16]>.<format>` for a URL given directly in
+  `placement.image`. Marker volume `<base>.sha256` (raw, 64 bytes) holds the
+  digest of the verified upload.
+- **Ready check:** base volume exists *and* marker exists *and* (no `sha256`
+  configured, or marker content == configured `sha256`). Read the marker with
+  `StorageVolDownload`. Ready → return `StorageVolGetPath(base)`.
+- **Not ready:** delete both volumes if present (partial upload from a crash,
+  or a changed checksum), then `GET url` with the ctx, require a
+  `Content-Length` (`ErrPermanent` otherwise), `StorageVolCreateXML` the base
+  with `<capacity unit="bytes">Content-Length</capacity>` and the configured
+  format, `StorageVolUpload(base, io.TeeReader(body, sha256), 0, 0, 0)` (length
+  0 = until EOF, so nothing is buffered in the operator), then compare the
+  digest with the configured one (mismatch → delete base, `ErrPermanent`),
+  `StoragePoolRefresh` so libvirt re-reads the real qcow2 capacity, create the
+  marker volume and upload the digest last.
+- HTTP 4xx → `ErrPermanent`; 5xx, network, or upload transport errors →
+  `ErrTransient` (partial volumes are cleaned on the next attempt).
+- A per-`(host, image)` `sync.Mutex` map serialises concurrent Creates for the
+  same image on the same host; the second caller finds it ready.
+- **Judgment call:** the fetch runs synchronously inside `Create`, so the first
+  Proxy for a given image on a given host holds a reconcile worker for the
+  download (LAN speed, a few hundred MB, once). This keeps `Create` idempotent
+  and needs no background state. Async fetch with a `Provisioning`-style
+  progress status is a possible later improvement, noted in the docs.
+
+### Create (idempotent, fire-and-forget after the image is cached)
+
+1. Resolve host, machine type, pool, and image source (`images:` key or URL;
+   else `ErrPermanent`). `ensureImage` → base path.
+2. Overlay volume `<name>.qcow2`: `StorageVolCreateXML` with
+   `<capacity unit="G">diskSizeGb</capacity>`, `<target><format type="qcow2"/></target>`,
+   `<backingStore><path>base</path><format type="qcow2"/></backingStore>`.
+   `ErrStorageVolExist` → reuse.
+3. Seed volume `<name>-seed.iso`: build ISO in memory (`user-data` = req.CloudInit,
+   `meta-data` = `instance-id: <name>\nlocal-hostname: <name>\n`, `vendor-data` =
+   cfg or default), `StorageVolCreateXML` with exact capacity + `<format type="raw"/>`,
+   then `StorageVolUpload`. Exists → skip upload (content is deterministic per spec hash;
+   spec changes recreate the whole instance anyway).
+4. `DomainLookupByName(name)`; if `ErrNoDomain` → `DomainDefineXML(buildDomainXML(...))`.
+5. `DomainGetState`; if not running → `DomainCreate`. Return providerID.
+
+`buildDomainXML` is a pure function over `encoding/xml` structs: `type="kvm"`,
+q35 machine, `host-passthrough` CPU, vcpus/memory from the profile, virtio disk
+(overlay) + sata cdrom (seed), `interface type="bridge"` with virtio model,
+`org.qemu.guest_agent.0` virtio channel, serial console, virtio rng, no
+graphics, and the `<metadata>` block. Unit-tested field by field like
+`buildInsertRequest`.
+
+### Get
+
+`parseProviderID` → host → `DomainLookupByName` (`ErrNoDomain` → `ErrNotFound`)
+→ `DomainGetState` + `DomainGetMetadata` + `DomainInterfaceAddresses`
+(ARP, then agent; first IPv4 wins; agent errors while the agent isn't up yet
+are swallowed). Mapping:
+
+| libvirt state | IP | Instance.State |
+|---|---|---|
+| running | yes | Running |
+| running | no | Provisioning |
+| shutoff / crashed / paused / pmsuspended | – | Stopped |
+| no domain | – | `ErrNotFound` |
+
+### Delete (idempotent)
+
+Destroy if running (`ErrOperationInvalid` on a stopped domain is fine) →
+`DomainUndefineFlags(managed-save | snapshots-metadata | nvram)` → delete
+`<name>.qcow2` and `<name>-seed.iso` (`ErrNoStorageVol` fine). Volumes are
+deleted by name even when the domain is already gone, so a crash between
+volume creation and define still gets cleaned up on the next reconcile.
+Known gap (documented, not solved now): volumes orphaned with no domain *and*
+no CR are invisible to `ListByTag`.
+
+### Errors (`internal/provider/kvm/errors.go`)
+
+`classify(err)` on `libvirt.Error.Code`: `ErrNoDomain`, `ErrNoStorageVol` →
+`ErrNotFound`; `ErrNoStoragePool`, `ErrXMLError`, `ErrConfigUnsupported`,
+`ErrNoSupport`, `ErrAuthFailed`, `ErrInvalidArg` → `ErrPermanent`; SSH dial /
+host-key / transport errors and everything else → `ErrTransient`. Image
+download: HTTP 4xx, missing Content-Length, checksum mismatch → `ErrPermanent`;
+5xx and network errors → `ErrTransient`. Quota is not detectable from libvirt;
+left as transient (revisit with a capacity check later).
+
+### Wiring & manifests
+
+- `cmd/main.go`: add `"kvm": kvm.New` to the builtins map.
+- `config/manager/manager.yaml`: mount Secret `kvm-ssh` (optional) at
+  `/etc/proxy-operator/kvm` (keys `id_ed25519`, `known_hosts`); document the
+  `kubectl create secret generic kvm-ssh --from-file=...` command.
+- `config/samples/providers-config.yaml`: add the `kvm-home` block with the
+  `images:` map (Debian 13 and Ubuntu 24.04 official cloud-image URLs);
+  `config/samples/proxy_kvm.yaml`: Proxy with `placement: {zone: nas1,
+  machineType: small, image: debian-13}` and the same squid cloud-init as the
+  GCP sample plus `ssh_authorized_keys`.
+- `hack/providers-dev.yaml` stays kubernetes-only (needs no secrets).
+- Deps: `github.com/digitalocean/go-libvirt` (latest pseudo-version),
+  `github.com/kdomanski/iso9660 v0.4.0`, promote `golang.org/x/crypto` to direct.
+
+### Docs
+
+- New `docs/kvm-provider.md`: hypervisor prerequisites (user in `libvirt`
+  group, a bridge to the LAN, a storage pool with enough space), the config
+  reference including `images:`, how the image cache works (`base-*` and
+  `*.sha256` volumes in the pool, when a re-download happens, how to evict
+  with `virsh vol-delete`), the placement table, how to log into a VM
+  (`ssh_authorized_keys` in user-data, `virsh console` on the host), and the
+  known gaps (no quota detection, orphan volumes without a domain,
+  synchronous first-time image fetch).
+- Update the provider tables in `README.md`, `docs/architecture.md` §5,
+  `docs/testing.md` (integration test section).
+
+### Towards the generic runner (not built now, just kept open)
+
+Provider knows nothing about squid. Future hooks: `vendor-data` for injecting an
+agent, `QEMUDomainAgentCommand` for reading exit status/output, and a future
+`Placement`-independent "job" mode in the CRD. None of this changes the KVM
+provider's shape.
+
+## Steps
+
+0. **Save plan + branch.** Copy this file to `docs/plans/<ts>-kvm-provider.md`,
+   commit alone, then `git checkout -b feat/kvm-provider` off `feat/proxy-operator`
+   (the working main for this project).
+1. **Config.** `KVMConfig` & friends, `validate()` case, table tests.
+2. **Pure builders.** `internal/provider/kvm/{domainxml.go,volume.go,seed.go,id.go,metadata.go}`
+   + tests: domain XML fields, volume XML, seed contents (unpack the ISO with
+   the same library's reader and assert the three files + label `cidata`),
+   providerID round-trip, metadata marshal/parse.
+3. **Transport + errors.** `conn.go` (SSH dial, host-key policy, per-host cache,
+   reconnect), `errors.go` + classification table test. SSH dialer tested
+   against an in-process `x/crypto/ssh` server that serves a unix socket
+   forward (cheap, keeps the dial code covered).
+3b. **Image cache.** `image.go` + `image_test.go`: `httptest.Server` serving a
+   small fake image over the `fakeAPI` (which records uploaded bytes);
+   cases: fresh fetch, already ready, marker missing (partial upload → redo),
+   checksum changed → redo, checksum mismatch → Permanent + cleanup, 404 →
+   Permanent, 503 → Transient, two concurrent callers → one download.
+4. **Provider methods.** `kvm.go`: `New`, `newWithDialer`, `Create/Get/Delete/ListByTag`
+   over `libvirtAPI`; `kvm_test.go` with a hand-written `fakeAPI` (domains and
+   volumes in maps, per-call error injection) covering idempotent re-Create
+   after each partial step, state/IP mapping, delete-when-gone, ListByTag
+   across two hosts and with one host down.
+5. **Wiring, manifests, samples, docs, CHANGELOG.**
+6. **Integration test** (`//go:build integration`, `kvm_integration_test.go`):
+   skipped unless `KVM_TEST_HOST`, `KVM_TEST_USER`, `KVM_TEST_SSH_KEY`,
+   `KVM_TEST_BRIDGE`, `KVM_TEST_IMAGE_URL` are set; fetches the image (or finds
+   it cached), creates, waits for Running+IP, deletes, asserts per-VM volumes
+   gone and the base image still cached. Deviation from the testcontainers convention,
+   called out in `docs/testing.md`: libvirt+KVM can't run inside a container
+   without `/dev/kvm`, so this targets a real hypervisor.
+7. **Manual validation** on one real hypervisor, results in
+   `docs/plans-executions/` (like `docs/gcp-vm-validation.md`), then MR via `tea`.
+
+## Verification
+
+- `make test` green (`-race`, envtest suite untouched but must still pass).
+- `go test -short ./internal/provider/...` covers builders, fake-API provider, errors, config.
+- Integration: `KVM_TEST_HOST=nas1.home.lan … go test -tags=integration -run TestKVM ./internal/provider/kvm/`.
+- End to end: deploy with the `kvm-ssh` Secret and `kvm-home` config against a
+  host with no cached image, apply `config/samples/proxy_kvm.yaml`, watch the
+  operator log the image fetch and `virsh vol-list` show `base-debian-13.qcow2`
+  plus its `.sha256` marker, watch `kubectl get px -w` reach `Ready` with a LAN
+  IP, apply a second Proxy and confirm no second download, confirm the VM shows up in `GET /v1/proxies` and works with
+  `hack/` demo egress-IP check, delete the Proxy and confirm `virsh list --all`
+  and `virsh vol-list default` are clean.

docs/plans/2026-09-10-0805-kvm-libvirt-tracing.md

@@ -0,0 +1,186 @@
+# Plan: Trace every libvirt call in the KVM provider
+**Created:** 2026-09-10 08:05
+
+## Context
+
+Step 7 of the KVM provider plan (live validation on `beelink`) has gone through
+three fixes found only on the real host, and the current failure (guest hangs at
+GRUB's "Booting" line) is being diagnosed blind: the provider logs at V(1)/V(2)
+but nothing shows them. The integration test never installs a logger, so
+controller-runtime prints "log.SetLogger(...) was never called" and swallows
+everything; and even with a logger, the provider only logs per *operation*
+(Create, Get…), not per libvirt RPC, so the sequence and timing of what was
+actually sent to the hypervisor is invisible.
+
+Goal: one log line per libvirt call at debug level (`--zap-log-level=debug`, logr
+V(1) in this repo's scheme: V(1) = outcome per operation, V(2) = curated detail,
+V(5) = raw wire, per the GCP verbose-logging plans), with method, host, domain or
+volume, duration and the libvirt error code on failure; and those lines visible in
+`make test-integration-kvm` output.
+
+Assumption: this lands on the current `feat/kvm-provider` branch, since it exists
+to finish that branch's validation, and the plan file plus execution log follow
+CLAUDE.md conventions (plan committed alone first, execution summary alongside
+each step). Say so if you want a separate branch instead.
+
+## Design
+
+- **Decorator, not call-site logging.** New `tracingAPI` implements the 17-method
+  `libvirtAPI` seam (`internal/provider/kvm/conn.go:26-44`), wrapping the real
+  client. Every method logs one V(1) line after the inner call returns. Applied in
+  `hostPool.call`, so every provider method, `image.go`'s cache and the
+  integration test's own `p.hosts.call` are traced with no per-site changes.
+- **Logger passed explicitly.** `call(ctx, log, host, fn)` and `get(ctx, log, host)`
+  gain a `logr.Logger` parameter, matching how this package already threads `log`
+  into every helper. The five call sites (`kvm.go:169, 322, 458, 530`,
+  `kvm_integration_test.go:220`) already hold the right logger with
+  provider/host/name values. `call` does `log = log.WithName("libvirt")`, so trace
+  lines are named `kvm.libvirt`.
+- **Wrap only when `log.V(1).Enabled()`.** Zero cost at info level; the
+  controller-runtime unfulfilled root sink reports disabled, so unit tests and
+  `conn_test.go`'s `hostClient{api: nil}` cases are unaffected.
+- **One line per RPC at every level.** At V(2) the same line gains extra keys
+  (`xml`, `metadata`, `addresses`, `names`) instead of a second line. Upload and
+  download log offset/length/duration only, never bytes (the seed ISO carries
+  cloud-init). Domain and volume XML contain no secrets, so V(2) is fine for them.
+- **Connection events traced too** in the pool: dial with duration, dial failure,
+  dead cached client redial, and both drop reasons (context cancelled, transport
+  error). `dialSSH`/`connectLibvirt`/`keepalive` stay untouched: changing
+  `hostDialer`'s signature would ripple into every test dialer, and the pool
+  already observes every event.
+- **Rendering.** Handles by name only (`domain`, `pool`, `volume`), never UUID/Key.
+  `libvirt.DomainState` and `DomainInterfaceAddressesSource` have stringers, pass
+  them typed. Bitmask flags don't stringify, log as integers only when non-zero.
+  `libvirt.OptString` → first element or `""` under `uri`.
+- **awaitOrCancel.** `done` runs inside fn's goroutine; on the normal path it logs
+  before `call` returns. On cancel, the abandoned RPC logs one late line when the
+  closed connection makes it fail: desirable in production, and in the integration
+  test only reachable in an already-failing timeout run (testr panics on a log after
+  the test ends; note that in the header, don't engineer around it).
+
+## Steps
+
+### Step 0 — Save plan
+Copy this file to `docs/plans/2026-09-10-0805-kvm-libvirt-tracing.md`, commit alone.
+
+### Step 1 — `errors.go`: shared error fields
+Factor `libvirtErrorFields(err error) []any` out of `logAPIError` (`errors.go:62-72`):
+returns `libvirtCode` (uint32), `libvirtCodeName` (`libvirt.ErrorNumber(code)`,
+e.g. `ErrNoDomain`), `libvirtMessage`; nil for non-libvirt errors; works through
+`%w` wrapping via `errors.As`. `logAPIError` keeps its message and gains
+`libvirtCodeName`. Test `TestLibvirtErrorFields` (table: libvirt.Error, wrapped,
+plain error) in `errors_test.go`.
+
+### Step 2 — new `trace.go`
+```go
+type tracingAPI struct { inner libvirtAPI; log logr.Logger }
+type tracedCall struct { t *tracingAPI; method string; start time.Time; kv []any }
+func (t *tracingAPI) begin(method string, kv ...any) *tracedCall
+func (c *tracedCall) detail(kv ...any) *tracedCall // appended only if V(2) enabled
+func (c *tracedCall) done(err error, kv ...any)    // one V(1) line
+```
+`done` emits `log.V(1).Info("libvirt call", ...)` with keys in order: `method`,
+begin args, result kv, `duration` (rounded to µs), then on error `error` plus
+`libvirtErrorFields`. Per-method keys:
+
+| method | V(1) args | V(1) results | V(2) extra |
+|---|---|---|---|
+| DomainLookupByName | domain | | |
+| DomainDefineXML | xmlBytes | domain | xml |
+| DomainCreate, DomainDestroy | domain | | |
+| DomainUndefineFlags | domain, flags | | |
+| DomainGetState | domain | state, reason | |
+| DomainGetMetadata | domain, type, uri | metadataBytes | metadata |
+| DomainInterfaceAddresses | domain, source | interfaces (count) | addresses |
+| ConnectListAllDomains | | count | names |
+| StoragePoolLookupByName, StoragePoolRefresh | pool | | |
+| StorageVolLookupByName | pool, volume | | |
+| StorageVolGetPath | pool, volume | path | |
+| StorageVolCreateXML | pool, xmlBytes | volume | xml |
+| StorageVolUpload, StorageVolDownload | pool, volume, offset, length | | (never content) |
+| StorageVolDelete | pool, volume | | |
+
+Return values pass through untouched.
+
+### Step 3 — `conn.go`: pool logging and wrapping
+- `get(ctx, log, host)`: dead cached client → `"libvirt connection lost, redialing"`;
+  time the dial; `"libvirt host connected"` with `address`, `duration`; failure →
+  `"libvirt host dial failed"` with `address`, `duration`, `error`.
+- `call(ctx, log, host, fn)`: `log = log.WithName("libvirt")`; wrap when
+  `log.V(1).Enabled()`; abort closure logs `"libvirt connection dropped"`
+  `reason="context cancelled"` before `drop`; transport-error branch logs the same
+  message with `reason="transport error"`. Update `call`'s doc comment.
+
+### Step 4 — call sites
+`kvm.go` Create/Get/Delete pass `log`, ListByTag passes `hlog`;
+`kvm_integration_test.go:220` passes `p.logger(ctx).WithValues("host", host.Name)`;
+`conn_test.go` (12 `p.call(` sites) gets `var discard = logr.Discard()`.
+
+### Step 5 — integration test output
+- `integrationLogger(t)`: `KVM_TEST_LOG_LEVEL` (default `1`, `strconv.Atoi`,
+  `t.Fatalf` on garbage) → `testr.NewWithOptions(t, testr.Options{LogTimestamp:
+  true, Verbosity: n})` (`github.com/go-logr/logr/testr`, present in logr v1.4.3,
+  no go.mod change); `logf.SetLogger(log)` once so nothing falls back to the
+  unfulfilled root.
+- `TestKVM_lifecycle`: `ctx = logr.NewContext(ctx, log)` after `WithTimeout`; the
+  cleanup ctx gets the same logger.
+- Header comment: document `KVM_TEST_LOG_LEVEL` (1 = one line per RPC, 2 adds
+  XML/metadata/addresses, 0 = only `t.Log`) and the late-line-after-timeout caveat.
+
+### Step 6 — unit tests
+- `kvm_test.go`: `captureContext(verbosity)` like `gcp_test.go:277-286` but with a
+  mutex around the slice; `TestLogging_verbosityTiers` (mirror `gcp_test.go:322-399`):
+  V0 → no lines; V1 → `"msg"="libvirt call"`, `"method"="DomainDefineXML"`,
+  `"method"="StorageVolUpload"`, `"msg"="libvirt host connected"`, provider/host/name
+  values, `"libvirtCodeName"="ErrNoDomain"` from the first lookup, no `"xml"=`;
+  V2 → `"xml"="<domain` and `"metadata"=`; a cloud-init sentinel absent at every tier.
+- `trace_test.go`: `TestTracingAPI_logsEveryMethod` (table of 17 calls over a
+  `readyHost()` fake, exactly one line each, plus
+  `reflect.TypeOf((*libvirtAPI)(nil)).Elem().NumMethod() == len(cases)` so a new
+  interface method without tracing fails); `TestTracingAPI_passesResultsThrough`;
+  `TestTracingAPI_errorFields` (libvirt error → code/name/message; `io.EOF` →
+  `"error"="EOF"`, no code); `TestTracingAPI_detailOnlyAtV2`;
+  `TestTracingAPI_uploadLogsSizeNotContent`.
+- `conn_test.go`: `fakeDialer` gains an `api` field; `TestHostPool_call_tracesAtV1`
+  (V1 → connected + method lines; V0 → no lines and fn receives the bare
+  `*fakeAPI`); `TestHostPool_call_logsDropReason` (transport error, cancel).
+
+### Step 7 — docs, execution log, CHANGELOG
+- `docs/kvm-provider.md`: `## Logging` section before "Known gaps":
+  `--zap-log-level=debug` = one `libvirt call` line per RPC plus connect/drop lines;
+  `--zap-log-level=2` adds XML, metadata, addresses; cloud-init never logged; no
+  V(5) wire tier because go-libvirt exposes no hook. Mention `KVM_TEST_LOG_LEVEL`
+  in its Testing paragraph.
+- `docs/testing.md`: `KVM_TEST_LOG_LEVEL` in the optional vars; `-v` now shows every
+  RPC; extend the `kvm_test.go` table row.
+- `docs/plans-executions/2026-09-10-0805-kvm-libvirt-tracing.md` with the checklist
+  and per-step entries; also a short cross-reference line in the KVM provider
+  execution log's Step 7 section.
+- `CHANGELOG.md` entry once the user confirms the live run shows the trace.
+
+## Verification
+
+```bash
+go build ./... && go vet ./... && go vet -tags=integration ./internal/provider/kvm/
+go test -race -count=1 ./internal/provider/kvm/
+go test -short ./...
+```
+Live, with the usual env: `make test-integration-kvm`. Expect, interleaved with
+`t.Log` lines and attributed to `trace.go:<line>`, lines like
+
+```
+kvm.libvirt: "level"=1 "msg"="libvirt call" "provider"="kvm-integration" "host"="kvm-test" "name"="proxy-…" "method"="DomainLookupByName" "domain"="proxy-…" "duration"="2.1ms" "error"="…" "libvirtCode"=42 "libvirtCodeName"="ErrNoDomain" "libvirtMessage"="…"
+```
+
+plus one `libvirt host connected` line with `duration`, a `StorageVolUpload` line
+whose duration is the seed upload, and no "log.SetLogger(...) was never called"
+text. `KVM_TEST_LOG_LEVEL=2` shows `"xml"="<domain` and `"addresses"=`;
+`KVM_TEST_LOG_LEVEL=0` shows only `t.Log` output. No line at any level contains
+`#cloud-config`.
+
+## Critical files
+- `internal/provider/kvm/conn.go`, `trace.go` (new), `trace_test.go` (new)
+- `internal/provider/kvm/errors.go`, `errors_test.go`
+- `internal/provider/kvm/kvm.go`, `kvm_test.go`, `conn_test.go`
+- `internal/provider/kvm/kvm_integration_test.go`
+- `docs/kvm-provider.md`, `docs/testing.md`

docs/plans/2026-09-10-2222-kvm-review-fixes.md

@@ -0,0 +1,195 @@
+# Plan: Fix the five code-review findings on `feat/kvm-provider`
+**Created:** 2026-09-10 22:22 UTC
+
+## Context
+
+PR #5 (`feat/kvm-provider` → `main`, 19 commits) is open and awaiting merge. A
+`/code-review high` pass over the branch returned five confirmed findings: two that can
+pin a Proxy's finalizer forever, one startup crash-loop, and two in the guest-agent merge
+committed earlier today. All five are fixed on the same branch so the PR updates in place,
+then re-validated against the real hypervisor before merging.
+
+Four decisions, made by the user during planning:
+
+1. **Wedged deletes** — attempt `Delete` even when `Get` fails permanently, make an
+   unconfigured KVM host a no-op delete, and add an annotation that abandons cleanup for
+   whatever still cannot finish.
+2. **Missing `kvm-ssh` Secret** — read the SSH material lazily, so only KVM Proxies fail.
+3. **Guest agent** — stop rewriting the user's cloud-config. Ship the bootstrap as a
+   vendor-data shell script instead.
+4. Validation is unit tests, the integration test against beelink, and the kind e2e. The
+   uncommitted dev-only files stay exactly as they are and out of the PR.
+
+## Why the guest-agent fix is a rewrite, not a patch
+
+Both cloud-init findings come from the same root cause: this operator edits the user's
+document. Reading cloud-init 26.1's sources settles that it never has to.
+
+Vendor-data is consumed by its **own** handler chain with separate script and config paths
+(`stages.py:_default_vendordata_handlers`). A vendor-data **shell script** is therefore
+written to the vendor script directory and run by `cc_scripts_vendor`, never passing
+through the config merger that drops colliding keys. On the final-stage module list it
+runs after `package_update_upgrade_install` and before the user's own `runcmd`, so the
+agent comes up early regardless of what the user's config does.
+
+That also closes the door the previous approach left open. The merge cannot be rescued
+from inside the documents: `merge_how` in the user-data governs only the first source, and
+`merge_how` in vendor-data is popped and discarded by `dict_extract_mergers` before the
+document is ever written. There is no supported way to make vendor-data cloud-config keys
+survive a user-data that defines the same key.
+
+Consequences: user-data reaches the seed byte for byte in any format, so Jinja templates,
+MIME archives, gzip payloads and shell scripts stop being special cases (finding 4), and
+nothing re-serialises YAML, so unquoted scalars keep their type (finding 5). The cost is
+that the script picks the package manager itself where cloud-init's `packages` key was
+distro-agnostic.
+
+## Why the delete fix is shaped this way
+
+KVM is the only provider that resolves a providerID segment against operator config. GCP
+and Kubernetes let the backend answer "not found", so retiring a location yields
+`ErrNotFound` and the finalizer drops. `Provider.host()`
+([kvm.go:118-139](internal/provider/kvm/kvm.go#L118-L139)) returns `ErrPermanent`, and
+`reconcileDelete` ([proxy_controller.go:281-291](internal/controller/proxy_controller.go#L281-L291))
+drops the finalizer only on `ErrNotFound`. `deletionFailure`
+([:327-336](internal/controller/proxy_controller.go#L327-L336)) says in its own comment
+that it retries "until an operator intervenes", and no intervention mechanism exists. The
+annotation is that mechanism, and it also rescues the unconfigured-provider wedge at
+[:276-279](internal/controller/proxy_controller.go#L276-L279).
+
+`Get` keeps returning `ErrPermanent` for an unknown host deliberately. Turning it into
+`ErrNotFound` would make a typo in the host list look like a missing VM on the live path,
+and the reconciler would recreate every affected Proxy elsewhere while the originals keep
+running where garbage collection cannot see them.
+
+## Steps
+
+One commit per step on `feat/kvm-provider`, each ending with
+`Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>`.
+
+### Step 0 — Save this plan into the repo
+Copy to `docs/plans/2026-09-10-2222-kvm-review-fixes.md` with the same header timestamp and
+commit it alone, before any code change. Start
+`docs/plans-executions/2026-09-10-2222-kvm-review-fixes.md` with the status checklist when
+Step 1 lands.
+
+### Step 1 — Unwedge deletion (finding 1)
+- `api/v1alpha1/proxy_types.go`: add `AnnotationSkipCleanup = "crawl.example.com/skip-cleanup"`
+  beside `AnnotationSpecHash` ([:59-63](api/v1alpha1/proxy_types.go#L59-L63)), documented as
+  abandoning the backend instance.
+- `internal/controller/proxy_controller.go`, `reconcileDelete`: honour the annotation after
+  the empty-providerID branch and **before** the provider lookup, so it also covers a
+  provider that is no longer configured. Log the abandoned providerID at Info.
+  `ProxyReconciler` has no `EventRecorder`; do not add one for this.
+- Same function: replace the two `if`s after `Get` with a switch. On
+  `provider.Class(err) == provider.ErrPermanent`, call `Delete` anyway, since it is
+  idempotent, and remove the finalizer when it succeeds rather than requeueing into a `Get`
+  that can never converge. On failure, `deletionFailure(errors.Join(getErr, delErr))`.
+  Guard on `err != nil`, since `Class(nil)` is nil.
+- `internal/provider/kvm/kvm.go`, `Delete`: an unconfigured host segment means there is
+  nothing this operator can clean, so log it and return success, mirroring `removeVolumes`
+  ([kvm.go:506-514](internal/provider/kvm/kvm.go#L506-L514)) for a missing pool. `Get` is
+  unchanged.
+- Tests: extend the deletion table in `internal/controller/reconcile_test.go`
+  ([:357-393](internal/controller/reconcile_test.go#L357-L393)) with permanent-Get plus
+  succeeding Delete, permanent-Get plus failing Delete, the annotation with the provider
+  never called, and the annotation on an unconfigured provider. In `kvm_test.go`, the
+  `"host not configured"` case at [:737](internal/provider/kvm/kvm_test.go#L737) flips from
+  `ErrPermanent` to success; add a named test beside `TestDelete_missingPoolIsNotAnError`
+  ([:715](internal/provider/kvm/kvm_test.go#L715)).
+- Docs: the annotation in `README.md` (~line 163) and `docs/architecture.md` (~line 487);
+  a *Known gaps* entry in `docs/kvm-provider.md` saying a domain on a host removed from
+  config can no longer be cleaned up by the operator, the Proxy still finishes deleting,
+  and the VM must be removed by hand because garbage collection only sweeps configured
+  hosts.
+
+### Step 2 — Validate names that become identifiers (finding 2)
+- `internal/provider/config_kvm.go`: add a name regexp beside `sha256Hex`
+  ([:6](internal/provider/config_kvm.go#L6)) allowing letters, digits, dot, dash and
+  underscore. Apply it to `hosts[].name` after the existing empty check
+  ([:129-131](internal/provider/config_kvm.go#L129-L131)) and to each `images` map key,
+  which becomes the cached volume name `base-<key>.<format>`
+  ([image.go:49-55](internal/provider/kvm/image.go#L49-L55)) and half of the image-cache
+  lock key ([image.go:96](internal/provider/kvm/image.go#L96)). `machineTypes` keys are only
+  compared, never embedded, so leave them.
+- Keep the file's first-error-wins style and `fmt.Sprintf` field paths. The host message
+  should name the consequence: the value becomes part of `hosts/<name>/domains/<vm>`, which
+  `parseProviderID` splits on `/` ([id.go:11-21](internal/provider/kvm/id.go#L11-L21)).
+- Tests: two entries in the `TestLoadConfig_kvmInvalid` table
+  ([config_kvm_test.go:76](internal/provider/config_kvm_test.go#L76)), existing
+  name/yaml/wantErrSub shape.
+
+### Step 3 — Read the SSH material lazily (finding 3)
+- `internal/provider/kvm/conn.go`: keep `newSSHAuth`, and add a lazy dialer that builds it
+  on first dial and caches it once it succeeds, so a `kvm-ssh` Secret created after the
+  manager starts heals on the next reconcile with no restart.
+- `internal/provider/kvm/kvm.go`, `New`: stop calling `newSSHAuth`; pass the lazy dialer.
+  Use the context parameter it currently ignores to log a warning when either file is
+  unreadable at startup, keeping the diagnostic without the exit.
+- Error classes: a missing or unreadable file stays unclassified, so `Class` defaults it to
+  transient and the Proxy retries until the Secret appears; an unparseable key or malformed
+  known_hosts wraps `ErrPermanent`. Distinguish with `errors.Is(err, fs.ErrNotExist)`.
+- Fix the two doc comments promising startup failure
+  ([kvm.go:79-81](internal/provider/kvm/kvm.go#L79-L81),
+  [conn.go:182-185](internal/provider/kvm/conn.go#L182-L185)) and the matching claim in
+  `docs/kvm-provider.md` (~lines 107-109). `DISCOVERY_TOKEN` already sets the house
+  convention for an optional mount: degrade with a loud warning.
+- Tests in `conn_test.go`, which already writes key files: `New` succeeds with no key
+  present, the first dial fails transiently, a dial after the file appears succeeds, and a
+  corrupt key is permanent.
+
+### Step 4 — Guest agent via vendor-data script (findings 4 and 5)
+- `internal/provider/kvm/seed.go`: delete `withGuestAgent`, `isCloudConfig`, `appendUnique`
+  and the `sigs.k8s.io/yaml` import. User-data goes onto the seed exactly as the Proxy
+  supplied it.
+- Add the bootstrap as a `#!/bin/sh` vendor-data script: skip when the agent is already
+  present, otherwise install `qemu-guest-agent` through whichever of `apt-get`, `dnf`,
+  `yum`, `zypper` or `apk` exists, then `systemctl enable --now`. Run `apt-get update`
+  before installing, since nothing else guarantees a fresh index when the user's config has
+  no `packages` key. Keep it idempotent and let it log to stdout, which lands in
+  `cloud-init-output.log`.
+- When the provider config also sets `vendorData`, wrap both into a MIME multipart
+  vendor-data: the user's document as one part, the script as another. Cloud-init detects
+  multipart by finding `mime-version:` in the first 4096 bytes, so the header must lead.
+- `internal/provider/config_kvm.go`: add `installGuestAgent`, defaulting to on. Off means
+  the seed carries only the user's own vendor-data and IP discovery falls back to the host
+  ARP table. Nothing fails permanently any more, so the error path in `writeSeed`
+  ([kvm.go:252-255](internal/provider/kvm/kvm.go#L252-L255)) narrows to MIME assembly.
+- Tests in `seed_test.go`: user-data byte-identical for cloud-config, Jinja, MIME, gzip and
+  script inputs; the script present in vendor-data; the opt-out; the multipart case
+  including the `mime-version:` position; keep the existing ISO tests.
+- Docs: rewrite the *What lands on the host* passage added earlier today, the `vendorData`
+  row and a new `installGuestAgent` row in the config table, and the ARP note in
+  *Known gaps*. Cite the cloud-init behaviour that forced the change.
+
+### Step 5 — Validate, record, push
+- `gofmt -l internal/`, `go vet ./...`, `go test -short ./...`.
+- `make test-integration-kvm` against beelink, environment as recorded in the KVM execution
+  log.
+- Rebuild the image, `kind load`, restart the manager, delete and recreate both Proxies,
+  curl a test URL through each.
+- `CHANGELOG.md` entry at the top with a real `date` timestamp, noting it supersedes the
+  merge approach described in today's earlier entry. Add a Run 8 section to
+  `docs/plans-executions/2026-09-04-1404-kvm-provider.md` plus this plan's own execution
+  file. Push with the token helper; PR #5 updates in place.
+
+## Verification
+- Unit: the controller cases prove a permanent `Get` no longer pins the finalizer and that
+  the annotation short-circuits before the provider is touched. `TestDelete_*` proves an
+  unconfigured host is a clean delete while `TestGet_*` stays permanent, which is what keeps
+  the live path from recreating VMs.
+- Seed: assert the uploaded user-data equals the Proxy's `cloudInit` byte for byte, which is
+  the property the old merge could not offer.
+- Integration on beelink: Create, IP via the guest agent, ListByTag, Delete. In the guest,
+  `cloud-init status --long` clean and `systemctl is-active qemu-guest-agent` active.
+- In-cluster: both Proxies reach Ready with an IP and return 204 through squid. Their
+  cloud-config defines its own `packages`, which is the exact case that failed in Run 7.
+- Negative check in kind: rename the `kvm-ssh` Secret away and restart. The pod must stay
+  up, the Kubernetes Proxies must stay Ready, and only the KVM Proxies degrade. Restore the
+  Secret and confirm they recover without a restart.
+
+## Out of scope
+- The two items the review noted but did not report: deleting a cached base image while
+  running VMs back onto it, and an ad-hoc image URL always being treated as qcow2. Both are
+  documented known gaps and neither is a regression from this branch.
+- The uncommitted dev-only files, by the user's decision.

docs/plans/2026-09-11-1526-provider-health-checks.md

@@ -0,0 +1,354 @@
+# Plan: Provider health checks — see whether the cluster can reach each provider
+**Created:** 2026-09-11 15:26 UTC
+
+## Context
+
+Today the operator can't say whether a provider's backend is reachable. The `Provider`
+interface ([provider.go:89-109](internal/provider/provider.go#L89-L109)) only has
+Create/Get/Delete/ListByTag, and `/healthz` and `/readyz` are bare `healthz.Ping`. A
+broken KVM path only shows up later: as a Proxy stuck in Provisioning, or as an orphan-GC
+sweep logging an error. Several failures stay invisible until a Proxy needs that host:
+
+- a missing `kvm-ssh` Secret;
+- a rotated host key;
+- a Tailscale route that is down;
+- a storage pool that is inactive or full.
+
+The goal is a periodic sanity check of every configured provider from inside the cluster,
+with results you can see through `kubectl` and Prometheus.
+
+Decisions made by the user during planning:
+
+1. **Surface:** a new operator-owned, read-only, cluster-scoped **`ProviderHealth`** CRD,
+   with one object per configured provider, a `Ready` condition and per-target detail.
+   Also Prometheus gauges, and a log line whenever a target goes up or down. No
+   EventRecorder, matching the earlier decision.
+2. **Scope:** real checks for all three providers.
+   - **kvm:** checked per host.
+   - **kubernetes:** confirms the API server and RBAC work.
+   - **gcp:** confirms credentials and the API work.
+3. **KVM depth:** reachability plus the storage pool.
+   - Over the existing pooled SSH+libvirt connection, call the libvirt version and
+     hostname RPCs.
+   - Confirm the configured pool is running and report its free space.
+   - No bridge check.
+
+Not affected: the Proxy reconcile behaviour, and the manager's `/healthz` and `/readyz`.
+A dead hypervisor must never make the manager unready.
+
+## Design
+
+### Contract: `internal/provider/check.go`
+
+```go
+type TargetStatus struct {
+    Name, Address   string
+    Ready           bool
+    Reason, Message string
+    Latency         time.Duration
+    Details         map[string]string
+}
+// Check never errors; each target carries its own verdict. Must return promptly once ctx ends.
+type Checker interface{ Check(ctx context.Context) []TargetStatus }
+```
+
+- **Shared reasons:** `Reachable`, `Timeout`, `Unreachable`, `Unauthorized`, `Forbidden`,
+  `APIError`.
+- **Compile-time checks:** each provider's `check.go` has `var _ provider.Checker = (*Provider)(nil)`.
+- **Bypass the wrappers.** `cmd/main.go:197-201` wraps every provider in
+  `WithTracing(WithMetrics(...))`, so the Checkers are collected from the **raw** providers
+  map, before that loop. As a result, checks don't inflate `provider_requests_total`.
+
+### The checks
+
+**kvm** (`internal/provider/kvm/check.go`)
+- **Concurrency:** one goroutine per `cfg.Hosts` entry (`wg.Go`), writing to `out[i]` so
+  results stay in config order.
+- **Inside `p.hosts.call`:** `ConnectGetLibVersion`, `ConnectGetHostname`,
+  `StoragePoolLookupByName(host.Pool)`, then `StoragePoolGetInfo`.
+- **Rules for `fn`:**
+  - It returns only raw RPC errors. `call` drops the shared connection on any other error
+    ([conn.go:160](internal/provider/kvm/conn.go#L160)), so "pool not running" is decided
+    after `call` returns, never returned from `fn`.
+  - It fills a captured result struct only after all four calls succeed. That struct is
+    read only when `call` returned nil, which avoids a race with an abandoned goroutine.
+- **Details:** `hostname`, `libvirtVersion` (decoded as major.minor.patch), `pool`,
+  `poolState`, `poolCapacity` and `poolAvailable` (in GiB).
+- **New RPCs:** add the three to `libvirtAPI` ([conn.go:29-47](internal/provider/kvm/conn.go#L29-L47)),
+  `tracingAPI` (`trace.go`) and `fakeAPI` (`fake_test.go`). `fakeAPI` gets a hostname, a
+  fixed version, and a per-pool state map that defaults to Running.
+- **Telling failure stages apart without matching strings:** add
+  `type dialError struct{ stage dialStage; err error }` in `conn.go`, with `Error()`
+  returning the inner message unchanged and `Unwrap()`. It wraps at four places:
+  - `lazySSH.dial`, when loading the key material fails (`ssh-material`)
+  - `dialSSH`, at the dial step (`tcp`)
+  - `dialSSH`, at the handshake (`ssh-handshake`)
+  - `connectLibvirt` (`libvirt-connect`)
+
+  Because `Unwrap` is kept, `classify` and `isClassified` still see `ErrPermanent` and
+  `fs.ErrNotExist`, so reconcile behaviour doesn't change.
+- **`checkReason(err)`, tested in order:**
+
+  | Condition | Reason |
+  |---|---|
+  | stage ssh-material | `SSHMaterialUnavailable` |
+  | stage tcp | `Unreachable` |
+  | stage ssh-handshake | `SSHHandshakeFailed` |
+  | stage libvirt-connect | `LibvirtConnectFailed` |
+  | ctx deadline or cancel | `Timeout` |
+  | `ErrNoStoragePool` | `PoolNotFound` |
+  | any other libvirt error | `LibvirtError` |
+  | anything else | `Unreachable` |
+  | success, pool not running | `PoolNotActive` |
+  | success | `Reachable` |
+- **Accepted behaviour:**
+  - A check that hits its timeout drops the shared connection. That already happens for
+    any cancelled reconcile context, and it only fires when the host isn't answering.
+  - The leader now keeps one SSH connection per host open from startup, with keepalives
+    every 15s. Before, the connection only opened with the first KVM reconcile.
+  - No forced periodic redial. `lazySSH` caches the key material forever, so a redial
+    would not pick up a rotated Secret anyway; that gap goes into the docs under
+    "Known gaps".
+
+**kubernetes** (`internal/provider/kubernetes/check.go`)
+- **Call:** `List` pods with the managed-label selector and `client.Limit(1)`,
+  cluster-wide. It uses the same uncached client, verb and RBAC as `ListByTag` and GC.
+  There is no namespace setting in the kubernetes config.
+- **Target:** one target named `apiserver`, with the address from a new `apiHost` field
+  set in `NewWithTransportWrapper`.
+- **Reasons:**
+  - `Timeout`: the context ended, or the error is `IsTimeout`/`IsServerTimeout`.
+  - `Unauthorized`, `Forbidden`.
+  - `APIError`: any other API status.
+  - `Unreachable`: anything else.
+
+**gcp** (`internal/provider/gcp/check.go`, plus a seam change in `gcp.go`)
+- **Why a new seam method:** the existing `AggregatedList` seam reads every page
+  ([gcp.go:63-78](internal/provider/gcp/gcp.go#L63-L78)), and MaxResults is only the page
+  size. So add `Probe(ctx, *computepb.AggregatedListInstancesRequest) error`, which makes
+  one `it.Next()` call and treats `iterator.Done` as nil.
+- **Request:** `Project`, `MaxResults: 1`, `ReturnPartialSuccess: true`, and the same
+  managed-label filter `ListByTag` uses.
+- **Target:** one per project, `Name: cfg.Project`, `Address: compute.googleapis.com`.
+  GCP config has no zones, and reading them from Proxy placements would tie the check to
+  the Proxy cache.
+- **Reasons:**
+  - `Timeout`: the context ended.
+  - For `*googleapi.Error`: 401 is `Unauthorized`, 403 is `Forbidden` (this also covers a
+    disabled API), anything else is `APIError`.
+  - `Unreachable`: anything else, including a failure to get a token.
+
+### The `ProviderHealth` CRD: `api/v1alpha1/providerhealth_types.go`
+
+- **Written by hand**, since kubebuilder isn't installed; `PROJECT` is edited by hand too.
+- **Scope and markers:** cluster-scoped, `shortName=ph`, status subresource, no spec.
+- **Scheme registration:** add the type to the existing `AddKnownTypes` call in
+  [proxy_types.go:362](api/v1alpha1/proxy_types.go#L362), not a second `init()`.
+- **Status:**
+  - `type`
+  - `conditions` (`Ready`; `ConditionReady` constant)
+  - `targets[]`, as a list-map keyed by `name`, each with: `name`, `address`, `ready`,
+    `reason`, `message` (at most 1024 characters), `latencyMillis`,
+    `lastTransitionTime`, `details`
+  - `targetsReady` (for example `"1/2"`, used by a printer column)
+  - `lastCheckTime`
+- **Printer columns:** Type, Ready, Targets, Last Check, Age.
+- **Viewer role:** `config/rbac/providerhealth_viewer_role.yaml`. No editor role, since the
+  operator owns the object.
+
+### Monitor: `internal/providerhealth`
+
+- **Fields:** `Monitor{Client, Reader, Targets []Target{Name, Type, Checker}, Interval, Timeout}`.
+- **Scheduling:**
+  - `NeedLeaderElection() true`.
+  - Checks once immediately, then on a ticker. Copy the ticker loop from
+    [gc.go:54-80](internal/gc/gc.go#L54-L80).
+- **Each cycle:**
+  - Runs every provider concurrently under `context.WithTimeout(Timeout)`, inside one
+    `tracing.Start` span.
+  - Stores the snapshot under a mutex; `Snapshot()` returns a deep copy.
+  - Logs changes. Going down logs `WARNING: provider target not ready` with provider,
+    target, address, reason and message. Coming back logs `Provider target ready`. A
+    target that is ready on first sight logs at V(1) only.
+- **Writes:**
+  - Get through `mgr.GetAPIReader()`, not the cache. That avoids a cluster-wide informer
+    and the watch permission it would need, whatever `--proxy-namespace` is set to.
+  - Create the object if it is missing, labelled
+    `app.kubernetes.io/managed-by=egress-proxies-operator`.
+  - Then `Status().Patch(MergeFrom(base))` when the status changed, following
+    [status.go:114-121](internal/controller/status.go#L114-L121).
+  - This means one write per provider per interval, because `lastCheckTime` moves each
+    time. That's acceptable given the 10s minimum interval.
+- **`buildStatus(prev, result, now)` is a pure function:**
+  - A target keeps its `lastTransitionTime` unless `ready` flipped.
+  - The `Ready` condition is set through `apimeta.SetStatusCondition`:
+    - `True/AllTargetsReady` when every target is ready
+    - `False/TargetsNotReady` otherwise, with the failing targets in the message
+    - `Unknown/ChecksNotImplemented` when the provider has no Checker
+    - `Unknown/NoTargets` when there are zero targets
+- **Cleanup, once at `Start`:** delete labelled objects whose name isn't in the config.
+  The config is loaded once per process, and unlabelled objects are never touched.
+- **Interval 0 (disabled):** run the cleanup with nothing kept, then return nil, so no
+  object is left showing a stale `Ready=True`.
+- **Invalid provider names:** provider names aren't validated as DNS-1123 names. For an
+  invalid name, warn once and skip the object writes; metrics are still emitted.
+- **`ValidateSchedule(interval, timeout)`** is called in `main.go` before the manager
+  starts. Interval must be 0 or at least 10s; timeout must be in (0, interval].
+- **RBAC markers on `Monitor`:**
+  - `providerhealths`: get, list, create, delete
+  - `providerhealths/status`: get, patch
+  - `make manifests` regenerates `config/rbac/role.yaml`.
+
+### Metrics and wiring
+
+- **New function:** `metrics.RegisterProviderTargets(reg, read func() []ProviderTarget) error`.
+  `Register`'s signature is left alone.
+- **Gauges:** a scrape-time collector, in the style of `constCollector`, reads the Monitor
+  snapshot and emits three gauges labelled `{provider,target}`:
+  - `proxy_operator_provider_target_up`
+  - `proxy_operator_provider_check_duration_seconds`
+  - `proxy_operator_provider_check_timestamp_seconds`
+- **Conversion closure:** the one that turns the Monitor snapshot into gauges lives in
+  `main.go`, next to `proxyPhaseCounts`. So `providerhealth` doesn't import `metrics`.
+- **No pool-bytes gauge:** that value is in status, and node-exporter is the right source
+  for disk alerts.
+- **`cmd/main.go`:**
+  - New flags `--provider-check-interval` (1m) and `--provider-check-timeout` (15s).
+  - Build the `Targets` from `cfg.Providers` in config order, with
+    `c, _ := raw[pc.Name].(provider.Checker)`.
+  - `mgr.Add(&providerhealth.Monitor{Client: mgr.GetClient(), Reader: mgr.GetAPIReader(), ...})`,
+    then register the collector.
+
+## Steps (one commit each, execution-log entry per step)
+
+**Step 0 — Save the plan.**
+- Create branch `feat/provider-health` off `main`.
+- Copy this plan to `docs/plans/2026-09-11-1526-provider-health-checks.md` and commit it
+  alone.
+- The uncommitted `.claude/settings.json` change stays out of every commit.
+
+**Step 1 — ProviderHealth API and CRD.**
+- `api/v1alpha1/providerhealth_types.go`
+- `zz_generated.deepcopy.go`, from `make generate`
+- `config/crd/bases/crawl.example.com_providerhealths.yaml`, from `make manifests`
+- the CRD and RBAC `kustomization.yaml` files, plus the viewer role
+- `PROJECT`
+
+**Step 2 — Contract plus the kubernetes and GCP checks.**
+- `internal/provider/check.go`
+- `internal/provider/kubernetes/{kubernetes.go,check.go,check_test.go}`
+- `internal/provider/gcp/{gcp.go,check.go,check_test.go,gcp_test.go}`
+
+**Step 3 — KVM check.**
+- `internal/provider/kvm/{conn.go,trace.go,check.go,check_test.go,fake_test.go,conn_test.go,trace_test.go,kvm_integration_test.go}`
+
+**Step 4 — Monitor.**
+- `internal/providerhealth/{monitor.go,status.go,monitor_test.go,status_test.go,envtest_test.go}`
+- `config/rbac/role.yaml`, from `make manifests`
+
+**Step 5 — Metrics and wiring.**
+- `internal/metrics/{metrics.go,metrics_test.go}`
+- `cmd/main.go`
+- `config/manager/manager.yaml`: a comment for the new flags
+- `test/e2e/e2e_test.go`
+
+**Step 6 — Docs.**
+- `docs/architecture.md`: a component row plus a provider-health section
+- `docs/kvm-provider.md`: the reason table, the shared-connection behaviour, and the
+  Secret-rotation gap under "Known gaps"
+- `docs/testing.md`: the new tests and the commands
+- `CLAUDE.md`: its Architecture and Gotchas sections still say "empty scaffold"; point
+  them at `docs/architecture.md`
+
+**Step 7 — Validate, record, push, open the PR.**
+- Run `go vet ./...`, `go test -short ./...`, `make test` and `make test-e2e`.
+- Hand the user the KVM commands below, and record their results in the execution log.
+- Add a CHANGELOG entry once the user confirms.
+- Push, using the token-file credential helper from memory.
+- Open the PR against `main` through the Gitea API (`tea` isn't installed).
+- Don't merge.
+
+## Tests
+
+**Provider checks** (table-driven, `Test<Func>_<scenario>`):
+- kubernetes:
+  - reachable
+  - list options: `Limit==1`, the label selector, no namespace
+  - failure reasons: forbidden, unauthorized, timeout, cancelled, internal, and a
+    network `*url.Error`
+- gcp:
+  - reachable, asserting the request fields
+  - failure reasons: 401, 403, 500, deadline, network
+  - `TestRealInstances_Probe_fetchesOnePage` against `httptest`: exactly one request with
+    `maxResults=1`, even though the page has a `nextPageToken`
+- kvm (using `hostFakes`/`newTestProvider` from `kvm_test.go`):
+  - reachable, with details filled in
+  - per-host verdicts, order preserved
+  - pool inactive, pool missing
+  - hosts checked concurrently (a sequential check would deadlock)
+  - a pool verdict keeps the connection (the dial count stays 1)
+  - a `checkReason` table
+  - `TestSSHAuth_dial_stages`, using the in-process SSH server in `conn_test.go`
+  - three new rows in the tracing table
+
+**Monitor:**
+- Setup: the fake client is built with a status subresource and a RESTMapper that
+  registers ProviderHealth as cluster-scoped.
+- Tests:
+  - `buildStatus` table
+  - objects are created with the label, and existing ones are patched
+  - the timeout bounds a checker that blocks
+  - a nil checker gives Unknown
+  - only transitions are logged (`funcr` logger)
+  - cleanup deletes only labelled objects that aren't configured
+  - the disabled path cleans up and returns
+  - an invalid name skips the writes
+  - `ValidateSchedule` table
+- An envtest round-trip, skipped under `-short`, which proves the generated schema is
+  accepted by a real apiserver.
+
+**Metrics:** `TestRegisterProviderTargets_scrape`, using a fresh registry and
+`GatherAndCompare`; an empty snapshot produces no series.
+
+**e2e (kind):** `providerhealth kubernetes` reaches `Ready=True`, which proves the
+generated RBAC works end to end.
+
+## Verification
+
+**Automated (Claude runs):** `go vet ./...`, `go test -short ./...`, `make test` (envtest
+with `-race`) and `make test-e2e`.
+
+**Against the real hypervisor (handed to the user, not run by Claude)**, with beelink as
+the host:
+
+```bash
+KVM_TEST_HOST=beelink.tail8c69c.ts.net:22 KVM_TEST_USER=novakj KVM_TEST_SSH_KEY=~/.ssh/id_ed25519 \
+KVM_TEST_KNOWN_HOSTS=~/.ssh/known_hosts KVM_TEST_BRIDGE=<bridge> KVM_TEST_IMAGE_URL=<image-url> \
+  go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/
+
+make install && go run ./cmd/main.go --providers-config <scratch copy of kvm-home config, local key paths> \
+  --metrics-bind-address :8080 --metrics-secure=false --provider-check-interval=15s
+kubectl get ph -w
+kubectl get ph kvm-home -o yaml
+curl -s localhost:8080/metrics | grep -E 'proxy_operator_provider_(target_up|check_)'
+```
+
+Each negative case is a one-line edit to the scratch config; nothing changes on the host.
+
+| Edit | Expected reason |
+|---|---|
+| `pool: nope` | `PoolNotFound` |
+| `address: …:2` | `Unreachable` |
+| `knownHostsFile: /dev/null` or `user: nobody` | `SSHHandshakeFailed` |
+| `sshKeyFile: /nonexistent` | `SSHMaterialUnavailable` |
+| `socket: /nonexistent` | `LibvirtConnectFailed` |
+
+## Out of scope
+
+- Gating Proxy creates on a host that is known to be down.
+- Tying `/readyz` to provider health.
+- Bridge checks.
+- SelfSubjectAccessReview checks of create/delete rights.
+- Hot-reloading the providers config.
+- Picking up a rotated `kvm-ssh` Secret without a restart. This is an existing gap; it
+  gets documented.

docs/plans/2026-09-16-0841-provider-blueprint.md

@@ -0,0 +1,119 @@
+# Plan: Provider blueprint (docs/providers.md)
+**Created:** 2026-09-16 08:41
+
+## Context
+
+The provider contract is spread across `docs/prompts/__initial-prompt.md` §5,
+the comments in `internal/provider/{provider,errors,check}.go`, `docs/architecture.md`
+§5 and §11, `docs/kvm-provider.md`, and plan Step 2/3/8. Many rules exist only in
+code: providerID must be self-contained, UID/CreatedAt must be read back, Create
+must survive a crash partway through, calls must stop when ctx is cancelled,
+decorators are wired in `cmd/main.go`, and so on. With "~5 providers soon", the
+next provider author needs one blueprint. It should say what a provider MUST,
+SHOULD or MAY do, show which features each existing provider (kubernetes, gcp,
+kvm) actually implements (with file:line evidence), give a checklist for adding
+one, and list the gaps.
+
+Decisions (user): store in `docs/providers.md`; scope = current state plus a
+separate "Gaps and proposed features" section (nothing implemented); wait for
+`feat/provider-health` to merge, then branch off `main`.
+
+## Step 0 — Save the plan; wait for the merge
+
+- Right after ExitPlanMode, copy this plan to `docs/plans/2026-09-16-0841-provider-blueprint.md`
+  and commit it on its own. The current branch is `feat/provider-health`, so the
+  plan reaches `main` with that merge. CLAUDE.md requires the plan commit to
+  come before any branching.
+- Stop and wait for the user to confirm that `feat/provider-health` is merged.
+  Then `git switch main && git pull`, and `git switch -c feat/provider-blueprint`.
+
+## Step 1 — Collect evidence (read-only, in parallel)
+
+Run 3 Explore agents at the same time, one each for `internal/provider/kubernetes`,
+`internal/provider/gcp` and `internal/provider/kvm`, including their tests. Each
+answers the **same questionnaire**, citing `file:line` for every answer and writing
+"not implemented" explicitly when that's the case:
+
+1. **Identity**: providerID format and parsing; what it carries (zone, host, namespace).
+2. **Create**: how it stays idempotent (name lookup, 409, DomainLookupByName);
+   whether it returns before the VM is up or waits; what happens when an earlier
+   attempt crashed partway; which Placement fields are used or ignored; how
+   cloud-init is delivered; how the port is used; how GC labels/UID are attached.
+3. **Get**: native state → InstanceState mapping; IP discovery rules; UID and
+   CreatedAt readback; how NotFound is detected.
+4. **Delete**: idempotency; what gets removed (disks, volumes, seeds); returns
+   before or after the resource is gone; unknown/malformed providerID behaviour.
+5. **ListByTag**: scope (namespaces, zones, hosts); partial-failure behaviour; how
+   managed resources are recognised.
+6. **Error classification**: table of backend error → NotFound/Quota/Transient/
+   Permanent; whether quota is detected at all.
+7. **Health check** (`Checker`): targets, provider-specific Reasons, Details keys.
+8. **Config**: YAML section, defaults, validation (`internal/provider/config*.go`),
+   credentials/Secrets, which settings are read once per process.
+9. **Connections and cancellation**: pooling, reconnects, timeouts, ctx honouring.
+10. **Observability**: log keys and verbosity levels, spans (`kvm/trace.go`, gcp
+    `wirelog.go`), anything beyond the shared metrics/tracing decorators.
+11. **Testing**: fakes, envtest/kind, `//go:build integration` tests, what is
+    not covered.
+12. **Known gaps**: TODOs, stale comments, documented limitations.
+
+Meanwhile, read the shared layer myself: `internal/provider/{provider,errors,check,
+name,config,metrics,tracing}.go`, `internal/provider/registry/registry.go`,
+`cmd/main.go` (registry wiring, decorator order, Checker collection), and the
+places that consume the contract, so every MUST rule traces to a caller that depends on it:
+`internal/controller` (error-class branching, ErrNotFound → replace/adopt),
+`internal/gc` (UID, CreatedAt age skip, ListByTag errors),
+`internal/providerhealth`. Then cross-check against the existing docs listed in Context.
+
+## Step 2 — Write `docs/providers.md`
+
+Sections:
+
+1. **What a provider is**: a few sentences plus links to architecture.md §5/§11.
+2. **The contract**: per method (Create/Get/Delete/ListByTag), MUST/SHOULD/MAY
+   rules, each followed by *why*: the reconciler or GC behaviour it protects,
+   linked to code.
+3. **Error taxonomy**: what each class makes the reconciler do; how to map backend
+   errors; "unclassified = transient" pitfall.
+4. **Identity and GC labels**: providerID guidance, `LabelManaged`/`LabelUID`,
+   CreatedAt, deterministic `NameFromUID`.
+5. **Optional interfaces**: `Checker` (shared reasons, never fails as a whole,
+   collected before decorators).
+6. **Feature matrix**: one row per questionnaire item, columns kubernetes/gcp/kvm,
+   cells `yes`/`partial`/`no` plus a short note. Put the file:line evidence in a
+   compact per-provider notes subsection under the table, not in the cells.
+7. **Adding a provider: checklist**: package layout, config section and validation,
+   registry constructor, `cmd/main.go` wiring, Checker, error mapping with tests,
+   `name`/label tests, integration test behind a build tag, provider doc page,
+   architecture.md rows, demo script (optional).
+8. **Gaps and proposed features**: current-state gaps per provider (e.g. no kvm
+   quota detection, orphan volumes invisible to GC, Secret rotation needs a
+   restart, ListByTag partial-failure inconsistency, stale `mock` references in
+   `provider.go` comments), plus proposals that are clearly marked as not
+   implemented. Only list them; don't design them.
+
+Also: link it from `docs/architecture.md` (Components table → provider contract row)
+and from the CLAUDE.md "Architecture" section. Leave per-provider docs where they are.
+
+Follow the plan-execution convention: create `docs/plans-executions/2026-09-16-0841-provider-blueprint.md`
+with the status checklist and step entries, and commit it with each step.
+
+## Step 3 — Verify, then open the MR
+
+- A verification agent (Explore) re-checks each matrix cell and each MUST rule
+  against the cited file:line and reports any mismatch. Fix what it finds.
+- Check that every relative link in the new and edited docs resolves (grep the
+  `](` targets and `ls` each one).
+- `go build ./... && go test -short ./...` as a sanity check (docs-only change,
+  expected no-op).
+- Push `feat/provider-blueprint` and open the MR against `main` via the Gitea API,
+  following the project's push/MR memory note. After the user confirms, add a CHANGELOG.md entry.
+
+## Critical files
+
+- New: `docs/providers.md`, `docs/plans/2026-09-16-0841-provider-blueprint.md`,
+  `docs/plans-executions/2026-09-16-0841-provider-blueprint.md`
+- Edited: `docs/architecture.md` (link), `CLAUDE.md` (link)
+- Read-only sources: `internal/provider/**`, `cmd/main.go`, `internal/controller`,
+  `internal/gc`, `internal/providerhealth`, `docs/prompts/__initial-prompt.md`,
+  `docs/kvm-provider.md`, `docs/gcp-vm-validation.md`

docs/plans/2026-09-16-1108-provider-docs-split.md

@@ -0,0 +1,99 @@
+# Plan: Split the provider blueprint into contract, appendix, template and index
+
+**Created:** 2026-09-16 11:08
+
+## Context
+
+`docs/providers.md` is one 731-line blueprint that mixes the generic provider contract with how kubernetes, gcp and kvm happen to meet it, plus gaps and proposals. The user wants to hand an agent a self-contained set of rules for building a new provider (Proxmox is next), so the generic part must stand alone, state explicitly where providers are allowed to differ (a pod is ready at once, a VM has a provisioning window, kvm blocks in Create, gcp does not), and leave implementation detail to a separate appendix. A third file, a fill-in spec template, lets the user write the new provider's specifics in the same vocabulary. `docs/providers.md` becomes a short index so every existing link keeps working.
+
+Decisions already made by the user: two new files plus a template; `providers.md` becomes an index; the contract is fully abstract (never names an existing provider); work continues on the checked-out branch `feat/provider-blueprint`.
+
+Conventions for all four files: code references are `path` plus function or symbol name, never line numbers; rule keywords MUST / SHOULD / MAY as today, and every MUST names the caller and what breaks; no em-dashes in prose. The contract cites only shared-layer files (`internal/provider/*.go`, `internal/provider/registry`, `internal/controller`, `internal/gc`, `internal/providerhealth`, `internal/tracing`, `cmd/main.go`, `config/manager`, `api/v1alpha1`) and never `internal/provider/{kubernetes,gcp,kvm}`.
+
+## Facts verified against the code that the new contract must carry
+
+- Callers put no deadline on the context. Only `Monitor.checkOnce` (`internal/providerhealth/monitor.go`) uses `WithTimeout`; the reconcile and GC contexts are cancel-only. `Sweeper.sweep` (`internal/gc/gc.go`) loops providers sequentially, so one hung `ListByTag` stalls orphan GC for every provider.
+- The reconciler's Stopped/Terminated branch in `reconcileManaged` (`internal/controller/proxy_controller.go`) deletes and requeues without clearing `ProviderID`, so a backend that reports a freshly created, not-yet-started instance as Stopped gets it deleted right after creation. kvm avoids this only by blocking `Create` through start.
+- `Sweeper.sweep` treats a zero `CreatedAt` as collectable now, skips `UID == ""`, and skips anything younger than `MinAge` (10m).
+- `cmd/main.go` type-asserts `provider.Checker` on the unwrapped provider before `WithTracing(WithMetrics(...))`; `registry.Build` fails the whole manager on a constructor error, so constructors must not dial.
+- Manager runtime: non-root, read-only rootfs, `/tmp` emptyDir 64Mi, 128Mi limit, distroless static image with CA roots (`config/manager/manager.yaml`, `Dockerfile`).
+- Corrections to earlier notes, for the appendix: kvm Checker `Details` also carry `user` and `pool`; the kubernetes provider has no seam interface (it injects `client.Client` via `newWithClient` and tests use the fake client); kvm `classify` has an `isClassified` pass-through for errors already wrapped with a class.
+
+## Files
+
+### 1. `docs/provider-contract.md` (new, the generic contract)
+
+Every behaviour axis gets a stable kebab-case code in backticks, used verbatim by the appendix table and the template so the three files can be cross-checked by grep.
+
+Sections:
+
+1. **Preamble**: audience, reading order (contract, template, appendix), rule keywords, "path plus symbol, never line numbers", where the rest lives.
+2. **What a provider is and who calls it**: the types in `provider.go`, `name.go`; callers and entry points (`reconcileManaged`, `replaceInstance`, `reconcileDelete`, `providerFailure`, `deletionFailure`, `placementFrom`, poll intervals, `MaxConcurrentReconciles`; `Sweeper.sweep` and `MinAge`; `Monitor.checkOnce`; decorators; `registry.Build`); what the instance is for (health probe CONNECT in `internal/health`, discovery handing out ip and port); leader-only vs every-replica; concurrency (MUST be safe, including two calls for one name); runtime envelope; time and cancellation (no caller deadline, MUST honour cancel, SHOULD bound each backend call, what breaks: a pinned worker, a stalled GC sweep, a wedged check round).
+3. **The contract, method by method**: current §2 rewritten abstractly, plus the new points. Create: idempotent by name, mark on create or within a documented window shorter than `MinAge`, self-contained ID, ctx, blocking vs submit, reject-early with `ErrPermanent` naming field and value, catalogue changes are outside the spec hash (`spechash.go`), MAY ignore Placement fields but MUST document, SHOULD deliver CloudInit, MUST NOT log it, and an eventually consistent backend MUST NOT let `Get` answer NotFound for an ID `Create` just returned. Get: NotFound only when gone, keep answering during delete, Running only with IP, Stopped/Terminated only when it will not recover, the "defined but not started" rule (block Create through start, or report Provisioning using written metadata or a task handle), behaviour during async operations and locks (last known state or Transient, never NotFound or Permanent), UID and CreatedAt read back, Permanent for an unresolvable ID. Delete: idempotent, re-entrant while deleting, removes secondary resources, MAY return before gone, SHOULD succeed where Get fails permanently. ListByTag: marked instances only with ID, UID, CreatedAt; error rather than an unvouched list; MAY omit unreadable parts; SHOULD cover the whole creatable scope; no installation identity.
+4. **Behaviour profiles**: one subsection per axis with the question, allowed profiles, what callers tolerate, obligations per profile. Axes: `readiness`, `create-return`, `delete-return`, `ip-source`, `gc-mark`, `created-at`, `list-partial`, `quota`, `not-found`, `primary-key`, `provisioning-state`, `async-ops`, `secondary-resources`, `credentials`, `client-kind`, `timeouts`, `logging`, `child-spans`, `backend-test`. Closing table: code, profiles, tolerated by which caller, template entry.
+5. **Error taxonomy**: `Wrap`, `Class`, `Error` fields, `Op` values, configured name not type; the class-to-reaction table; pitfalls; NotFound detection guidance (404, error code, list-and-miss, and why list-and-miss is forbidden with a partial list); the quota-vs-permission 403 case stated abstractly.
+6. **Identity, names, marks, CreatedAt**: `NameFromUID` shape, providerID strict parsing, marks and written metadata, provider name as DNS-1123 (`Monitor` skip, `spec.provider` MaxLength), primary key not the name.
+7. **Credentials, TLS and secret hygiene** (new): source and read time (constructor MUST NOT dial, SHOULD warn on a missing optional mount and fail at first use), never in logs, spans, error strings or `Error.Err`, TLS trust options and the naming of an insecure toggle, scratch only under `/tmp`.
+8. **Optional interfaces: Checker**: current §5 rules; `status.type` comment in `api/v1alpha1/providerhealth_types.go` must list the new type and the CRD be regenerated.
+9. **Observability**: the three layers, span and metric names, ctx and logger rules, child-span strategies by `client-kind`, no secrets or CloudInit in attributes, log tiers.
+10. **Configuration and wiring**: `ProviderConfig`, `setBlocks`, the allow-list in `Config.validate`, `UnmarshalStrict`, `config_<type>.go`, identifiers pasted into IDs, `registry.Constructor`, the map in `cmd/main.go`, decorator order, the duplicated type list.
+11. **Testing obligations**: fake behind a seam; the contract test list (current checklist item 5 plus Get during an async operation, the not-yet-started state, NotFound after Create is forbidden); classify table test; Checker tests; sentinel tests for CloudInit and credentials; span recorder test; integration test behind `//go:build integration` with `<TYPE>_TEST_*` variables, `make test-integration-<type>`, a section in `docs/testing.md`.
+12. **Checklist for adding a provider**: current §8 generalised, plus: Makefile target, testing.md section, `hack/providers-dev.yaml` decision, `ProviderHealth` type comment and `make manifests`, RBAC diff, CHANGELOG, appendix row and section, the filled template committed as `docs/<type>-provider-spec.md`, README row, architecture.md rows.
+13. **Known limits of the contract**: adoption unverified; Stopped and Terminated indistinguishable; config read once; no installation identity in GC; unconfigured provider latches; type list duplicated; Delete resubmitted every poll; no caller deadline; no conformance suite.
+14. **Related documents**.
+
+### 2. `docs/provider-spec-template.md` (new, the questionnaire)
+
+Preamble: copy to `docs/<type>-provider-spec.md`, fill every Answer line, hand it to the implementing agent with the contract and appendix; unanswered entries block implementation. Header block: type, example configured name, backend and API version, SDK or thin client, author, date.
+
+Fixed entry format: Question; Allowed answers (profile names linking to the contract §4 subsection); What each answer obliges; `Answer:` blank; `Evidence:` blank.
+
+Entries: identity and providerID shape; primary key (`primary-key`); placement mapping (Region, Zone, MachineType, Image, ignored fields); image source; cloud-init delivery; machine type mapping; network and IP discovery (`ip-source`, `readiness`); GC marks and CreatedAt (`gc-mark`, `created-at`); state mapping table incl. not yet started (`provisioning-state`); Create blocking and task tracking (`create-return`, `async-ops`); Delete and secondary resources (`delete-return`, `secondary-resources`); ListByTag scope and partial failure (`list-partial`); error classification table (`not-found`, `quota`); credentials and TLS (`credentials`); config block fields, defaults, validation; Checker targets, reasons, Details; SDK vs thin client and child spans (`client-kind`, `child-spans`); timeouts (`timeouts`); concurrency; logging tiers (`logging`); tests (`backend-test`); deployment (mounts, samples, Makefile helper, RBAC, dev config); docs; known limits accepted at design time.
+
+### 3. `docs/providers-existing.md` (new, the appendix)
+
+Sources: current `providers.md` §6 table, §7 matrix and per-provider notes, §9 per-provider gaps and proposed features, `docs/kvm-provider.md`, the gcp and kubernetes package facts. Links to architecture.md §5, §11 and Decisions, and to the README table, rather than duplicating them.
+
+Sections: preamble; **profile table** with exactly one row per contract axis code and one column per existing provider, each cell a profile name plus a symbol as evidence; **other differences** (matrix rows that are not axes: Placement used, Port used, CloudInit delivered, states produced, Error.Provider correctness, credential rotation, connection handling, Checker Details, fakes, proxy software, reachability, distinct egress, adopted-UID check); **observability status** (current §6 table); one section each for **kubernetes**, **gcp**, **kvm** with the same sub-headings: package layout and seam, config block, ID format, Create, Get and state mapping table, IP source, Delete, ListByTag, classify table, Check (targets, reasons, Details), logging, tracing layer, tests and fakes, deployment bits, known gaps; **gaps shared by the existing providers**; **proposed features** (current list, unchanged).
+
+### 4. `docs/providers.md` (rewritten in place as the index)
+
+One paragraph on the split; one line each for contract, template, appendix; the per-provider guides (`docs/kvm-provider.md`, `docs/gcp-vm-validation.md`, `docs/gcp-in-specific-project.md`); the architecture.md sections; reading order for a new provider; a note that the pre-split blueprint is in git history under this path.
+
+### 5. One-line link edits
+
+- `README.md` "### Adding a provider": point at the contract, appendix and template, with `docs/providers.md` named as the index. Keep the "Start there" sentence.
+- `CLAUDE.md` "## Architecture": same replacement for the providers.md sentence.
+- `docs/architecture.md` components row "Provider contract": rules in `provider-contract.md`, profile table in `providers-existing.md`.
+- Historical files under `docs/plans*` keep their links; they resolve to the index.
+
+## Steps (one commit per step, on `feat/provider-blueprint`)
+
+- **Step 0**: copy this plan to `docs/plans/<ts>-provider-docs-split.md` with the real timestamp and commit it alone, before anything else.
+- **Step 1**: write `docs/provider-contract.md`; freeze the axis code list. Create `docs/plans-executions/<ts>-provider-docs-split.md` with the status checklist.
+- **Step 2**: write `docs/provider-spec-template.md` from contract §4 and §12.
+- **Step 3**: write `docs/providers-existing.md`, replacing every `file:line` from the old blueprint with `file` plus symbol.
+- **Step 4**: rewrite `docs/providers.md` as the index; apply the three link edits.
+- **Step 5**: run the verification below; fix findings.
+- **Step 6**: final execution summary entry. CHANGELOG entry at the top only after the user confirms. Push and MR only when the user asks (per memory, push uses the token helper and the MR goes through the Gitea API).
+
+## Verification (read-only shell, from the repo root)
+
+1. Every MUST names a caller: `grep -n MUST docs/provider-contract.md` and confirm each bullet cites a shared-layer path or symbol and states what breaks.
+2. Every cited path exists: extract `(internal|cmd|config|docs|api|hack|test)/...` tokens from the four files and `test -e` each.
+3. Every cited symbol exists: extract backticked identifiers, drop paths, keywords and YAML keys, and `grep -rqw` each across `*.go`, `Makefile`, `*.yaml`.
+4. Contract is abstract: `grep -niE "kubernetes|gcp|kvm|libvirt|squid|google|compute engine|internal/provider/(kubernetes|gcp|kvm)" docs/provider-contract.md` is empty.
+5. Profiles line up 1:1: for each axis code from contract §4 headings, it appears in the appendix table and in the template; the appendix table has one row per code.
+6. Links resolve: extract `](target)` from the four files plus README, CLAUDE.md and architecture.md; check file targets exist and anchors match slugified headings.
+7. No em-dashes in the four files or in the added lines of the three edited files.
+8. No line references: `grep -nE '\.go:[0-9]+|:[0-9]+-[0-9]+' <four files>` is empty.
+9. Nothing lost: compare the gap and proposal bullets of `git show HEAD:docs/providers.md` against the appendix and contract §13 by keyword.
+10. `go build ./... && go test -short ./...` still pass (docs-only change).
+
+## Execution summary must record
+
+- Step 1: the frozen axis list; the two contract additions found in verification (sequential GC sweep; the Stopped branch re-deleting a not-yet-started instance); per-call timeout facts as verified for each existing provider.
+- Step 3: the fact corrections applied (kvm Details keys, kubernetes seam, kvm `isClassified`); matrix rows dropped in favour of the profile table.
+- Step 4: the exact link edits.
+- Step 5: the verification commands as run with trimmed output.
+- Worth noting: CHANGELOG headings keep the repo's own em-dash format; the stale `mock` comments in `provider.go`, `registry.go` and `proxy_types.go` stay listed as gaps and are not fixed here.

docs/provider-contract.md

@@ -0,0 +1,860 @@
+# The provider contract
+
+This document is the specification a provider must meet. It is written for
+whoever builds the next one, human or agent, and it deliberately says nothing
+about how any existing provider is built. Those details are in
+[providers-existing.md](providers-existing.md); the questionnaire a new
+provider's design must answer is [provider-spec-template.md](provider-spec-template.md).
+Read the three in that order: this contract, then the template, then the
+appendix for worked examples.
+
+Rule keywords: **MUST** means breaking the rule breaks a caller, and every MUST
+below names the caller and what breaks. **SHOULD** means every current provider
+does it and a new one needs a written reason not to. **MAY** means it is
+allowed and the choice must be recorded in the provider's spec.
+
+Code references are a file path plus a function or symbol name, never a line
+number. Where a rule is enforced by a caller, that caller is one of:
+
+- the **reconciler**, `internal/controller/proxy_controller.go`;
+- **orphan GC**, `internal/gc/gc.go`;
+- the **provider health monitor**, `internal/providerhealth/monitor.go`;
+- the **decorators**, `internal/provider/metrics.go` and
+  `internal/provider/tracing.go`;
+- the **composition root**, `cmd/main.go`.
+
+## 1. What a provider is and who calls it
+
+### 1.1 The types
+
+A provider is a Go type implementing `provider.Provider`
+(`internal/provider/provider.go`): four methods over one VM-like instance,
+identified by an opaque `providerID` string.
+
+- `Create(ctx, CreateRequest) (providerID, error)`
+- `Get(ctx, providerID) (*Instance, error)`
+- `Delete(ctx, providerID) error`
+- `ListByTag(ctx) ([]Instance, error)`
+
+`CreateRequest` carries `Name` (from `NameFromUID` in `internal/provider/name.go`,
+22 characters, `proxy-` plus base32), `UID` (the owning Proxy's UID),
+`Namespace`, `ProxyName`, `Placement` (`Region`, `Zone`, `MachineType`,
+`Image`), `CloudInit` (already resolved from inline text or a Secret) and
+`Port`.
+
+`Instance` carries `ID`, `IP`, `State` (one of `StateProvisioning`,
+`StateRunning`, `StateStopped`, `StateTerminated`), `UID` and `CreatedAt`.
+
+The GC marks are the constants `LabelManaged` (value `LabelManagedYes`) and
+`LabelUID`, also in `provider.go`.
+
+Providers are named instances of a type: two entries of one type with
+different config blocks are two providers. The configured name is what
+`Proxy.spec.provider` refers to, and it is immutable, so a Proxy never moves
+between providers.
+
+### 1.2 The callers
+
+Nothing above the provider watches backend resources. Everything the operator
+knows about an instance comes from polling `Get`.
+
+- **The reconciler** calls `Create`, `Get` and `Delete` from `reconcileManaged`,
+  `replaceInstance` and `reconcileDelete`, and branches only on the error class
+  (`providerFailure`, `deletionFailure`) and on `Instance.State`. It maps the
+  CRD's placement to `Placement` one to one in `placementFrom`. Its poll
+  intervals are set in `applyDefaults`: `ProvisioningPoll` and `DeletionPoll`
+  10s, `DriftPoll` 2m, `RequeueNow` 1s, `QuotaRetry` 5m. Up to 3 Proxies
+  reconcile at once (`MaxConcurrentReconciles` in `SetupWithManager`).
+- **Orphan GC** calls `ListByTag` and `Delete` from `Sweeper.sweep`, every
+  `Interval` (10m), and never deletes an instance younger than `MinAge` (10m).
+  It loops over providers one after another in a single goroutine.
+- **The provider health monitor** calls the optional `Checker` from
+  `Monitor.checkOnce`, every `--provider-check-interval`, all providers
+  concurrently under one timeout.
+- **The composition root** builds every provider through `registry.Build`
+  (`internal/provider/registry/registry.go`) before the manager starts, on
+  every replica, then wraps each one in `WithTracing(WithMetrics(...))`.
+
+**Where the calls come from.** The reconciler, GC, the health engine and the
+monitor run on the leader only. The constructors run on every replica at
+startup.
+
+**Concurrency.** Up to 3 reconciles, a GC sweep and a health-check round can
+call the same provider at the same time, and GC's `Delete` can overlap the
+reconciler's `Delete` for the same instance. A provider **MUST be safe for
+concurrent use, including two calls for the same instance name**. Nothing
+above the provider serialises them.
+
+### 1.3 What the instance is for
+
+The instance is a forward proxy with its own egress path. Two parties connect
+to `http://<Instance.IP>:<spec.port>`:
+
+- the health engine, from the operator pod, which sends a real `CONNECT`
+  through it (`probe` in `internal/health/probe.go`);
+- clients, which get `ip` and `port` from the discovery API (`viewOf` in
+  `internal/discovery/handlers.go`).
+
+So the IP a provider reports **MUST be reachable on that port from the
+operator's pod network and from the clients**. Whatever that takes on the
+backend (firewall rules, security groups, routing) is either done by the
+provider or documented as a prerequisite in the provider's guide.
+
+The provider does not know which proxy software runs. The proxy is installed
+and configured by the Proxy's `cloudInit`; building install logic into the
+operator is a stated non-goal (`docs/prompts/__initial-prompt.md`, non-goals).
+A backend that cannot take user-data (see `readiness` in §4) documents that
+it ignores the field and how the proxy gets configured instead.
+
+### 1.4 Where it runs
+
+The manager container (`config/manager/manager.yaml`) runs as non-root with a
+read-only root filesystem, a 64Mi `emptyDir` at `/tmp` and a 128Mi memory
+limit, from a distroless static image (`Dockerfile`) that carries the public
+CA roots and nothing else. So a provider:
+
+- **MUST stream large payloads** rather than buffering them in memory;
+- **MUST write scratch files only under `/tmp`**, and delete them;
+- gets credentials either as mounted files (the providers ConfigMap at
+  `/etc/proxy-operator`, optional Secret mounts) or from the environment or
+  the pod's own identity. See §7.
+
+### 1.5 Time and cancellation
+
+**No caller puts a deadline on the context.** The reconcile and GC contexts
+are cancelled on shutdown and on loss of leadership, and otherwise never.
+The only bounded context is the health monitor's (`Monitor.checkOnce`, one
+timeout per round).
+
+- **MUST honour `ctx`**: return promptly once it is cancelled, on every
+  method.
+- **SHOULD bound every backend call itself**: a dial timeout, a per-request
+  timeout, a cap on any wait loop. What breaks otherwise: a hung `Create`,
+  `Get` or `Delete` pins one of the 3 reconcile workers until the manager
+  stops; a hung `ListByTag` stalls orphan GC for every provider, because
+  `Sweeper.sweep` runs them sequentially; a `Check` that ignores its context
+  wedges a monitor round. Choose budgets against the polls: a `Get` that
+  routinely outlives `ProvisioningPoll` gains nothing.
+
+## 2. The contract, method by method
+
+### 2.1 Create(ctx, CreateRequest) (providerID, error)
+
+- **MUST be idempotent by `req.Name`.** A repeat call for a name that already
+  exists returns that instance's providerID with a nil error, creates no
+  duplicate and returns no error.
+  *Caller:* the reconciler records the ID in status only after `Create`
+  returns (`reconcileManaged`). A crash between the two, or a lost status
+  write, runs `Create` again for the same Proxy; `replaceInstance` recovers
+  from a crash between its two writes the same way.
+- **MUST mark the instance as managed**, with `LabelManaged=LabelManagedYes`
+  and `LabelUID=req.UID` or the backend's equivalent, in the same call that
+  creates it. When the backend's create call cannot carry the mark, the
+  window between creation and marking **MUST be far shorter than `MinAge`**
+  and documented in the provider's spec (`gc-mark` in §4).
+  *Caller:* orphan GC finds instances only through the mark and never deletes
+  one whose UID is empty (`Sweeper.sweep`). An instance that exists without
+  the mark leaks if the operator crashes before marking it.
+- **MUST return a providerID that is self-contained.** `Get` and `Delete`
+  receive only the ID, and the config or spec may have changed since
+  `Create` (a zone edit during a replacement). Put whatever `Get` needs to
+  find the instance into the ID, and validate config values that end up in
+  the ID so they cannot contain the separator. See `primary-key` in §4 for
+  backends that allocate their own identifier.
+- **MUST NOT let `Get` answer NotFound for an ID that `Create` just
+  returned.** A backend with eventual consistency between create and read
+  needs a read-after-write guarantee or a retry inside `Create`.
+  *Caller:* on NotFound the reconciler clears the ID and creates again
+  (`reconcileManaged`), so a lagging read makes a duplicate.
+- **MUST honour `ctx`** (§1.5).
+- **MAY return once the request is submitted, or block until the instance is
+  running** (`create-return` in §4). Submit-and-return is preferred: readiness
+  is discovered by polling `Get`, and a blocking `Create` holds one of the 3
+  workers for its whole duration. A provider that blocks MUST still honour
+  cancellation and SHOULD cap the wait.
+- **SHOULD reject a request it can never satisfy** with `ErrPermanent`
+  before touching the backend: a missing or unknown zone, machine type or
+  image, or any placement value not in its catalogue. The CRD cannot validate
+  `placement`, because it is provider-agnostic, so this error is the user's
+  only feedback. The message SHOULD name the offending field and value.
+- **Catalogue changes do not replace instances.** The spec hash
+  (`specHashInput` in `internal/controller/spechash.go`) covers `placement`,
+  the resolved `cloudInit` and `port`, not the provider config. When a
+  placement value names a config entry, changing what that entry means affects
+  only new instances. The provider's guide MUST say so and tell users to add a
+  new key rather than edit one in use.
+- **MAY ignore `Placement` fields that do not apply.** If a field is ignored,
+  the provider's guide MUST say so, because the field is still part of the
+  spec hash: changing it replaces the instance and has no other effect.
+- **SHOULD deliver `CloudInit` as user-data**, byte for byte, without parsing
+  or re-rendering it. A provider that cannot documents that it ignores the
+  field.
+- **MUST NOT log `CloudInit` contents**, put them in span attributes, or
+  include them in error messages. It can hold secrets. Log its byte count.
+  Test this with a sentinel string (§11).
+
+### 2.2 Get(ctx, providerID) (*Instance, error)
+
+- **MUST return `ErrNotFound` when, and only when, the instance is gone.**
+  *Caller:* `ErrNotFound` triggers three actions. In normal reconciling the
+  reconciler clears the ID and creates again (`reconcileManaged`). During
+  replacement it is the signal to create the new instance
+  (`replaceInstance`). During deletion it releases the finalizer
+  (`reconcileDelete`). A false NotFound during an outage creates a duplicate
+  or loses the instance. "Unreachable", "wrong credentials" and "cannot tell"
+  are Transient or Permanent, never NotFound.
+- **MUST keep returning the instance, not NotFound, while a delete is in
+  progress** and the name is still taken.
+  *Caller:* the replacement instance gets the same name. `replaceInstance`
+  waits for NotFound before creating it; a NotFound too early makes `Create`
+  adopt the dying instance.
+- **MUST report `StateRunning` only with a non-empty `IP`.** Running without
+  an address yet is `StateProvisioning`.
+  *Caller:* the IP from a Running answer goes straight into status and to the
+  health engine (`reconcileManaged`).
+- **MUST report `StateStopped` or `StateTerminated` only when the instance
+  will not recover by itself.** SHOULD map states it does not recognise to
+  `StateStopped`.
+  *Caller:* the reconciler reacts to both states identically: it calls
+  `Delete` and requeues after `DeletionPoll` without clearing the ID, then
+  polls `Get` and deletes again until NotFound, then creates
+  (`reconcileManaged`). Instances are disposable, so this is the safe answer
+  for anything unknown, but it makes the next rule critical.
+- **MUST NOT report a freshly created instance that has not been started yet
+  as `StateStopped`.** A backend with no provisioning state of its own, where
+  "defined but not yet started" and "stopped" look the same, MUST resolve this
+  in one of two ways (`provisioning-state` in §4): block `Create` until the
+  instance is started, or record enough at creation (written metadata, a task
+  handle) for `Get` to answer `StateProvisioning`.
+  *Caller:* the same branch of `reconcileManaged` would delete the instance
+  right after it was created, forever.
+- **During an asynchronous operation or a backend lock** (a clone or a delete
+  still running, an instance locked by the backend), `Get` MUST return either
+  the last known state or `ErrTransient`, never NotFound and never
+  Permanent (`async-ops` in §4).
+- **MUST fill `UID` and `CreatedAt` by reading them back from the backend**,
+  never from memory.
+  *Caller:* GC compares `UID` with the live Proxies and deletes anything whose
+  `CreatedAt` is older than `MinAge`. **A zero `CreatedAt` counts as very
+  old** (`Sweeper.sweep`), so a provider that forgets it makes instances still
+  being created collectable. An unparsable timestamp giving the zero time is
+  accepted: it never shields an orphan forever.
+- **SHOULD return `ErrPermanent` for an ID it can never resolve**: a malformed
+  ID, a location no longer in the config.
+  *Caller:* `reconcileDelete` then tries `Delete` anyway and releases the
+  finalizer if that works. If the provider returned Transient, deletion would
+  poll forever.
+
+### 2.3 Delete(ctx, providerID) error
+
+- **MUST be idempotent.** Deleting something already gone returns nil.
+  *Caller:* GC and the reconciler race, and a crash repeats the call.
+- **MUST accept being called again while an earlier delete is still in
+  progress.** Treat "already deleting" as success, and SHOULD keep that cheap.
+  *Caller:* while `Get` still returns the instance, `reconcileDelete` and the
+  Stopped branch of `reconcileManaged` call `Delete` again every
+  `DeletionPoll` until `Get` returns NotFound.
+- **MUST remove everything `Create` made**: disks, volumes, seeds, addresses,
+  uploaded files. Either delete them explicitly or rely on the backend
+  cascading (`secondary-resources` in §4).
+  *Caller:* GC sees only what `ListByTag` returns, so secondary resources
+  have no other cleanup path.
+- **MAY return before the instance is gone** (`delete-return` in §4), as long
+  as `Get` keeps showing it until it is.
+- **SHOULD succeed for an ID whose `Get` fails permanently** when there is
+  nothing left the provider could reach, and log that it did nothing.
+  *Caller:* on the deletion path a permanent error is not latched; it retries
+  forever (`deletionFailure`), and the Proxy is then only released through the
+  `AnnotationSkipCleanup` annotation (`api/v1alpha1/proxy_types.go`).
+
+### 2.4 ListByTag(ctx) ([]Instance, error)
+
+- **MUST return only instances that carry the managed mark**, each with an
+  `ID` that `Delete` accepts, plus `UID` and `CreatedAt`. `State` and `IP`
+  are not used by GC.
+  *Caller:* GC deletes every listed instance whose UID is not live and whose
+  age is past `MinAge` (`Sweeper.sweep`). Listing something without the mark
+  is how an operator deletes an instance it does not own.
+- **MUST return an error rather than a list it cannot vouch for.** An error
+  makes GC skip that provider for this sweep (`Sweeper.sweep`).
+- **MAY leave out instances it could not read.** Leaving something out is
+  always safe, because GC never deletes what is not listed. Pick a policy
+  (`list-partial` in §4) and document it.
+- **SHOULD cover the whole scope the provider can create in**: every
+  namespace, zone, node or host. Anything outside that scope is never
+  collected.
+- **Scope caveat:** GC protects instances by UID, not by installation. Two
+  operator installs sharing one backend delete each other's instances (§13).
+  Two provider entries of one install that share a scope are safe, because
+  they share the live-UID set.
+
+## 3. What a container-style backend and a VM-style backend have in common
+
+The contract above is the same for both, but the profiles in §4 differ.
+A backend that runs a container has an instance that is ready seconds after
+`Create`, gets its address from the platform, is deleted in one call and has
+no secondary resources; a backend that boots a VM has a provisioning window
+of minutes, may discover the address only after the guest reports it, may
+need several deletes, and leaves disks and seeds behind unless told
+otherwise. Neither is preferred. What matters is that the provider's spec
+names its profile on every axis, because the appendix table and the test list
+are keyed by those names.
+
+## 4. Behaviour profiles
+
+Each axis below has a stable code. The provider's spec (the template) answers
+every axis; the appendix maps each existing provider to its profile on every
+axis. A profile is not a permission to skip a MUST in §2; it says which of
+the allowed ways of meeting it the provider chose.
+
+### 4.1 `readiness`
+
+*When is an instance usable after `Create`?*
+
+- **immediate**: the platform starts it at once; `Get` reports Running
+  within a poll or two.
+- **provisioning-window**: minutes of booting; `Get` reports Provisioning
+  for that time.
+
+Callers tolerate both: the reconciler polls at `ProvisioningPoll` either
+way. An immediate provider still MUST report Provisioning until the IP is
+known.
+
+### 4.2 `create-return`
+
+*Does `Create` return once the request is submitted or once the instance is
+running?*
+
+- **submit**: preferred. Readiness comes from `Get`.
+- **blocking**: allowed. Costs one of 3 workers for the duration, and any
+  first-use setup (an image download) runs inside it. MUST honour `ctx`,
+  SHOULD cap the wait, and MUST keep every step idempotent so a cut-off
+  attempt resumes.
+
+### 4.3 `delete-return`
+
+*Does `Delete` return before the instance is gone?*
+
+- **returns-early**: `Get` keeps returning the instance until it is gone
+  (§2.2), and repeated `Delete` calls are cheap.
+- **waits**: `Delete` returns nil only when nothing is left. Bound the wait
+  and honour `ctx`.
+
+### 4.4 `ip-source`
+
+*Where does `Instance.IP` come from?*
+
+- **assigned-at-create**: the backend reports it with the instance.
+- **discovered**: an agent in the guest, a DHCP lease, a neighbour table.
+  Discovery failures mean "no address yet", never an error; `Get` returns
+  Provisioning. The provider's guide MUST say what the guest needs (an agent
+  package, outbound access to install it) and what happens without it.
+
+### 4.5 `gc-mark`
+
+*Where do `LabelManaged` and `LabelUID` live?*
+
+- **native-labels**: labels or tags the backend indexes; `ListByTag` can
+  filter server-side.
+- **written-metadata**: a description, an annotation or a metadata element
+  written by the provider; `ListByTag` lists everything and filters
+  client-side.
+- **mark-window**: the create call cannot carry the mark and a second call
+  writes it. Allowed only when the window is a few seconds, documented, and
+  the unmarked instance is cleaned by the same Proxy's next reconcile (§2.1).
+
+### 4.6 `created-at`
+
+*Where does `CreatedAt` come from?*
+
+- **backend-timestamp**: the backend's own creation time.
+- **written-at-create**: the provider writes the time into the mark
+  metadata in the create call. A backend with no timestamp MUST do this.
+
+### 4.7 `list-partial`
+
+*What does `ListByTag` do when part of its scope is unreachable?*
+
+- **fail-whole**: one unreachable part fails the list; GC skips the provider
+  this sweep. Nothing is collected until every part answers.
+- **omit-unreachable**: unreachable parts are left out and the rest is
+  returned. Orphans in the reachable parts are collected; the provider MUST
+  log what it left out.
+
+Both are safe (§2.4). A single unreadable instance MAY always be skipped.
+
+### 4.8 `quota`
+
+*Can the provider tell "no capacity" from other failures?*
+
+- **detected**: a recognisable quota, rate-limit or out-of-resources signal
+  maps to `ErrQuotaExceeded`.
+- **absent**: capacity errors map to `ErrTransient`. The guide MUST say so,
+  and the `Checker` SHOULD expose a capacity figure in `Details` instead.
+
+### 4.9 `not-found`
+
+*How does the provider know an instance is gone?*
+
+- **status-404**: a distinct not-found status or error code from the
+  backend.
+- **error-code**: a code or message the backend uses for "does not exist",
+  matched exactly and covered by the classify table test with real captured
+  responses.
+- **list-and-miss**: absence from a listing. Allowed only when the listing
+  can never be partial; otherwise a missing part reads as "gone" and §2.2 is
+  broken.
+
+### 4.10 `primary-key`
+
+*Is `req.Name` the backend's key?*
+
+- **name-is-key**: the backend addresses instances by name. `Create` looks
+  the name up, the ID is `<location>/<name>`.
+- **allocated-id**: the backend allocates an identifier and the name is a
+  label. `Create` MUST first find an existing instance by name (idempotency),
+  then allocate. Allocation MUST be safe against a concurrent `Create` for
+  another Proxy: a deterministic identifier derived from `req.UID`, or an
+  allocate-then-verify loop. The ID SHOULD embed the allocated key and the
+  name; `Get` and `Delete` use the allocated key. If instances can move
+  between locations (migration), the ID MUST NOT depend on the location.
+
+### 4.11 `provisioning-state`
+
+*Does the backend have a state for "being created"?*
+
+- **native**: the backend reports a provisioning or staging state, mapped to
+  `StateProvisioning`.
+- **create-blocks**: no such state; `Create` blocks until the instance is
+  started, so `Get` never sees "defined but not started".
+- **marker**: no such state and `Create` returns early; the provider records
+  at creation (metadata or a task handle) what lets `Get` answer
+  `StateProvisioning` for an instance that was never started, and
+  `StateStopped` for one that stopped afterwards.
+
+### 4.12 `async-ops`
+
+*Does the backend run create and delete as asynchronous tasks?*
+
+- **none**: calls complete synchronously.
+- **task-handle**: the backend returns a task the provider can poll. The
+  provider SHOULD keep the handle where `Get` can find it (metadata or the
+  ID) so a failed create can be reported instead of read as NotFound.
+- **backend-lock**: the backend locks the instance during a task. `Get`
+  returns the last known state or Transient while locked (§2.2); `Delete`
+  on a locked instance returns Transient, not Permanent.
+
+### 4.13 `secondary-resources`
+
+*What else does `Create` make, and who removes it?*
+
+- **none**.
+- **cascade**: the backend deletes them with the instance (auto-delete
+  flags). The provider MUST set those flags at create.
+- **explicit**: the provider deletes them by name in `Delete`, tolerating
+  their absence. Resources that can exist without an instance (a crash
+  between creating them and creating the instance) are a documented GC blind
+  spot unless `ListByTag` covers them.
+
+### 4.14 `credentials`
+
+*Where do credentials and trust material come from?*
+
+- **mounted-files**: paths in the config block, pointing into an optional
+  Secret mount (§7).
+- **environment**: standard SDK environment discovery.
+- **workload-identity**: the pod's own identity.
+
+TLS trust for an HTTPS backend with a private CA is one of **system-roots**,
+**ca-file** (a path in the config block) or **insecure** (a boolean whose
+name starts with `insecure`, warned about at startup).
+
+### 4.15 `client-kind`
+
+*What talks to the backend?*
+
+- **vendor-sdk**: put it behind a small interface (the seam) so a fake can
+  stand in.
+- **thin-client**: a `net/http` client written in the provider package.
+  Preferred when no maintained SDK exists (`CLAUDE.md`: stdlib over
+  third-party). It MUST set transport timeouts, MUST carry the backend's
+  status and body in one error type that `classify` reads, and SHOULD wrap
+  its transport for child spans (§9).
+- **rpc-over-ssh** or another non-HTTP transport: one pooled connection per
+  target, a keepalive, and a documented answer to "what does a cancelled call
+  do to the shared connection".
+
+### 4.16 `timeouts`
+
+*How is each backend call bounded?* (§1.5)
+
+- **per-call**: every request has its own deadline, independent of `ctx`.
+- **ctx-only**: the provider relies on `ctx`. Allowed only for backends
+  whose client already enforces a timeout; the spec MUST cite it.
+
+### 4.17 `logging`
+
+*Which log tiers does the provider produce?* The logger MUST come from
+`logf.FromContext(ctx)` at call time (§9).
+
+- **tiered**: V(1) one line per operation outcome and per failed backend
+  call; V(2) request and response detail; V(5) raw wire if there is one.
+- **outcome-only**: V(1) only.
+- **none**: allowed only for a backend whose client logs on its own.
+
+### 4.18 `child-spans`
+
+*How do backend calls appear under the operation span?* (§9)
+
+- **sdk-instrumentation**: the SDK emits spans itself.
+- **transport-wrapper**: a parent-gated `otelhttp` transport.
+- **span-decorator**: a decorator over the seam that starts one span per
+  call.
+- **none**: a documented gap, with log correlation through `traceID` only.
+
+### 4.19 `backend-test`
+
+*How is the provider proven against the real backend?*
+
+- **cluster-e2e**: the kind end-to-end suite.
+- **integration-tag**: a test behind `//go:build integration`, skipped
+  without its environment variables, with a `make test-integration-<type>`
+  target and a section in `docs/testing.md`.
+- **manual-runbook**: a documented by-hand run. Allowed only when the
+  backend cannot be reached from CI or a developer machine.
+
+### 4.20 Summary
+
+| Axis | Profiles | Who tolerates the difference | Template entry |
+|---|---|---|---|
+| `readiness` | immediate, provisioning-window | reconciler polls either way | 7 |
+| `create-return` | submit, blocking | reconciler; worker budget | 10 |
+| `delete-return` | returns-early, waits | reconciler re-calls Delete | 11 |
+| `ip-source` | assigned-at-create, discovered | reconciler gates Running on IP | 7 |
+| `gc-mark` | native-labels, written-metadata, mark-window | GC | 8 |
+| `created-at` | backend-timestamp, written-at-create | GC `MinAge` | 8 |
+| `list-partial` | fail-whole, omit-unreachable | GC | 12 |
+| `quota` | detected, absent | reconciler `QuotaRetry` | 13 |
+| `not-found` | status-404, error-code, list-and-miss | reconciler | 13 |
+| `primary-key` | name-is-key, allocated-id | reconciler idempotency | 2 |
+| `provisioning-state` | native, create-blocks, marker | reconciler Stopped branch | 9 |
+| `async-ops` | none, task-handle, backend-lock | reconciler | 10 |
+| `secondary-resources` | none, cascade, explicit | GC blind spots | 11 |
+| `credentials` | mounted-files, environment, workload-identity | deployment | 14 |
+| `client-kind` | vendor-sdk, thin-client, rpc-over-ssh | tests, tracing | 17 |
+| `timeouts` | per-call, ctx-only | worker and GC budget | 18 |
+| `logging` | tiered, outcome-only, none | operators | 20 |
+| `child-spans` | sdk-instrumentation, transport-wrapper, span-decorator, none | tracing | 17 |
+| `backend-test` | cluster-e2e, integration-tag, manual-runbook | CI | 21 |
+
+## 5. Error taxonomy
+
+Every error a provider method returns **MUST be built with
+`provider.Wrap(class, op, providerName, id, err)`** (`internal/provider/errors.go`).
+`op` is one of `create`, `get`, `delete` or `list`, and `providerName` is the
+**configured name** (`ProviderConfig.Name`), not the type.
+*Caller:* the decorators label metrics and spans with the class
+(`resultLabel`), and the reconciler branches on `provider.Class`.
+
+`Wrap` keeps both `errors.Is(err, class)` and `errors.As` to the backend's
+error type working (`Error.Unwrap` returns both).
+
+| Class | Use it for | During normal reconciling | While deleting |
+|---|---|---|---|
+| `ErrNotFound` | the instance does not exist | Get: recreate or advance (§2.2); other ops: returned as the error, so exponential backoff | Get: release the finalizer |
+| `ErrQuotaExceeded` | quota, rate limit, out of capacity | `Provisioned=False` with reason `QuotaExceeded`, requeue after `QuotaRetry`, not counted as an error (`providerFailure`) | requeue after `QuotaRetry` (`deletionFailure`) |
+| `ErrTransient` | network, 5xx, timeouts, cancellation, anything unknown | error returned, so workqueue exponential backoff | same |
+| `ErrPermanent` | bad config, bad argument, auth or permission denied, malformed ID | **latched**: `Provisioned=False` with reason `PermanentError`, no more provider calls until `metadata.generation` changes (`reconcileManaged`) | error returned and retried forever; a Permanent `Get` also triggers `Delete` (`reconcileDelete`) |
+
+Pitfalls:
+
+- **Unclassified means Transient.** `provider.Class` returns `ErrTransient`
+  for any error not built with `Wrap`. That is the safe default, but a
+  permanent failure that was never classified retries forever, and a quota
+  error backs off far too fast.
+- **Permanent is sticky.** Use it only for errors that editing the Proxy spec
+  or fixing the config can resolve. A permission-denied status that the
+  backend also uses for quota is the hard case: read the backend's reason if
+  it gives one, and if it does not, pick Permanent and document it.
+- **Do not return NotFound from `Create`.** The reconciler treats it as
+  Transient there, but it misleads the metrics and anyone reading logs.
+- **NotFound detection** follows the `not-found` profile (§4.9). A backend
+  that reports "does not exist" as a generic server error needs the exact
+  message or code matched, with real captured responses in the test table.
+- **Classify in one `classify` function** with a table-driven test covering
+  every class, including "not a backend error at all, so Transient".
+
+## 6. Identity, names, marks, CreatedAt
+
+- **Names:** `NameFromUID(uid)` gives `proxy-` plus 16 lowercase base32
+  characters. It is a valid DNS label. A backend with stricter rules SHOULD
+  map it deterministically rather than invent a new scheme, because
+  idempotency depends on the name being the same every time. A backend that
+  keys on an allocated identifier follows `primary-key` (§4.10).
+- **providerID:** opaque to everyone except the provider. Keep it
+  `<location>/<name>`-shaped or `<location>/<allocated-key>`-shaped, parse
+  it strictly, and return `ErrPermanent` for a malformed one (§2.2).
+- **Marks:** `LabelManaged` with value `LabelManagedYes`, and `LabelUID`.
+  Use native labels or tags where the backend has them; otherwise write them
+  into metadata in the create call (§4.5).
+- **`CreatedAt`:** the backend's creation timestamp, or written into the mark
+  metadata at creation (§4.6).
+- **Provider names:** a configured `name` MUST be a lowercase DNS-1123 label
+  of at most 63 characters (`spec.provider` in
+  `api/v1alpha1/proxy_types.go`). The `ProviderHealth` object is named after
+  the provider; `Monitor.Start` warns and `Monitor.publish` skips the object
+  for a name that is not a valid object name. Config loading does not check
+  this yet (§13).
+
+## 7. Credentials, TLS and secret hygiene
+
+- **The constructor MUST NOT dial the backend.** `registry.Build` aborts the
+  whole provider set on a constructor error, and the manager exits: one
+  unreachable backend must not take the other providers down. Fail in the
+  constructor only on config errors.
+- **Credential files SHOULD be read lazily.** A mount marked `optional: true`
+  is empty when the Secret is missing; the constructor logs a warning and the
+  first call fails Transient until the Secret lands. A file that exists but
+  does not parse is Permanent.
+- **Credentials MUST NOT appear in logs, span attributes, error strings or
+  `Error.Err`.** Backend errors that echo a request URL or header with a
+  token in it MUST be sanitised before wrapping. The same holds for
+  `CloudInit` (§2.1). Test both with a sentinel (§11).
+- **TLS trust** follows `credentials` (§4.14). An insecure toggle MUST be
+  named so that it reads as insecure, default off, and MUST be logged as a
+  warning at startup when on.
+- **Rotation.** State in the spec whether rotating the Secret takes effect
+  without a restart. Reading files on every dial makes rotation free; caching
+  them after first load needs a restart and must say so.
+- **Scratch files** only under `/tmp` (§1.4), removed when the call ends.
+
+## 8. Optional interfaces: `Checker`
+
+`provider.Checker` is `Check(ctx) []TargetStatus` (`internal/provider/check.go`).
+`cmd/main.go` type-asserts it on the **unwrapped** provider before adding the
+decorators, and the monitor treats a nil Checker as unsupported, not failed
+(`Monitor.run`).
+
+- **MUST never fail as a whole.** Every failure is a `TargetStatus` with
+  `Ready=false`.
+  *Caller:* `Monitor.checkOnce` has no error path per provider.
+- **MUST return promptly once `ctx` ends.** The monitor gives the whole round
+  one timeout.
+- **MUST return one entry per target, in a stable order.** Targets are what
+  can fail independently: an apiserver, a project, each host or node.
+  *Caller:* the monitor logs transitions per target (`logTransitions`) and
+  the gauges are labelled `{provider,target}`.
+- **MUST NOT disrupt reconciles.** A check that shares a connection with
+  provisioning must not tear that connection down when it times out.
+- **SHOULD use the shared reasons** (`ReasonReachable`, `ReasonTimeout`,
+  `ReasonUnreachable`, `ReasonUnauthorized`, `ReasonForbidden`,
+  `ReasonAPIError`). Add provider-specific reasons only for failures an admin
+  would fix differently, and document each one in the guide.
+- **SHOULD check with the same credentials, verb and scope that `ListByTag`
+  and `Create` use**, and be cheap: one page, one object.
+- **MAY fill `Details`** with facts useful when triaging: versions, capacity,
+  the account in use.
+- The `status.type` field of `ProviderHealth`
+  (`api/v1alpha1/providerhealth_types.go`) lists the known types in its doc
+  comment; add the new type there and run `make manifests`.
+
+## 9. Observability
+
+Tracing is OpenTelemetry, set up in `internal/tracing` from the standard
+`OTEL_*` environment variables. With no exporter configured, spans are no-ops
+and logs are unchanged ([architecture.md §10](architecture.md#10-tracing-internaltracing)).
+
+**Layer 1, operation spans and metrics: nothing to write.** `cmd/main.go`
+wraps every provider in `WithTracing(WithMetrics(...))`:
+
+- `WithTracing` opens one client span per call, named `provider.create`,
+  `provider.get`, `provider.delete` or `provider.list`, with
+  `provider.name`, `provider.id` and `provider.result` (the error class) as
+  attributes. `ErrNotFound` keeps the span status Ok, because polling to
+  NotFound is expected.
+- `WithMetrics` counts each call as
+  `proxy_operator_provider_requests_total{provider,op,result}`
+  (`internal/metrics/metrics.go`). A `Checker` feeds
+  `proxy_operator_provider_target_up`,
+  `proxy_operator_provider_check_duration_seconds` and
+  `proxy_operator_provider_check_timestamp_seconds`, labelled
+  `{provider,target}`.
+
+Rules:
+
+- **MUST NOT open its own per-operation span or count its own requests.**
+  They would duplicate the decorators.
+- **MUST pass the caller's `ctx` to every backend call and take the logger
+  from `logf.FromContext(ctx)` at call time.** `tracing.StartSpan`
+  (`internal/tracing/logger.go`) puts a logger carrying `traceID` and
+  `spanID` into `ctx`. A logger cached at construction, or a backend call
+  made with `context.Background()`, loses the trace.
+- **Health checks bypass the decorators.** A `Checker` runs under the
+  monitor's `providerhealth.check` span, and the same `ctx` rules apply.
+
+**Layer 2, child spans per backend call: SHOULD.** One operation span says
+that `Create` took 40 seconds; child spans say which backend call took them.
+Follow `child-spans` (§4.18):
+
+- HTTP backends: wrap the transport with `otelhttp`, gated on an existing
+  parent span as `tracing.RestConfigWrapper` does
+  (`internal/tracing/transport.go`), so background traffic (token refreshes,
+  keepalives) does not become root spans. Check first whether the SDK already
+  does this.
+- Non-HTTP backends: start child spans yourself with
+  `tracing.Start(ctx, "<type>.<call>")` around each call and around long
+  steps such as a dial or a download, in a decorator over the seam, not at
+  call sites.
+- **MUST NOT put secrets or CloudInit in span attributes.** Put identifiers,
+  sizes and error codes there.
+
+**Layer 3, logs.** Follow `logging` (§4.17). Log lines join a trace through
+the `traceID` key from layer 1.
+
+## 10. Configuration and wiring
+
+The providers file is parsed by `LoadConfig` (`internal/provider/config.go`)
+with `UnmarshalStrict`, so unknown keys already fail.
+
+- Add a `<Type>Config` struct and a `ProviderConfig` field for it.
+- Add the block to `setBlocks`, so a block that does not match `Type` is
+  rejected.
+- Add the type to the allow-list in `Config.validate`. The same list exists a
+  second time as the constructor map in `cmd/main.go`; both must change
+  (§13).
+- Add required-field validation in its own `config_<type>.go` when it is
+  more than a few lines, with rows in `config_test.go`.
+- **Validate every config value that is pasted into an identifier**: a
+  location name that becomes part of the providerID must not contain the
+  separator; a catalogue key that becomes a file or volume name must be a
+  safe filename.
+- The constructor has the signature `registry.Constructor`:
+  `func(ctx, provider.ProviderConfig) (provider.Provider, error)`. Register
+  it in the map in `cmd/main.go`. Flags a provider needs are defined there
+  too and passed through a `NewWith...` variant, never read from the
+  environment inside the package.
+- Decorators are applied by the composition root, in the order
+  `WithTracing(WithMetrics(p))`, after the Checker has been collected from
+  the unwrapped provider.
+
+## 11. Testing obligations
+
+`CLAUDE.md` applies: stdlib `testing`, table-driven, tests next to source,
+`t.Parallel()` where safe.
+
+- **A fake behind the seam.** Every unit test runs against an in-memory
+  fake of the backend, never the network.
+- **Contract tests**, each against the fake:
+  - repeat `Create` returns the same ID and creates nothing;
+  - `Get` after `Delete` returns `ErrNotFound`;
+  - `Delete` twice succeeds;
+  - malformed ID is `ErrPermanent` from `Get` and `Delete`;
+  - an unknown backend state maps to `StateStopped`;
+  - `StateRunning` requires an IP, and a running instance without one is
+    `StateProvisioning`;
+  - `UID` and `CreatedAt` are read back from the fake, not remembered;
+  - `ListByTag` excludes unmarked instances and instances with an empty UID
+    are still listed with `UID == ""`;
+  - `Get` during an in-progress delete still returns the instance;
+  - `Get` during an asynchronous operation or lock returns the last state or
+    Transient (`async-ops`);
+  - a created but not yet started instance is never `StateStopped`
+    (`provisioning-state`);
+  - a cancelled `ctx` makes every method return promptly.
+- **Classify**: one table test covering every class, every backend code the
+  provider matches, and "not a backend error at all, so Transient". Use
+  captured real responses for message matching.
+- **Checker**: a table test for the reasons, a test that `Check` returns
+  promptly with an expired `ctx`, and a test that it returns one entry per
+  target in config order.
+- **Sentinel tests**: a `CloudInit` containing a sentinel string and a
+  credential containing another never appear in any log line, span attribute
+  or error string.
+- **Tracing**: with an in-memory span recorder
+  (`sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracetest.NewSpanRecorder()))`,
+  as in `internal/provider/tracing_test.go`), the child spans are children
+  of the operation span.
+- **Real-backend test** per `backend-test` (§4.19). It runs the whole
+  lifecycle: create, create again, poll until Running, list, delete, delete
+  again, confirm it is gone and that no secondary resource remains. Keep it
+  compiling with `go vet -tags=integration ./...`.
+
+## 12. Checklist for adding a provider
+
+1. **Spec first.** Copy `docs/provider-spec-template.md` to
+   `docs/<type>-provider-spec.md` and answer every entry. Unanswered entries
+   block implementation.
+2. **Package** `internal/provider/<type>/` with the provider type, a
+   compile-time assertion for `provider.Provider` and one for
+   `provider.Checker`, the seam interface, and a logger built from
+   `logf.FromContext(ctx).WithName("<type>").WithValues("provider", name)`.
+3. **Config** per §10, including `config_test.go` rows.
+4. **Constructor** `New(ctx, provider.ProviderConfig)` registered in
+   `cmd/main.go`; no dialing.
+5. **Contract** per §2, tested per §11.
+6. **Errors** per §5, `classify` plus its table test, `ProviderConfig.Name`
+   in every `Wrap`.
+7. **Checker** per §8, with the `status.type` comment updated and
+   `make manifests` run.
+8. **Tracing and logging** per §9 and §4.17, §4.18.
+9. **Real-backend test** per §4.19, with a `make test-integration-<type>`
+   target and a section in `docs/testing.md`.
+10. **RBAC**: if the provider talks to the cluster's own API, add
+    `+kubebuilder:rbac` markers in their own comment block, never inside a
+    doc comment, then `make manifests` and check
+    `git diff config/rbac/role.yaml`.
+11. **Deployment**: an `optional: true` credentials mount in
+    `config/manager/manager.yaml`; a commented entry in
+    `config/manager/providers_config.yaml` and
+    `config/samples/providers-config.yaml`; a `config/samples/proxy_<type>.yaml`
+    whose cloud-init installs a proxy on the sample image; a Makefile helper
+    for the Secret if there is one; `hack/providers-dev.yaml` only if the
+    type needs no credentials.
+12. **Docs**: `docs/<type>-provider.md` (prerequisites, config reference,
+    placement mapping, what lands on the backend, the health check, logging,
+    known gaps); a row and a section in `providers-existing.md`; a row in the
+    `architecture.md` components table and a block in its §5 and §11
+    (§10 too if the provider adds spans); a row in the README "Implemented
+    providers" table; a `CHANGELOG.md` entry once the user has confirmed it
+    works.
+13. **Optional:** a demo script under `docs/demo/`.
+
+## 13. Known limits of the contract
+
+These are facts a new provider has to live with today. None is designed or
+scheduled.
+
+- **Adoption is not verified.** `Create` adopts any existing instance with
+  the right name without checking its `LabelUID`.
+- **`StateStopped` and `StateTerminated` are indistinguishable** to the
+  reconciler; both mean delete and recreate.
+- **Config is read once per process.** Adding a provider or a location, or
+  rotating credentials a provider caches, needs a manager restart.
+- **GC has no installation identity.** Two installs sharing one backend
+  delete each other's instances, because GC only knows its own live UIDs.
+- **An unconfigured provider latches the Proxy.** A Proxy naming a provider
+  missing from the config is latched `PermanentError`; adding the provider
+  and restarting does not change `metadata.generation`, and `spec.provider`
+  is immutable, so the Proxy stays latched until it is recreated.
+- **Provider names are not validated** as DNS-1123 at config load.
+- **The type list exists twice**: `Config.validate` and the constructor map
+  in `cmd/main.go`.
+- **`Delete` is resubmitted on every deletion poll** until `Get` says
+  NotFound.
+- **Callers set no deadline** (§1.5); each provider bounds itself.
+- **There is no conformance suite.** The §11 contract tests are written per
+  provider; `internal/provider/providertest` is a proposal in the appendix.
+- **Stale comments** in `internal/provider/provider.go`,
+  `internal/provider/registry/registry.go` and `api/v1alpha1/proxy_types.go`
+  still describe a `mock` provider that no longer exists.
+
+## 14. Related documents
+
+- [providers.md](providers.md): the index.
+- [provider-spec-template.md](provider-spec-template.md): the questionnaire.
+- [providers-existing.md](providers-existing.md): how the existing providers
+  meet this contract, and their gaps.
+- [architecture.md §5](architecture.md#5-what-provider-calls-do-in-the-outside-world),
+  [§10](architecture.md#10-tracing-internaltracing) and
+  [§11](architecture.md#11-provider-health-checks-internalproviderhealth).
+- [testing.md](testing.md).

docs/provider-spec-template.md

@@ -0,0 +1,455 @@
+# Provider spec template
+
+Copy this file to `docs/<type>-provider-spec.md`, fill in every **Answer**
+line, and hand the result to whoever implements the provider together with
+[provider-contract.md](provider-contract.md) and
+[providers-existing.md](providers-existing.md). An unanswered entry blocks
+implementation: it is a decision the contract leaves to the provider, and the
+implementer must not guess it.
+
+Where an entry names an axis in backticks, the allowed answers are the
+profiles of that axis in contract §4, and the entry number matches the last
+column of the contract's §4.20 table. **Evidence** is where the answer was
+confirmed: a link into the backend's API documentation, or the experiment
+that was run. "Verify against the target version" is a valid interim
+answer only for the Evidence line, never for the Answer.
+
+## Header
+
+- **Type** (the `type:` value in the providers config, lowercase):
+- **Example configured name**:
+- **Backend and API version this spec was written against**:
+- **Client** (`vendor-sdk` with the module path, or `thin-client`):
+- **Author and date**:
+
+## 1. Identity and providerID
+
+**Question.** What is the providerID's shape, which location parts does it
+embed, and what validates the config values pasted into it?
+
+**Allowed answers.** `<location>/<name>`, `<location>/<allocated-key>`, or a
+location-free `<allocated-key>` (contract §6; required when instances can
+migrate, see entry 2).
+
+**Obligations.** Strict parsing; `ErrPermanent` for a malformed ID from `Get`
+and `Delete`; a regexp on every config value that becomes an ID segment, in
+`config_<type>.go`.
+
+**Answer:**
+
+**Evidence:**
+
+## 2. Primary key (`primary-key`)
+
+**Question.** Is `req.Name` the backend's key, or does the backend allocate an
+identifier?
+
+**Allowed answers.** `name-is-key`; `allocated-id`.
+
+**Obligations.** `allocated-id`: `Create` finds an existing instance by name
+before allocating; allocation is safe against a concurrent `Create` for
+another Proxy (deterministic from `req.UID`, or allocate-then-verify); `Get`
+and `Delete` use the allocated key; state how a collision with an instance
+the operator does not own is detected and reported.
+
+**Answer:**
+
+**Evidence:**
+
+## 3. Placement mapping
+
+**Question.** What do `Placement.Region`, `Zone`, `MachineType` and `Image`
+mean for this backend, which are required, which are ignored, and what
+happens when a value is unknown?
+
+**Allowed answers.** Per field: a config catalogue key, a backend identifier
+passed through, or ignored (contract §2.1, MAY ignore with documentation).
+
+**Obligations.** Unknown values are rejected with `ErrPermanent` naming the
+field and value before any backend call; ignored fields are listed in the
+provider's guide because they are still in the spec hash; an empty value with
+a single-choice catalogue may default, and the guide says so.
+
+**Answer:**
+
+**Evidence:**
+
+## 4. Image source
+
+**Question.** Where does the boot image come from, and what does a catalogue
+edit do to running instances?
+
+**Allowed answers.** A catalogue in the config block (key to URL or backend
+image ID); a backend template or image referenced directly by
+`Placement.Image`; a URL fetched by the provider into the backend on first
+use.
+
+**Obligations.** If the provider fetches or converts images: streaming, no
+buffering (contract §1.4), a completeness marker so a cut-off fetch is redone,
+concurrency control so two Creates fetch once, and a written warning that
+replacing an image in use may corrupt instances that depend on it. If the
+backend owns the image: the prerequisite is documented in the guide.
+
+**Answer:**
+
+**Evidence:**
+
+## 5. Cloud-init delivery
+
+**Question.** How does `CreateRequest.CloudInit` reach the guest, byte for
+byte?
+
+**Allowed answers.** User-data metadata on the instance; a NoCloud seed
+built in-process and attached as media; a file placed on the backend and
+referenced from the instance config; not delivered (documented, with how the
+proxy gets configured instead).
+
+**Obligations.** Size limits of the channel stated; whether vendor-data or a
+second document can ride along; the sentinel test (contract §11); the channel
+must be fully reachable through the API the provider uses (no step that needs
+shell access on the backend unless the spec says the provider has it).
+
+**Answer:**
+
+**Evidence:**
+
+## 6. Machine type mapping
+
+**Question.** How does `Placement.MachineType` become CPU, memory and disk?
+
+**Allowed answers.** A catalogue of named profiles in the config block; a
+backend size name passed through.
+
+**Obligations.** A catalogue needs validation of each profile; disk size
+default and its config field named.
+
+**Answer:**
+
+**Evidence:**
+
+## 7. Network and IP discovery (`readiness`, `ip-source`)
+
+**Question.** Which network does the instance attach to, where does its
+address come from, how long until it is usable, and what makes it reachable
+from the operator pod and from clients on `spec.port`?
+
+**Allowed answers.** `readiness`: `immediate`; `provisioning-window`.
+`ip-source`: `assigned-at-create`; `discovered` (name the mechanism: guest
+agent, DHCP lease, neighbour table, and the order if several).
+
+**Obligations.** `discovered`: what the guest must run, how it gets
+installed, what happens without it; discovery failure is Provisioning, not an
+error; only routable IPv4 is reported (state the filter). Reachability:
+firewall or routing steps the provider performs, or the prerequisite the
+guide states. Whether `Port` is used by the provider at all.
+
+**Answer:**
+
+**Evidence:**
+
+## 8. GC marks and CreatedAt (`gc-mark`, `created-at`)
+
+**Question.** Where do `LabelManaged` and `LabelUID` live, where does
+`CreatedAt` come from, and which secondary resources are marked?
+
+**Allowed answers.** `gc-mark`: `native-labels`; `written-metadata`;
+`mark-window`. `created-at`: `backend-timestamp`; `written-at-create`.
+
+**Obligations.** `mark-window`: the window's length in seconds, why it is
+far below `MinAge`, and how the same Proxy's next reconcile cleans an
+unmarked instance. Any value-format constraint of the backend's labels
+(allowed characters, length) and how a Proxy UID fits it.
+
+**Answer:**
+
+**Evidence:**
+
+## 9. State mapping (`provisioning-state`)
+
+**Question.** How does every backend state map to `StateProvisioning`,
+`StateRunning`, `StateStopped`, `StateTerminated`, and how does `Get` tell
+"defined but never started" from "stopped"?
+
+**Allowed answers.** A table of backend state (plus lock or task flags) to
+provider state, with the unknown-state row mapping to `StateStopped`.
+`provisioning-state`: `native`; `create-blocks`; `marker`.
+
+**Obligations.** Running requires an IP; `marker` names what is recorded at
+creation and how `Get` reads it; `create-blocks` names the cap on the wait.
+
+**Answer:**
+
+| Backend state | Lock or task | Provider state |
+|---|---|---|
+| | | |
+
+**Evidence:**
+
+## 10. Create: blocking and task tracking (`create-return`, `async-ops`)
+
+**Question.** What does `Create` wait for before returning, and how are
+asynchronous backend tasks tracked?
+
+**Allowed answers.** `create-return`: `submit`; `blocking`. `async-ops`:
+`none`; `task-handle`; `backend-lock`.
+
+**Obligations.** `task-handle`: where the handle is kept and how a failed
+task surfaces (in `Get`, as which class, with which message). `backend-lock`:
+what `Get` and `Delete` return while locked. Every step of `Create` is
+idempotent so a cut-off attempt resumes; list the steps and what each finds
+on a retry.
+
+**Answer:**
+
+**Evidence:**
+
+## 11. Delete and secondary resources (`delete-return`, `secondary-resources`)
+
+**Question.** In what order does `Delete` tear things down, what does it do
+when called again mid-delete, and what does `Get` return meanwhile?
+
+**Allowed answers.** `delete-return`: `returns-early`; `waits`.
+`secondary-resources`: `none`; `cascade` (name the flags set at create);
+`explicit` (name each resource and how it is found by name).
+
+**Obligations.** Idempotent at every step; "already deleting" is success and
+cheap; a location no longer in the config is a logged no-op success; any
+resource that can outlive the instance without being listable is a documented
+GC blind spot.
+
+**Answer:**
+
+**Evidence:**
+
+## 12. ListByTag scope and partial failure (`list-partial`)
+
+**Question.** What is the full scope `ListByTag` enumerates, and what happens
+when part of it is unreachable?
+
+**Allowed answers.** `fail-whole`; `omit-unreachable`.
+
+**Obligations.** The scope covers everything the provider can create in;
+server-side filtering on the mark where the backend allows it; a single
+unreadable instance may be skipped with a log line; the cost of one list at
+fleet size.
+
+**Answer:**
+
+**Evidence:**
+
+## 13. Error classification (`not-found`, `quota`)
+
+**Question.** Which backend responses map to which of the four classes, and
+which signal means "does not exist"?
+
+**Allowed answers.** A table of backend status or code (with message pattern
+where needed) to class. `not-found`: `status-404`; `error-code`;
+`list-and-miss`. `quota`: `detected`; `absent`.
+
+**Obligations.** One `classify`, a table test with captured real responses
+for every matched message; the permission-denied-versus-quota case decided;
+authentication failures, host-key or certificate failures, and cancellation
+each assigned a class with a reason.
+
+**Answer:**
+
+| Backend response | Class | Why |
+|---|---|---|
+| | | |
+
+**Evidence:**
+
+## 14. Credentials and TLS (`credentials`)
+
+**Question.** What credential does the operator use, where does it come from,
+when is it read, and how is the backend's TLS certificate trusted?
+
+**Allowed answers.** `mounted-files` (Secret name, mount path, file names,
+`optional: true`); `environment`; `workload-identity`. TLS: `system-roots`;
+`ca-file`; `insecure` (the flag's exact name).
+
+**Obligations.** The minimum privilege set the credential needs, listed;
+rotation behaviour (free, or restart needed); the sentinel test for the
+credential (contract §7); the constructor never dials.
+
+**Answer:**
+
+**Evidence:**
+
+## 15. Config block
+
+**Question.** What are the fields of `<Type>Config`, their defaults, and their
+validation?
+
+**Allowed answers.** A table: field, required, default, validation, and which
+identifier or path the value ends up in.
+
+**Obligations.** Contract §10: `setBlocks`, the type allow-list in
+`Config.validate` and the constructor map in `cmd/main.go`, `config_<type>.go`
+with its tests, regexps for values pasted into IDs or file names.
+
+**Answer:**
+
+| Field | Required | Default | Validation | Ends up in |
+|---|---|---|---|---|
+| | | | | |
+
+**Evidence:**
+
+## 16. Checker
+
+**Question.** What are the check targets, what does one check do, which
+reasons can it report, and what goes into `Details`?
+
+**Allowed answers.** One target per thing that fails independently; the
+shared reasons from `internal/provider/check.go` first, provider-specific
+reasons only with a stated "what the admin does differently"; `Details` keys
+listed.
+
+**Obligations.** Same credential, verb and scope as `ListByTag` and `Create`;
+cheap (one page, one object); never disrupts a reconcile that shares a
+connection; returns promptly on `ctx`; the `status.type` comment in
+`api/v1alpha1/providerhealth_types.go` updated.
+
+**Answer:**
+
+**Evidence:**
+
+## 17. Client and child spans (`client-kind`, `child-spans`)
+
+**Question.** What talks to the backend, and how do its calls appear under the
+operation span?
+
+**Allowed answers.** `client-kind`: `vendor-sdk` (module path, and whether it
+is maintained); `thin-client`; `rpc-over-ssh`. `child-spans`:
+`sdk-instrumentation`; `transport-wrapper`; `span-decorator`; `none`.
+
+**Obligations.** The seam interface's name and the fake that implements it;
+`thin-client`: one error type carrying status and body, transport timeouts,
+parent-gated `otelhttp`; `none` is a documented gap; the span recorder test
+(contract §11).
+
+**Answer:**
+
+**Evidence:**
+
+## 18. Timeouts (`timeouts`)
+
+**Question.** What bounds each backend call, the dial, and any long step such
+as a download or a wait loop, and what does a cancelled `ctx` do to a shared
+connection?
+
+**Allowed answers.** `per-call` (the budgets, per operation); `ctx-only`
+(cite the client's own timeout).
+
+**Obligations.** Contract §1.5: no caller sets a deadline; a hung call pins a
+worker or stalls GC. Cancellation behaviour on a shared connection stated
+(abandon the call, or close the connection).
+
+**Answer:**
+
+**Evidence:**
+
+## 19. Concurrency
+
+**Question.** What is shared between concurrent calls (connections, caches,
+locks), and what happens when two calls for the same instance name overlap?
+
+**Allowed answers.** A description of the shared state and its locking.
+
+**Obligations.** Safe for 3 reconciles, a GC sweep and a check round at once;
+two `Delete` calls for one instance overlap safely; one fetch per image per
+location when several Creates race.
+
+**Answer:**
+
+**Evidence:**
+
+## 20. Logging (`logging`)
+
+**Question.** Which tiers does the provider log, with which keys on every
+line?
+
+**Allowed answers.** `tiered`; `outcome-only`; `none` (cite the client's own
+logging).
+
+**Obligations.** Logger from `logf.FromContext(ctx)` on every call; never
+`CloudInit` or credentials; `provider` and the instance name on every line
+that concerns an instance.
+
+**Answer:**
+
+**Evidence:**
+
+## 21. Tests (`backend-test`)
+
+**Question.** What is the fake, which contract tests exist, and how is the
+provider proven against the real backend?
+
+**Allowed answers.** `backend-test`: `cluster-e2e`; `integration-tag` (the
+`<TYPE>_TEST_*` variables, the `make test-integration-<type>` target); `manual-runbook`.
+
+**Obligations.** Every item of the contract §11 list, ticked here:
+
+- [ ] repeat `Create`
+- [ ] `Get` after `Delete`
+- [ ] `Delete` twice
+- [ ] malformed ID
+- [ ] unknown state maps to `StateStopped`
+- [ ] `StateRunning` requires an IP
+- [ ] `UID` and `CreatedAt` read back
+- [ ] `ListByTag` excludes unmarked instances
+- [ ] `Get` during an in-progress delete
+- [ ] `Get` during an asynchronous operation or lock
+- [ ] never started is never `StateStopped`
+- [ ] cancelled `ctx` returns promptly
+- [ ] `classify` table with captured responses
+- [ ] Checker reasons, expired `ctx`, stable order
+- [ ] sentinel tests for `CloudInit` and the credential
+- [ ] child spans under the operation span
+- [ ] real-backend lifecycle test
+
+**Answer:**
+
+**Evidence:**
+
+## 22. Deployment
+
+**Question.** What changes in `config/` and the Makefile?
+
+**Allowed answers.** The list: the `optional: true` mount in
+`config/manager/manager.yaml`; the commented entries in
+`config/manager/providers_config.yaml` and
+`config/samples/providers-config.yaml`; `config/samples/proxy_<type>.yaml`
+with a cloud-init that installs a proxy on the sample image; a Makefile
+helper for the Secret; RBAC markers if the provider talks to the cluster's
+own API; whether `hack/providers-dev.yaml` gets an entry.
+
+**Answer:**
+
+**Evidence:**
+
+## 23. Docs
+
+**Question.** Which documents are written or updated?
+
+**Allowed answers.** `docs/<type>-provider.md` with its section list; the row
+and section in `providers-existing.md`; the `architecture.md` rows (components
+table, §5, §11, and §10 if spans are added); the README row; the
+`docs/testing.md` section; the `CHANGELOG.md` entry after confirmation.
+
+**Answer:**
+
+**Evidence:**
+
+## 24. Known limits accepted at design time
+
+**Question.** Which of the contract's §13 limits bite this provider in
+practice, and which new gaps does this design accept?
+
+**Allowed answers.** A list, each with the consequence for an operator and
+the workaround.
+
+**Answer:**
+
+**Evidence:**

docs/providers-existing.md

@@ -0,0 +1,392 @@
+# The existing providers
+
+This is the appendix to [provider-contract.md](provider-contract.md): how the
+three providers that exist today (kubernetes, gcp, kvm) meet the contract,
+which profile each one chose on every axis, and what is still missing. It is
+reference material for whoever writes the next provider, not a user guide;
+the user guides are [kvm-provider.md](kvm-provider.md),
+[gcp-vm-validation.md](gcp-vm-validation.md) and
+[gcp-in-specific-project.md](gcp-in-specific-project.md). The README's
+"Implemented providers" table stays the short public summary.
+
+The per-provider call flows are drawn in
+[architecture.md §5](architecture.md#5-what-provider-calls-do-in-the-outside-world),
+the health checks are tabulated in
+[§11](architecture.md#11-provider-health-checks-internalproviderhealth), and the
+judgment calls behind several rows below are recorded under its "Decisions"
+list. They are linked, not repeated.
+
+Code references are a file path plus a function or symbol name.
+
+## 1. Profile table
+
+One row per axis of contract §4, in the same order, with the profile name
+and the symbol that shows it.
+
+| Axis | kubernetes | gcp | kvm |
+|---|---|---|---|
+| `readiness` | immediate: a Pod is scheduled within seconds (`stateFromPod`) | provisioning-window: minutes of boot (`mapState`) | provisioning-window: boot plus, on first use, an image download (`imageCache.ensure`) |
+| `create-return` | submit: returns after `client.Create` | submit: `Insert` is never awaited, `Operation.Wait` is not called | **blocking**: waits through `DomainCreate` (`ensureRunning`), and the first image fetch runs inside (`imageCache.fetch`) |
+| `delete-return` | returns-early: `client.Delete`, Pod terminates on its own | returns-early: `Delete` submits an operation each time | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) |
+| `ip-source` | assigned-at-create: `PodIP` (`instanceFromPod`) | assigned-at-create: the external NAT IP of the first NIC (`toInstance`) | discovered: ARP table first, then the guest agent, first routable IPv4 (`address`, `firstIPv4`) |
+| `gc-mark` | native-labels: Pod labels (`buildPod`) | native-labels: instance labels (`buildInsertRequest`) | written-metadata: a namespaced `<metadata>` element written in `DomainDefineXML` (`metadata.go`) |
+| `created-at` | backend-timestamp: `CreationTimestamp` | backend-timestamp: `CreationTimestamp`, parse error gives zero | written-at-create: `createdAt` in the metadata element (`newInstanceMetadata`) |
+| `list-partial` | n/a: one call, all namespaces | omit-unreachable: `ReturnPartialSuccess` on `AggregatedList` | fail-whole per host, one unreadable domain skipped (`ListByTag`, `managedInstance`) |
+| `quota` | absent: a quota 403 is Permanent (`classify`) | detected: 429, and 403 with reason `quotaExceeded` or `rateLimitExceeded` (`classify`, `hasReason`) | absent (`classify`) |
+| `not-found` | status-404: `apierrors.IsNotFound` | status-404: `isNotFound` | error-code: `ErrNoDomain`, `ErrNoStorageVol` (`hasCode`) |
+| `primary-key` | name-is-key: `namespace/name` | name-is-key: `zones/<zone>/instances/<name>` | name-is-key: `hosts/<host>/domains/<name>` |
+| `provisioning-state` | native: `PodPending` | native: `PROVISIONING`, `STAGING`, `REPAIRING` | create-blocks: a domain is defined and started in one `Create`; `nostate` maps to Provisioning |
+| `async-ops` | none | task-handle, but the handle is discarded: an insert that fails after acceptance is never read | none |
+| `secondary-resources` | none | cascade: `AutoDelete` on the boot disk (`buildInsertRequest`) | explicit: overlay and seed removed by name (`removeVolumes`); base image kept |
+| `credentials` | workload-identity: in-cluster config or kubeconfig (`NewWithTransportWrapper`) | environment: Application Default Credentials | mounted-files: SSH key and known_hosts from the optional `kvm-ssh` Secret, read lazily (`newLazySSH`) |
+| `client-kind` | vendor-sdk: controller-runtime `client.Client`, no seam interface | vendor-sdk: `cloud.google.com/go/compute` behind `instancesAPI` | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind `libvirtAPI` (`conn.go`) |
+| `timeouts` | ctx-only: no `rest.Config.Timeout` is set | ctx-only: SDK defaults, with SDK retries on Get and List | per-call for the dial only (`defaultHandshakeTimeout` in `dialSSH`); RPCs are ctx-only and a cancelled `call` closes the connection |
+| `logging` | none | tiered: V(1) outcomes and `logAPIError`, V(2) detail, V(5) wire log (`wirelog.go`) | tiered: V(1) outcomes and one `libvirt call` line per RPC (`trace.go`), V(2) XML and addresses; no wire tier |
+| `child-spans` | transport-wrapper: `tracing.RestConfigWrapper()` on its own `rest.Config`, parent-gated | sdk-instrumentation: the SDK adds `otelhttp` itself; not parent-gated, but every call site has a parent | **none**: `trace.go` is a log decorator, not spans |
+| `backend-test` | cluster-e2e: kind (`test/e2e`) | manual-runbook: [gcp-vm-validation.md](gcp-vm-validation.md) | integration-tag: `kvm_integration_test.go`, `make test-integration-kvm` |
+
+## 2. Other differences
+
+Rows that are not profile axes but that a new provider author compares
+against.
+
+| Feature | kubernetes | gcp | kvm |
+|---|---|---|---|
+| Create verifies the adopted instance's UID | no | no | no |
+| Placement used | none | Zone, MachineType, Image | Zone (host), MachineType, Image |
+| CloudInit delivered | **no**: ignored | yes: `user-data` metadata | yes: NoCloud seed ISO built in-process (`buildSeedISO`) |
+| Port used | yes: Squid config and containerPort (`squidConf`) | no: firewall is manual | no |
+| States produced | Provisioning, Running, Terminated | all four | Provisioning, Running, Stopped |
+| Deleting instance still reports its state | yes: a Pod being deleted reports its phase | n/a | n/a |
+| Error.Provider is the configured name | **no**: hardcoded `"kubernetes"` | yes | yes |
+| Checker target | `apiserver` | the project | each host, concurrently |
+| Checker Details | none | none | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacity`, `poolAvailable` |
+| Credential rotation without restart | n/a | via ADC | no: cached after first successful load |
+| Connection handling | client-go | SDK client, never closed | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostPool`) |
+| Unit-test double | controller-runtime fake client | `fakeAPI` behind `instancesAPI`, plus `httptest` for the real SDK client in `check_test.go` | `fakeAPI` behind `libvirtAPI`, plus an in-process `x/crypto/ssh` server in `conn_test.go` |
+| Proxy software | generated Squid config (`pod.go`) | from `cloudInit`; the image must run cloud-init | from `cloudInit`, plus a vendor-data script that installs the guest agent (`seed.go`) |
+| IP reachable from the operator | Pod network, so in-cluster only (not from `make run-dev` on a laptop) | external IP, once a firewall rule allows the cluster's egress IP | LAN address on the host's bridge; the operator needs a route to that LAN |
+| Distinct egress IP per proxy | no: the cluster's egress | yes: ephemeral external IP | whatever the host's LAN routes through; not per VM by itself |
+
+## 3. Observability status
+
+| Layer | kubernetes | gcp | kvm |
+|---|---|---|---|
+| 1. Operation spans and metrics | yes: decorators | yes: decorators | yes: decorators |
+| 2. Child spans per backend call | yes: `tracing.RestConfigWrapper()`, parent-gated (`cmd/main.go`, `NewWithTransportWrapper`) | yes: the SDK's own `otelhttp`; not parent-gated | **no** |
+| 3. Logs carry `traceID` and `spanID` | n/a: the package does not log | yes, including the V(5) wire log | yes: every `libvirt call` line at V(1) |
+| Tested against a real collector | yes: the kind e2e checks `provider.create` and `provider.delete` in Tempo (`test/e2e/tracing_test.go`), operation spans only | no | no |
+
+## 4. kubernetes (`internal/provider/kubernetes`)
+
+- **Package and seam.** `kubernetes.go`, `pod.go`, `check.go`. There is no
+  seam interface: `newWithClient` injects a controller-runtime
+  `client.Client`, and tests use the fake client. `New` calls
+  `NewWithTransportWrapper`, which builds its own uncached client from
+  `ctrl.GetConfig()` with an optional `transport.WrapperFunc` for tracing.
+- **Config.** `KubernetesConfig.Image` only, defaulting to the Squid image
+  constant in `kubernetes.go`.
+- **ID.** `providerID(namespace, name)` gives `namespace/name`;
+  `parseProviderID` uses `cache.SplitMetaNamespaceKey` and rejects an empty
+  namespace with `ErrPermanent`. An empty name part is not rejected.
+- **Create.** `buildPod` (pure) then `client.Create`; `IsAlreadyExists` is
+  success. Only `Name`, `Namespace`, `UID` and `Port` are used. The Squid
+  config is generated by `squidConf` from the port and passed through the
+  `SQUID_CONF` environment variable, so there is exactly one object per
+  proxy.
+- **Get and state.** `stateFromPod`:
+
+  | Pod phase | PodIP | Provider state |
+  |---|---|---|
+  | Pending | any | Provisioning |
+  | Running | empty | Provisioning |
+  | Running | set | Running |
+  | Succeeded, Failed, Unknown | any | Terminated |
+
+  `instanceFromPod` sets `IP` only when the state is Running. A Pod with a
+  deletion timestamp still reports its phase. `UID` from the `LabelUID`
+  label, `CreatedAt` from `CreationTimestamp`.
+- **Delete.** `client.Delete`, `IsNotFound` ignored.
+- **ListByTag.** One `client.List` across all namespaces with
+  `MatchingLabels{LabelManaged: LabelManagedYes}`; this is what the
+  cluster-wide Pod RBAC is for.
+- **Errors.** `classify`: `IsNotFound` NotFound; `IsTooManyRequests`,
+  `IsServerTimeout`, `IsTimeout` Transient; `IsForbidden`, `IsInvalid`,
+  `IsBadRequest`, `IsUnauthorized` Permanent; default Transient. Nothing is
+  Quota, because a quota 403 and an RBAC 403 cannot be told apart.
+- **Check.** Lists at most one managed Pod cluster-wide (`client.Limit(1)`),
+  proving read and list access, not create. Target `apiserver`, address the
+  rest config host. `checkReason`: context errors and timeouts
+  `ReasonTimeout`; `IsUnauthorized`; `IsForbidden`; any `APIStatus`
+  `ReasonAPIError`; else `ReasonUnreachable`.
+- **Logging and tracing.** No log lines in the package. Child spans come
+  from the parent-gated transport wrapper.
+- **Tests.** `kubernetes_test.go`, `pod_test.go`, `check_test.go` over the
+  fake client; the kind e2e (`test/e2e`) is the real-backend run.
+- **Deployment.** RBAC markers for Pods live next to the reconciler's in
+  `internal/controller/proxy_controller.go`. `hack/providers-dev.yaml` holds
+  only this provider, for `make run-dev`.
+- **Known gaps.**
+  - `Error.Provider` is hardcoded to `"kubernetes"`, so errors and logs show
+    the wrong name when the configured name differs.
+  - `CloudInit` and all of `Placement` are ignored.
+  - No quota class: a quota 403 is Permanent.
+  - A Pod being deleted still reports its phase, so it can read as Running.
+  - No securityContext, no resources, and `http_access allow all` in
+    `pod.go`.
+  - No logging in the package, and no Checker `Details`.
+  - No `OwnerReference` on Pods: a bypassed finalizer leaves the Pod behind.
+  - The Pod RBAC is always installed, even when no kubernetes provider is
+    configured.
+
+## 5. gcp (`internal/provider/gcp`)
+
+- **Package and seam.** `gcp.go`, `insert.go`, `errors.go`, `check.go`,
+  `wirelog.go`. The seam is `instancesAPI` (`Insert`, `Get`, `Delete`,
+  `AggregatedList`, `Probe`); `realInstances` adapts the SDK client and
+  `newWithAPI` is the test constructor. `New` calls `NewWithWireOptions`,
+  which takes the `--gcp-wire-log-full-payloads` flag from `cmd/main.go`.
+- **Config.** `GCPConfig`: `project` required; `network`, `networkTag`,
+  `diskSizeGb` defaulted in `withDefaults`.
+- **ID.** `formatProviderID(zone, name)` gives
+  `zones/<zone>/instances/<name>`; `parseProviderID` rejects anything else
+  with `ErrPermanent`. The project comes from the config.
+- **Create.** Validates `Zone`, `MachineType` and `Image` up front and
+  returns `ErrPermanent` naming the missing field. `buildInsertRequest`
+  (pure) builds one NIC with an ephemeral external IP (`ONE_TO_ONE_NAT`) and
+  a boot disk with `AutoDelete`, labelled with both marks. `Insert` is
+  submitted and never awaited; 409 (`isAlreadyExists`) is success.
+- **Get and state.** `mapState(status, ip)`:
+
+  | Instance status | NAT IP | Provider state |
+  |---|---|---|
+  | PROVISIONING, STAGING, REPAIRING | any | Provisioning |
+  | RUNNING | empty | Provisioning |
+  | RUNNING | set | Running |
+  | STOPPING, STOPPED, SUSPENDING, SUSPENDED | any | Stopped |
+  | TERMINATED | any | Terminated |
+  | anything else | any | Stopped |
+
+  `toInstance` reads the NAT IP of the first access config of the first
+  NIC, `UID` from the label, and `CreatedAt` from `CreationTimestamp` with
+  the parse error dropped (zero time on failure).
+- **Delete.** Submits a delete operation and returns; 404 (`isNotFound`) is
+  success. Every deletion poll submits a new operation.
+- **ListByTag.** `AggregatedList` with `managedFilter` on the managed label
+  only and `ReturnPartialSuccess`, so an unreachable zone is left out
+  silently; the zone is recovered from the URL by `lastPathSegment`.
+- **Errors.** `classify`: non-`googleapi` errors Transient; 404 NotFound;
+  429 Quota; 403 with reason `quotaExceeded` or `rateLimitExceeded`
+  (`hasReason`) Quota; 400, 401 and other 403s Permanent; other codes
+  Transient. `logAPIError` records the HTTP status and reasons at V(1)
+  because `classify` collapses them.
+- **Check.** `Probe` makes one `AggregatedList` page with `maxResults=1`
+  and the managed filter: credentials, the Compute API and
+  `compute.instances.list` in one call. Target is the project, address
+  `compute.googleapis.com`. `checkReason`: context errors `ReasonTimeout`;
+  non-`googleapi` errors `ReasonUnreachable`; 401; 403; else
+  `ReasonAPIError`.
+- **Logging.** `logger(ctx)` from the context, named `gcp` with
+  `provider`. V(2) for the curated insert request (never the user-data),
+  V(1) per operation outcome, V(2) per listed instance. The wire log in
+  `wirelog.go` runs at V(5) with payloads elided unless the flag is set.
+- **Tracing.** The SDK adds `otelhttp` itself unless telemetry is disabled;
+  not parent-gated.
+- **Tests.** `insert_test.go` (pure builder), `errors_test.go` (code
+  table), `gcp_test.go` (IDs, 409, state mapping, list filter and partial
+  success) over `fakeAPI`; `check_test.go` also drives the real SDK client
+  against `httptest` to prove exactly one request with `maxResults=1`.
+  Real backend: the manual runbook in [gcp-vm-validation.md](gcp-vm-validation.md).
+- **Deployment.** Credentials by ADC; the runbook mounts a service-account
+  key and points `GOOGLE_APPLICATION_CREDENTIALS` at it. A commented entry
+  in both providers configs, `config/samples/proxy_gcp.yaml`, and demo
+  scripts under `docs/demo/`.
+- **Known gaps.**
+  - Asynchronous insert failures are never read: an insert the API accepts
+    but that fails later (quota, bad image) is seen only as NotFound on the
+    next `Get`, and the reconciler creates again, so the reason never
+    reaches status. [gcp-vm-validation.md](gcp-vm-validation.md#gotchas)
+    still says such errors do reach status.
+  - External IP only: a VM without an external NAT IP stays Provisioning
+    forever.
+  - No firewall management: the rule for the proxy port is a manual
+    prerequisite, and `req.Port` is unused.
+  - The SDK client is never closed, and `Placement.Region` is unused.
+  - Wrong label value in the docs: `gcp-vm-validation.md` filters on
+    `labels.proxy-operator-managed=yes`; the value is `true`.
+  - The sample cannot work: `config/samples/proxy_gcp.yaml` uses a
+    `debian-cloud` image, which has no cloud-init, so its Squid install never
+    runs ([gcp-in-specific-project.md](gcp-in-specific-project.md) uses
+    Ubuntu for that reason).
+
+## 6. kvm (`internal/provider/kvm`)
+
+The full user guide is [kvm-provider.md](kvm-provider.md); this section is
+the author's view.
+
+- **Package and seam.** `kvm.go` (the four methods), `conn.go` (SSH dial,
+  `hostPool`, `libvirtAPI`), `image.go` (`imageCache`), `seed.go` (NoCloud
+  ISO and vendor-data), `domainxml.go`, `volume.go`, `metadata.go`, `id.go`,
+  `errors.go`, `check.go`, `trace.go`. The seam is `libvirtAPI`, the twenty
+  go-libvirt calls the provider uses; `fake_test.go` implements it in memory
+  and `newWithDialer` injects the dialer and the HTTP client.
+- **Config.** `KVMConfig` in `internal/provider/config_kvm.go`:
+  `sshKeyFile`, `knownHostsFile` or `insecureIgnoreHostKey`, `diskSizeGb`,
+  `vendorData`, `installGuestAgent`, `images` (`url`, `sha256`, `format`,
+  `firmware`), `machineTypes` (`vcpus`, `memoryMiB`), `hosts` (`name`,
+  `address`, `user`, `socket`, `pool`, `bridge`). `validateKVM` restricts
+  image keys and host names with `kvmIdentifier` because they become a volume
+  name and an ID segment.
+- **ID.** `formatProviderID(host, name)` gives `hosts/<host>/domains/<name>`.
+- **Create.** `host` resolves `Placement.Zone` (optional with one host),
+  the machine type must be a catalogue key, `resolveImage` accepts a
+  catalogue key or an http(s) URL. Then, inside one `hostPool.call`:
+  `DomainLookupByName`, and if found `ensureRunning`; otherwise
+  `imageCache.ensure` (download on first use, streamed through the operator
+  into the pool with a sha256 marker volume), `ensureOverlay` (reused if it
+  exists), `writeSeed` (always rebuilt: a cut-off upload must not be
+  trusted), `defineDomain` with the metadata element, `ensureRunning`.
+  `Create` returns only when the domain is running.
+- **Get and state.** `mapState(state, ip)`:
+
+  | libvirt state | IP | Provider state |
+  |---|---|---|
+  | nostate | any | Provisioning |
+  | running, blocked | empty | Provisioning |
+  | running, blocked | set | Running |
+  | paused, shutdown, shutoff, crashed, pmsuspended | any | Stopped |
+
+  `address` asks the ARP table first, then the guest agent, and
+  `firstIPv4` drops loopback, link-local and unspecified addresses. Metadata
+  is read back by `metadata` and parsed by `parseInstanceMetadata`; a domain
+  without the element reports an empty UID.
+- **Delete.** `removeDomain` (destroy, tolerating not running or gone; then
+  `DomainUndefineFlags` with NVRAM, managed save and snapshot metadata) and
+  `removeVolumes` (overlay and seed by name, a missing pool is a no-op). A
+  host no longer in the config is a logged no-op success.
+- **ListByTag.** `ConnectListAllDomains` on every host; `managedInstance`
+  keeps domains whose metadata says managed and skips a domain whose
+  metadata or state cannot be read. One unreachable host fails the whole
+  list.
+- **Errors.** `classify`: `isClassified` errors keep their class;
+  `ErrNoDomain`, `ErrNoStorageVol` NotFound; XML, unsupported,
+  invalid-argument, auth and access errors, and a missing pool, Permanent;
+  everything else Transient, including SSH handshake and auth failures.
+  Nothing is Quota. `logAPIError` adds `libvirtErrorFields`.
+- **Check.** `checkHost` per host concurrently through `hostPool.probe`,
+  not `call`, so a check that runs out of time abandons its RPCs instead of
+  closing the connection a reconcile may be using. Two cheap RPCs
+  (`ConnectGetLibVersion`, `ConnectGetHostname`) then the pool must exist
+  and be running. Reasons: the shared ones plus `SSHMaterialUnavailable`,
+  `SSHHandshakeFailed`, `LibvirtConnectFailed`, `LibvirtError`,
+  `PoolNotFound`, `PoolNotActive`, chosen by `checkReason` from the
+  `dialError` stage. Details: `user`, `pool`, `hostname`, `libvirtVersion`,
+  `poolState`, `poolCapacity`, `poolAvailable`.
+- **Connections.** `hostPool` keeps one SSH plus libvirt connection per
+  host, dialled lazily (`lazySSH.dial`, `sshAuth.dial`, `dialSSH`,
+  `connectLibvirt`), with a 15s `keepalive`. `hostEntry.acquire` is a
+  one-slot semaphore so waiting can end with the caller's context. A
+  cancelled `call` closes the shared connection to unblock the RPC;
+  `probe` abandons it and leaves the connection open. The SSH material is
+  read once by `lazySSH.load` and cached.
+- **Logging.** Logger from the context, named `kvm`, with `provider`,
+  `host` and `name`. V(1): one line per operation outcome and one
+  `libvirt call` line per RPC from the `tracingAPI` decorator in `trace.go`
+  (rpc name, identifiers, duration, error fields). V(2): domain and volume
+  XML, metadata, addresses. Upload and download bodies are never logged.
+- **Tracing.** No child spans; `trace.go` is a log decorator.
+- **Tests.** Pure builders (`domainxml_test.go`, `volume_test.go`,
+  `seed_test.go`, `id_test.go`, `metadata_test.go`); `errors_test.go`,
+  `conn_test.go` (pool behaviour and a real in-process SSH server);
+  `image_test.go`, `kvm_test.go` (Create resumes after every partial step,
+  every Delete already-gone case, ListByTag policies, logging tiers, a
+  cloud-init sentinel never logged); `trace_test.go`; `check_test.go`. Real
+  backend: `kvm_integration_test.go` behind `//go:build integration`, driven
+  by `KVM_TEST_*` variables and `make test-integration-kvm`, described in
+  [testing.md](testing.md#the-kvm-integration-test-real-hypervisor).
+- **Deployment.** The optional `kvm-ssh` Secret mounted at
+  `/etc/proxy-operator/kvm` (`config/manager/manager.yaml`),
+  `make kvm-ssh-secret`, a commented entry in both providers configs,
+  `config/samples/proxy_kvm.yaml`.
+- **Known gaps** (see also [kvm-provider.md § Known gaps](kvm-provider.md#known-gaps)).
+  - No quota or capacity detection.
+  - `Create` blocks, including the first image download on each host.
+  - GC blind spots: volumes without a domain are invisible to GC, and so are
+    VMs on a host removed from the config, which `Delete` treats as done.
+  - SSH material is cached after the first successful load, so rotating
+    keys needs a restart.
+  - Cancelling one operation cuts off the others on that host: a cancelled
+    `call` closes the shared per-host connection.
+  - SSH auth failures retry forever: SSH authentication and host-key
+    failures are Transient, while libvirt auth errors are Permanent.
+  - `Create` can return `ErrNotFound` when a volume or domain disappears
+    during the call.
+  - No reused-overlay check: an existing overlay is reused without checking
+    its backing image or size.
+  - Sample mismatch: `config/samples/proxy_kvm.yaml` places on host `nas1`,
+    while both providers configs define only `beelink`.
+  - Doc drift in kvm-provider.md: it says ARP is the fallback, but the code
+    tries ARP first; it says `host` and `name` are on every log line, but
+    the check and list paths lack them.
+
+## 7. Gaps shared by the existing providers
+
+- **Adoption is not verified.** None of the three checks `LabelUID` when
+  `Create` finds an existing instance with the right name.
+- **The Create comment does not match kvm.** The interface comment in
+  `internal/provider/provider.go` says `Create` returns as soon as the
+  request is submitted; kvm blocks until the domain has started.
+- **`Stopped` and `Terminated` are produced inconsistently.** kubernetes
+  never produces Stopped, kvm never produces Terminated, and the reconciler
+  treats them the same.
+- **Different `ListByTag` partial-failure policies.** gcp leaves failed
+  parts out, kvm fails the whole list. Both are safe.
+- **Stale `mock` comments** in `internal/provider/provider.go`,
+  `internal/provider/registry/registry.go` and `api/v1alpha1/proxy_types.go`
+  (and so the generated CRD's `spec.provider` description).
+- **GC does not know about installations.** Two clusters sharing one project
+  or hypervisor delete each other's instances. This was a merge-blocker in
+  the 2026-08-10 PR review
+  ([docs/reviews](reviews/2026-08-10-1134-proxy-operator-pr-review.md)) and
+  is still open.
+- **Tracing.** kvm has no child spans, so a kvm trace shows `provider.create`
+  and nothing under it; correlation works only through `traceID` on its V(1)
+  log lines. Only kubernetes is tested against a real collector, and that
+  test checks only the operation spans; nothing checks that gcp's SDK spans
+  or the kubernetes API spans nest under `provider.*`.
+
+## 8. Proposed features
+
+Listed only. Nothing below is designed or implemented.
+
+- **Child spans in kvm:** extend the `tracingAPI` decorator in
+  `kvm/trace.go`, which already wraps every libvirt call, to also start a
+  span per RPC, and add spans for the SSH dial and the image download. The
+  seam's methods take no `ctx` today (`hostPool.call` hands `fn` only the
+  API), so the span context has to be plumbed in.
+- **Tempo e2e checks for child spans**, and for gcp and kvm through their
+  real backend runbooks.
+- **A shared conformance suite:** for example `internal/provider/providertest`,
+  which runs the contract §11 tests against any provider plus a fake
+  backend, so a new provider gets the contract tests for free.
+- **Verify the UID when `Create` adopts an instance.** Return `ErrPermanent`
+  on a mismatch instead of adopting an instance that belongs to someone else.
+- **Report asynchronous create failures:** gcp keeps the operation name, and
+  `Get` reports the failure instead of NotFound.
+- **Capacity or quota signals** for kvm, such as pool free space and host
+  memory, mapped to `ErrQuotaExceeded`.
+- **GC for secondary resources:** mark volumes and similar resources, and
+  let `ListByTag` or a second listing method cover them.
+- **Reload credentials and config without a restart.**
+- **Checker `Details`** for kubernetes (server version) and gcp (quota
+  headroom).
+- **Internal-IP mode** for gcp, and a firewall rule managed per provider.
+- **An installation-identity mark**, set on create and filtered on in
+  `ListByTag`, so installations sharing a backend do not collect each
+  other's instances.
+- **A permanent-failure latch that sees provider availability**, so adding
+  a missing provider releases the Proxies waiting on it.
+- **Validate provider names at load**, and derive the config type
+  allow-list from the constructor map.
+- **Per-provider RBAC** as an optional kustomize component, and an
+  `OwnerReference` on kubernetes Pods.

docs/providers.md

@@ -0,0 +1,39 @@
+# Providers
+
+The provider documentation is split into three files so that a new provider
+can be specified and built from the generic rules alone, without reading how
+the existing ones happen to work.
+
+- **[provider-contract.md](provider-contract.md)**: what every provider MUST,
+  SHOULD and MAY do, which caller depends on each rule, the behaviour profiles
+  where providers are allowed to differ, the error taxonomy, credential
+  hygiene, testing obligations, the checklist for adding a provider, and the
+  contract's known limits. It never names an existing provider.
+- **[provider-spec-template.md](provider-spec-template.md)**: the questionnaire
+  a new provider's design answers, one entry per decision the contract leaves
+  open. Copy it to `docs/<type>-provider-spec.md`, fill it in, and hand it to
+  the implementer with the other two files.
+- **[providers-existing.md](providers-existing.md)**: how kubernetes, gcp and
+  kvm meet the contract, a table mapping each of them to every profile, their
+  per-provider reference notes, shared gaps and proposed features.
+
+Per-provider user guides: [kvm-provider.md](kvm-provider.md),
+[gcp-vm-validation.md](gcp-vm-validation.md) and
+[gcp-in-specific-project.md](gcp-in-specific-project.md).
+
+Related sections of [architecture.md](architecture.md): the components table,
+[§5 what provider calls do](architecture.md#5-what-provider-calls-do-in-the-outside-world),
+[§10 tracing](architecture.md#10-tracing-internaltracing),
+[§11 provider health checks](architecture.md#11-provider-health-checks-internalproviderhealth)
+and the "Decisions" list. The test layout is in [testing.md](testing.md).
+
+## Reading order for a new provider
+
+1. The contract, end to end.
+2. The template, filled in for the new backend; every unanswered entry is a
+   design decision still to make.
+3. The appendix, for a worked example of each profile the spec chose.
+4. The user guide of the existing provider closest to the new one.
+
+The single-file blueprint that preceded this split is in the git history of
+this path.

docs/reviews/2026-08-10-1134-proxy-operator-pr-review.md

@@ -0,0 +1,131 @@
+# PR review findings: feat/proxy-operator
+
+**Created:** 2026-08-10 11:34
+**Scope:** `origin/main...feat/proxy-operator` (merge-base 076bc66, 25 commits, ~80 files)
+**Reviewers:** `go-operator-reviewer` + `operator-reviewer` agents; findings consolidated, most severe first. Check off items as they're processed.
+
+Both reviewers rated the core reconcile architecture sound: single status writer with one
+deferred patch, finalizer added before any provider call, Get-before-RemoveFinalizer on
+delete, CEL immutability rules correctly split to avoid the oldSelf-on-CREATE trap,
+leader-election gating on destructive runnables, GC tombstone rules (MinAge, UID-less
+instances never deleted).
+
+## Merge-blockers
+
+- [ ] **Discovery leases proxies with an empty IP** — found independently by both reviewers.
+  `internal/discovery/handlers.go:151`, `internal/controller/proxy_controller.go:226`
+  During instance replacement (and the Get→NotFound recovery path) the reconciler clears
+  `status.ip` but only the create branch removes the `Healthy` condition, and the health
+  engine prunes state for empty-host proxies so nothing refreshes it. For the whole
+  delete→recreate window (minutes on GCP), `isHealthy` still returns true and
+  `handleAcquireLease` grants `201 Created` with `"ip": ""`, burning a `MaxLeases` slot.
+  **Fix:** add `EffectiveHost() != ""` to `isHealthy` (covers list + acquire), and
+  remove/downgrade `Healthy` wherever `status.IP` is cleared.
+
+- [ ] **Orphan GC deletes other installations' fleets in a shared GCP project.**
+  `internal/provider/gcp/insert.go:57`, `internal/gc/gc.go:93`
+  Instances are tagged only `proxy-operator-managed=true` + CR UID; the sweeper deletes any
+  tagged instance whose UID isn't in *its own cluster's* Proxy list. Two clusters sharing a
+  GCP project delete each other's VMs every GC interval in a permanent loop.
+  **Fix:** add an installation-identity label (cluster/deployment ID) set by both providers
+  and filtered on in `ListByTag`.
+
+- [ ] **Permanent-error latch wedges proxies on failures that aren't spec-caused.**
+  `internal/controller/proxy_controller.go:126`
+  Latch keys on `observedGeneration == generation`, but two failure inputs live outside the
+  spec: an unconfigured provider (config fix + restart doesn't bump generation, and
+  `spec.provider` is CEL-immutable → stuck `Failed` short of deleting the CR) and resolved
+  Secret content (Secret fix enqueues a reconcile that short-circuits at the latch before
+  re-resolving cloud-init).
+  **Fix:** latch should also consider current spec-hash / provider availability.
+
+## Worth fixing
+
+- [ ] **Deletion-path failures invisible in status** — flagged by both reviewers.
+  `internal/controller/proxy_controller.go:319`, `:266`
+  `deletionFailure` swallows `ErrQuotaExceeded` (nil error, no status write); unconfigured
+  provider returns a bare error forever. A Proxy wedged in `Deleting` shows nothing in
+  `kubectl describe`. Stage `setProvisioned(p, False, ReasonDeleting, ...)` before returning.
+  Also: `Delete` is resubmitted on every `DeletionPoll` pass, churning GCP quota — a state
+  check on the `Get` result would avoid it.
+
+- [ ] **Lost providerID on `setSpecHash` conflict.**
+  `internal/controller/proxy_controller.go:161`
+  On Update conflict the function returns before `p.Status.ProviderID = id`, so the deferred
+  patch persists an empty providerID for a just-created instance. Self-heals via GC.
+  **Fix:** set `p.Status.ProviderID = id` before returning the error (one line).
+
+- [ ] **Terminating pods still report `StateRunning`.**
+  `internal/provider/kubernetes/kubernetes.go:151`
+  A pod with a deletionTimestamp keeps `phase=Running` + `PodIP` while terminating, so drift
+  reconcile republishes `Provisioned=True` and discovery keeps leasing a dying pod.
+  **Fix:** map non-zero `pod.DeletionTimestamp` to `StateTerminated` in `instanceFromPod`.
+
+- [ ] **Stale-cache spec-hash race deletes the freshly created replacement instance.**
+  `internal/controller/proxy_controller.go:176`
+  Instance name derives from CR UID, so old and new instances share a providerID. A reconcile
+  served a cached object from before a just-completed replacement re-enters `replaceInstance`
+  and deletes the *new* healthy instance. Converges, but destroys a good instance.
+  **Fix:** re-read uncached before the destructive branch, or compare `inst.CreatedAt`
+  against the annotation-update time.
+
+- [ ] **`observedGeneration` written before the generation is actually processed.**
+  `internal/controller/status.go:114`
+  Set unconditionally in `patchStatusIfChanged`, including on the finalizer-add pass and
+  `resolveCloudInit` failures — misleads kstatus-style tooling. Set it only once the state
+  machine has genuinely evaluated the spec.
+
+- [ ] **No event filtering on the Proxy watch.**
+  `internal/controller/proxy_controller.go:402`
+  Every self-inflicted status patch triggers a follow-up reconcile with an extra cloud `Get`,
+  roughly doubling provider read traffic. Caution: a plain `GenerationChangedPredicate`
+  breaks the finalizer flow (relies on its own Update event to re-enter) — needs a
+  status-only/resourceVersion-only filter or an explicit requeue in the finalizer pass.
+
+- [ ] **Unlabelled cloud-init Secrets produce a misleading NotFound with endless backoff.**
+  `internal/controller/proxy_controller.go:344`, `cmd/main.go:210`
+  The label-restricted cache turns "exists but missing `crawl.example.com/cloud-init=true`"
+  into `CloudInitError: not found`. Mention the label requirement in the condition message,
+  or read via uncached `APIReader` and validate the label explicitly.
+
+- [ ] **RBAC over-grant.**
+  `config/rbac/role.yaml:25`
+  `create;delete` on `proxies` is scaffold residue (controller never creates/deletes CRs);
+  cluster-wide `pods create/delete` and `secrets get/list/watch` apply even when only the GCP
+  provider is configured — pod rules belong in an optional kustomize component.
+
+## Simplifications
+
+- [ ] **Delete `internal/provider/registry`** — 14 lines of logic, one caller
+  (`cmd/main.go:148`); fold `Build`/`Constructor` into the composition root. Also fixes the
+  two-sources-of-truth problem: `internal/provider/config.go:84` hardcodes
+  `"kubernetes"`/`"gcp"` while `registry.Build` dispatches through a caller-supplied map —
+  validate against the constructor map instead. Net −1 package, −44 lines, −92 test lines.
+
+- [ ] **Collapse `LeaseStore` interface to `*lease.Store`.**
+  `internal/discovery/server.go:28`
+  Single implementation and not a test seam (tests wire the real `lease.NewStore`).
+  Keep `HealthSnapshotter` and `instancesAPI` — those are genuine seams.
+
+- [ ] **Replace metrics nil-guards with no-op defaults.**
+  `internal/health/engine.go:68`, `internal/discovery/server.go:38`,
+  `internal/provider/metrics.go:8`
+  Keep the interfaces (legit "no prometheus in domain packages" rationale) but default the
+  fields to a no-op impl — `provider.WithMetrics` already dereferences unconditionally, so
+  the guards are inconsistent anyway.
+
+## Nice-to-have
+
+- [ ] Add an `OwnerReference` to provider pods (`internal/provider/kubernetes/pod.go:24`) —
+  free cascading deletion if the finalizer is ever bypassed; `CreateRequest` already carries
+  Namespace/ProxyName/UID.
+- [ ] `Close()` the GCP `*compute.InstancesClient` (`internal/provider/gcp/gcp.go:89`) —
+  harmless today, a leak the moment providers are rebuilt on config reload.
+- [ ] Fix `.golangci` config: it references a missing `logcheck` plugin, so the linter only
+  runs with the project config disabled.
+- [ ] External-mode endpoint edits don't reset health-engine counters
+  (`internal/health/engine.go:206` keys on name+UID): flipping `endpoint.host` keeps the old
+  host's `Healthy=True` for `failureThreshold × interval`. Arguably a replacement, not a flap.
+- [ ] `init()` funcs at `api/v1alpha1/proxy_types.go:353` and `cmd/main.go:67` conflict with
+  the repo's "no `init()`" convention; kubebuilder-idiomatic, but scheme registration could
+  use the scaffold's `SchemeBuilder.Register` at package var scope.

docs/testing.md

@@ -0,0 +1,233 @@
+# Testing
+
+What tests exist, how to run them, and where the boundaries of the
+automated suites are. The philosophy throughout: **test against the most
+real double available** — a real envtest API server over a fake client, a
+real CONNECT-capable proxy stub over a mocked HTTP client, the real lease
+store under the discovery handlers — and leave the gaps that only real
+infrastructure can close to the kind verification run, documented at the
+bottom.
+
+## Running
+
+```sh
+make test              # THE canonical run: codegen + fmt + vet + envtest setup,
+                       # then `go test -race` on every package, with coverage
+go test -short ./...   # skip the envtest suite (runs in <1s per package)
+go tool cover -html=cover.out   # browse coverage from the last make test
+```
+
+Targeted runs:
+
+```sh
+go test -race ./internal/lease/                  # one package
+go test -race -run TestAcquire ./internal/lease/ # one test (or a prefix)
+go test -race -count=2 ./internal/health/        # flake-shaking: run twice
+```
+
+**Gotcha — plain `go test ./...` fails the controller package** with an
+error like `/usr/local/kubebuilder/bin/etcd: no such file`. That is not a
+code problem: the envtest suite needs `KUBEBUILDER_ASSETS` pointing at the
+API-server/etcd binaries, which only `make test` sets up (via
+`setup-envtest`). Either use `make test`, or export it once:
+
+```sh
+KUBEBUILDER_ASSETS="$PWD/bin/k8s/<version>-<os>-<arch>" go test -race ./...
+```
+
+Everything runs with `-race` — `make test` enforces it; keep the flag on
+manual runs too.
+
+## The suites, package by package
+
+| Package | Files | What is covered | Test double / technique |
+|---|---|---|---|
+| `api/v1alpha1` | `helpers_test.go` | `EffectivePort/EffectiveHost/HealthCheckOrDefault/MaxLeasesOrDefault` | pure table-driven; CEL rules are **not** unit-testable — see envtest below |
+| `internal/controller` | `reconcile_test.go` | every row of the reconciler's action table: create, poll, publish IP, replace, adopt, NotFound recovery, delete/finalizer, quota/permanent/transient errors, cloud-init resolution | `controller-runtime/pkg/client/fake` + an in-test `stubProvider` |
+| | `status_test.go` | `computePhase` truth table (10 rows) | pure |
+| | `spechash_test.go` | hash stability, nil/empty normalization, sensitivity to every replacement-triggering field | pure |
+| | `health_test.go` | Healthy-condition representation: Ready/Unhealthy phases, no-verdict, stale-verdict cleanup on replacement, nil snapshotter | fake client + `fakeSnapshotter` |
+| | `proxy_controller_test.go` (envtest, ginkgo) | full lifecycles against a **real API server**: provision→Running, spec-change replacement, finalizer deletion, External tracking, Ready-through-health, quota-vs-permanent, adoption; **all CEL rules** (six invalid creates, mode/provider immutability incl. removal) and the `default={}` materialization | envtest apiserver + stub provider; the only place CEL and structural defaulting actually execute |
+| `internal/provider` | `name_test.go` | deterministic naming: idempotency, `^proxy-[a-z2-7]{16}$`, 10k-UID distinctness | pure |
+| | `errors_test.go` | taxonomy: `Class()` mapping, `errors.Is` **and** `errors.As` through the multi-unwrap | pure |
+| | `config_test.go` | providers-config parsing + fail-fast validation | pure |
+| | `metrics_test.go` | `WithMetrics` decorator: classified result labels, error passthrough | fake recorder + static provider |
+| `internal/provider/kubernetes` | `kubernetes_test.go`, `pod_test.go` | Create/Get/Delete/ListByTag over real Pod objects; pure `buildPod`/`squidConf` incl. the `max_filedescriptors` OOM-regression assertion | `client/fake` (real `corev1.Pod`s, no kubelet) |
+| | `check_test.go` | health check: a single cluster-wide list with `Limit=1` and the managed-label selector; the reason for every API error class | `client/fake` + interceptor |
+| `internal/provider/gcp` | `insert_test.go`, `errors_test.go`, `gcp_test.go` | field-by-field `buildInsertRequest`; HTTP-code classification table; zone-qualified IDs, 409-is-success, 9-row state mapping, ListByTag filter + `ReturnPartialSuccess` | pure builder needs **no fake**; the rest uses the flattened `instancesAPI` seam |
+| | `check_test.go` | health check: request shape and the reason per HTTP code or network error; the **real SDK client** makes exactly one request with `maxResults=1`, even when a next page exists | fake seam + `httptest` |
+| `internal/provider/kvm` | `domainxml_test.go`, `volume_test.go`, `seed_test.go`, `id_test.go`, `metadata_test.go` | pure builders: domain/volume XML unmarshalled back and checked field by field, seed ISO reopened with the same library, providerID round-trip, metadata in both namespace spellings | pure |
+| | `errors_test.go`, `conn_test.go` | libvirt-code classification table; connection pool (reuse, redial, drop on transport error, cancel releases a blocked RPC, one dial per host under concurrency); SSH dial, host-key policy, keepalive | fake dialer; a **real in-process `x/crypto/ssh` server** forwarding a unix socket |
+| | `image_test.go`, `kvm_test.go` | image cache (fresh fetch, ready, marker missing, checksum change/mismatch, 4xx/5xx, concurrent callers); Create resume after every partial step, leftover truncated seed, state/IP mapping, every Delete already-gone case, ListByTag across hosts / one down / unreadable domain; logging tiers (V0 silent, V1 one line per RPC, V2 XML) with cloud-init never logged | `httptest` image server + in-memory `fakeAPI` behind the `libvirtAPI` seam |
+| | `trace_test.go`, `conn_test.go`, `errors_test.go` | every `libvirtAPI` method traced exactly once (guarded by reflection on the interface), results and errors pass through, libvirt error fields, detail keys only at V2, upload content never logged; pool connect/drop lines, no wrapping below V1 | `fakeAPI` + `funcr` capture |
+| | `check_test.go`, `conn_test.go` | health check: details, per-host verdicts in config order, hosts checked concurrently, a timeout that keeps the connection and refuses a second check while the first still waits, no waiting behind a reconcile's dial, the reason table; `probe` abandons without closing, but still drops on a transport error; waiting for a host's slot ends with the caller's deadline; an inactive or missing pool and daemon errors keep the connection; every dial stage tagged through the real SSH server, with classification unchanged | `fakeAPI`, fake dialers, in-process SSH server |
+| | `kvm_integration_test.go` (`-tags=integration`) | one real VM: image fetch, Create, idempotent re-Create, Running with IP, ListByTag, Delete, pool state afterwards; `TestKVMCheck_realHost` runs the health check against the host and creates nothing | a **real hypervisor** — see below |
+| `internal/health` | `probe_test.go` | probes through a **real CONNECT-capable proxy stub** to a real TLS server: tunnel success, refused CONNECT, wrong status, dead proxy, plain-http forward | `httptest` + hijacked bidirectional tunnel |
+| | `engine_test.go` | thresholds, first-verdict, latency suppression matrix, dropped-event retry, stale-probe UID guard, tick scheduling/seeding/pruning, `Start` end-to-end | fake `client.Reader` + injected `probeFn` |
+| `internal/lease` | `store_test.go` | capacity, `MaxLeases=0`, all three selection tie-breaks, target-scoped vs global cooldowns, expiry via fake clock, report-on-retained-lease, idempotent release, **40 concurrent acquires never exceeding capacity** | injectable clock; the concurrency case is why `-race` matters |
+| `internal/discovery` | `server_test.go` | auth matrix (incl. `/healthz` exemption), list filtering, grant shape, full 409 arithmetic, invalid TTL/body/result, idempotent release, report→cooldown→409 round trip, `Start`/shutdown | `httptest` over the real handler chain, fake cache reader, **real** `lease.Store` |
+| `internal/gc` | `gc_test.go` | the true-orphan matrix (live/deleting/young/unlabelled all kept), broken-provider isolation, list-failure skips sweep, namespace guard, sweep loop | fake reader + canned-list provider |
+| `internal/providerhealth` | `status_test.go`, `monitor_test.go` | `buildStatus` (the Ready condition, per-target transition times, truncation); objects created with the label and patched, the round timeout, no-checker Unknown, invalid names, transition-only logging, cleanup of labelled unconfigured objects, the checks-off path, `ValidateSchedule` | fake client with a cluster-scoped RESTMapper + `funcr` capture |
+| | `envtest_test.go` | the generated `ProviderHealth` schema against a real API server: cluster scope, status subresource, required fields, list-map keys, the 1024-character message limit | envtest (skipped under `-short`) |
+| `internal/metrics` | `metrics_test.go` | scrape-time collectors (`GatherAndCompare`) incl. the provider-target gauges, per-proxy series cleanup via `ForgetProxy`, label counts, fresh-registry-per-test property | `prometheus/client_golang/testutil` |
+
+Conventions (from `CLAUDE.md`): stdlib `testing`, table-driven by default,
+`t.Parallel()` where safe, tests next to source, ginkgo only in the
+envtest suite the scaffold generated.
+
+## What the automated suites deliberately do NOT cover
+
+- **`New()` constructors that dial real infrastructure**: the kubernetes
+  provider's `New` (connects to whatever your kubeconfig points at), the
+  GCP provider's `New` and its `realInstances` SDK adapter (ADC + real
+  Google endpoints). Both are thin; both are exercised by the kind run /
+  real deployments. Their 0% coverage is by design — do not "fix" it.
+- **CEL and structural defaulting under the fake client**: the fake
+  client runs neither. Every unit test that relies on defaults calls the
+  `*OrDefault` helpers; every CEL rule is asserted in the envtest suite
+  instead.
+- **A container actually starting and serving**: envtest has no kubelet,
+  so a Pod created there sits Pending forever. Whether Squid really comes
+  up and tunnels CONNECT is provable only on a real cluster — that is the
+  kind verification's job, and it is exactly what caught the Squid OOM
+  bug (below).
+- **Live GCP**: no test talks to Google. The seam boundary
+  (`buildInsertRequest` + classification) is tested exhaustively instead.
+- **Live libvirt, by default**: the KVM unit tests run over an in-memory
+  fake of the 20 libvirt calls the provider uses. Whether libvirt accepts
+  the domain XML, whether a raw qcow2 upload plus pool refresh yields a
+  usable backing image, and whether the guest actually boots and reports
+  an address are provable only on a real host — that is the opt-in
+  integration test below.
+
+## The KVM integration test (real hypervisor)
+
+`internal/provider/kvm/kvm_integration_test.go` compiles only under
+`-tags=integration` and skips itself unless the environment names a
+host. It deviates from the "testcontainers for real infra" convention on
+purpose: libvirt needs `/dev/kvm` and a bridge to a DHCP network, and a
+container has neither. So it targets whichever hypervisor you point it
+at and leaves that host's pool with one more cached base image than it
+had.
+
+```sh
+export KVM_TEST_HOST=nas1.home.lan:22          # SSH endpoint
+export KVM_TEST_USER=virt                      # must be in the libvirt group
+export KVM_TEST_SSH_KEY=$HOME/.ssh/id_ed25519  # private key for that user
+export KVM_TEST_KNOWN_HOSTS=$HOME/.ssh/kvm_known_hosts  # the hypervisors' host keys
+export KVM_TEST_BRIDGE=br0                     # bridge to a network with DHCP
+export KVM_TEST_IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+make test-integration-kvm
+```
+
+`KVM_TEST_KNOWN_HOSTS` is technically optional, but leaving it out turns
+host-key verification **off** for the run (the log says so); keep a
+`kvm_known_hosts` holding just the hypervisors rather than pointing the
+test at your everyday `known_hosts`, so a re-keyed hypervisor is a
+one-line fix in a file nothing else uses.
+
+Also optional: `KVM_TEST_IMAGE_SHA256`, `KVM_TEST_POOL`
+(default `default`), `KVM_TEST_SOCKET`, `KVM_TEST_TIMEOUT` (default
+`20m`, must cover the first-time image download), `KVM_TEST_KEEP`
+to leave the VM behind for `virsh console` when something goes wrong,
+and `KVM_TEST_LOG_LEVEL` (default `1`) for how much of the provider's
+logging the `-v` output shows: at `1` every libvirt RPC is one compact
+line (`HH:MM:SS.mmm libvirt virDomainLookupByName domain=... duration=...
+error=... libvirtError=ErrNoDomain`), interleaved with the test's own
+`t.Log` lines; `2` adds the domain and volume XML, metadata and
+addresses; `0` leaves only `t.Log`. The per-run constants (provider,
+host, domain name) are left off each line. See the Logging section of
+[kvm-provider.md](kvm-provider.md#logging).
+
+What one run asserts, in order: `New` dials the host; `Create` returns a
+`hosts/<host>/domains/<name>` ID (downloading the image on a cold host —
+the log prints how long it took); a second `Create` returns the same ID;
+`Get` reaches `Running` with an IPv4 within the budget, and the UID and
+`CreatedAt` survive the metadata round-trip through libvirt; `ListByTag`
+lists it; `Delete` removes it, `Get` says `ErrNotFound`, a second
+`Delete` is a no-op, and the pool holds the base image and its marker
+but no per-VM overlay or seed. It also tries a TCP connect to port 22 on
+the guest, but only logs the outcome — the machine running the test may
+not route to the bridge network.
+
+`TestKVMCheck_realHost` runs the provider health check
+([kvm-provider.md](kvm-provider.md#health-check)) against the same host.
+It must find the host reachable and the pool running, and it creates
+nothing. It needs the same variables; to run only this test:
+
+```sh
+go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/
+```
+
+The test is not part of `make test` and not run in CI. Keep it compiling
+with `go vet -tags=integration ./...` when the provider's internals move.
+
+## The scaffolded `make test-e2e` suite
+
+`test/e2e/` is the kubebuilder-generated smoke suite (build image → kind
+cluster → deploy → assert the manager pod runs, publishes a Ready
+`ProviderHealth` for the `kubernetes` provider, and serves metrics that
+include that provider's `proxy_operator_provider_target_up` series). It
+compiles only under `-tags=e2e`, manages its own kind cluster
+(`make test-e2e` / `make cleanup-test-e2e`), and has been kept compiling
+(`go vet -tags=e2e ./...` is part of the routine) but is **not part of
+`make test` and was not used for the release verification** — the manual
+kind run below covers strictly more.
+
+### The OTel tracing spec (Tempo-gated)
+
+`test/e2e/tracing_test.go` proves the full tracing pipeline against a
+**real Tempo**: deploy in kind with tracing enabled, create two
+kubernetes-provider proxies, delete them, and assert in Tempo that the
+traces exist with the documented span topology (`Reconcile Proxy` →
+`reconcile.managed` / `provider.create` / `status.patch`, and
+`reconcile.delete` / `provider.delete` on the way out).
+
+It **skips unless both env vars are set** (so the rest of the suite runs
+anywhere). Homelab invocation:
+
+```bash
+TEMPO_URL=http://192.168.0.30:3200 \
+OTLP_ENDPOINT=http://192.168.0.30:4318 \
+make test-e2e
+```
+
+Worth knowing:
+
+- Every span of a run carries the resource attribute
+  `test.run.id=e2e-<nanos>` (injected via `OTEL_RESOURCE_ATTRIBUTES`, no
+  code involved); the run ID is printed in the test log, and
+  `{resource.test.run.id="<id>"}` in Grafana shows exactly that run.
+- The spec preflights the OTLP endpoint **from inside the cluster** with a
+  curl pod and fails fast with a clear message if it's unreachable —
+  export failures are otherwise only visible at `-zap-log-level=1`.
+- Proxy CRs are created in `default`, not the operator namespace: the
+  squid pods carry no securityContext and the operator namespace enforces
+  restricted PSS.
+
+## The kind verification run (the real end-to-end)
+
+The README quickstart **is** the e2e test, run by hand before the MR:
+deploy the operator in-cluster on a throwaway kind cluster, watch a real
+Squid pod reach `Ready` via a real CONNECT probe, then exercise the whole
+lease lifecycle through the discovery API (list → lease → report
+`rate_limited` → same-target lease answered 409 `inCooldown:1` → release
+204 twice) and delete the CR to watch the finalizer remove the pod.
+
+Worth knowing about that run (full detail in
+[plans-executions/2026-08-07-1747-proxy-operator.md](plans-executions/2026-08-07-1747-proxy-operator.md),
+"Verification" section): it was not a clean pass-through. It caught two
+real bugs the automated suites structurally could not —
+
+1. `make run-dev` on a laptop can never produce a `Ready`
+   kubernetes-provider proxy: health probes originate on the host, which
+   cannot route to kind's pod network. The quickstart was rewritten to
+   deploy in-cluster.
+2. Squid was OOM-killed at startup: it sizes file-descriptor tables from
+   `RLIMIT_NOFILE`, which containerd under kind sets effectively
+   unlimited. Fixed with `max_filedescriptors 1024` in the generated
+   config, plus a regression assertion in `pod_test.go`.
+
+That is the pattern to keep: when the kind run finds something, the fix
+lands **with a unit-level regression test**, so the manual run stays a
+discovery tool rather than a recurring gate.

go.mod

@@ -3,8 +3,23 @@ module gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator
 go 1.26.0
 
 require (
+	cloud.google.com/go/compute v1.65.0
+	github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858
+	github.com/go-logr/logr v1.4.4
+	github.com/kdomanski/iso9660 v0.4.0
 	github.com/onsi/ginkgo/v2 v2.27.4
 	github.com/onsi/gomega v1.39.0
+	github.com/prometheus/client_golang v1.23.2
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
+	go.opentelemetry.io/otel v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0
+	go.opentelemetry.io/otel/sdk v1.45.0
+	go.opentelemetry.io/otel/trace v1.45.0
+	golang.org/x/crypto v0.54.0
+	google.golang.org/api v0.292.0
+	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
 	k8s.io/api v0.36.0
 	k8s.io/apimachinery v0.36.0
 	k8s.io/client-go v0.36.0
@@ -13,7 +28,10 @@ require (
 )
 
 require (
-	cel.dev/expr v0.25.1 // indirect
+	cel.dev/expr v0.25.2 // indirect
+	cloud.google.com/go/auth v0.22.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -23,10 +41,9 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/felixge/httpsnoop v1.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
-	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
@@ -37,17 +54,20 @@ require (
 	github.com/google/gnostic-models v0.7.0 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect
+	github.com/googleapis/gax-go/v2 v2.23.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.19.2 // indirect
@@ -56,33 +76,28 @@ require (
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
-	go.opentelemetry.io/otel v1.41.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.41.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.40.0 // indirect
-	go.opentelemetry.io/otel/trace v1.41.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
+	go.opentelemetry.io/otel/metric v1.45.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
-	golang.org/x/mod v0.32.0 // indirect
-	golang.org/x/net v0.49.0 // indirect
-	golang.org/x/oauth2 v0.34.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.40.0 // indirect
-	golang.org/x/term v0.39.0 // indirect
-	golang.org/x/text v0.33.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
-	golang.org/x/tools v0.41.0 // indirect
+	golang.org/x/mod v0.37.0 // indirect
+	golang.org/x/net v0.57.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/term v0.45.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	golang.org/x/tools v0.47.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
-	google.golang.org/grpc v1.79.3 // indirect
-	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
+	google.golang.org/grpc v1.83.0 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -1,5 +1,15 @@
-cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
-cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
+cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs=
+cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
+cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
+cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
+cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ=
+cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s=
+cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
+cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
+cloud.google.com/go/compute v1.65.0 h1:K0a3NRvazE7sZn5qswwI6BtlaZv1fgR5wFop5LZCLz8=
+cloud.google.com/go/compute v1.65.0/go.mod h1:vFq+Ztj9Rzhc8zf1t6hGp/6NdrEVG1GakkyVRQPRgKc=
+cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
+cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
@@ -18,14 +28,16 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858 h1:8xCFt73OddCD5WXxoYidJBWp3bJMC+CpE7Zwf5tcwk8=
+github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858/go.mod h1:qb0Ofa71d3oXARQf633h2tNaeBxLsVxuDp+jcsVO2+4=
 github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
 github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
 github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
 github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
-github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
-github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
+github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
@@ -37,8 +49,8 @@ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6O
 github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
 github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
-github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
+github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
@@ -68,10 +80,16 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
+github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
+github.com/googleapis/enterprise-certificate-proxy v0.3.19 h1:mMOE7DN2+p76/EdIrmAy9B9bH+yC4563vmnJ34QR8i4=
+github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k=
+github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE=
+github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -80,6 +98,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE
 github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg=
+github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -154,24 +174,28 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0=
-go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
-go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
-go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
-go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
-go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
-go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
-go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
-go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
-go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
-go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
-go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
-go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
+go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
+go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=
+go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
+go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
+go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
+go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
+go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
+go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
+go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
+go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
+go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
+go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -182,36 +206,42 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
 go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=
 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
-golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
-golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
-golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
-golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
-golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
-golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
-golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
-golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
-golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
-golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
-golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
-golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
-golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
-gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
-google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
-google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
-google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
+google.golang.org/api v0.292.0 h1:Ewiwo/GTtiaPZSNAZQUcWLh8AYDEoPmIXyJfeoTSMHU=
+google.golang.org/api v0.292.0/go.mod h1:07kjmMnFGm2RQuCza2EZM/5N68G/fVvFb1xKjWqoFA0=
+google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
+google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
+google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

hack/providers-dev.yaml

@@ -0,0 +1,6 @@
+# Providers config for `make run-dev`: local development against the
+# current kubeconfig context (e.g. a kind cluster). Only the
+# kubernetes-pod provider — no cloud credentials needed.
+providers:
+  - name: kubernetes
+    type: kubernetes

internal/controller/proxy_controller.go

@@ -41,6 +41,7 @@ import (
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // HealthSnapshotter provides the current probe verdict for a proxy. The
@@ -82,6 +83,9 @@ type ProxyReconciler struct {
 // +kubebuilder:rbac:groups=crawl.example.com,resources=proxies/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=crawl.example.com,resources=proxies/finalizers,verbs=update
 // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
+// The pod verbs are for the kubernetes-pod provider; cluster-scoped, since
+// its ListByTag enumerates the operator's Pods across all namespaces.
+// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;delete
 
 // Reconcile fetches the Proxy named by req into p (r.Get fills the struct
 // through the pointer), dispatches to the delete/external/managed state
@@ -93,9 +97,14 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 	}
 	base := p.DeepCopy()
 	defer func() {
+		// Runs inside the root reconcile span (this defer fires before the
+		// tracing.NewReconciler wrapper sees the return), and its error is
+		// folded into err, which that wrapper records.
+		pctx, span := tracing.Start(ctx, "status.patch")
+		defer span.End()
 		// NotFound is expected when this reconcile just removed the last
 		// finalizer and the object is already gone.
-		if perr := r.patchStatusIfChanged(ctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
+		if perr := r.patchStatusIfChanged(pctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
 			err = errors.Join(err, perr)
 		}
 	}()
@@ -111,6 +120,8 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 }
 
 func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.managed")
+	defer span.End()
 	log := logf.FromContext(ctx)
 
 	if controllerutil.AddFinalizer(p, crawlv1alpha1.FinalizerName) {
@@ -221,6 +232,8 @@ func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1
 // exists" — hence: delete, poll to NotFound, only then advance the hash and
 // let the create branch run.
 func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.Proxy, prov provider.Provider, hash string) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.replaceInstance")
+	defer span.End()
 	_, err := prov.Get(ctx, p.Status.ProviderID)
 	if provider.Class(err) == provider.ErrNotFound {
 		// Old instance is gone. The Update inside setSpecHash refreshes p
@@ -249,14 +262,22 @@ func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.
 }
 
 func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.delete")
+	defer span.End()
 	if !controllerutil.ContainsFinalizer(p, crawlv1alpha1.FinalizerName) {
 		return ctrl.Result{}, nil
 	}
 	if p.Status.ProviderID == "" {
 		// Nothing was ever recorded as created; orphan GC reaps any stray
 		// instance a crashed create might have left behind.
-		controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
-		return ctrl.Result{}, r.Update(ctx, p)
+		return r.releaseFinalizer(ctx, p)
+	}
+	if p.Annotations[crawlv1alpha1.AnnotationSkipCleanup] == "true" {
+		// Checked before the provider lookup so it also releases a Proxy whose
+		// provider is gone from the config. The instance is left running.
+		logf.FromContext(ctx).Info("skip-cleanup annotation set; abandoning the instance",
+			"providerID", p.Status.ProviderID, "provider", p.Spec.Provider)
+		return r.releaseFinalizer(ctx, p)
 	}
 	prov, ok := r.Providers[p.Spec.Provider]
 	if !ok {
@@ -264,11 +285,22 @@ func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.
 			"provider %q is not configured; cannot clean up instance %s", p.Spec.Provider, p.Status.ProviderID)
 	}
 	_, err := prov.Get(ctx, p.Status.ProviderID)
-	if provider.Class(err) == provider.ErrNotFound {
-		controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
-		return ctrl.Result{}, r.Update(ctx, p)
+	switch {
+	case err == nil:
+	case provider.Class(err) == provider.ErrNotFound:
+		return r.releaseFinalizer(ctx, p)
+	case provider.Class(err) == provider.ErrPermanent:
+		// Get will never answer for this instance — a host retired from the
+		// config, a providerID the provider can no longer resolve. Delete is
+		// idempotent and may still finish, so try it instead of polling a Get
+		// that cannot converge, and release the object when it does.
+		if delErr := prov.Delete(ctx, p.Status.ProviderID); delErr != nil {
+			return r.deletionFailure(errors.Join(err, delErr))
 		}
-	if err != nil {
+		logf.FromContext(ctx).Info("instance unreachable but cleanup succeeded; removing the finalizer",
+			"providerID", p.Status.ProviderID, "error", err.Error())
+		return r.releaseFinalizer(ctx, p)
+	default:
 		return r.deletionFailure(err)
 	}
 	if err := prov.Delete(ctx, p.Status.ProviderID); err != nil {
@@ -279,6 +311,11 @@ func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.
 	return ctrl.Result{RequeueAfter: r.DeletionPoll}, nil
 }
 
+func (r *ProxyReconciler) releaseFinalizer(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
+	controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
+	return ctrl.Result{}, r.Update(ctx, p)
+}
+
 func (r *ProxyReconciler) reconcileExternal(_ context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
 	if p.Spec.Endpoint == nil {
 		// CEL guarantees an endpoint on any object that went through the API
@@ -406,7 +443,10 @@ func (r *ProxyReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	if r.HealthEvents != nil {
 		b = b.WatchesRawSource(source.Channel(r.HealthEvents, &handler.EnqueueRequestForObject{}))
 	}
-	return b.Complete(r)
+	// Root span per reconcile; sub-reconcilers and the status flush hang
+	// their spans off it. Tests calling r.Reconcile directly bypass the
+	// wrapper and see no-op spans — the global tracer is never set there.
+	return b.Complete(tracing.NewReconciler("Proxy", r))
 }
 
 func (r *ProxyReconciler) applyDefaults() {

internal/controller/proxy_controller_test.go

@@ -28,6 +28,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
 )
 
@@ -233,6 +234,105 @@ var _ = Describe("Proxy controller", func() {
 		Expect(apierrors.IsNotFound(err)).To(BeTrue(), "proxy should be fully deleted")
 	})
 
+	It("reaches Ready once the health engine has a verdict", func() {
+		const name = "e2e-ready"
+		stub := &stubProvider{createID: "inst-rdy"}
+		r := newEnvtestReconciler(stub)
+		DeferCleanup(func() { cleanup(r, stub, name) })
+
+		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec:       managedSpec(),
+		})).To(Succeed())
+		_, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		stub.getInst = &provider.Instance{ID: "inst-rdy", IP: "10.3.3.3", State: provider.StateRunning}
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseProvisioning),
+			"no health verdict yet — must not be Ready")
+
+		By("supplying a healthy snapshot")
+		r.Health = fakeSnapshotter{ok: true, snap: health.Snapshot{
+			Healthy: true, Latency: 21 * time.Millisecond, LastProbe: time.Now(),
+		}}
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		p := fetch(name)
+		Expect(p.Status.Phase).To(Equal(crawlv1alpha1.PhaseReady))
+		Expect(p.Status.LatencyMillis).To(Equal(int64(21)))
+	})
+
+	It("treats quota exhaustion as a wait and a permanent error as Failed", func() {
+		const name = "e2e-errors"
+		stub := &stubProvider{
+			createErr: provider.Wrap(provider.ErrQuotaExceeded, "create", "stub", "", nil),
+		}
+		r := newEnvtestReconciler(stub)
+		DeferCleanup(func() { cleanup(r, stub, name) })
+
+		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec:       managedSpec(),
+		})).To(Succeed())
+		_, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("quota: condition set, slow requeue, phase NOT Failed")
+		res, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred(), "quota must not count as an error (stays off the backoff curve)")
+		Expect(res.RequeueAfter).To(Equal(r.QuotaRetry))
+		p := fetch(name)
+		cond := apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionProvisioned)
+		Expect(cond.Reason).To(Equal(ReasonQuotaExceeded))
+		Expect(p.Status.Phase).NotTo(Equal(crawlv1alpha1.PhaseFailed))
+
+		By("permanent: phase Failed and no further provider calls")
+		stub.createErr = provider.Wrap(provider.ErrPermanent, "create", "stub", "", nil)
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseFailed))
+		callsAfterLatch := stub.createCalls
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(stub.createCalls).To(Equal(callsAfterLatch), "the latch must stop provider calls")
+	})
+
+	It("adopts an instance when the spec-hash annotation is stripped", func() {
+		const name = "e2e-adopt"
+		stub := &stubProvider{createID: "inst-adopt"}
+		r := newEnvtestReconciler(stub)
+		DeferCleanup(func() { cleanup(r, stub, name) })
+
+		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec:       managedSpec(),
+		})).To(Succeed())
+		_, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		stub.getInst = &provider.Instance{ID: "inst-adopt", IP: "10.4.4.4", State: provider.StateRunning}
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		originalHash := fetch(name).Annotations[crawlv1alpha1.AnnotationSpecHash]
+		Expect(originalHash).NotTo(BeEmpty())
+
+		By("stripping the annotation, as an operator-version upgrade with a changed hash input would")
+		p := fetch(name)
+		delete(p.Annotations, crawlv1alpha1.AnnotationSpecHash)
+		Expect(k8sClient.Update(ctx, p)).To(Succeed())
+
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		p = fetch(name)
+		Expect(p.Annotations[crawlv1alpha1.AnnotationSpecHash]).To(Equal(originalHash), "hash must be restored")
+		Expect(p.Status.ProviderID).To(Equal("inst-adopt"), "adoption must keep the instance")
+		Expect(stub.deleteCalls).To(BeZero(), "adoption must never replace")
+	})
+
 	It("tracks an External proxy without touching providers", func() {
 		const name = "e2e-external"
 		stub := &stubProvider{}
@@ -264,3 +364,157 @@ var _ = Describe("Proxy controller", func() {
 		Expect(apierrors.IsNotFound(err)).To(BeTrue())
 	})
 })
+
+// These specs assert the CRD's CEL rules and structural defaulting against
+// the real envtest API server — the fake client runs neither, which is the
+// documented caveat on the action-table unit tests.
+var _ = Describe("Proxy CRD validation (CEL)", func() {
+	const ns = "default"
+
+	managed := func(name string) *crawlv1alpha1.Proxy {
+		return &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec: crawlv1alpha1.ProxySpec{
+				Mode:     crawlv1alpha1.ModeManaged,
+				Provider: "stub",
+			},
+		}
+	}
+	external := func(name string) *crawlv1alpha1.Proxy {
+		return &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec: crawlv1alpha1.ProxySpec{
+				Mode:     crawlv1alpha1.ModeExternal,
+				Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
+			},
+		}
+	}
+	mustCreate := func(p *crawlv1alpha1.Proxy) {
+		GinkgoHelper()
+		Expect(k8sClient.Create(ctx, p)).To(Succeed())
+		DeferCleanup(func() { _ = k8sClient.Delete(ctx, p) })
+	}
+
+	It("rejects invalid creates", func() {
+		invalid := []struct {
+			about string
+			spec  crawlv1alpha1.ProxySpec
+			want  string
+		}{
+			{
+				about: "Managed without provider",
+				spec:  crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged},
+				want:  "provider is required when mode is Managed",
+			},
+			{
+				about: "External with provider",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeExternal, Provider: "stub",
+					Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
+				},
+				want: "provider must not be set when mode is External",
+			},
+			{
+				about: "External without endpoint",
+				spec:  crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeExternal},
+				want:  "endpoint is required when mode is External",
+			},
+			{
+				about: "Managed with endpoint",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
+					Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
+				},
+				want: "endpoint must not be set when mode is Managed",
+			},
+			{
+				about: "cloudInit with both inline and secretRef",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
+					CloudInit: &crawlv1alpha1.CloudInitSpec{
+						Inline:    "#cloud-config",
+						SecretRef: &crawlv1alpha1.SecretKeySelector{Name: "s"},
+					},
+				},
+				want: "exactly one of inline or secretRef",
+			},
+			{
+				about: "cloudInit with neither inline nor secretRef",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
+					CloudInit: &crawlv1alpha1.CloudInitSpec{},
+				},
+				want: "exactly one of inline or secretRef",
+			},
+		}
+		for _, tc := range invalid {
+			p := &crawlv1alpha1.Proxy{
+				ObjectMeta: metav1.ObjectMeta{Name: "cel-invalid", Namespace: ns},
+				Spec:       tc.spec,
+			}
+			err := k8sClient.Create(ctx, p)
+			Expect(err).To(HaveOccurred(), tc.about)
+			Expect(err.Error()).To(ContainSubstring(tc.want), tc.about)
+		}
+	})
+
+	It("rejects mode mutation", func() {
+		p := external("cel-mode-immutable")
+		mustCreate(p)
+		p.Spec = crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged, Provider: "stub"}
+		err := k8sClient.Update(ctx, p)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("mode is immutable"))
+	})
+
+	It("rejects provider mutation and removal", func() {
+		p := managed("cel-provider-immutable")
+		mustCreate(p)
+
+		p.Spec.Provider = "other"
+		err := k8sClient.Update(ctx, p)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("provider is immutable"))
+
+		// Removal must also be rejected — the has()==has() form exists
+		// exactly because a field-level rule would not fire on absence.
+		// (Dropping provider alone would also trip the required-iff rule,
+		// so flip mode too and check the immutability rules win.)
+		fresh := fetchProxy(ns, "cel-provider-immutable")
+		fresh.Spec.Provider = ""
+		fresh.Spec.Endpoint = &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"}
+		fresh.Spec.Mode = crawlv1alpha1.ModeExternal
+		err = k8sClient.Update(ctx, fresh)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("immutable"))
+	})
+
+	It("materializes every nested healthCheck default when healthCheck is omitted", func() {
+		p := managed("cel-defaults")
+		mustCreate(p)
+
+		got := fetchProxy(ns, "cel-defaults")
+		// The +kubebuilder:default={} assertion: structural defaulting only
+		// descends into values that exist, so without it a nil healthCheck
+		// would get none of these.
+		hc := got.Spec.HealthCheck
+		Expect(hc).NotTo(BeNil())
+		Expect(hc.ProbeURL).To(Equal(crawlv1alpha1.DefaultProbeURL))
+		Expect(hc.IntervalSeconds).To(Equal(crawlv1alpha1.DefaultHealthCheckIntervalSeconds))
+		Expect(hc.TimeoutSeconds).To(Equal(crawlv1alpha1.DefaultHealthCheckTimeoutSeconds))
+		Expect(hc.FailureThreshold).To(Equal(crawlv1alpha1.DefaultFailureThreshold))
+		Expect(hc.SuccessThreshold).To(Equal(crawlv1alpha1.DefaultSuccessThreshold))
+		Expect(hc.ExpectedStatusCodes).To(Equal(crawlv1alpha1.DefaultExpectedStatusCodes))
+
+		Expect(got.Spec.Port).To(Equal(crawlv1alpha1.DefaultPort))
+		Expect(got.Spec.MaxLeases).NotTo(BeNil())
+		Expect(*got.Spec.MaxLeases).To(Equal(crawlv1alpha1.DefaultMaxLeases))
+	})
+})
+
+func fetchProxy(ns, name string) *crawlv1alpha1.Proxy {
+	GinkgoHelper()
+	p := &crawlv1alpha1.Proxy{}
+	Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, p)).To(Succeed())
+	return p
+}

internal/controller/reconcile_test.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"context"
+	"slices"
 	"testing"
 	"time"
 
@@ -78,6 +79,10 @@ func notFoundErr() error {
 	return provider.Wrap(provider.ErrNotFound, "get", "stub", "some-id", nil)
 }
 
+func permanentErr() error {
+	return provider.Wrap(provider.ErrPermanent, "get", "stub", "some-id", nil)
+}
+
 func testScheme(t *testing.T) *runtime.Scheme {
 	t.Helper()
 	s := runtime.NewScheme()
@@ -125,6 +130,15 @@ func withSpecHashAnnotation(hash string) func(*crawlv1alpha1.Proxy) {
 	}
 }
 
+func withSkipCleanup() func(*crawlv1alpha1.Proxy) {
+	return func(p *crawlv1alpha1.Proxy) {
+		if p.Annotations == nil {
+			p.Annotations = map[string]string{}
+		}
+		p.Annotations[crawlv1alpha1.AnnotationSkipCleanup] = "true"
+	}
+}
+
 func deleting() func(*crawlv1alpha1.Proxy) {
 	return func(p *crawlv1alpha1.Proxy) {
 		now := metav1.Now()
@@ -375,6 +389,59 @@ func TestReconcile_actionTable(t *testing.T) {
 				assertProxyGone(t, r)
 			},
 		},
+		{
+			// A permanent Get (a retired KVM host, a providerID the provider
+			// cannot resolve) used to retry forever with the finalizer on.
+			name:       "deletion with an unreachable instance still deletes and releases",
+			proxy:      managedProxy(deleting(), withProviderID("stub-id-1")),
+			stub:       &stubProvider{getErr: permanentErr()},
+			wantResult: ctrl.Result{},
+			verify: func(t *testing.T, r *ProxyReconciler, stub *stubProvider) {
+				assertProxyGone(t, r)
+				if stub.deleteCalls != 1 {
+					t.Errorf("deleteCalls = %d, want 1 — Delete must be tried when Get cannot answer", stub.deleteCalls)
+				}
+			},
+		},
+		{
+			name:  "deletion keeps the finalizer when cleanup itself fails",
+			proxy: managedProxy(deleting(), withProviderID("stub-id-1")),
+			stub: &stubProvider{
+				getErr:    permanentErr(),
+				deleteErr: provider.Wrap(provider.ErrPermanent, "delete", "stub", "stub-id-1", nil),
+			},
+			wantResult: ctrl.Result{},
+			wantErr:    true,
+			verify: func(t *testing.T, r *ProxyReconciler, _ *stubProvider) {
+				// getProxy fails the test if the object is gone, which is what
+				// finalizer removal would have caused.
+				if p := getProxy(t, r); !slices.Contains(p.Finalizers, crawlv1alpha1.FinalizerName) {
+					t.Error("finalizer removed although cleanup failed; the instance would leak silently")
+				}
+			},
+		},
+		{
+			name:       "skip-cleanup annotation abandons the instance",
+			proxy:      managedProxy(deleting(), withProviderID("stub-id-1"), withSkipCleanup()),
+			stub:       &stubProvider{getInst: &provider.Instance{ID: "stub-id-1", State: provider.StateRunning}},
+			wantResult: ctrl.Result{},
+			verify: func(t *testing.T, r *ProxyReconciler, stub *stubProvider) {
+				assertProxyGone(t, r)
+				if stub.getCalls != 0 || stub.deleteCalls != 0 {
+					t.Errorf("getCalls/deleteCalls = %d/%d, want 0/0 — the provider must not be asked", stub.getCalls, stub.deleteCalls)
+				}
+			},
+		},
+		{
+			name: "skip-cleanup annotation releases a proxy whose provider is gone",
+			proxy: managedProxy(deleting(), withProviderID("stub-id-1"), withSkipCleanup(),
+				func(p *crawlv1alpha1.Proxy) { p.Spec.Provider = "retired" }),
+			stub:       &stubProvider{},
+			wantResult: ctrl.Result{},
+			verify: func(t *testing.T, r *ProxyReconciler, _ *stubProvider) {
+				assertProxyGone(t, r)
+			},
+		},
 		{
 			name:  "deletion deletes the instance and polls",
 			proxy: managedProxy(deleting(), withProviderID("stub-id-1")),

internal/discovery/handlers.go

@@ -0,0 +1,245 @@
+package discovery
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"slices"
+	"strings"
+	"time"
+
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+)
+
+// proxyView is the wire shape of one proxy in list and lease responses.
+type proxyView struct {
+	ID            string            `json:"id"` // namespace/name
+	IP            string            `json:"ip"`
+	Port          int32             `json:"port"`
+	Attributes    map[string]string `json:"attributes,omitempty"`
+	Phase         string            `json:"phase"`
+	Healthy       bool              `json:"healthy"`
+	LatencyMillis int64             `json:"latencyMillis"`
+	ActiveLeases  int               `json:"activeLeases"`
+	MaxLeases     int32             `json:"maxLeases"`
+}
+
+func viewOf(p *crawlv1alpha1.Proxy, activeLeases int) proxyView {
+	return proxyView{
+		ID:            client.ObjectKeyFromObject(p).String(),
+		IP:            p.EffectiveHost(),
+		Port:          p.EffectivePort(),
+		Attributes:    p.Spec.Attributes,
+		Phase:         string(p.Status.Phase),
+		Healthy:       isHealthy(p),
+		LatencyMillis: p.Status.LatencyMillis,
+		ActiveLeases:  activeLeases,
+		MaxLeases:     p.MaxLeasesOrDefault(),
+	}
+}
+
+func isHealthy(p *crawlv1alpha1.Proxy) bool {
+	return apimeta.IsStatusConditionTrue(p.Status.Conditions, crawlv1alpha1.ConditionHealthy)
+}
+
+// matchesAttributes is spec.attributes equality: every selector pair must
+// be present verbatim.
+func matchesAttributes(p *crawlv1alpha1.Proxy, selector map[string]string) bool {
+	for k, v := range selector {
+		if p.Spec.Attributes[k] != v {
+			return false
+		}
+	}
+	return true
+}
+
+// GET /v1/proxies?attr.<key>=<value>&healthy=true|false
+func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
+	selector := map[string]string{}
+	var healthyFilter *bool
+	for key, values := range r.URL.Query() {
+		switch {
+		case key == "healthy":
+			switch values[0] {
+			case "true":
+				healthyFilter = ptr(true)
+			case "false":
+				healthyFilter = ptr(false)
+			default:
+				writeError(w, http.StatusBadRequest, "invalid_query",
+					fmt.Sprintf("healthy must be true or false, got %q", values[0]))
+				return
+			}
+		case strings.HasPrefix(key, "attr."):
+			selector[strings.TrimPrefix(key, "attr.")] = values[0]
+		}
+	}
+
+	var list crawlv1alpha1.ProxyList
+	if err := s.Reader.List(r.Context(), &list); err != nil {
+		// The ctx logger is s.log enriched with this request's
+		// traceID/spanID by the tracing middleware (logr sinks can't read
+		// ctx at log time, so per-request values ride on the logger).
+		logf.FromContext(r.Context()).Error(err, "listing proxies")
+		writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
+		return
+	}
+	counts := s.Store.Counts()
+
+	views := []proxyView{}
+	for i := range list.Items {
+		p := &list.Items[i]
+		if !p.DeletionTimestamp.IsZero() || !matchesAttributes(p, selector) {
+			continue
+		}
+		v := viewOf(p, counts[client.ObjectKeyFromObject(p).String()])
+		if healthyFilter != nil && v.Healthy != *healthyFilter {
+			continue
+		}
+		views = append(views, v)
+	}
+	slices.SortFunc(views, func(a, b proxyView) int { return strings.Compare(a.ID, b.ID) })
+
+	writeJSON(w, http.StatusOK, map[string]any{"proxies": views, "count": len(views)})
+}
+
+type leaseRequest struct {
+	Selector   map[string]string `json:"selector"`
+	TTLSeconds int64             `json:"ttlSeconds"`
+	Target     string            `json:"target"`
+}
+
+type leaseResponse struct {
+	LeaseID    string    `json:"leaseID"`
+	Proxy      proxyView `json:"proxy"`
+	ExpiresAt  time.Time `json:"expiresAt"`
+	TTLSeconds int64     `json:"ttlSeconds"`
+}
+
+// POST /v1/leases
+func (s *Server) handleAcquireLease(w http.ResponseWriter, r *http.Request) {
+	var req leaseRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid_body", err.Error())
+		return
+	}
+
+	ttl := time.Duration(req.TTLSeconds) * time.Second
+	if req.TTLSeconds == 0 {
+		ttl = defaultTTL
+	}
+	if ttl < 0 || ttl > s.MaxLeaseTTL {
+		writeError(w, http.StatusBadRequest, "invalid_ttl",
+			fmt.Sprintf("ttlSeconds must be between 1 and %d", int64(s.MaxLeaseTTL.Seconds())))
+		return
+	}
+
+	var list crawlv1alpha1.ProxyList
+	if err := s.Reader.List(r.Context(), &list); err != nil {
+		logf.FromContext(r.Context()).Error(err, "listing proxies for lease")
+		writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
+		return
+	}
+
+	// The store gets only healthy, live candidates; unhealthy matches are
+	// counted here because the store never sees them.
+	var candidates []lease.Candidate
+	byKey := map[string]*crawlv1alpha1.Proxy{}
+	unhealthy := 0
+	for i := range list.Items {
+		p := &list.Items[i]
+		if !p.DeletionTimestamp.IsZero() || !matchesAttributes(p, req.Selector) {
+			continue
+		}
+		if !isHealthy(p) {
+			unhealthy++
+			continue
+		}
+		key := client.ObjectKeyFromObject(p).String()
+		byKey[key] = p
+		candidates = append(candidates, lease.Candidate{
+			Proxy:     key,
+			MaxLeases: p.MaxLeasesOrDefault(),
+			Latency:   time.Duration(p.Status.LatencyMillis) * time.Millisecond,
+		})
+	}
+
+	granted, stats, err := s.Store.Acquire(r.Context(), lease.AcquireRequest{
+		Candidates: candidates,
+		Target:     req.Target,
+		TTL:        ttl,
+	})
+	if errors.Is(err, lease.ErrNoMatch) {
+		if s.Metrics != nil {
+			s.Metrics.LeaseRequest("no_match")
+		}
+		writeJSON(w, http.StatusConflict, map[string]any{
+			"error":      "no_match",
+			"message":    "no healthy proxy with free capacity matched the selector",
+			"considered": stats.Considered + unhealthy,
+			"atCapacity": stats.AtCapacity,
+			"inCooldown": stats.InCooldown,
+			"unhealthy":  unhealthy,
+		})
+		return
+	}
+	if err != nil {
+		writeError(w, http.StatusBadRequest, "invalid_request", err.Error())
+		return
+	}
+
+	if s.Metrics != nil {
+		s.Metrics.LeaseRequest("granted")
+	}
+	writeJSON(w, http.StatusCreated, leaseResponse{
+		LeaseID:    granted.ID,
+		Proxy:      viewOf(byKey[granted.Proxy], s.Store.Counts()[granted.Proxy]),
+		ExpiresAt:  granted.ExpiresAt,
+		TTLSeconds: int64(ttl.Seconds()),
+	})
+}
+
+// DELETE /v1/leases/{id} — early release, always 204: releasing an unknown
+// or already expired lease is not an error.
+func (s *Server) handleReleaseLease(w http.ResponseWriter, r *http.Request) {
+	s.Store.Release(r.Context(), r.PathValue("id"))
+	w.WriteHeader(http.StatusNoContent)
+}
+
+type reportRequest struct {
+	Result string `json:"result"`
+	Target string `json:"target"`
+}
+
+// POST /v1/leases/{id}/report
+func (s *Server) handleReportLease(w http.ResponseWriter, r *http.Request) {
+	var req reportRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid_body", err.Error())
+		return
+	}
+	result, ok := lease.ParseResult(req.Result)
+	if !ok {
+		writeError(w, http.StatusBadRequest, "invalid_result",
+			fmt.Sprintf("result must be one of ok, rate_limited, banned; got %q", req.Result))
+		return
+	}
+	if err := s.Store.Report(r.Context(), r.PathValue("id"), result, req.Target); err != nil {
+		if errors.Is(err, lease.ErrUnknownLease) {
+			writeError(w, http.StatusNotFound, "unknown_lease", "no such lease")
+			return
+		}
+		logf.FromContext(r.Context()).Error(err, "reporting lease", "leaseID", r.PathValue("id"))
+		writeError(w, http.StatusInternalServerError, "internal", "report failed")
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func ptr[T any](v T) *T { return &v }

internal/discovery/server.go

@@ -0,0 +1,238 @@
+// Package discovery implements the HTTP API crawler clients use to find
+// and lease proxies: list healthy proxies filtered by attributes, acquire a
+// TTL-based lease, release it early, and report how a target treated the
+// proxy. Reads go through the manager's informer cache; lease state lives
+// in the injected store.
+package discovery
+
+import (
+	"context"
+	"crypto/subtle"
+	"encoding/json"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/go-logr/logr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+// LeaseStore is what the handlers need from a lease backend. Defined here,
+// consumer-side, so a CRD- or Redis-backed store can replace the in-memory
+// one (which internal/lease's *Store satisfies) without touching handlers.
+type LeaseStore interface {
+	Acquire(ctx context.Context, req lease.AcquireRequest) (*lease.Lease, lease.AcquireStats, error)
+	Release(ctx context.Context, id string)
+	Report(ctx context.Context, id string, result lease.Result, target string) error
+	Counts() map[string]int
+}
+
+// LeaseMetrics counts lease acquisitions by outcome. Implemented by
+// internal/metrics; defined here so this package carries no metrics
+// dependency.
+type LeaseMetrics interface {
+	LeaseRequest(outcome string)
+}
+
+const (
+	defaultAddr     = ":8090"
+	defaultTTL      = 5 * time.Minute
+	defaultMaxTTL   = time.Hour
+	maxBodyBytes    = 64 << 10
+	shutdownGrace   = 10 * time.Second
+	readHeadTimeout = 5 * time.Second
+)
+
+// Server serves the discovery API as a manager Runnable.
+type Server struct {
+	// Reader lists Proxies from the manager's cache.
+	Reader client.Reader
+	// Store is the lease backend.
+	Store LeaseStore
+	// Addr is the listen address (default ":8090"; --discovery-addr).
+	Addr string
+	// Token is the static bearer token from DISCOVERY_TOKEN. Empty
+	// disables auth — allowed for the prototype, but loudly warned about
+	// at startup, because in-cluster that is a silent security hole.
+	Token string
+	// MaxLeaseTTL caps requested lease TTLs (default 1h; --max-lease-ttl).
+	MaxLeaseTTL time.Duration
+	// Metrics, when non-nil, counts lease requests by outcome.
+	Metrics LeaseMetrics
+
+	log logr.Logger
+
+	mu        sync.Mutex
+	boundAddr string
+}
+
+// NeedLeaderElection is false, and the deployment ships replicas: 1.
+// Verified against controller-runtime's runnable ordering: caches start and
+// sync before non-leader-election runnables, so cache reads here are safe.
+// If this were leader-elected, non-leader replicas would refuse connections
+// while still being Service endpoints. The 1-replica constraint comes from
+// lease state being per-process — both facts are README caveats.
+func (s *Server) NeedLeaderElection() bool { return false }
+
+// BoundAddr returns the actual listen address once Start has bound it —
+// meaningful when Addr uses port 0 (tests).
+func (s *Server) BoundAddr() string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.boundAddr
+}
+
+// Start listens and serves until ctx ends, then shuts down gracefully with
+// a 10-second grace period.
+func (s *Server) Start(ctx context.Context) error {
+	if s.Addr == "" {
+		s.Addr = defaultAddr
+	}
+	if s.MaxLeaseTTL == 0 {
+		s.MaxLeaseTTL = defaultMaxTTL
+	}
+	s.log = logf.FromContext(ctx).WithName("discovery")
+	if s.Token == "" {
+		s.log.Info("WARNING: DISCOVERY_TOKEN is empty — the discovery API is served without authentication")
+	}
+
+	ln, err := net.Listen("tcp", s.Addr)
+	if err != nil {
+		return err
+	}
+	s.mu.Lock()
+	s.boundAddr = ln.Addr().String()
+	s.mu.Unlock()
+
+	srv := &http.Server{
+		Handler:           s.handler(),
+		ReadHeaderTimeout: readHeadTimeout,
+		ReadTimeout:       10 * time.Second,
+		WriteTimeout:      10 * time.Second,
+		IdleTimeout:       60 * time.Second,
+	}
+
+	errCh := make(chan error, 1)
+	go func() { errCh <- srv.Serve(ln) }()
+	s.log.Info("discovery API listening", "addr", s.boundAddr)
+
+	select {
+	case err := <-errCh:
+		return err
+	case <-ctx.Done():
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownGrace)
+		defer cancel()
+		if err := srv.Shutdown(shutdownCtx); err != nil {
+			return err
+		}
+		<-errCh // always http.ErrServerClosed after a clean Shutdown
+		return nil
+	}
+}
+
+// handler assembles the mux and the middleware chain, outermost first:
+// recover → tracing (server span + request logger) → request-log →
+// body-size cap → bearer auth. Everything inside the tracing layer logs via
+// logf.FromContext(r.Context()) and so carries traceID/spanID.
+func (s *Server) handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok\n"))
+	})
+	mux.HandleFunc("GET /v1/proxies", s.handleListProxies)
+	mux.HandleFunc("POST /v1/leases", s.handleAcquireLease)
+	mux.HandleFunc("DELETE /v1/leases/{id}", s.handleReleaseLease)
+	mux.HandleFunc("POST /v1/leases/{id}/report", s.handleReportLease)
+
+	var h http.Handler = mux
+	h = s.authMiddleware(h)
+	h = maxBytesMiddleware(h)
+	h = s.logMiddleware(h)
+	h = tracing.HTTPMiddleware("discovery", s.log)(h)
+	h = s.recoverMiddleware(h)
+	return h
+}
+
+func (s *Server) recoverMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if p := recover(); p != nil {
+				s.log.Error(nil, "panic in discovery handler", "panic", p, "path", r.URL.Path)
+				writeError(w, http.StatusInternalServerError, "internal", "internal server error")
+			}
+		}()
+		next.ServeHTTP(w, r)
+	})
+}
+
+// statusRecorder captures the response code for the request log.
+type statusRecorder struct {
+	http.ResponseWriter
+	status int
+}
+
+func (r *statusRecorder) WriteHeader(code int) {
+	r.status = code
+	r.ResponseWriter.WriteHeader(code)
+}
+
+func (s *Server) logMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/healthz" {
+			next.ServeHTTP(w, r) // probes are noise
+			return
+		}
+		rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
+		start := time.Now()
+		next.ServeHTTP(rec, r)
+		logf.FromContext(r.Context()).Info("request",
+			"method", r.Method, "path", r.URL.Path,
+			"status", rec.status, "duration", time.Since(start).String())
+	})
+}
+
+func maxBytesMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
+		next.ServeHTTP(w, r)
+	})
+}
+
+func (s *Server) authMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if s.Token == "" || r.URL.Path == "/healthz" {
+			next.ServeHTTP(w, r)
+			return
+		}
+		token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if !ok || subtle.ConstantTimeCompare([]byte(token), []byte(s.Token)) != 1 {
+			writeError(w, http.StatusUnauthorized, "unauthorized", "missing or invalid bearer token")
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+// errorBody is the shared error shape:
+// {"error":"<machine_code>","message":"<human>"}.
+type errorBody struct {
+	Error   string `json:"error"`
+	Message string `json:"message"`
+}
+
+func writeError(w http.ResponseWriter, status int, code, message string) {
+	writeJSON(w, status, errorBody{Error: code, Message: message})
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(v)
+}

internal/discovery/server_test.go

@@ -0,0 +1,386 @@
+package discovery
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+)
+
+func testProxy(name string, attrs map[string]string, healthy bool, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy {
+	p := &crawlv1alpha1.Proxy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name, Namespace: "default", UID: types.UID("uid-" + name),
+		},
+		Spec: crawlv1alpha1.ProxySpec{
+			Mode:       crawlv1alpha1.ModeExternal,
+			Endpoint:   &crawlv1alpha1.EndpointSpec{Host: "10.0.0.1", Port: 3128},
+			Attributes: attrs,
+		},
+	}
+	p.Status.IP = "10.0.0.1"
+	p.Status.Phase = crawlv1alpha1.PhaseReady
+	status := metav1.ConditionFalse
+	if healthy {
+		status = metav1.ConditionTrue
+	}
+	p.Status.Conditions = []metav1.Condition{{
+		Type: crawlv1alpha1.ConditionHealthy, Status: status,
+		Reason: "Probing", LastTransitionTime: metav1.Now(),
+	}}
+	for _, m := range mut {
+		m(p)
+	}
+	return p
+}
+
+func withMaxLeases(n int32) func(*crawlv1alpha1.Proxy) {
+	return func(p *crawlv1alpha1.Proxy) { p.Spec.MaxLeases = &n }
+}
+
+func withLatency(ms int64) func(*crawlv1alpha1.Proxy) {
+	return func(p *crawlv1alpha1.Proxy) { p.Status.LatencyMillis = ms }
+}
+
+// newTestServer wires the handler chain to a fake cache reader and a real
+// lease store, served over httptest.
+func newTestServer(t *testing.T, token string, proxies ...*crawlv1alpha1.Proxy) (*httptest.Server, *Server) {
+	t.Helper()
+	s := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("scheme: %v", err)
+	}
+	builder := fake.NewClientBuilder().WithScheme(s)
+	for _, p := range proxies {
+		builder = builder.WithObjects(p)
+	}
+	srv := &Server{
+		Reader:      builder.Build(),
+		Store:       lease.NewStore(15 * time.Minute),
+		Token:       token,
+		MaxLeaseTTL: time.Hour,
+	}
+	ts := httptest.NewServer(srv.handler())
+	t.Cleanup(ts.Close)
+	return ts, srv
+}
+
+type response struct {
+	status int
+	body   map[string]any
+}
+
+func do(t *testing.T, ts *httptest.Server, method, path, token string, body any) response {
+	t.Helper()
+	var reader io.Reader
+	if body != nil {
+		if s, ok := body.(string); ok {
+			reader = bytes.NewBufferString(s)
+		} else {
+			b, err := json.Marshal(body)
+			if err != nil {
+				t.Fatalf("marshaling request body: %v", err)
+			}
+			reader = bytes.NewBuffer(b)
+		}
+	}
+	req, err := http.NewRequestWithContext(context.Background(), method, ts.URL+path, reader)
+	if err != nil {
+		t.Fatalf("building request: %v", err)
+	}
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
+	}
+	resp, err := ts.Client().Do(req)
+	if err != nil {
+		t.Fatalf("%s %s: %v", method, path, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	out := response{status: resp.StatusCode}
+	raw, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("reading response: %v", err)
+	}
+	if len(raw) > 0 && resp.Header.Get("Content-Type") == "application/json" {
+		if err := json.Unmarshal(raw, &out.body); err != nil {
+			t.Fatalf("decoding response %q: %v", raw, err)
+		}
+	}
+	return out
+}
+
+func TestAuth(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "sekrit", testProxy("p1", nil, true))
+
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "", nil); got.status != http.StatusUnauthorized {
+		t.Errorf("no token: status %d, want 401", got.status)
+	}
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "wrong", nil); got.status != http.StatusUnauthorized {
+		t.Errorf("wrong token: status %d, want 401", got.status)
+	}
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "sekrit", nil); got.status != http.StatusOK {
+		t.Errorf("correct token: status %d, want 200", got.status)
+	}
+	if got := do(t, ts, http.MethodGet, "/healthz", "", nil); got.status != http.StatusOK {
+		t.Errorf("healthz without token: status %d, want 200 (always unauthenticated)", got.status)
+	}
+}
+
+func TestAuth_disabledWithEmptyToken(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "", nil); got.status != http.StatusOK {
+		t.Errorf("status %d, want 200 with auth disabled", got.status)
+	}
+}
+
+func TestListProxies(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "",
+		testProxy("eu-healthy", map[string]string{"geo": "eu", "purpose": "crawl"}, true),
+		testProxy("eu-sick", map[string]string{"geo": "eu"}, false),
+		testProxy("us-healthy", map[string]string{"geo": "us"}, true),
+	)
+
+	tests := []struct {
+		name      string
+		query     string
+		wantCount int
+		wantFirst string
+	}{
+		{name: "no filter returns everything", query: "", wantCount: 3, wantFirst: "default/eu-healthy"},
+		{name: "healthy filter", query: "?healthy=true", wantCount: 2},
+		{name: "unhealthy filter", query: "?healthy=false", wantCount: 1, wantFirst: "default/eu-sick"},
+		{name: "attribute filter", query: "?attr.geo=eu", wantCount: 2},
+		{name: "attribute and health combined", query: "?attr.geo=eu&healthy=true", wantCount: 1, wantFirst: "default/eu-healthy"},
+		{name: "two attributes must both match", query: "?attr.geo=eu&attr.purpose=crawl", wantCount: 1, wantFirst: "default/eu-healthy"},
+		{name: "no matches is 200 with count 0", query: "?attr.geo=mars", wantCount: 0},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got := do(t, ts, http.MethodGet, "/v1/proxies"+tc.query, "", nil)
+			if got.status != http.StatusOK {
+				t.Fatalf("status %d, want 200", got.status)
+			}
+			count := int(got.body["count"].(float64))
+			proxies := got.body["proxies"].([]any)
+			if count != tc.wantCount || len(proxies) != tc.wantCount {
+				t.Fatalf("count = %d (len %d), want %d", count, len(proxies), tc.wantCount)
+			}
+			if tc.wantFirst != "" {
+				first := proxies[0].(map[string]any)
+				if first["id"] != tc.wantFirst {
+					t.Errorf("first id = %v, want %s", first["id"], tc.wantFirst)
+				}
+			}
+		})
+	}
+
+	t.Run("invalid healthy value is 400", func(t *testing.T) {
+		t.Parallel()
+		if got := do(t, ts, http.MethodGet, "/v1/proxies?healthy=maybe", "", nil); got.status != http.StatusBadRequest {
+			t.Errorf("status %d, want 400", got.status)
+		}
+	})
+}
+
+func TestAcquireLease_grantShape(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "",
+		testProxy("eu1", map[string]string{"geo": "eu"}, true, withLatency(30)),
+		testProxy("eu2", map[string]string{"geo": "eu"}, true, withLatency(10)),
+	)
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
+		"selector": map[string]string{"geo": "eu"},
+	})
+	if got.status != http.StatusCreated {
+		t.Fatalf("status %d (%v), want 201", got.status, got.body)
+	}
+	if got.body["leaseID"] == "" || got.body["leaseID"] == nil {
+		t.Error("empty leaseID")
+	}
+	if got.body["ttlSeconds"].(float64) != 300 {
+		t.Errorf("ttlSeconds = %v, want the 300 default", got.body["ttlSeconds"])
+	}
+	proxy := got.body["proxy"].(map[string]any)
+	if proxy["id"] != "default/eu2" {
+		t.Errorf("granted %v, want default/eu2 (lower latency at equal load)", proxy["id"])
+	}
+	if proxy["activeLeases"].(float64) != 1 {
+		t.Errorf("activeLeases = %v, want 1 (this grant included)", proxy["activeLeases"])
+	}
+	if _, err := time.Parse(time.RFC3339, got.body["expiresAt"].(string)); err != nil {
+		t.Errorf("expiresAt %v is not RFC3339: %v", got.body["expiresAt"], err)
+	}
+}
+
+func TestAcquireLease_noMatchBody(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "",
+		testProxy("eu-tiny", map[string]string{"geo": "eu"}, true, withMaxLeases(1)),
+		testProxy("eu-sick", map[string]string{"geo": "eu"}, false),
+	)
+
+	body := map[string]any{"selector": map[string]string{"geo": "eu"}}
+	if got := do(t, ts, http.MethodPost, "/v1/leases", "", body); got.status != http.StatusCreated {
+		t.Fatalf("first acquire: status %d, want 201", got.status)
+	}
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", body)
+	if got.status != http.StatusConflict {
+		t.Fatalf("second acquire: status %d, want 409", got.status)
+	}
+	want := map[string]float64{"considered": 2, "atCapacity": 1, "inCooldown": 0, "unhealthy": 1}
+	for k, v := range want {
+		if got.body[k].(float64) != v {
+			t.Errorf("%s = %v, want %v (body %v)", k, got.body[k], v, got.body)
+		}
+	}
+	if got.body["error"] != "no_match" {
+		t.Errorf("error = %v, want no_match", got.body["error"])
+	}
+}
+
+func TestAcquireLease_badRequests(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
+
+	tests := []struct {
+		name     string
+		body     any
+		wantCode string
+	}{
+		{name: "ttl above the cap", body: map[string]any{"ttlSeconds": 999999}, wantCode: "invalid_ttl"},
+		{name: "negative ttl", body: map[string]any{"ttlSeconds": -5}, wantCode: "invalid_ttl"},
+		{name: "malformed json", body: "{not json", wantCode: "invalid_body"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got := do(t, ts, http.MethodPost, "/v1/leases", "", tc.body)
+			if got.status != http.StatusBadRequest || got.body["error"] != tc.wantCode {
+				t.Errorf("= %d/%v, want 400/%s", got.status, got.body["error"], tc.wantCode)
+			}
+		})
+	}
+}
+
+func TestReleaseLease_alwaysNoContent(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{})
+	if got.status != http.StatusCreated {
+		t.Fatalf("acquire: status %d, want 201", got.status)
+	}
+	id := got.body["leaseID"].(string)
+
+	for _, path := range []string{"/v1/leases/" + id, "/v1/leases/" + id, "/v1/leases/never-existed"} {
+		if got := do(t, ts, http.MethodDelete, path, "", nil); got.status != http.StatusNoContent {
+			t.Errorf("DELETE %s: status %d, want 204", path, got.status)
+		}
+	}
+}
+
+func TestReportLease(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", map[string]string{"geo": "eu"}, true))
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
+		"selector": map[string]string{"geo": "eu"}, "target": "example.com",
+	})
+	if got.status != http.StatusCreated {
+		t.Fatalf("acquire: status %d, want 201", got.status)
+	}
+	id := got.body["leaseID"].(string)
+	reportPath := fmt.Sprintf("/v1/leases/%s/report", id)
+
+	if got := do(t, ts, http.MethodPost, reportPath, "", map[string]any{"result": "rate_limited", "target": "example.com"}); got.status != http.StatusNoContent {
+		t.Fatalf("report: status %d, want 204", got.status)
+	}
+
+	// The cooldown from the report now blocks same-target acquisition.
+	got = do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
+		"selector": map[string]string{"geo": "eu"}, "target": "example.com",
+	})
+	if got.status != http.StatusConflict || got.body["inCooldown"].(float64) != 1 {
+		t.Errorf("post-report acquire = %d/%v, want 409 with inCooldown 1", got.status, got.body)
+	}
+
+	t.Run("invalid result value", func(t *testing.T) {
+		got := do(t, ts, http.MethodPost, reportPath, "", map[string]any{"result": "throttled"})
+		if got.status != http.StatusBadRequest || got.body["error"] != "invalid_result" {
+			t.Errorf("= %d/%v, want 400/invalid_result", got.status, got.body["error"])
+		}
+	})
+	t.Run("unknown lease", func(t *testing.T) {
+		got := do(t, ts, http.MethodPost, "/v1/leases/never-existed/report", "", map[string]any{"result": "ok"})
+		if got.status != http.StatusNotFound || got.body["error"] != "unknown_lease" {
+			t.Errorf("= %d/%v, want 404/unknown_lease", got.status, got.body["error"])
+		}
+	})
+}
+
+func TestStart_servesAndShutsDown(t *testing.T) {
+	t.Parallel()
+	s := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("scheme: %v", err)
+	}
+	srv := &Server{
+		Reader: fake.NewClientBuilder().WithScheme(s).Build(),
+		Store:  lease.NewStore(time.Minute),
+		Addr:   "127.0.0.1:0",
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- srv.Start(ctx) }()
+
+	var addr string
+	deadline := time.After(5 * time.Second)
+	for addr == "" {
+		select {
+		case <-deadline:
+			t.Fatal("server never bound")
+		case <-time.After(5 * time.Millisecond):
+			addr = srv.BoundAddr()
+		}
+	}
+
+	resp, err := http.Get("http://" + addr + "/healthz")
+	if err != nil {
+		t.Fatalf("healthz: %v", err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("healthz status %d, want 200", resp.StatusCode)
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start returned %v, want nil after graceful shutdown", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}

internal/gc/gc.go

@@ -0,0 +1,137 @@
+// Package gc implements orphan garbage collection: a periodic sweep that
+// deletes provider instances tagged by this operator whose owning Proxy CR
+// no longer exists — the safety net for crashes between a provider Create
+// and the status write that records it.
+package gc
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"go.opentelemetry.io/otel/attribute"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+// Sweeper is the manager Runnable running the sweep loop.
+type Sweeper struct {
+	// Reader lists Proxies from the manager's cache to establish the live
+	// UID set.
+	Reader client.Reader
+	// Providers are the configured backends; each is swept independently.
+	Providers map[string]provider.Provider
+
+	// Interval between sweeps (default 10m). The first sweep runs one full
+	// interval after start, not immediately — right after startup the
+	// cache is coldest and an in-flight create is most likely.
+	Interval time.Duration
+	// MinAge exempts young instances (default 10m): an instance mid-create
+	// may not have its status write landed yet; deleting it would race the
+	// reconciler.
+	MinAge time.Duration
+
+	// NamespaceRestricted must be set when the manager cache is limited to
+	// one namespace. Then the live-UID set is incomplete, and a sweep
+	// would delete VMs owned by Proxies the cache cannot see — so Start
+	// refuses unless AllowNamespaced (--gc-allow-namespaced) is explicit.
+	NamespaceRestricted bool
+	AllowNamespaced     bool
+
+	now func() time.Time
+}
+
+// NeedLeaderElection is true: the sweep is destructive and must have a
+// single writer.
+func (s *Sweeper) NeedLeaderElection() bool { return true }
+
+// Start runs the sweep loop until ctx ends.
+func (s *Sweeper) Start(ctx context.Context) error {
+	if s.NamespaceRestricted && !s.AllowNamespaced {
+		return errors.New(
+			"orphan GC refuses to run against a namespace-restricted cache: proxies outside the namespace " +
+				"would count as orphans and their instances would be deleted; pass --gc-allow-namespaced to override")
+	}
+	if s.Interval == 0 {
+		s.Interval = 10 * time.Minute
+	}
+	if s.MinAge == 0 {
+		s.MinAge = 10 * time.Minute
+	}
+	if s.now == nil {
+		s.now = time.Now
+	}
+
+	ticker := time.NewTicker(s.Interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			s.sweep(ctx)
+		}
+	}
+}
+
+// sweep deletes tagged instances whose UID matches no existing Proxy CR.
+// A CR with a deletionTimestamp still counts as live: its finalizer owns
+// that deletion, and GC racing it would double-delete. A UID becomes
+// orphan-eligible only once the object is fully gone.
+func (s *Sweeper) sweep(ctx context.Context) {
+	ctx, span := tracing.Start(ctx, "gc.sweep")
+	defer span.End()
+	log := logf.FromContext(ctx).WithName("orphan-gc")
+
+	var list crawlv1alpha1.ProxyList
+	if err := s.Reader.List(ctx, &list); err != nil {
+		// Without the live set nothing can be proven orphaned; skip the
+		// whole sweep rather than guess.
+		log.Error(err, "listing proxies; skipping this sweep")
+		return
+	}
+	live := make(map[string]bool, len(list.Items))
+	for i := range list.Items {
+		live[string(list.Items[i].UID)] = true
+	}
+
+	deleted := 0
+	for name, prov := range s.Providers {
+		instances, err := prov.ListByTag(ctx)
+		if err != nil {
+			// One broken provider must not abort the sweep for the rest.
+			log.Error(err, "listing instances; skipping this provider", "provider", name)
+			continue
+		}
+		for _, inst := range instances {
+			switch {
+			case inst.UID == "":
+				// Managed label without a UID label shouldn't exist for
+				// anything this operator created; without ownership proof,
+				// never delete.
+				continue
+			case live[inst.UID]:
+				continue
+			case s.now().Sub(inst.CreatedAt) < s.MinAge:
+				continue
+			}
+			log.Info("WARNING: deleting orphaned instance",
+				"provider", name, "providerID", inst.ID, "uid", inst.UID)
+			if err := prov.Delete(ctx, inst.ID); err != nil {
+				log.Error(err, "deleting orphaned instance",
+					"provider", name, "providerID", inst.ID, "uid", inst.UID)
+				continue
+			}
+			deleted++
+		}
+	}
+	span.SetAttributes(
+		attribute.Int("gc.providers", len(s.Providers)),
+		attribute.Int("gc.deleted", deleted),
+	)
+}

internal/gc/gc_test.go

@@ -0,0 +1,214 @@
+package gc
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// listProvider serves a canned instance list and records deletions.
+type listProvider struct {
+	mu        sync.Mutex
+	instances []provider.Instance
+	listErr   error
+	deleted   []string
+}
+
+func (l *listProvider) Create(context.Context, provider.CreateRequest) (string, error) {
+	return "", errors.New("not used")
+}
+
+func (l *listProvider) Get(context.Context, string) (*provider.Instance, error) {
+	return nil, provider.ErrNotFound
+}
+
+func (l *listProvider) Delete(_ context.Context, id string) error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.deleted = append(l.deleted, id)
+	return nil
+}
+
+func (l *listProvider) ListByTag(context.Context) ([]provider.Instance, error) {
+	return l.instances, l.listErr
+}
+
+func (l *listProvider) deletedIDs() []string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return append([]string(nil), l.deleted...)
+}
+
+func proxyWithUID(name, uid string, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy {
+	p := &crawlv1alpha1.Proxy{
+		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", UID: types.UID(uid)},
+		Spec: crawlv1alpha1.ProxySpec{
+			Mode:     crawlv1alpha1.ModeManaged,
+			Provider: "stub",
+		},
+	}
+	for _, m := range mut {
+		m(p)
+	}
+	return p
+}
+
+func newReader(t *testing.T, objs ...client.Object) client.Reader {
+	t.Helper()
+	s := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("scheme: %v", err)
+	}
+	return fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build()
+}
+
+func oldInstance(id, uid string) provider.Instance {
+	return provider.Instance{ID: id, UID: uid, State: provider.StateRunning,
+		CreatedAt: time.Now().Add(-time.Hour)}
+}
+
+func newSweeper(reader client.Reader, providers map[string]provider.Provider) *Sweeper {
+	return &Sweeper{
+		Reader:    reader,
+		Providers: providers,
+		Interval:  10 * time.Minute,
+		MinAge:    10 * time.Minute,
+		now:       time.Now,
+	}
+}
+
+func TestSweep_deletesOnlyTrueOrphans(t *testing.T) {
+	t.Parallel()
+
+	deletingCR := proxyWithUID("deleting", "uid-deleting", func(p *crawlv1alpha1.Proxy) {
+		now := metav1.Now()
+		p.DeletionTimestamp = &now
+		p.Finalizers = []string{crawlv1alpha1.FinalizerName}
+	})
+	prov := &listProvider{instances: []provider.Instance{
+		oldInstance("inst-live", "uid-live"),
+		oldInstance("inst-orphan", "uid-orphan"),
+		oldInstance("inst-deleting", "uid-deleting"),
+		{ID: "inst-young", UID: "uid-young-orphan", State: provider.StateRunning,
+			CreatedAt: time.Now().Add(-time.Minute)},
+		oldInstance("inst-unlabelled", ""),
+	}}
+	s := newSweeper(
+		newReader(t, proxyWithUID("live", "uid-live"), deletingCR),
+		map[string]provider.Provider{"stub": prov},
+	)
+
+	s.sweep(context.Background())
+
+	got := prov.deletedIDs()
+	if len(got) != 1 || got[0] != "inst-orphan" {
+		t.Errorf("deleted %v, want exactly [inst-orphan]:\n"+
+			"live CR's instance must stay; a deleting CR still owns its instance (finalizer, not GC);\n"+
+			"young instances may be mid-create; unlabelled instances have no ownership proof", got)
+	}
+}
+
+func TestSweep_providerErrorDoesNotAbortOthers(t *testing.T) {
+	t.Parallel()
+	broken := &listProvider{listErr: errors.New("cloud is down")}
+	working := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
+	s := newSweeper(newReader(t), map[string]provider.Provider{
+		"broken":  broken,
+		"working": working,
+	})
+
+	s.sweep(context.Background())
+
+	if got := working.deletedIDs(); len(got) != 1 {
+		t.Errorf("working provider deletions = %v, want the orphan despite the broken provider", got)
+	}
+}
+
+// errReader fails every List: without the live set nothing can be proven
+// orphaned, so the sweep must delete nothing.
+type errReader struct{ client.Reader }
+
+func (errReader) List(context.Context, client.ObjectList, ...client.ListOption) error {
+	return errors.New("cache broken")
+}
+
+func TestSweep_listFailureSkipsSweep(t *testing.T) {
+	t.Parallel()
+	prov := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
+	s := newSweeper(errReader{}, map[string]provider.Provider{"stub": prov})
+
+	s.sweep(context.Background())
+
+	if got := prov.deletedIDs(); len(got) != 0 {
+		t.Errorf("deleted %v with an unreadable live set, want nothing", got)
+	}
+}
+
+func TestStart_namespaceGuard(t *testing.T) {
+	t.Parallel()
+
+	s := newSweeper(newReader(t), nil)
+	s.NamespaceRestricted = true
+	err := s.Start(context.Background())
+	if err == nil || !strings.Contains(err.Error(), "--gc-allow-namespaced") {
+		t.Errorf("Start with restricted cache = %v, want refusal naming the override flag", err)
+	}
+
+	s2 := newSweeper(newReader(t), nil)
+	s2.NamespaceRestricted = true
+	s2.AllowNamespaced = true
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- s2.Start(ctx) }()
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start with override = %v, want it to run until cancel", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}
+
+func TestStart_sweepsOnIntervalAndStops(t *testing.T) {
+	t.Parallel()
+	prov := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
+	s := newSweeper(newReader(t), map[string]provider.Provider{"stub": prov})
+	s.Interval = 5 * time.Millisecond
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- s.Start(ctx) }()
+
+	deadline := time.After(5 * time.Second)
+	for len(prov.deletedIDs()) == 0 {
+		select {
+		case <-deadline:
+			t.Fatal("no sweep ran")
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start = %v, want nil", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}

internal/health/engine.go

@@ -10,6 +10,9 @@ import (
 	"sync"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -18,6 +21,7 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // Snapshot is the engine's current verdict for one proxy, read by the
@@ -62,6 +66,14 @@ type probeJob struct {
 	interval time.Duration
 }
 
+// ProbeMetrics receives every probe result and the retirement of a
+// proxy's series. Implemented by internal/metrics; defined here so this
+// package carries no metrics dependency.
+type ProbeMetrics interface {
+	ObserveProbe(proxy string, latency time.Duration, success bool)
+	ForgetProxy(proxy string)
+}
+
 // Engine runs the probe scheduler and worker pool as a manager Runnable. It
 // never writes Proxy status itself — keeping the reconciler the single
 // status writer — and instead emits a GenericEvent per status-affecting
@@ -87,6 +99,14 @@ type Engine struct {
 	// ProbeTLSConfig overrides TLS verification for https probe URLs; nil
 	// means system roots. Needed for private CAs (and tests).
 	ProbeTLSConfig *tls.Config
+	// Metrics, when non-nil, is fed on every probe — the status writes are
+	// transition-only by design, so metrics are where high-frequency
+	// signal (true probe recency, every latency sample) lives.
+	Metrics ProbeMetrics
+	// TraceProbes emits one root span per probe (--trace-health-probes).
+	// Off by default: probes run about once per second per proxy and would
+	// dominate trace volume.
+	TraceProbes bool
 
 	probeFn func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult
 
@@ -143,8 +163,7 @@ func (e *Engine) Start(ctx context.Context) error {
 				case <-ctx.Done():
 					return
 				case job := <-jobs:
-					res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
-					e.record(job, res, time.Now())
+					e.record(job, e.runProbe(ctx, job), time.Now())
 				}
 			}
 		})
@@ -220,6 +239,11 @@ func (e *Engine) tick(ctx context.Context, now time.Time, jobs chan<- probeJob)
 	for key := range e.states {
 		if _, ok := probeable[key]; !ok {
 			delete(e.states, key)
+			if e.Metrics != nil {
+				// Retire the per-proxy series with the state, or series
+				// for deleted proxies leak forever.
+				e.Metrics.ForgetProxy(key.String())
+			}
 		}
 	}
 }
@@ -247,11 +271,38 @@ func newState(p *crawlv1alpha1.Proxy, now time.Time, interval time.Duration) *st
 	return st
 }
 
+// runProbe executes one probe, inside its own root span when TraceProbes is
+// on. The span ends before record folds the result (record stays ctx-free),
+// and the probe transport is deliberately uninstrumented — no traceparent
+// must ever leak through a proxy toward external targets.
+func (e *Engine) runProbe(ctx context.Context, job probeJob) probeResult {
+	if !e.TraceProbes {
+		return e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
+	}
+	ctx, span := tracing.Start(ctx, "health.probe",
+		trace.WithSpanKind(trace.SpanKindClient),
+		trace.WithAttributes(attribute.String("proxy", job.key.String())))
+	defer span.End()
+	res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
+	span.SetAttributes(
+		attribute.Bool("probe.ok", res.ok),
+		attribute.Int64("probe.latency_ms", res.latency.Milliseconds()),
+	)
+	if !res.ok {
+		span.SetStatus(codes.Error, res.err.Error())
+	}
+	return res
+}
+
 // record folds one probe result into the proxy's threshold state and emits
 // an event when the result is status-affecting: a first-ever verdict, a
 // threshold-crossing flip, or a material latency change (beyond
 // max(LatencyFloor, 50% of reported) and rate-limited by MinReportInterval).
 func (e *Engine) record(job probeJob, res probeResult, now time.Time) {
+	if e.Metrics != nil {
+		e.Metrics.ObserveProbe(job.key.String(), res.latency, res.ok)
+	}
+
 	e.mu.Lock()
 	defer e.mu.Unlock()
 

internal/lease/store.go

@@ -0,0 +1,319 @@
+// Package lease implements the in-memory lease store behind the discovery
+// API: TTL-based proxy assignment with server-side usage tracking and
+// per-(proxy, target) cooldowns. Accepted prototype limitation, documented
+// in the README: state is per-process, so an operator restart drops all
+// leases and cooldowns — clients must tolerate a lease vanishing (their
+// requests still work; they just re-lease).
+package lease
+
+import (
+	"cmp"
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Result is a client's report of how a leased proxy behaved against a
+// target. ResultRateLimited and ResultBanned record a cooldown; ResultOK is
+// an acknowledgement and records nothing.
+type Result string
+
+const (
+	ResultOK          Result = "ok"
+	ResultRateLimited Result = "rate_limited"
+	ResultBanned      Result = "banned"
+)
+
+// ParseResult maps a wire value to a Result; ok is false for anything
+// unknown, which the API layer turns into a 400.
+func ParseResult(s string) (Result, bool) {
+	switch r := Result(s); r {
+	case ResultOK, ResultRateLimited, ResultBanned:
+		return r, true
+	default:
+		return "", false
+	}
+}
+
+var (
+	// ErrNoMatch means no candidate could take a lease; AcquireStats says
+	// why, and the API layer turns both into the 409 body.
+	ErrNoMatch = errors.New("lease: no candidate available")
+	// ErrUnknownLease means the lease ID does not resolve (404). Reports on
+	// recently expired leases do NOT hit this — see the retention note on
+	// Store.
+	ErrUnknownLease = errors.New("lease: unknown lease id")
+)
+
+// Candidate is one leasable proxy as seen by the caller at selection time.
+// The store itself knows nothing about Proxy objects — the discovery layer
+// filters for health/attributes and passes what selection needs.
+type Candidate struct {
+	// Proxy is the opaque proxy key ("namespace/name").
+	Proxy string
+	// MaxLeases caps concurrent leases; 0 means unleasable.
+	MaxLeases int32
+	// Latency is the proxy's last reported latency, used as the tie-break.
+	Latency time.Duration
+}
+
+// Lease is a granted assignment. Values returned by the store are copies;
+// mutating them does not affect the store.
+type Lease struct {
+	ID        string
+	Proxy     string
+	Target    string
+	ExpiresAt time.Time
+}
+
+// AcquireRequest carries the candidate set and lease parameters. Acquire
+// deliberately takes the whole candidate set, not a pre-chosen proxy:
+// selection and insertion must happen under one lock, or two concurrent
+// requests both see "3 of 5 used" and overcommit.
+type AcquireRequest struct {
+	Candidates []Candidate
+	// Target scopes the cooldown check; empty means the global pool.
+	Target string
+	TTL    time.Duration
+}
+
+// AcquireStats explains an ErrNoMatch (and is returned on success too):
+// every candidate is either leased, at capacity, or in cooldown.
+type AcquireStats struct {
+	Considered int
+	AtCapacity int
+	InCooldown int
+}
+
+type cooldownKey struct{ proxy, target string }
+
+// Store is the in-memory lease store. One mutex guards everything: at tens
+// of proxies and human-rate QPS, sharding would be premature complexity.
+//
+// Retention: an expired lease is kept for CooldownWindow past its TTL so a
+// Report arriving just after expiry still resolves — which matters most
+// exactly when a proxy is being rate-limited. Acquire and the counts ignore
+// retained leases; only the sweep finally drops them.
+type Store struct {
+	// CooldownWindow is how long a reported proxy/target pair is excluded
+	// from selection (default 15m; --lease-cooldown in Step 10).
+	CooldownWindow time.Duration
+	// SweepInterval is how often the expiry sweep runs (default 30s).
+	SweepInterval time.Duration
+
+	now func() time.Time
+
+	mu        sync.Mutex
+	byID      map[string]*Lease
+	byProxy   map[string]map[string]*Lease
+	cooldowns map[cooldownKey]time.Time
+}
+
+// NewStore returns a ready Store. A non-positive cooldownWindow selects the
+// 15-minute default.
+func NewStore(cooldownWindow time.Duration) *Store {
+	if cooldownWindow <= 0 {
+		cooldownWindow = 15 * time.Minute
+	}
+	return &Store{
+		CooldownWindow: cooldownWindow,
+		SweepInterval:  30 * time.Second,
+		now:            time.Now,
+		byID:           map[string]*Lease{},
+		byProxy:        map[string]map[string]*Lease{},
+		cooldowns:      map[cooldownKey]time.Time{},
+	}
+}
+
+// Acquire selects the least-loaded eligible candidate (ties: lowest
+// latency, then name, so selection is deterministic and testable) and
+// grants a lease on it.
+func (s *Store) Acquire(_ context.Context, req AcquireRequest) (*Lease, AcquireStats, error) {
+	stats := AcquireStats{Considered: len(req.Candidates)}
+	if req.TTL <= 0 {
+		return nil, stats, fmt.Errorf("lease: non-positive TTL %v", req.TTL)
+	}
+	now := s.now()
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	type eligible struct {
+		cand   Candidate
+		active int
+	}
+	var elig []eligible
+	for _, c := range req.Candidates {
+		if s.inCooldownLocked(c.Proxy, req.Target, now) {
+			stats.InCooldown++
+			continue
+		}
+		active := s.activeCountLocked(c.Proxy, now)
+		if int32(active) >= c.MaxLeases {
+			stats.AtCapacity++
+			continue
+		}
+		elig = append(elig, eligible{cand: c, active: active})
+	}
+	if len(elig) == 0 {
+		return nil, stats, ErrNoMatch
+	}
+
+	slices.SortFunc(elig, func(a, b eligible) int {
+		if c := cmp.Compare(a.active, b.active); c != 0 {
+			return c
+		}
+		if c := cmp.Compare(a.cand.Latency, b.cand.Latency); c != 0 {
+			return c
+		}
+		return strings.Compare(a.cand.Proxy, b.cand.Proxy)
+	})
+
+	l := &Lease{
+		ID:        rand.Text(),
+		Proxy:     elig[0].cand.Proxy,
+		Target:    req.Target,
+		ExpiresAt: now.Add(req.TTL),
+	}
+	s.byID[l.ID] = l
+	if s.byProxy[l.Proxy] == nil {
+		s.byProxy[l.Proxy] = map[string]*Lease{}
+	}
+	s.byProxy[l.Proxy][l.ID] = l
+
+	granted := *l
+	return &granted, stats, nil
+}
+
+// Release drops a lease early. Idempotent: releasing an unknown or already
+// expired lease is a no-op, so the API's DELETE can always answer 204.
+func (s *Store) Release(_ context.Context, id string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.dropLocked(id)
+}
+
+// Report records the outcome of using a lease. Rate-limited and banned
+// results put the (proxy, target) pair in cooldown — target taken from the
+// report, falling back to the lease's own target, falling back to the
+// global pool. Reports on recently expired leases still resolve (see the
+// retention note on Store).
+func (s *Store) Report(_ context.Context, id string, result Result, target string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	l, ok := s.byID[id]
+	if !ok {
+		return ErrUnknownLease
+	}
+	if result == ResultOK {
+		return nil
+	}
+	if target == "" {
+		target = l.Target
+	}
+	s.cooldowns[cooldownKey{proxy: l.Proxy, target: target}] = s.now().Add(s.CooldownWindow)
+	return nil
+}
+
+// ActiveCount returns the number of unexpired leases held on one proxy.
+func (s *Store) ActiveCount(proxy string) int {
+	now := s.now()
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.activeCountLocked(proxy, now)
+}
+
+// Counts returns the active-lease count per proxy, for the discovery list
+// endpoint and the metrics collector. Proxies with no active leases are
+// absent from the map.
+func (s *Store) Counts() map[string]int {
+	now := s.now()
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	counts := make(map[string]int, len(s.byProxy))
+	for proxy := range s.byProxy {
+		if n := s.activeCountLocked(proxy, now); n > 0 {
+			counts[proxy] = n
+		}
+	}
+	return counts
+}
+
+// Start runs the expiry sweep until ctx ends; it satisfies
+// manager.Runnable so cmd/main.go can mgr.Add the store directly.
+func (s *Store) Start(ctx context.Context) error {
+	ticker := time.NewTicker(s.SweepInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			s.sweep(s.now())
+		}
+	}
+}
+
+// NeedLeaderElection is false: lease state is per-process and the discovery
+// API serves wherever this process runs, so the sweep must run there too.
+func (s *Store) NeedLeaderElection() bool { return false }
+
+// sweep drops leases past their retention window and elapsed cooldowns.
+// Correctness never depends on sweep timing — every read path checks
+// expiry against the clock — so this is purely garbage collection.
+func (s *Store) sweep(now time.Time) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for id, l := range s.byID {
+		if now.After(l.ExpiresAt.Add(s.CooldownWindow)) {
+			s.dropLocked(id)
+		}
+	}
+	for k, until := range s.cooldowns {
+		if now.After(until) {
+			delete(s.cooldowns, k)
+		}
+	}
+}
+
+func (s *Store) dropLocked(id string) {
+	l, ok := s.byID[id]
+	if !ok {
+		return
+	}
+	delete(s.byID, id)
+	delete(s.byProxy[l.Proxy], id)
+	if len(s.byProxy[l.Proxy]) == 0 {
+		delete(s.byProxy, l.Proxy)
+	}
+}
+
+func (s *Store) activeCountLocked(proxy string, now time.Time) int {
+	n := 0
+	for _, l := range s.byProxy[proxy] {
+		if now.Before(l.ExpiresAt) {
+			n++
+		}
+	}
+	return n
+}
+
+// inCooldownLocked: the global cooldown (empty target) always applies; a
+// target-scoped cooldown additionally applies to acquisitions for that
+// target. An acquisition without a target sees only the global pool — a
+// proxy rate-limited by one site is still fine for everyone else.
+func (s *Store) inCooldownLocked(proxy, target string, now time.Time) bool {
+	if until, ok := s.cooldowns[cooldownKey{proxy: proxy}]; ok && now.Before(until) {
+		return true
+	}
+	if target == "" {
+		return false
+	}
+	until, ok := s.cooldowns[cooldownKey{proxy: proxy, target: target}]
+	return ok && now.Before(until)
+}

internal/lease/store_test.go

@@ -0,0 +1,365 @@
+package lease
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+)
+
+// fakeClock is an injectable, manually advanced clock.
+type fakeClock struct {
+	mu  sync.Mutex
+	cur time.Time
+}
+
+func newFakeClock() *fakeClock {
+	return &fakeClock{cur: time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC)}
+}
+
+func (c *fakeClock) Now() time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.cur
+}
+
+func (c *fakeClock) Advance(d time.Duration) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.cur = c.cur.Add(d)
+}
+
+func newTestStore() (*Store, *fakeClock) {
+	s := NewStore(15 * time.Minute)
+	clock := newFakeClock()
+	s.now = clock.Now
+	return s, clock
+}
+
+func candidate(proxy string, maxLeases int32, latency time.Duration) Candidate {
+	return Candidate{Proxy: proxy, MaxLeases: maxLeases, Latency: latency}
+}
+
+func mustAcquire(t *testing.T, s *Store, req AcquireRequest) *Lease {
+	t.Helper()
+	l, _, err := s.Acquire(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Acquire: %v", err)
+	}
+	return l
+}
+
+func TestAcquire_capacity(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 2, 0)}, TTL: time.Minute}
+
+	l1 := mustAcquire(t, s, req)
+	l2 := mustAcquire(t, s, req)
+	if l1.ID == l2.ID {
+		t.Fatal("two leases share an ID")
+	}
+	if got := s.ActiveCount("ns/p1"); got != 2 {
+		t.Fatalf("ActiveCount = %d, want 2", got)
+	}
+
+	_, stats, err := s.Acquire(context.Background(), req)
+	if !errors.Is(err, ErrNoMatch) {
+		t.Fatalf("third acquire error = %v, want ErrNoMatch", err)
+	}
+	want := AcquireStats{Considered: 1, AtCapacity: 1}
+	if stats != want {
+		t.Errorf("stats = %+v, want %+v", stats, want)
+	}
+
+	// Early release frees the slot again.
+	s.Release(context.Background(), l1.ID)
+	mustAcquire(t, s, req)
+}
+
+func TestAcquire_maxLeasesZeroIsUnleasable(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+		Candidates: []Candidate{candidate("ns/p1", 0, 0)},
+		TTL:        time.Minute,
+	})
+	if !errors.Is(err, ErrNoMatch) {
+		t.Fatalf("err = %v, want ErrNoMatch", err)
+	}
+	if stats.AtCapacity != 1 {
+		t.Errorf("stats = %+v, want the unleasable proxy counted AtCapacity", stats)
+	}
+}
+
+func TestAcquire_selectionOrder(t *testing.T) {
+	t.Parallel()
+
+	t.Run("least loaded wins", func(t *testing.T) {
+		t.Parallel()
+		s, _ := newTestStore()
+		mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{candidate("ns/a", 5, 10*time.Millisecond)}, TTL: time.Minute,
+		})
+		l := mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{
+				candidate("ns/a", 5, 10*time.Millisecond), // 1 active, lower latency
+				candidate("ns/b", 5, 90*time.Millisecond), // 0 active
+			},
+			TTL: time.Minute,
+		})
+		if l.Proxy != "ns/b" {
+			t.Errorf("chose %s, want the least-loaded ns/b", l.Proxy)
+		}
+	})
+
+	t.Run("latency breaks the load tie", func(t *testing.T) {
+		t.Parallel()
+		s, _ := newTestStore()
+		l := mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{
+				candidate("ns/a", 5, 90*time.Millisecond),
+				candidate("ns/b", 5, 10*time.Millisecond),
+			},
+			TTL: time.Minute,
+		})
+		if l.Proxy != "ns/b" {
+			t.Errorf("chose %s, want the lower-latency ns/b", l.Proxy)
+		}
+	})
+
+	t.Run("name breaks a full tie deterministically", func(t *testing.T) {
+		t.Parallel()
+		s, _ := newTestStore()
+		l := mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{
+				candidate("ns/b", 5, 10*time.Millisecond),
+				candidate("ns/a", 5, 10*time.Millisecond),
+			},
+			TTL: time.Minute,
+		})
+		if l.Proxy != "ns/a" {
+			t.Errorf("chose %s, want ns/a (lexicographic tie-break)", l.Proxy)
+		}
+	})
+}
+
+func TestAcquire_cooldownScoping(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, "example.com"); err != nil {
+		t.Fatalf("Report: %v", err)
+	}
+
+	// Same target: excluded.
+	_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+		Candidates: cands, Target: "example.com", TTL: time.Minute,
+	})
+	if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
+		t.Errorf("same-target acquire = (%v, %+v), want ErrNoMatch with InCooldown=1", err, stats)
+	}
+
+	// Different target: fine.
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "other.org", TTL: time.Minute})
+
+	// No target (global pool): a target-scoped cooldown does not apply.
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+}
+
+func TestAcquire_globalCooldownBlocksEverything(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	// A lease without a target, reported banned without a target: the
+	// cooldown lands on the global pool.
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+	if err := s.Report(context.Background(), l.ID, ResultBanned, ""); err != nil {
+		t.Fatalf("Report: %v", err)
+	}
+
+	for _, target := range []string{"", "example.com"} {
+		_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+			Candidates: cands, Target: target, TTL: time.Minute,
+		})
+		if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
+			t.Errorf("acquire(target=%q) = (%v, %+v), want global cooldown to block", target, err, stats)
+		}
+	}
+}
+
+func TestAcquire_cooldownExpires(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); err != nil {
+		t.Fatalf("Report: %v", err)
+	}
+	if _, _, err := s.Acquire(context.Background(), AcquireRequest{Candidates: cands, TTL: time.Minute}); !errors.Is(err, ErrNoMatch) {
+		t.Fatal("expected cooldown to block immediately after the report")
+	}
+
+	clock.Advance(15*time.Minute + time.Second)
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+}
+
+func TestExpiry_freesCapacityWithoutSweep(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 1, 0)}, TTL: time.Minute}
+
+	mustAcquire(t, s, req)
+	if _, _, err := s.Acquire(context.Background(), req); !errors.Is(err, ErrNoMatch) {
+		t.Fatal("capacity 1 not enforced")
+	}
+
+	clock.Advance(2 * time.Minute)
+	// No sweep has run; expiry must still free capacity and zero the counts.
+	if got := s.ActiveCount("ns/p1"); got != 0 {
+		t.Fatalf("ActiveCount after TTL = %d, want 0", got)
+	}
+	if counts := s.Counts(); len(counts) != 0 {
+		t.Fatalf("Counts after TTL = %v, want empty", counts)
+	}
+	mustAcquire(t, s, req)
+}
+
+func TestReport_expiredButRetainedLease(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+
+	// TTL lapses; the report arrives late — exactly when the proxy is being
+	// rate-limited, which is when the cooldown matters most.
+	clock.Advance(5 * time.Minute)
+	s.sweep(clock.Now())
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); err != nil {
+		t.Fatalf("Report on an expired-but-retained lease: %v", err)
+	}
+	// The cooldown fell back to the lease's own target.
+	_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+		Candidates: cands, Target: "example.com", TTL: time.Minute,
+	})
+	if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
+		t.Errorf("acquire = (%v, %+v), want cooldown from the late report", err, stats)
+	}
+
+	// Past the retention window the sweep finally drops it.
+	clock.Advance(15 * time.Minute)
+	s.sweep(clock.Now())
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); !errors.Is(err, ErrUnknownLease) {
+		t.Errorf("Report after retention = %v, want ErrUnknownLease", err)
+	}
+}
+
+func TestReport_okRecordsNothing(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+
+	if err := s.Report(context.Background(), l.ID, ResultOK, "example.com"); err != nil {
+		t.Fatalf("Report(ok): %v", err)
+	}
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+}
+
+func TestRelease_isIdempotent(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	l := mustAcquire(t, s, AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 1, 0)}, TTL: time.Minute})
+
+	s.Release(context.Background(), l.ID)
+	s.Release(context.Background(), l.ID)
+	s.Release(context.Background(), "never-existed")
+	if got := s.ActiveCount("ns/p1"); got != 0 {
+		t.Errorf("ActiveCount = %d, want 0", got)
+	}
+}
+
+func TestParseResult(t *testing.T) {
+	t.Parallel()
+	for _, valid := range []string{"ok", "rate_limited", "banned"} {
+		if _, ok := ParseResult(valid); !ok {
+			t.Errorf("ParseResult(%q) rejected a valid value", valid)
+		}
+	}
+	for _, invalid := range []string{"", "OK", "throttled", "rate-limited"} {
+		if _, ok := ParseResult(invalid); ok {
+			t.Errorf("ParseResult(%q) accepted an invalid value", invalid)
+		}
+	}
+}
+
+func TestAcquire_concurrentNeverOvercommits(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 5, 0)}, TTL: time.Minute}
+
+	const attempts = 40
+	var wg sync.WaitGroup
+	granted := make(chan *Lease, attempts)
+	for range attempts {
+		wg.Go(func() {
+			if l, _, err := s.Acquire(context.Background(), req); err == nil {
+				granted <- l
+			}
+		})
+	}
+	wg.Wait()
+	close(granted)
+
+	var n int
+	for range granted {
+		n++
+	}
+	if n != 5 {
+		t.Errorf("%d of %d concurrent acquires granted, want exactly MaxLeases=5", n, attempts)
+	}
+	if got := s.ActiveCount("ns/p1"); got != 5 {
+		t.Errorf("ActiveCount = %d, want 5", got)
+	}
+}
+
+func TestStart_sweepsAndStops(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	s.SweepInterval = time.Millisecond
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 5, 0)}, TTL: time.Minute})
+	clock.Advance(20 * time.Minute) // past TTL + retention
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- s.Start(ctx) }()
+
+	deadline := time.After(5 * time.Second)
+	for {
+		if err := s.Report(context.Background(), l.ID, ResultOK, ""); errors.Is(err, ErrUnknownLease) {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("sweep never dropped the lease")
+		case <-time.After(5 * time.Millisecond):
+		}
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start returned %v, want nil", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}

internal/metrics/metrics.go

@@ -0,0 +1,167 @@
+// Package metrics defines the operator's Prometheus metrics. Nothing here
+// registers itself — no init(), per house rules — the composition root
+// calls Register explicitly, which also lets every test use a fresh
+// registry.
+package metrics
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metrics holds the vector metrics the operator's components feed. The
+// consuming packages (health, discovery, provider) each define their own
+// small recorder interface; *Metrics satisfies all of them structurally,
+// so none of them import this package's prometheus surface.
+type Metrics struct {
+	healthcheckDuration *prometheus.HistogramVec
+	healthcheckFailures *prometheus.CounterVec
+	leaseRequests       *prometheus.CounterVec
+	providerRequests    *prometheus.CounterVec
+}
+
+// New builds the metric set, unregistered.
+func New() *Metrics {
+	return &Metrics{
+		healthcheckDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "proxy_operator_healthcheck_duration_seconds",
+			Help:    "Duration of through-the-proxy health probes.",
+			Buckets: prometheus.DefBuckets,
+		}, []string{"proxy"}),
+		healthcheckFailures: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_healthcheck_failures_total",
+			Help: "Failed health probes.",
+		}, []string{"proxy"}),
+		leaseRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_lease_requests_total",
+			Help: "Lease acquisition requests by outcome.",
+		}, []string{"outcome"}),
+		providerRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_provider_requests_total",
+			Help: "Provider API calls by operation and classified result.",
+		}, []string{"provider", "op", "result"}),
+	}
+}
+
+// Register registers the vectors plus the two scrape-time collectors.
+// proxyPhases and activeLeases are read at every scrape: gauges derived
+// from reconcile-time increments inevitably drift and leak series on
+// delete; reading the source of truth cannot.
+func (m *Metrics) Register(reg prometheus.Registerer, proxyPhases func() map[string]int, activeLeases func() int) error {
+	collectors := []prometheus.Collector{
+		m.healthcheckDuration,
+		m.healthcheckFailures,
+		m.leaseRequests,
+		m.providerRequests,
+		&constCollector{
+			desc: prometheus.NewDesc("proxy_operator_proxies",
+				"Proxy objects by phase.", []string{"phase"}, nil),
+			read: proxyPhases,
+		},
+		&constCollector{
+			desc: prometheus.NewDesc("proxy_operator_leases_active",
+				"Currently active leases.", nil, nil),
+			read: func() map[string]int { return map[string]int{"": activeLeases()} },
+		},
+	}
+	for _, c := range collectors {
+		if err := reg.Register(c); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ObserveProbe records one health probe. Called on every probe — metrics
+// are the home for high-frequency signal that must never touch status.
+func (m *Metrics) ObserveProbe(proxy string, latency time.Duration, success bool) {
+	m.healthcheckDuration.WithLabelValues(proxy).Observe(latency.Seconds())
+	if !success {
+		m.healthcheckFailures.WithLabelValues(proxy).Inc()
+	}
+}
+
+// ForgetProxy drops the per-proxy series when the health engine prunes its
+// state — without this, series for deleted proxies leak forever.
+func (m *Metrics) ForgetProxy(proxy string) {
+	m.healthcheckDuration.DeleteLabelValues(proxy)
+	m.healthcheckFailures.DeleteLabelValues(proxy)
+}
+
+// LeaseRequest records a lease acquisition outcome ("granted"|"no_match").
+func (m *Metrics) LeaseRequest(outcome string) {
+	m.leaseRequests.WithLabelValues(outcome).Inc()
+}
+
+// ProviderRequest records one provider API call with its classified result.
+func (m *Metrics) ProviderRequest(provider, op, result string) {
+	m.providerRequests.WithLabelValues(provider, op, result).Inc()
+}
+
+// constCollector reads a label→value map at scrape time and emits one
+// gauge sample per entry. An empty-string label key means "no labels".
+type constCollector struct {
+	desc *prometheus.Desc
+	read func() map[string]int
+}
+
+func (c *constCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.desc }
+
+func (c *constCollector) Collect(ch chan<- prometheus.Metric) {
+	for label, value := range c.read() {
+		if label == "" {
+			ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(value))
+			continue
+		}
+		ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(value), label)
+	}
+}
+
+// ProviderTarget is one provider health-check target as the gauges see it.
+type ProviderTarget struct {
+	Provider, Target string
+	Up               bool
+	Duration         time.Duration
+	CheckedAt        time.Time
+}
+
+// RegisterProviderTargets registers the provider-check gauges. Like the
+// other scrape-time collectors they are read from the source at every
+// scrape, so a target that goes away stops being reported rather than
+// leaving a stale series behind.
+func RegisterProviderTargets(reg prometheus.Registerer, read func() []ProviderTarget) error {
+	labels := []string{"provider", "target"}
+	return reg.Register(&providerTargetCollector{
+		up: prometheus.NewDesc("proxy_operator_provider_target_up",
+			"Whether the target passed its latest provider health check (1) or not (0).", labels, nil),
+		duration: prometheus.NewDesc("proxy_operator_provider_check_duration_seconds",
+			"Duration of the target's latest provider health check.", labels, nil),
+		checked: prometheus.NewDesc("proxy_operator_provider_check_timestamp_seconds",
+			"Unix time of the target's latest provider health check.", labels, nil),
+		read: read,
+	})
+}
+
+type providerTargetCollector struct {
+	up, duration, checked *prometheus.Desc
+	read                  func() []ProviderTarget
+}
+
+func (c *providerTargetCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.up
+	ch <- c.duration
+	ch <- c.checked
+}
+
+func (c *providerTargetCollector) Collect(ch chan<- prometheus.Metric) {
+	for _, t := range c.read() {
+		up := 0.0
+		if t.Up {
+			up = 1
+		}
+		ch <- prometheus.MustNewConstMetric(c.up, prometheus.GaugeValue, up, t.Provider, t.Target)
+		ch <- prometheus.MustNewConstMetric(c.duration, prometheus.GaugeValue, t.Duration.Seconds(), t.Provider, t.Target)
+		ch <- prometheus.MustNewConstMetric(c.checked, prometheus.GaugeValue, float64(t.CheckedAt.UnixNano())/1e9, t.Provider, t.Target)
+	}
+}

internal/metrics/metrics_test.go

@@ -0,0 +1,155 @@
+package metrics
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+// register wires a fresh registry — the reason Register exists instead of
+// init()-time self-registration.
+func register(t *testing.T, m *Metrics, phases map[string]int, active int) *prometheus.Registry {
+	t.Helper()
+	reg := prometheus.NewRegistry()
+	err := m.Register(reg,
+		func() map[string]int { return phases },
+		func() int { return active },
+	)
+	if err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+	return reg
+}
+
+func TestRegister_scrapeTimeCollectors(t *testing.T) {
+	t.Parallel()
+	m := New()
+	reg := register(t, m, map[string]int{"Ready": 3, "Provisioning": 1}, 7)
+
+	expected := `
+# HELP proxy_operator_leases_active Currently active leases.
+# TYPE proxy_operator_leases_active gauge
+proxy_operator_leases_active 7
+# HELP proxy_operator_proxies Proxy objects by phase.
+# TYPE proxy_operator_proxies gauge
+proxy_operator_proxies{phase="Provisioning"} 1
+proxy_operator_proxies{phase="Ready"} 3
+`
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(expected),
+		"proxy_operator_proxies", "proxy_operator_leases_active"); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestObserveProbe_andForget(t *testing.T) {
+	t.Parallel()
+	m := New()
+	reg := register(t, m, nil, 0)
+
+	m.ObserveProbe("default/p1", 30*time.Millisecond, true)
+	m.ObserveProbe("default/p1", 40*time.Millisecond, false)
+	m.ObserveProbe("default/p2", 10*time.Millisecond, true)
+
+	if got := testutil.CollectAndCount(m.healthcheckDuration); got != 2 {
+		t.Errorf("duration series = %d, want 2 (one per proxy)", got)
+	}
+	if got := testutil.ToFloat64(m.healthcheckFailures.WithLabelValues("default/p1")); got != 1 {
+		t.Errorf("p1 failures = %v, want 1 (only the failed probe)", got)
+	}
+
+	m.ForgetProxy("default/p1")
+	if got := testutil.CollectAndCount(m.healthcheckDuration); got != 1 {
+		t.Errorf("duration series after ForgetProxy = %d, want 1 — series must not leak", got)
+	}
+	if got := testutil.CollectAndCount(m.healthcheckFailures); got != 0 {
+		t.Errorf("failure series after ForgetProxy = %d, want 0", got)
+	}
+	_ = reg
+}
+
+func TestLeaseRequest(t *testing.T) {
+	t.Parallel()
+	m := New()
+	register(t, m, nil, 0)
+
+	m.LeaseRequest("granted")
+	m.LeaseRequest("granted")
+	m.LeaseRequest("no_match")
+
+	if got := testutil.ToFloat64(m.leaseRequests.WithLabelValues("granted")); got != 2 {
+		t.Errorf("granted = %v, want 2", got)
+	}
+	if got := testutil.ToFloat64(m.leaseRequests.WithLabelValues("no_match")); got != 1 {
+		t.Errorf("no_match = %v, want 1", got)
+	}
+}
+
+func TestProviderRequest(t *testing.T) {
+	t.Parallel()
+	m := New()
+	register(t, m, nil, 0)
+
+	m.ProviderRequest("gcp-eu", "create", "ok")
+	m.ProviderRequest("gcp-eu", "create", "quota_exceeded")
+
+	if got := testutil.ToFloat64(m.providerRequests.WithLabelValues("gcp-eu", "create", "ok")); got != 1 {
+		t.Errorf("ok = %v, want 1", got)
+	}
+	if got := testutil.ToFloat64(m.providerRequests.WithLabelValues("gcp-eu", "create", "quota_exceeded")); got != 1 {
+		t.Errorf("quota_exceeded = %v, want 1", got)
+	}
+}
+
+func TestRegister_freshRegistryPerTest(t *testing.T) {
+	t.Parallel()
+	// Registering the same metric set on two registries must both succeed —
+	// the property init()-style global registration would break.
+	m1, m2 := New(), New()
+	register(t, m1, nil, 0)
+	register(t, m2, nil, 0)
+}
+
+func TestRegisterProviderTargets_scrape(t *testing.T) {
+	t.Parallel()
+	checked := time.Unix(1757602800, 0)
+	targets := []ProviderTarget{
+		{Provider: "kvm-home", Target: "nas1", Up: true, Duration: 84 * time.Millisecond, CheckedAt: checked},
+		{Provider: "kvm-home", Target: "nas2", Up: false, Duration: 15 * time.Second, CheckedAt: checked},
+	}
+	reg := prometheus.NewRegistry()
+	if err := RegisterProviderTargets(reg, func() []ProviderTarget { return targets }); err != nil {
+		t.Fatalf("RegisterProviderTargets: %v", err)
+	}
+
+	expected := `
+# HELP proxy_operator_provider_check_duration_seconds Duration of the target's latest provider health check.
+# TYPE proxy_operator_provider_check_duration_seconds gauge
+proxy_operator_provider_check_duration_seconds{provider="kvm-home",target="nas1"} 0.084
+proxy_operator_provider_check_duration_seconds{provider="kvm-home",target="nas2"} 15
+# HELP proxy_operator_provider_check_timestamp_seconds Unix time of the target's latest provider health check.
+# TYPE proxy_operator_provider_check_timestamp_seconds gauge
+proxy_operator_provider_check_timestamp_seconds{provider="kvm-home",target="nas1"} 1.7576028e+09
+proxy_operator_provider_check_timestamp_seconds{provider="kvm-home",target="nas2"} 1.7576028e+09
+# HELP proxy_operator_provider_target_up Whether the target passed its latest provider health check (1) or not (0).
+# TYPE proxy_operator_provider_target_up gauge
+proxy_operator_provider_target_up{provider="kvm-home",target="nas1"} 1
+proxy_operator_provider_target_up{provider="kvm-home",target="nas2"} 0
+`
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(expected)); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestRegisterProviderTargets_emptySnapshotHasNoSeries(t *testing.T) {
+	t.Parallel()
+	reg := prometheus.NewRegistry()
+	if err := RegisterProviderTargets(reg, func() []ProviderTarget { return nil }); err != nil {
+		t.Fatalf("RegisterProviderTargets: %v", err)
+	}
+	if n, err := testutil.GatherAndCount(reg); err != nil || n != 0 {
+		t.Errorf("series before the first check = %d (%v), want 0", n, err)
+	}
+}

internal/provider/check.go

@@ -0,0 +1,46 @@
+package provider
+
+import (
+	"context"
+	"time"
+)
+
+// Reasons shared by every Checker, so a listing reads the same across
+// providers. Providers add their own for failures only they can tell
+// apart (KVM's SSHHandshakeFailed or PoolNotActive, say).
+const (
+	ReasonReachable    = "Reachable"
+	ReasonTimeout      = "Timeout"
+	ReasonUnreachable  = "Unreachable"
+	ReasonUnauthorized = "Unauthorized"
+	ReasonForbidden    = "Forbidden"
+	ReasonAPIError     = "APIError"
+)
+
+// TargetStatus is one target's verdict from a Checker.
+type TargetStatus struct {
+	// Name identifies the target within its provider: a KVM host name,
+	// "apiserver", a GCP project.
+	Name string
+	// Address is what the check connected to, for humans.
+	Address string
+	Ready   bool
+	// Reason is ReasonReachable when Ready, otherwise why not.
+	Reason string
+	// Message is human-readable detail, usually the error.
+	Message string
+	Latency time.Duration
+	// Details are provider-specific facts gathered by the check.
+	Details map[string]string
+}
+
+// Checker is the optional interface of a provider that can verify its
+// backend is reachable and usable without touching any instance. The
+// provider health monitor type-asserts for it on the unwrapped providers,
+// so it stays out of the Provider contract.
+type Checker interface {
+	// Check returns one verdict per target, in a stable order. It never
+	// fails as a whole — a failure is a verdict — and must return promptly
+	// once ctx ends.
+	Check(ctx context.Context) []TargetStatus
+}

internal/provider/config.go

@@ -21,6 +21,23 @@ type ProviderConfig struct {
 	Type       string            `json:"type"`
 	Kubernetes *KubernetesConfig `json:"kubernetes,omitempty"`
 	GCP        *GCPConfig        `json:"gcp,omitempty"`
+	KVM        *KVMConfig        `json:"kvm,omitempty"`
+}
+
+// setBlocks names the type-specific blocks that are non-nil, so validate can
+// reject a block that doesn't match Type without a pairwise check per type.
+func (p ProviderConfig) setBlocks() []string {
+	var out []string
+	if p.Kubernetes != nil {
+		out = append(out, "kubernetes")
+	}
+	if p.GCP != nil {
+		out = append(out, "gcp")
+	}
+	if p.KVM != nil {
+		out = append(out, "kvm")
+	}
+	return out
 }
 
 // KubernetesConfig configures the kubernetes-pod provider, which creates
@@ -81,22 +98,28 @@ func (c *Config) validate() error {
 		}
 		seen[p.Name] = true
 
+		prefix := fmt.Sprintf("providers[%d] %q", i, p.Name)
 		switch p.Type {
-		case "kubernetes":
-			if p.GCP != nil {
-				return fmt.Errorf("providers[%d] %q: type is kubernetes but a gcp block is set", i, p.Name)
-			}
-		case "gcp":
-			if p.Kubernetes != nil {
-				return fmt.Errorf("providers[%d] %q: type is gcp but a kubernetes block is set", i, p.Name)
-			}
-			if p.GCP == nil || p.GCP.Project == "" {
-				return fmt.Errorf("providers[%d] %q: gcp.project is required", i, p.Name)
-			}
+		case "kubernetes", "gcp", "kvm":
 		case "":
-			return fmt.Errorf("providers[%d] %q: type is required", i, p.Name)
+			return fmt.Errorf("%s: type is required", prefix)
 		default:
-			return fmt.Errorf("providers[%d] %q: unknown provider type %q", i, p.Name, p.Type)
+			return fmt.Errorf("%s: unknown provider type %q", prefix, p.Type)
+		}
+		for _, block := range p.setBlocks() {
+			if block != p.Type {
+				return fmt.Errorf("%s: type is %s but a %s block is set", prefix, p.Type, block)
+			}
+		}
+		switch p.Type {
+		case "gcp":
+			if p.GCP == nil || p.GCP.Project == "" {
+				return fmt.Errorf("%s: gcp.project is required", prefix)
+			}
+		case "kvm":
+			if err := validateKVM(prefix, p.KVM); err != nil {
+				return err
+			}
 		}
 	}
 	return nil

internal/provider/config_kvm.go

@@ -0,0 +1,184 @@
+package provider
+
+import (
+	"fmt"
+	"net/url"
+	"regexp"
+)
+
+// KVMConfig configures a kvm provider instance: one or more libvirt/KVM
+// hypervisors reached over SSH (the libvirt unix socket is forwarded through
+// the SSH connection, so hosts need no libvirtd TCP/TLS listener).
+// placement.zone on a Proxy selects a host by name; with a single host it may
+// be omitted.
+type KVMConfig struct {
+	// SSHKeyFile is the path to the private key used for every host. Required.
+	SSHKeyFile string `json:"sshKeyFile"`
+	// KnownHostsFile is an OpenSSH known_hosts file used to verify host keys.
+	// Required unless InsecureIgnoreHostKey is set.
+	KnownHostsFile string `json:"knownHostsFile,omitempty"`
+	// InsecureIgnoreHostKey skips host-key verification entirely.
+	InsecureIgnoreHostKey bool `json:"insecureIgnoreHostKey,omitempty"`
+	// DiskSizeGB is the capacity of each VM's overlay disk. Default 10. The
+	// guest's cloud-init growpart expands the root filesystem into it.
+	DiskSizeGB int64 `json:"diskSizeGb,omitempty"`
+	// VendorData is optional cloud-init vendor-data placed on every seed next
+	// to the Proxy's own user-data. Note that cloud-init drops any top-level
+	// key of a vendor-data cloud-config that the user-data also defines.
+	// When InstallGuestAgent is on, this document and the agent script are
+	// carried as separate parts of a MIME multipart vendor-data.
+	VendorData string `json:"vendorData,omitempty"`
+	// InstallGuestAgent adds a vendor-data script that installs and starts
+	// qemu-guest-agent, which is how the provider learns a VM's address
+	// before the host's ARP table has seen traffic from it. Default true.
+	// Turn it off for images that already ship the agent, or that cannot
+	// reach a package mirror; discovery then depends on ARP alone.
+	InstallGuestAgent *bool `json:"installGuestAgent,omitempty"`
+	// Images are named base images; placement.image names a key here (or is
+	// an http(s) URL directly). The provider downloads each image into a
+	// host's storage pool on first use and caches it there.
+	Images map[string]KVMImage `json:"images,omitempty"`
+	// MachineTypes are named size profiles; placement.machineType names a
+	// key here. Required, at least one.
+	MachineTypes map[string]KVMMachineType `json:"machineTypes"`
+	// Hosts are the hypervisors. Required, at least one, unique names.
+	Hosts []KVMHost `json:"hosts"`
+}
+
+// KVMImage is one downloadable base image (a distro cloud image with
+// cloud-init preinstalled, e.g. Debian's generic qcow2).
+type KVMImage struct {
+	// URL is where to fetch the image from. Required, http or https.
+	URL string `json:"url"`
+	// SHA256 is the expected hex digest of the file. Optional; when set it is
+	// verified after download and a later change triggers a re-download.
+	SHA256 string `json:"sha256,omitempty"`
+	// Format is the image's disk format, "qcow2" (default) or "raw".
+	Format string `json:"format,omitempty"`
+	// Firmware is how VMs from this image boot: "uefi" (default; OVMF with
+	// secure boot off) or "bios" (SeaBIOS) for images with no EFI system
+	// partition. Current Debian, Ubuntu, Fedora and Rocky cloud images all
+	// boot UEFI, and Debian 13's BIOS path was found not to boot at all.
+	Firmware string `json:"firmware,omitempty"`
+}
+
+// KVMMachineType is a VM size profile.
+type KVMMachineType struct {
+	VCPUs     int32 `json:"vcpus"`
+	MemoryMiB int64 `json:"memoryMiB"`
+}
+
+// KVMHost is one hypervisor.
+type KVMHost struct {
+	// Name is the placement.zone value that selects this host. Required.
+	Name string `json:"name"`
+	// Address is the SSH endpoint, host:port. Required.
+	Address string `json:"address"`
+	// User is the SSH user; it must be allowed to use the libvirt socket
+	// (typically a member of the libvirt group). Required.
+	User string `json:"user"`
+	// Socket is the libvirt unix socket path on the host.
+	// Default /var/run/libvirt/libvirt-sock.
+	Socket string `json:"socket,omitempty"`
+	// Pool is the libvirt storage pool for base images, overlay disks and
+	// seed ISOs. Default "default".
+	Pool string `json:"pool,omitempty"`
+	// Bridge is the host bridge the VM's NIC attaches to; it must lead to a
+	// network the cluster can reach and that hands out DHCP leases. Required.
+	Bridge string `json:"bridge"`
+}
+
+// Accepted KVMImage.Format and KVMImage.Firmware values.
+const (
+	KVMFormatQcow2  = "qcow2"
+	KVMFormatRaw    = "raw"
+	KVMFirmwareUEFI = "uefi"
+	KVMFirmwareBIOS = "bios"
+)
+
+// IsHTTPURL reports whether s is an absolute http(s) URL with a host, the
+// only kind of image source the kvm provider can fetch.
+func IsHTTPURL(s string) bool {
+	u, err := url.Parse(s)
+	return err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != ""
+}
+
+var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
+
+// kvmIdentifier constrains the config names that are pasted into
+// identifiers elsewhere: a host name goes into the providerID
+// hosts/<name>/domains/<vm>, which is split on "/" to parse, and an image
+// key becomes the file name of the cached base volume. A slash in either
+// produces an ID or a path that never resolves again, so the VM leaks with
+// nothing able to address it.
+var kvmIdentifier = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`)
+
+func validateKVM(prefix string, c *KVMConfig) error {
+	if c == nil {
+		return fmt.Errorf("%s: kvm block is required", prefix)
+	}
+	if c.SSHKeyFile == "" {
+		return fmt.Errorf("%s: kvm.sshKeyFile is required", prefix)
+	}
+	if c.KnownHostsFile == "" && !c.InsecureIgnoreHostKey {
+		return fmt.Errorf("%s: kvm.knownHostsFile is required unless kvm.insecureIgnoreHostKey is true", prefix)
+	}
+	if c.DiskSizeGB < 0 {
+		return fmt.Errorf("%s: kvm.diskSizeGb must not be negative", prefix)
+	}
+	for name, img := range c.Images {
+		p := fmt.Sprintf("%s: kvm.images[%q]", prefix, name)
+		if !kvmIdentifier.MatchString(name) {
+			return fmt.Errorf("%s: key must be letters, digits, dots, dashes or underscores"+
+				" — it becomes the cached volume name base-<key>.<format>", p)
+		}
+		if !IsHTTPURL(img.URL) {
+			return fmt.Errorf("%s.url must be an http(s) URL (got %q)", p, img.URL)
+		}
+		if img.SHA256 != "" && !sha256Hex.MatchString(img.SHA256) {
+			return fmt.Errorf("%s.sha256 must be 64 hex characters", p)
+		}
+		if img.Format != "" && img.Format != KVMFormatQcow2 && img.Format != KVMFormatRaw {
+			return fmt.Errorf("%s.format must be %s or %s (got %q)", p, KVMFormatQcow2, KVMFormatRaw, img.Format)
+		}
+		if img.Firmware != "" && img.Firmware != KVMFirmwareUEFI && img.Firmware != KVMFirmwareBIOS {
+			return fmt.Errorf("%s.firmware must be %s or %s (got %q)", p, KVMFirmwareUEFI, KVMFirmwareBIOS, img.Firmware)
+		}
+	}
+	if len(c.MachineTypes) == 0 {
+		return fmt.Errorf("%s: kvm.machineTypes must define at least one profile", prefix)
+	}
+	for name, mt := range c.MachineTypes {
+		if mt.VCPUs <= 0 || mt.MemoryMiB <= 0 {
+			return fmt.Errorf("%s: kvm.machineTypes[%q] needs vcpus > 0 and memoryMiB > 0", prefix, name)
+		}
+	}
+	if len(c.Hosts) == 0 {
+		return fmt.Errorf("%s: kvm.hosts must list at least one host", prefix)
+	}
+	seen := make(map[string]bool, len(c.Hosts))
+	for i, h := range c.Hosts {
+		p := fmt.Sprintf("%s: kvm.hosts[%d]", prefix, i)
+		if h.Name == "" {
+			return fmt.Errorf("%s: name is required", p)
+		}
+		if !kvmIdentifier.MatchString(h.Name) {
+			return fmt.Errorf("%s: name %q must be letters, digits, dots, dashes or underscores"+
+				" — it becomes part of the providerID hosts/<name>/domains/<vm>", p, h.Name)
+		}
+		if seen[h.Name] {
+			return fmt.Errorf("%s: duplicate host name %q", p, h.Name)
+		}
+		seen[h.Name] = true
+		if h.Address == "" {
+			return fmt.Errorf("%s %q: address is required", p, h.Name)
+		}
+		if h.User == "" {
+			return fmt.Errorf("%s %q: user is required", p, h.Name)
+		}
+		if h.Bridge == "" {
+			return fmt.Errorf("%s %q: bridge is required", p, h.Name)
+		}
+	}
+	return nil
+}

internal/provider/config_kvm_test.go

@@ -0,0 +1,226 @@
+package provider
+
+import (
+	"strings"
+	"testing"
+)
+
+const validKVMYAML = `
+providers:
+  - name: kvm-home
+    type: kvm
+    kvm:
+      sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+      knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+      diskSizeGb: 20
+      images:
+        debian-13:
+          url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+          sha256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+        ubuntu:
+          url: http://mirror.local/noble.img
+          format: raw
+      machineTypes:
+        small: {vcpus: 1, memoryMiB: 1024}
+        medium: {vcpus: 2, memoryMiB: 2048}
+      hosts:
+        - name: nas1
+          address: nas1.home.lan:22
+          user: virt
+          bridge: br0
+        - name: lab2
+          address: 10.0.0.2:2222
+          user: root
+          socket: /run/libvirt/virtqemud-sock
+          pool: fast
+          bridge: br1
+`
+
+func TestLoadConfig_kvmValid(t *testing.T) {
+	t.Parallel()
+	cfg, err := LoadConfig([]byte(validKVMYAML))
+	if err != nil {
+		t.Fatalf("LoadConfig() error = %v, want nil", err)
+	}
+	k := cfg.Providers[0].KVM
+	if k == nil {
+		t.Fatal("providers[0].kvm = nil")
+	}
+	if k.SSHKeyFile != "/etc/proxy-operator/kvm/id_ed25519" || k.DiskSizeGB != 20 {
+		t.Errorf("kvm = %+v, want sshKeyFile and diskSizeGb parsed", k)
+	}
+	if got := k.Images["debian-13"].SHA256; !strings.HasPrefix(got, "0123") {
+		t.Errorf("images[debian-13].sha256 = %q", got)
+	}
+	if k.Images["ubuntu"].Format != "raw" {
+		t.Errorf("images[ubuntu].format = %q, want raw", k.Images["ubuntu"].Format)
+	}
+	if mt := k.MachineTypes["medium"]; mt.VCPUs != 2 || mt.MemoryMiB != 2048 {
+		t.Errorf("machineTypes[medium] = %+v, want {2 2048}", mt)
+	}
+	if len(k.Hosts) != 2 || k.Hosts[1].Socket != "/run/libvirt/virtqemud-sock" || k.Hosts[1].Pool != "fast" {
+		t.Errorf("hosts = %+v, want two hosts with lab2's socket and pool parsed", k.Hosts)
+	}
+}
+
+func TestLoadConfig_kvmInsecureHostKeyNeedsNoKnownHosts(t *testing.T) {
+	t.Parallel()
+	yaml := strings.Replace(validKVMYAML,
+		"knownHostsFile: /etc/proxy-operator/kvm/known_hosts",
+		"insecureIgnoreHostKey: true", 1)
+	if _, err := LoadConfig([]byte(yaml)); err != nil {
+		t.Fatalf("LoadConfig() error = %v, want nil", err)
+	}
+}
+
+func TestLoadConfig_kvmInvalid(t *testing.T) {
+	t.Parallel()
+	minimal := func(kvmBody string) string {
+		return "providers:\n  - name: kvm-home\n    type: kvm\n    kvm:\n" + indent(kvmBody, 6)
+	}
+	hostsAndTypes := `
+machineTypes:
+  small: {vcpus: 1, memoryMiB: 1024}
+hosts:
+  - {name: nas1, address: nas1:22, user: virt, bridge: br0}
+`
+	tests := []struct {
+		name       string
+		yaml       string
+		wantErrSub string
+	}{
+		{
+			name:       "missing kvm block",
+			yaml:       "providers:\n  - name: kvm-home\n    type: kvm\n",
+			wantErrSub: "kvm block is required",
+		},
+		{
+			name:       "missing sshKeyFile",
+			yaml:       minimal("knownHostsFile: /kh\n" + hostsAndTypes),
+			wantErrSub: "kvm.sshKeyFile is required",
+		},
+		{
+			name:       "missing knownHostsFile without insecure flag",
+			yaml:       minimal("sshKeyFile: /key\n" + hostsAndTypes),
+			wantErrSub: "kvm.knownHostsFile is required unless",
+		},
+		{
+			name:       "negative disk size",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\ndiskSizeGb: -1\n" + hostsAndTypes),
+			wantErrSub: "diskSizeGb must not be negative",
+		},
+		{
+			name:       "no machine types",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nhosts:\n  - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
+			wantErrSub: "machineTypes must define at least one",
+		},
+		{
+			name:       "machine type with zero memory",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1}\nhosts:\n  - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `machineTypes["small"] needs vcpus > 0 and memoryMiB > 0`,
+		},
+		{
+			name:       "no hosts",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\n"),
+			wantErrSub: "kvm.hosts must list at least one host",
+		},
+		{
+			name:       "duplicate host name",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, user: virt, bridge: br0}\n  - {name: nas1, address: b:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `duplicate host name "nas1"`,
+		},
+		{
+			// A slash would produce hosts/dc1/rack2/domains/<vm>, which
+			// parseProviderID can never split back, so the VM would be
+			// unreachable for Get, Delete and GC alike.
+			name:       "host name with a slash",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: dc1/rack2, address: a:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `name "dc1/rack2" must be letters, digits, dots, dashes or underscores`,
+		},
+		{
+			name:       "image key with a slash",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  ../etc/d: {url: https://x/y.qcow2}\n" + hostsAndTypes),
+			wantErrSub: `images["../etc/d"]: key must be letters, digits, dots, dashes or underscores`,
+		},
+		{
+			name:       "host missing address",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, user: virt, bridge: br0}\n"),
+			wantErrSub: "address is required",
+		},
+		{
+			name:       "host missing user",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, bridge: br0}\n"),
+			wantErrSub: "user is required",
+		},
+		{
+			name:       "host missing bridge",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, user: virt}\n"),
+			wantErrSub: "bridge is required",
+		},
+		{
+			name:       "image with non-http url",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: file:///tmp/x.qcow2}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].url must be an http(s) URL`,
+		},
+		{
+			name:       "image with bad sha256",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.qcow2, sha256: abc}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].sha256 must be 64 hex characters`,
+		},
+		{
+			name:       "image with unknown firmware",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.qcow2, firmware: coreboot}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].firmware must be uefi or bios`,
+		},
+		{
+			name:       "image with unknown format",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.vmdk, format: vmdk}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].format must be qcow2 or raw`,
+		},
+		{
+			name: "kvm type with gcp block",
+			yaml: `
+providers:
+  - name: p1
+    type: kvm
+    gcp:
+      project: my-project`,
+			wantErrSub: "type is kvm but a gcp block is set",
+		},
+		{
+			name: "gcp type with kvm block",
+			yaml: `
+providers:
+  - name: p1
+    type: gcp
+    gcp:
+      project: my-project
+    kvm:
+      sshKeyFile: /key`,
+			wantErrSub: "type is gcp but a kvm block is set",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			_, err := LoadConfig([]byte(tc.yaml))
+			if err == nil {
+				t.Fatalf("LoadConfig() error = nil, want error containing %q", tc.wantErrSub)
+			}
+			if !strings.Contains(err.Error(), tc.wantErrSub) {
+				t.Errorf("LoadConfig() error = %q, want substring %q", err.Error(), tc.wantErrSub)
+			}
+		})
+	}
+}
+
+func indent(s string, n int) string {
+	pad := strings.Repeat(" ", n)
+	lines := strings.Split(strings.TrimLeft(s, "\n"), "\n")
+	for i, l := range lines {
+		if l != "" {
+			lines[i] = pad + l
+		}
+	}
+	return strings.Join(lines, "\n")
+}

internal/provider/gcp/check.go

@@ -0,0 +1,78 @@
+package gcp
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"time"
+
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/iterator"
+	"google.golang.org/protobuf/proto"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+var _ provider.Checker = (*Provider)(nil)
+
+// computeAddress is the endpoint the SDK talks to, shown only to humans.
+const computeAddress = "compute.googleapis.com"
+
+// managedFilter selects the instances this operator created.
+const managedFilter = "labels." + provider.LabelManaged + " = " + provider.LabelManagedYes
+
+// Check asks the project for one page holding at most one managed
+// instance. That exercises the credentials, the Compute API and the
+// compute.instances.list permission orphan GC depends on, in a single
+// request whatever the project's size. The target is the project: the
+// provider config names no zones.
+func (p *Provider) Check(ctx context.Context) []provider.TargetStatus {
+	start := time.Now()
+	err := p.api.Probe(ctx, &computepb.AggregatedListInstancesRequest{
+		Project:              p.cfg.Project,
+		Filter:               proto.String(managedFilter),
+		MaxResults:           proto.Uint32(1),
+		ReturnPartialSuccess: proto.Bool(true),
+	})
+	ts := provider.TargetStatus{Name: p.cfg.Project, Address: computeAddress, Latency: time.Since(start)}
+	if err != nil {
+		logAPIError(p.logger(ctx), "check", err)
+		ts.Reason = checkReason(ctx, err)
+		ts.Message = err.Error()
+		return []provider.TargetStatus{ts}
+	}
+	ts.Ready = true
+	ts.Reason = provider.ReasonReachable
+	return []provider.TargetStatus{ts}
+}
+
+// Probe fetches the first page of an aggregated list and discards it. The
+// SDK treats MaxResults as the page size, so AggregatedList, which drains
+// the iterator, would still walk every page of the project.
+func (r *realInstances) Probe(ctx context.Context, req *computepb.AggregatedListInstancesRequest) error {
+	_, err := r.client.AggregatedList(ctx, req).Next()
+	if err == iterator.Done {
+		return nil
+	}
+	return err
+}
+
+// checkReason maps a failed probe to a reason. A token that cannot be
+// obtained never reaches the API, so it has no status code and lands on
+// Unreachable, with the detail in the message.
+func checkReason(ctx context.Context, err error) string {
+	var gerr *googleapi.Error
+	switch {
+	case ctx.Err() != nil, errors.Is(err, context.DeadlineExceeded):
+		return provider.ReasonTimeout
+	case !errors.As(err, &gerr):
+		return provider.ReasonUnreachable
+	case gerr.Code == http.StatusUnauthorized:
+		return provider.ReasonUnauthorized
+	case gerr.Code == http.StatusForbidden:
+		return provider.ReasonForbidden
+	default:
+		return provider.ReasonAPIError
+	}
+}

internal/provider/gcp/check_test.go

@@ -0,0 +1,165 @@
+package gcp
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"sync"
+	"syscall"
+	"testing"
+
+	compute "cloud.google.com/go/compute/apiv1"
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
+	"google.golang.org/protobuf/proto"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func TestCheck_reachable(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{}
+	p := newTestProvider(api)
+
+	got := p.Check(context.Background())
+
+	if len(got) != 1 {
+		t.Fatalf("Check() returned %d targets, want 1", len(got))
+	}
+	ts := got[0]
+	if !ts.Ready || ts.Reason != provider.ReasonReachable || ts.Message != "" {
+		t.Errorf("Check() = %+v, want ready and Reachable", ts)
+	}
+	if ts.Name != "my-project" || ts.Address != "compute.googleapis.com" {
+		t.Errorf("target = %q at %q, want my-project at compute.googleapis.com", ts.Name, ts.Address)
+	}
+	want := &computepb.AggregatedListInstancesRequest{
+		Project:              "my-project",
+		Filter:               proto.String("labels.proxy-operator-managed = true"),
+		MaxResults:           proto.Uint32(1),
+		ReturnPartialSuccess: proto.Bool(true),
+	}
+	if !proto.Equal(api.probeReq, want) {
+		t.Errorf("probe request = %v, want %v", api.probeReq, want)
+	}
+}
+
+func TestCheck_failureReasons(t *testing.T) {
+	t.Parallel()
+	ended, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	tests := []struct {
+		name string
+		ctx  context.Context
+		err  error
+		want string
+	}{
+		{"unauthorized", context.Background(), gerr(401), provider.ReasonUnauthorized},
+		{"forbidden", context.Background(), gerr(403, "forbidden"), provider.ReasonForbidden},
+		{"api disabled", context.Background(), gerr(403, "accessNotConfigured"), provider.ReasonForbidden},
+		{"server error", context.Background(), gerr(500), provider.ReasonAPIError},
+		{"deadline", context.Background(), &url.Error{
+			Op: "Get", URL: "https://compute.googleapis.com/", Err: context.DeadlineExceeded,
+		}, provider.ReasonTimeout},
+		{"context ended", ended, context.Canceled, provider.ReasonTimeout},
+		{"connection refused", context.Background(), &url.Error{
+			Op: "Get", URL: "https://compute.googleapis.com/", Err: syscall.ECONNREFUSED,
+		}, provider.ReasonUnreachable},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			p := newTestProvider(&fakeAPI{probeErr: tc.err})
+
+			got := p.Check(tc.ctx)
+
+			if len(got) != 1 {
+				t.Fatalf("Check() returned %d targets, want 1", len(got))
+			}
+			if got[0].Ready || got[0].Reason != tc.want || got[0].Message != tc.err.Error() {
+				t.Errorf("Check() = %+v, want not ready, reason %s, message %q", got[0], tc.want, tc.err.Error())
+			}
+		})
+	}
+}
+
+// TestRealInstances_Probe runs the real SDK client against a local server,
+// since the point of Probe is how many requests the SDK makes.
+func TestRealInstances_Probe(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name     string
+		status   int
+		body     string
+		wantCode int
+	}{
+		{
+			name:   "fetches only the first page",
+			status: http.StatusOK,
+			body:   `{"items":{"zones/europe-west1-b":{"instances":[{"name":"proxy-a"}]}},"nextPageToken":"page-2"}`,
+		},
+		{
+			name:   "empty project",
+			status: http.StatusOK,
+			body:   `{}`,
+		},
+		{
+			name:     "api error keeps its status",
+			status:   http.StatusForbidden,
+			body:     `{"error":{"code":403,"message":"Compute Engine API has not been used","errors":[{"reason":"accessNotConfigured"}]}}`,
+			wantCode: http.StatusForbidden,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			var (
+				mu   sync.Mutex
+				reqs []*url.URL
+			)
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				mu.Lock()
+				reqs = append(reqs, r.URL)
+				mu.Unlock()
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(tc.status)
+				_, _ = w.Write([]byte(tc.body))
+			}))
+			t.Cleanup(srv.Close)
+			ctx := context.Background()
+			c, err := compute.NewInstancesRESTClient(ctx, option.WithEndpoint(srv.URL), option.WithoutAuthentication())
+			if err != nil {
+				t.Fatalf("NewInstancesRESTClient: %v", err)
+			}
+			t.Cleanup(func() { _ = c.Close() })
+
+			err = (&realInstances{client: c}).Probe(ctx, &computepb.AggregatedListInstancesRequest{
+				Project:    "my-project",
+				MaxResults: proto.Uint32(1),
+			})
+
+			var gerr *googleapi.Error
+			switch {
+			case tc.wantCode == 0 && err != nil:
+				t.Fatalf("Probe() = %v, want nil", err)
+			case tc.wantCode != 0 && (!errors.As(err, &gerr) || gerr.Code != tc.wantCode):
+				t.Fatalf("Probe() = %v, want a googleapi error with code %d", err, tc.wantCode)
+			}
+			mu.Lock()
+			defer mu.Unlock()
+			if len(reqs) != 1 {
+				t.Fatalf("server saw %d requests, want 1", len(reqs))
+			}
+			if got := reqs[0].Query().Get("maxResults"); got != "1" {
+				t.Errorf("maxResults = %q, want 1", got)
+			}
+			if want := "/compute/v1/projects/my-project/aggregated/instances"; reqs[0].Path != want {
+				t.Errorf("path = %q, want %q", reqs[0].Path, want)
+			}
+		})
+	}
+}

internal/provider/gcp/errors.go

@@ -0,0 +1,81 @@
+package gcp
+
+import (
+	"errors"
+	"net/http"
+	"slices"
+
+	"github.com/go-logr/logr"
+	"google.golang.org/api/googleapi"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// classify maps a GCP API error onto the provider taxonomy:
+// 404 → NotFound; 429 and quota-flavored 403s → QuotaExceeded;
+// 400/401/other 403s → Permanent; 408/5xx and anything unrecognized
+// (network errors, context cancellation) → Transient, because retrying is
+// always safer than latching Failed on an error nobody taught this
+// function to recognize.
+func classify(err error) error {
+	var gerr *googleapi.Error
+	if !errors.As(err, &gerr) {
+		return provider.ErrTransient
+	}
+	switch {
+	case gerr.Code == http.StatusNotFound:
+		return provider.ErrNotFound
+	case gerr.Code == http.StatusTooManyRequests:
+		return provider.ErrQuotaExceeded
+	case gerr.Code == http.StatusForbidden && hasReason(gerr, "quotaExceeded", "rateLimitExceeded"):
+		return provider.ErrQuotaExceeded
+	case gerr.Code == http.StatusBadRequest,
+		gerr.Code == http.StatusUnauthorized,
+		gerr.Code == http.StatusForbidden:
+		return provider.ErrPermanent
+	default:
+		return provider.ErrTransient
+	}
+}
+
+func (p *Provider) wrapErr(op, id string, err error) error {
+	return provider.Wrap(classify(err), op, p.name, id, err)
+}
+
+// logAPIError records the raw googleapi error shape (HTTP status, reasons)
+// at V(1) — classify collapses it onto the coarser provider taxonomy, so
+// this line is the only place the original status survives.
+func logAPIError(log logr.Logger, op string, err error) {
+	if !log.V(1).Enabled() {
+		return
+	}
+	kv := []any{"op", op, "error", err.Error()}
+	var gerr *googleapi.Error
+	if errors.As(err, &gerr) {
+		reasons := make([]string, 0, len(gerr.Errors))
+		for _, item := range gerr.Errors {
+			reasons = append(reasons, item.Reason)
+		}
+		kv = append(kv, "httpStatus", gerr.Code, "reasons", reasons)
+	}
+	log.V(1).Info("GCP API call failed", kv...)
+}
+
+func hasReason(gerr *googleapi.Error, reasons ...string) bool {
+	for _, item := range gerr.Errors {
+		if slices.Contains(reasons, item.Reason) {
+			return true
+		}
+	}
+	return false
+}
+
+func isAlreadyExists(err error) bool {
+	var gerr *googleapi.Error
+	return errors.As(err, &gerr) && gerr.Code == http.StatusConflict
+}
+
+func isNotFound(err error) bool {
+	var gerr *googleapi.Error
+	return errors.As(err, &gerr) && gerr.Code == http.StatusNotFound
+}