Add KVM provider methods: Create/Get/Delete/ListByTag over the libvirt seam

Create looks the domain up first and only starts it when it exists;
otherwise it caches the image, reuses or creates the overlay, rebuilds
the seed ISO unconditionally (a cancelled upload can leave a truncated
seed that no later attempt would repair), defines and starts the domain.
Get maps libvirt state plus the first routable IPv4 from ARP or the
guest agent onto the provider states. Delete destroys, undefines and
removes the per-VM volumes by name, tolerating every already-gone case.
ListByTag keeps managed domains across all hosts, skipping domains it
cannot read and failing the sweep only for an unreachable host.

Also corrects the Step 2 seed determinism test: the iso9660 writer
stamps the build time into the volume descriptor, so builds differ by a
few bytes; the test now compares label, files and size.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

.claude/agents/operator-reviewer.md

@@ -0,0 +1,15 @@
+---
+name: operator-reviewer
+description: Reviews Kubernetes operator PRs for controller-runtime correctness, reconcile semantics, and API design
+tools: Read, Grep, Glob, Bash
+---
+You are a senior reviewer specializing in Kubernetes operators.
+Review with focus on:
+- Reconcile idempotency and requeue behavior; no state assumptions between reconciles
+- Informer cache reads vs direct API reads; stale-cache races
+- Finalizer handling, deletion flow, orphaned resources
+- CRD schema evolution, conversion webhooks, status subresource / conditions conventions
+- RBAC minimality vs what the controller actually touches
+- Leader election, watch predicates, event filtering for churn reduction
+- Go: context propagation, error wrapping, client.Object handling
+Output: findings ranked by severity, with file:line refs. No praise padding.

.claude/settings.json

@@ -63,7 +63,33 @@
       "Bash(grep -n 'func Channel' -A8 __CMDSUB_OUTPUT__/sigs.k8s.io/controller-runtime@v0.24.1/pkg/source/source.go)",
       "Bash(grep -n 'type GenericEvent' __CMDSUB_OUTPUT__/sigs.k8s.io/controller-runtime@v0.24.1/pkg/event/event.go)",
       "Bash(KUBEBUILDER_ASSETS=__TRACKED_VAR__/bin/k8s/1.36.2-darwin-arm64 go test -race ./...)",
-      "Bash(cat >> *)"
+      "Bash(cat >> *)",
+      "Bash(make run-dev *)",
+      "Bash(kubectl get *)",
+      "Bash(kubectl delete *)",
+      "Bash(make docker-build *)",
+      "Bash(kind load *)",
+      "Bash(make deploy *)",
+      "Bash(kubectl -n egress-proxies-operator-system rollout status deploy/egress-proxies-operator-controller-manager --timeout=120s)",
+      "Bash(kubectl -n egress-proxies-operator-system rollout restart deploy/egress-proxies-operator-controller-manager)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator add docs/plans/2026-08-11-1742-gcp-provider-verbose-logging.md)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator commit -m 'Add plan: verbose V-level logging in the GCP provider *)",
+      "Bash(echo \"exit: $?\")",
+      "Bash(echo \"tests exit: $?\")",
+      "Bash(./bin/manager --version)",
+      "Bash(./bin/manager-stamped --version)",
+      "Bash(./bin/manager-pkg --version)",
+      "Bash(docker run *)",
+      "Bash(kubectl -n egress-proxies-operator-system get pods -o wide)",
+      "Bash(kubectl -n egress-proxies-operator-system get deploy egress-proxies-operator-controller-manager -o jsonpath='{.spec.template.spec.containers[0].args}')",
+      "Bash(kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager)",
+      "Bash(python3 -c \"import json; d=json.load\\(open\\('docs/deploy/sa_key.json'\\)\\); print\\(d.get\\('type'\\), d.get\\('client_email'\\)\\)\")",
+      "Bash(git add *)",
+      "Bash(git commit -q -m 'Add plan: KVM/libvirt provider *)",
+      "Bash(go get *)",
+      "Bash(go build *)",
+      "Bash(go test *)",
+      "Bash(git commit *)"
     ],
     "additionalDirectories": [
       "/Users/jan.novak/srv/go/egress-proxies-operator/.claude",

.gitea/workflows/build.yaml

@@ -0,0 +1,76 @@
+name: Build and Push
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Image tag'
+        required: true
+        default: 'latest'
+  push:
+    tags:
+      - '*'
+
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Vet
+        run: go vet ./...
+      - name: Build
+        run: go build ./...
+      - name: Test (short)
+        run: go test -short ./...
+
+  build:
+    needs: check
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Compute image tags
+        id: meta
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            TAG="${{ inputs.tag }}"
+          else
+            TAG="${{ github.ref_name }}"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "sha=sha-$(echo '${{ github.sha }}' | cut -c1-12)" >> "$GITHUB_OUTPUT"
+
+      - name: Login to Gitea registry
+        run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login -u ${{ github.actor }} --password-stdin gitea.home.hrajfrisbee.cz
+
+      - name: Build and push
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker build \
+            --build-arg GIT_COMMIT=$(echo '${{ github.sha }}' | cut -c1-12) \
+            --label org.opencontainers.image.source=https://gitea.home.hrajfrisbee.cz/${{ github.repository }} \
+            --label org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
+            -t "$IMAGE:${{ steps.meta.outputs.tag }}" \
+            -t "$IMAGE:${{ steps.meta.outputs.sha }}" \
+            .
+          docker push "$IMAGE:${{ steps.meta.outputs.tag }}"
+          docker push "$IMAGE:${{ steps.meta.outputs.sha }}"
+
+      # Only real tag pushes move :latest — an ad-hoc dispatch of an old ref must not clobber it.
+      - name: Push latest (tag builds only)
+        if: github.event_name == 'push'
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker tag "$IMAGE:${{ steps.meta.outputs.tag }}" "$IMAGE:latest"
+          docker push "$IMAGE:latest"

.gitignore

@@ -28,3 +28,6 @@ go.work
 
 # Kubeconfig might contain secrets
 *.kubeconfig
+
+# GCP service-account keys (created per docs/gcp-in-specific-project.md)
+sa_key.json

CHANGELOG.md

@@ -1 +1,25 @@
 # Changelog
+
+## 2026-08-10 09:34 CEST — kind e2e verified; fix Squid OOM in containers; quickstart goes in-cluster
+
+- Full end-to-end pass on a throwaway kind cluster: Squid pod Ready with a real CONNECT
+  probe (89 ms), lease grant/report/cooldown-409/release through the discovery API,
+  finalizer cleanup on delete.
+- Fixed the kubernetes provider's generated squid.conf: `max_filedescriptors 1024`
+  (squid sizes FD tables from the container's effectively-unlimited RLIMIT_NOFILE and
+  was OOM-killed at startup under kind/containerd) + `cache_mem 16 MB`.
+- README quickstart now deploys the operator in-cluster: `make run-dev` on a laptop
+  cannot reach kind pod IPs, so health probes fail by construction there (documented).
+
+## 2026-08-09 17:27 CEST — Operator wired end to end: reconciler, health, leases, discovery, GC, two providers
+
+- `cmd/main.go` is now the full composition root: `--providers-config` (required, fail-fast),
+  `--discovery-addr`, `--proxy-namespace`, `--health-workers`, `--gc-interval`, `--gc-min-age`,
+  `--gc-allow-namespaced`, `--lease-cooldown`, `--max-lease-ttl`; wires the kubernetes + gcp
+  providers (metrics-instrumented), health engine, lease store, discovery API, orphan GC, and
+  Prometheus metrics onto one manager.
+- Deploy manifests: providers ConfigMap mount, optional `DISCOVERY_TOKEN` Secret env,
+  discovery port 8090 + Service; pods RBAC for the kubernetes provider.
+- Samples for all three proxy flavors + providers-config; `make run-dev` for local development.
+- README rewritten (kind quickstart, GCP setup, the two load-bearing caveats); architecture doc
+  completed with components table and the full decision log.

CLAUDE.md

@@ -199,8 +199,18 @@ Always append a `Co-Authored-By` trailer to indicate AI assistance:
 
     Co-Authored-By: Claude <noreply@anthropic.com>
 
-TODO: no `.gitea/workflows/` CI pipeline exists yet — add a CI/CD subsection here once
+### CI/CD
-one is set up.
+
+`.gitea/workflows/build.yaml` builds the manager image and pushes it to the Gitea
+registry. Triggers: any tag push, or manual `workflow_dispatch` with a `tag` input.
+A lightweight `check` job (`go vet` / `go build` / `go test -short`) gates the build.
+
+- Images: `gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator:<tag>` plus an
+  immutable `sha-<12-char-commit>` tag on every build; `:latest` moves only on real
+  tag pushes, never on manual dispatch.
+- Requires the `REGISTRY_TOKEN` repo secret (Gitea PAT with `write:package`),
+  same convention as the other projects on this Gitea instance.
+- The commit is baked into the binary via the `GIT_COMMIT` build arg (see Dockerfile).
 
 ## Gotchas
 

Dockerfile

@@ -2,6 +2,7 @@
 FROM golang:1.26 AS builder
 ARG TARGETOS
 ARG TARGETARCH
+ARG GIT_COMMIT=unknown
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -19,11 +20,15 @@ COPY . .
 # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a \
+    -ldflags "-X gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version.Commit=${GIT_COMMIT}" \
+    -o manager cmd/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
+ARG GIT_COMMIT=unknown
+LABEL org.opencontainers.image.revision="${GIT_COMMIT}"
 WORKDIR /
 COPY --from=builder /workspace/manager .
 USER 65532:65532

Makefile

@@ -2,6 +2,9 @@
 IMG ?= controller:latest
 # YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header.
 YEAR ?= $(shell date +%Y)
+# GIT_COMMIT is baked into the image (-ldflags in the Dockerfile); -dirty
+# covers staged and untracked changes too, which `git diff --quiet` misses.
+GIT_COMMIT ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)$(shell test -z "$$(git status --porcelain 2>/dev/null)" || echo -dirty)
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -61,7 +64,7 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet setup-envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test -race $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
 # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
 # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
@@ -110,18 +113,22 @@ lint-config: golangci-lint ## Verify golangci-lint linter configuration
 
 .PHONY: build
 build: manifests generate fmt vet ## Build manager binary.
-	go build -o bin/manager cmd/main.go
+	go build -o bin/manager ./cmd
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./cmd/main.go
 
+.PHONY: run-dev
+run-dev: manifests generate fmt vet ## Run locally against the current kubeconfig with the kubernetes-pod provider.
+	go run ./cmd/main.go --providers-config hack/providers-dev.yaml --metrics-bind-address :8080 --metrics-secure=false
+
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
 docker-build: ## Build docker image with the manager.
-	$(CONTAINER_TOOL) build -t ${IMG} .
+	$(CONTAINER_TOOL) build --build-arg GIT_COMMIT=$(GIT_COMMIT) -t ${IMG} .
 
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
@@ -140,7 +147,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform
 	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
 	- $(CONTAINER_TOOL) buildx create --name egress-proxies-operator-builder
 	$(CONTAINER_TOOL) buildx use egress-proxies-operator-builder
-	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
+	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --build-arg GIT_COMMIT=$(GIT_COMMIT) --tag ${IMG} -f Dockerfile.cross .
 	- $(CONTAINER_TOOL) buildx rm egress-proxies-operator-builder
 	rm Dockerfile.cross
 

README.md

@@ -1,135 +1,225 @@
 # egress-proxies-operator
-// TODO(user): Add simple overview of use/purpose
 
-## Description
+A Kubernetes operator that manages a fleet of HTTP egress proxies for
-// TODO(user): An in-depth paragraph about your project and overview of use
+crawling: each proxy is a `Proxy` custom resource that the operator
+provisions (or merely tracks), actively health-checks **through the proxy
+itself**, and hands out to crawler clients via an HTTP list/lease API.
 
-## Getting Started
+## Architecture in 60 seconds
 
-### Prerequisites
+- **`Proxy` CRD** (`crawl.example.com/v1alpha1`, namespaced, `kubectl get px`):
-- go version v1.24.6+
+  `Managed` proxies are provisioned by a configured provider; `External`
-- docker version 17.03+.
+  proxies exist elsewhere and are only tracked and health-checked.
-- kubectl version v1.11.3+.
+- **Reconciler** — a crash-safe state machine: every reconcile derives one
-- Access to a Kubernetes v1.11.3+ cluster.
+  action from (spec, status, provider Get). Proxies are **immutable
+  cattle**: any meaningful spec change (placement, cloud-init, port)
+  deletes and recreates the VM — never in-place mutation.
+- **Providers** behind one minimal interface: `kubernetes` (a real Squid
+  pod in this cluster — local dev/CI) and `gcp` (Compute Engine VMs with
+  ephemeral external IPs — the real egress fleet). Config is a YAML file
+  (`--providers-config`) with named instances (`gcp-eu`, `gcp-us`, ...).
+- **Health engine** probes every proxy by fetching a URL *through* it (a
+  real CONNECT tunnel — a proxy that accepts TCP but can't egress goes
+  Unhealthy), with threshold logic and transition-only status writes.
+- **Discovery API** (`:8090`): list healthy proxies filtered by
+  attributes, lease one (least-loaded, TTL-based), release, and report
+  rate-limiting — reports put the proxy in a per-target cooldown.
+- **Orphan GC** sweeps each provider for tagged instances whose owning CR
+  is gone — the safety net for crashes mid-create.
 
-### To Deploy on the cluster
+Details, diagrams, and recorded design decisions: [docs/architecture.md](docs/architecture.md).
-**Build and push your image to the location specified by `IMG`:**
+
+## Quickstart on kind (~5 minutes)
+
+Requires: kind, kubectl, docker, Go 1.26, jq (optional). The
+kubernetes-pod provider needs no cloud account — proxies are real
+`ubuntu/squid` pods in the kind cluster itself.
+
+The operator runs **in-cluster** for this quickstart. (Running it on your
+laptop with `make run-dev` provisions pods fine, but the health probe then
+originates on your machine, which cannot reach kind's pod IPs — the proxy
+would sit at `Unhealthy` forever. In-cluster, probes run where the pod
+network is routable.)
 
 ```sh
-make docker-build docker-push IMG=<some-registry>/egress-proxies-operator:tag
+kind create cluster --name proxy-operator-demo
+make install                                          # install the CRD
+make docker-build IMG=egress-proxies-operator:dev
+kind load docker-image egress-proxies-operator:dev --name proxy-operator-demo
+make deploy IMG=egress-proxies-operator:dev
+kubectl -n egress-proxies-operator-system rollout status deploy/egress-proxies-operator-controller-manager
 ```
 
-**NOTE:** This image ought to be published in the personal registry you specified.
+Create a proxy and watch it come up:
-And it is required to have access to pull the image from the working environment.
-Make sure you have the proper permission to the registry if the above commands don’t work.
-
-**Install the CRDs into the cluster:**
 
 ```sh
-make install
+kubectl apply -f config/samples/proxy_kubernetes.yaml
+kubectl get px -w
+# NAME                      MODE      PROVIDER     PHASE   IP           HEALTHY
+# proxy-kubernetes-sample   Managed   kubernetes   Ready   10.244.x.x   True
 ```
 
-**Deploy the Manager to the cluster with the image specified by `IMG`:**
+Once it's `Ready`, port-forward the discovery API and use it (full
+reference with schemas and error codes: [docs/api.md](docs/api.md)):
 
 ```sh
-make deploy IMG=<some-registry>/egress-proxies-operator:tag
+kubectl -n egress-proxies-operator-system port-forward \
+  svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090 &
 ```
 
-> **NOTE**: If you encounter RBAC errors, you may need to grant yourself cluster-admin
-privileges or be logged in as admin.
-
-**Create instances of your solution**
-You can apply the samples (examples) from the config/sample:
-
 ```sh
-kubectl apply -k config/samples/
+# List healthy proxies
+curl -s 'localhost:8090/v1/proxies?healthy=true' | jq
+
+# Lease one (5-minute TTL)
+curl -s -XPOST localhost:8090/v1/leases \
+  -d '{"selector":{"geo":"local"},"ttlSeconds":300}' | jq
+# → {"leaseID":"...", "proxy":{"id":"default/proxy-kubernetes-sample", "ip":..., ...}}
+
+# Actually crawl through it (from inside the cluster, or port-forward the pod)
+# curl -x http://<proxy-ip>:3128 https://example.com
+
+# Report the proxy got rate-limited by a site → 15-minute cooldown for that target
+curl -s -XPOST localhost:8090/v1/leases/<leaseID>/report \
+  -d '{"result":"rate_limited","target":"example.com"}'
+
+# Release early (idempotent — 204 both times)
+curl -si -XDELETE localhost:8090/v1/leases/<leaseID>
 ```
 
->**NOTE**: Ensure that the samples has default values to test it out.
+Tear down:
-
-### To Uninstall
-**Delete the instances (CRs) from the cluster:**
 
 ```sh
-kubectl delete -k config/samples/
+kubectl delete -f config/samples/proxy_kubernetes.yaml   # finalizer deletes the pod
+kind delete cluster --name proxy-operator-demo
 ```
 
-**Delete the APIs(CRDs) from the cluster:**
+## Deploying in-cluster
 
 ```sh
-make uninstall
+make docker-build IMG=<registry>/egress-proxies-operator:dev
+make deploy IMG=<registry>/egress-proxies-operator:dev
 ```
 
-**UnDeploy the controller from the cluster:**
+- Provider config comes from the `providers-config` ConfigMap
+  ([config/manager/providers_config.yaml](config/manager/providers_config.yaml));
+  the default ships only the kubernetes provider.
+- The discovery API is exposed by the
+  `controller-manager-discovery-service` Service on port 8090.
+- Auth: create the token Secret, or the API serves **unauthenticated**
+  (it warns loudly at startup):
 
   ```sh
-make undeploy
+  kubectl -n egress-proxies-operator-system create secret generic discovery-token \
+    --from-literal=token="$(openssl rand -hex 24)"
   ```
 
-## Project Distribution
+## GCP setup
 
-Following the options to release and provide this solution to the users.
+1. Add a `gcp` entry to the providers config (see
+   [config/samples/providers-config.yaml](config/samples/providers-config.yaml)) —
+   only `project` is required.
+2. Credentials are **Application Default Credentials**: workload identity
+   in-cluster, `gcloud auth application-default login` locally. No
+   key-file plumbing exists.
+3. The identity needs `roles/compute.instanceAdmin.v1` on the project —
+   plus `roles/iam.serviceAccountUser` if instances attach a service
+   account.
+4. Managed GCP proxies must set all of `placement.zone`,
+   `placement.machineType`, and `placement.image`
+   (see [config/samples/proxy_gcp.yaml](config/samples/proxy_gcp.yaml),
+   which also installs Squid via cloud-init). A missing field fails the
+   Proxy with a message naming it.
 
-### By providing a bundle with all YAML files
+Cloud-init from a Secret: the Secret **must** carry the label
+`crawl.example.com/cloud-init: "true"` — the operator's cache only holds
+labelled Secrets, so an unlabelled one is invisible (the Proxy reports
+`CloudInitError`). Rotating the Secret's content triggers VM replacement.
 
-1. Build the installer for the image built and published in the registry:
+## Caveats — read these two
+
+**Changing a proxy changes its IP.** Proxies are immutable cattle: editing
+`placement`, `cloudInit` (or rotating its Secret), or `port` deletes the
+VM and creates a replacement with the **same name but a new IP**. Clients
+discover the new address via the discovery API; anything that pinned the
+old IP breaks by design.
+
+**Operator restart drops all leases and cooldowns.** Lease state is
+in-memory (`replicas: 1` accordingly). Clients must tolerate a lease
+vanishing — requests through the proxy keep working; they just re-lease.
+The lease store sits behind an interface so a persistent backend can
+replace it without touching the API handlers.
+
+Smaller notes:
+
+- `status.lastHealthCheckTime` is the time of the last *status-affecting*
+  probe, not the most recent probe — status writes are transition-only by
+  design. True probe recency lives in the metrics
+  (`proxy_operator_healthcheck_*`).
+- The discovery API is served by every replica but is not leader-elected;
+  the operator ships with `replicas: 1` (see the lease caveat above).
+
+## Version pins
+
+Built and verified against the spec's pins with **no substitutions
+needed**: Go 1.26, kubebuilder v4.15.0, controller-runtime v0.24.1,
+k8s.io/* v0.36.3 (Kubernetes 1.36 API level), controller-tools v0.21.0,
+cloud.google.com/go/compute v1.65.0. envtest uses the 1.36.2 binary
+bundle (the latest 1.36 patch with published binaries — do not "fix" the
+Makefile's derived version to 1.36.3, which has none).
+
+## Gitea CI
+
+[.gitea/workflows/build.yaml](.gitea/workflows/build.yaml) builds the
+manager image and pushes it to this Gitea instance's container registry.
+It runs on **any tag push** or manually via **Run workflow** (with a `tag`
+input) — never on branch pushes. A lightweight `check` job (`go vet`,
+`go build`, `go test -short`) gates the build.
+
+Every build pushes two tags to
+`gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator`:
+
+- the human tag (the git tag, or the dispatch input), and
+- an immutable `sha-<12-char-commit>` tag — pin deployments to this one.
+
+`:latest` is additionally updated on real tag pushes only, so a manual
+dispatch of an old ref can never clobber it. The commit is baked into the
+binary (`internal/version.Commit`) via the `GIT_COMMIT` build arg.
+
+### Mandatory Gitea secrets
+
+Set under **Settings → Actions → Secrets** in this repo:
+
+| Secret           | Required by                   | What it is                               |
+| ---------------- | ----------------------------- | ---------------------------------------- |
+| `REGISTRY_TOKEN` | `build.yaml` (registry login) | Gitea PAT with the `write:package` scope |
+
+The token is paired with `${{ github.actor }}` as the username, so it
+must belong to the user triggering the workflow — same convention as the
+other projects on this instance.
+
+Without `REGISTRY_TOKEN` the `check` job still passes but the build job
+fails at the `docker login` step. No other secrets are needed — the
+workflow does not deploy anywhere.
+
+## Development
 
 ```sh
-make build-installer IMG=<some-registry>/egress-proxies-operator:tag
+make test        # unit + envtest suites, with -race (sets up envtest binaries itself)
+go test -short ./...   # skip the envtest suite
+make run-dev     # run against the current kubeconfig context
 ```
 
-**NOTE:** The makefile target mentioned above generates an 'install.yaml'
+`make run-dev` is for iterating on the operator itself: provisioning,
-file in the dist directory. This file contains all the resources built
+replacement, the discovery API, and External proxies all work from your
-with Kustomize, which are necessary to install this project without its
+laptop. Health checks against in-cluster pods do **not** (see the
-dependencies.
+quickstart note) — use the in-cluster deploy to see a kubernetes-provider
+proxy go `Ready`.
 
-2. Using the installer
+The full test inventory — what each suite covers, the deliberate gaps,
-
+and the manual kind verification procedure — is in
-Users can just run 'kubectl apply -f <URL for YAML BUNDLE>' to install
+[docs/testing.md](docs/testing.md).
-the project, i.e.:
-
-```sh
-kubectl apply -f https://raw.githubusercontent.com/<org>/egress-proxies-operator/<tag or branch>/dist/install.yaml
-```
-
-### By providing a Helm Chart
-
-1. Build the chart using the optional helm plugin
-
-```sh
-kubebuilder edit --plugins=helm/v2-alpha
-```
-
-2. See that a chart was generated under 'dist/chart', and users
-can obtain this solution from there.
-
-**NOTE:** If you change the project, you need to update the Helm Chart
-using the same command above to sync the latest changes. Furthermore,
-if you create webhooks, you need to use the above command with
-the '--force' flag and manually ensure that any custom configuration
-previously added to 'dist/chart/values.yaml' or 'dist/chart/manager/manager.yaml'
-is manually re-applied afterwards.
-
-## Contributing
-// TODO(user): Add detailed information on how you would like others to contribute to this project
-
-**NOTE:** Run `make help` for more information on all potential `make` targets
-
-More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html)
-
-## License
-
-Copyright 2026.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
 
+Project layout, reconcile-loop diagrams, and the decision log are in
+[docs/architecture.md](docs/architecture.md); the build history is in
+[docs/plans-executions/](docs/plans-executions/).

api/v1alpha1/proxy_types.go

@@ -61,6 +61,12 @@ const (
 	// provision. A mismatch against the freshly computed hash means the VM
 	// must be replaced.
 	AnnotationSpecHash = "crawl.example.com/spec-hash"
+
+	// LabelCloudInit must be set (to "true") on every Secret referenced by
+	// spec.cloudInit.secretRef: the manager's cache only holds Secrets
+	// carrying this label, so an unlabelled Secret is invisible to the
+	// operator — both to the resolve step and to the rotation watch.
+	LabelCloudInit = "crawl.example.com/cloud-init"
 )
 
 // Defaults, applied both by CRD structural defaulting (kubebuilder:default

cmd/main.go

@@ -14,28 +14,51 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package main is the composition root: it loads the provider config (fail
+// fast), assembles the provider registry, and wires the reconciler, health
+// engine, lease store, discovery API, orphan GC, and metrics onto one
+// controller-runtime manager.
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
+	"fmt"
 	"os"
+	goruntime "runtime"
+	"time"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/controller"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/discovery"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/gc"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/metrics"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -60,6 +83,17 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var tlsOpts []func(*tls.Config)
+
+	var providersConfig string
+	var discoveryAddr string
+	var proxyNamespace string
+	var healthWorkers int
+	var gcInterval, gcMinAge time.Duration
+	var gcAllowNamespaced bool
+	var leaseCooldown, maxLeaseTTL time.Duration
+	var showVersion bool
+	var gcpWireFullPayloads bool
+
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -74,13 +108,73 @@ func main() {
 	flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics server")
+
+	flag.StringVar(&providersConfig, "providers-config", "",
+		"Path to the providers config YAML. Required.")
+	flag.StringVar(&discoveryAddr, "discovery-addr", ":8090",
+		"Listen address of the discovery/lease HTTP API.")
+	flag.StringVar(&proxyNamespace, "proxy-namespace", "",
+		"Restrict the manager's cache to one namespace. Empty watches all namespaces. "+
+			"Restricting also disables orphan GC unless --gc-allow-namespaced is set.")
+	flag.IntVar(&healthWorkers, "health-workers", 8,
+		"Number of concurrent health-probe workers.")
+	flag.DurationVar(&gcInterval, "gc-interval", 10*time.Minute,
+		"Interval between orphan GC sweeps.")
+	flag.DurationVar(&gcMinAge, "gc-min-age", 10*time.Minute,
+		"Minimum instance age before orphan GC may delete it.")
+	flag.BoolVar(&gcAllowNamespaced, "gc-allow-namespaced", false,
+		"Allow orphan GC to run although the cache is namespace-restricted. Dangerous: proxies "+
+			"outside the namespace count as orphans and their instances get deleted.")
+	flag.DurationVar(&leaseCooldown, "lease-cooldown", 15*time.Minute,
+		"How long a reported proxy/target pair is excluded from lease selection.")
+	flag.DurationVar(&maxLeaseTTL, "max-lease-ttl", time.Hour,
+		"Maximum lease TTL a client may request.")
+	flag.BoolVar(&showVersion, "version", false,
+		"Print the commit the binary was built from and exit.")
+	flag.BoolVar(&gcpWireFullPayloads, "gcp-wire-log-full-payloads", false,
+		"Log GCP V(5) wire payloads verbatim instead of eliding fields larger than 1KiB.")
+
 	opts := zap.Options{
 		Development: true,
 	}
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
+	if showVersion {
+		fmt.Println(version.Resolve())
+		os.Exit(0)
+	}
+
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+	setupLog.Info("Starting egress-proxies-operator",
+		"commit", version.Resolve(), "goVersion", goruntime.Version())
+	ctx := ctrl.SetupSignalHandler()
+
+	// Providers load first and fail fast: a manager that comes up without
+	// its backends would just convert every Proxy into an error loop.
+	if providersConfig == "" {
+		setupLog.Error(nil, "--providers-config is required")
+		os.Exit(1)
+	}
+	cfg, err := provider.LoadConfigFile(providersConfig)
+	if err != nil {
+		setupLog.Error(err, "Failed to load providers config", "path", providersConfig)
+		os.Exit(1)
+	}
+	providers, err := registry.Build(ctx, cfg, map[string]registry.Constructor{
+		"kubernetes": kubernetes.New,
+		"gcp": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+			return gcp.NewWithWireOptions(ctx, pc, gcp.WireLogOptions{FullPayloads: gcpWireFullPayloads})
+		},
+	})
+	if err != nil {
+		setupLog.Error(err, "Failed to build providers")
+		os.Exit(1)
+	}
+	m := metrics.New()
+	for name, p := range providers {
+		providers[name] = provider.WithMetrics(name, p, m)
+	}
 
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
@@ -128,32 +222,91 @@ func main() {
 		metricsServerOptions.KeyName = metricsCertKey
 	}
 
+	// The Secret cache is restricted to labelled cloud-init Secrets: the
+	// operator has cluster-wide Secret read RBAC, and without the label
+	// selector it would cache every Secret in scope.
+	cacheOpts := cache.Options{
+		ByObject: map[client.Object]cache.ByObject{
+			&corev1.Secret{}: {
+				Label: labels.SelectorFromSet(labels.Set{crawlv1alpha1.LabelCloudInit: "true"}),
+			},
+		},
+	}
+	if proxyNamespace != "" {
+		cacheOpts.DefaultNamespaces = map[string]cache.Config{proxyNamespace: {}}
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
+		Cache:                  cacheOpts,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "b47711d1.example.com",
-		// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
-		// when the Manager ends. This requires the binary to immediately end when the
-		// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
-		// speeds up voluntary leader transitions as the new leader don't have to wait
-		// LeaseDuration time first.
-		//
-		// In the default scaffold provided, the program ends immediately after
-		// the manager stops, so would be fine to enable this option. However,
-		// if you are doing or is intended to do any operation such as perform cleanups
-		// after the manager stops then its usage might be unsafe.
-		// LeaderElectionReleaseOnCancel: true,
 	})
 	if err != nil {
 		setupLog.Error(err, "Failed to start manager")
 		os.Exit(1)
 	}
 
+	store := lease.NewStore(leaseCooldown)
+	if err := mgr.Add(store); err != nil {
+		setupLog.Error(err, "Failed to add lease store")
+		os.Exit(1)
+	}
+
+	engine := health.NewEngine(mgr.GetClient())
+	engine.Workers = healthWorkers
+	engine.Metrics = m
+	if err := mgr.Add(engine); err != nil {
+		setupLog.Error(err, "Failed to add health engine")
+		os.Exit(1)
+	}
+
+	if err := mgr.Add(&discovery.Server{
+		Reader:      mgr.GetClient(),
+		Store:       store,
+		Addr:        discoveryAddr,
+		Token:       os.Getenv("DISCOVERY_TOKEN"),
+		MaxLeaseTTL: maxLeaseTTL,
+		Metrics:     m,
+	}); err != nil {
+		setupLog.Error(err, "Failed to add discovery server")
+		os.Exit(1)
+	}
+
+	if err := mgr.Add(&gc.Sweeper{
+		Reader:              mgr.GetClient(),
+		Providers:           providers,
+		Interval:            gcInterval,
+		MinAge:              gcMinAge,
+		NamespaceRestricted: proxyNamespace != "",
+		AllowNamespaced:     gcAllowNamespaced,
+	}); err != nil {
+		setupLog.Error(err, "Failed to add orphan GC")
+		os.Exit(1)
+	}
+
+	if err := m.Register(ctrlmetrics.Registry,
+		proxyPhaseCounts(mgr.GetClient()),
+		func() int {
+			total := 0
+			for _, n := range store.Counts() {
+				total += n
+			}
+			return total
+		},
+	); err != nil {
+		setupLog.Error(err, "Failed to register metrics")
+		os.Exit(1)
+	}
+
 	if err := (&controller.ProxyReconciler{
 		Client:       mgr.GetClient(),
 		Scheme:       mgr.GetScheme(),
+		Providers:    providers,
+		Health:       engine,
+		HealthEvents: engine.Events,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "Failed to create controller", "controller", "proxy")
 		os.Exit(1)
@@ -170,8 +323,31 @@ func main() {
 	}
 
 	setupLog.Info("Starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "Failed to run manager")
 		os.Exit(1)
 	}
 }
+
+// proxyPhaseCounts reads phase counts from the cache at scrape time. Before
+// the cache has synced (or on any list error) it reports nothing rather
+// than something wrong.
+func proxyPhaseCounts(c client.Reader) func() map[string]int {
+	return func() map[string]int {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		var list crawlv1alpha1.ProxyList
+		if err := c.List(ctx, &list); err != nil {
+			return nil
+		}
+		counts := map[string]int{}
+		for i := range list.Items {
+			phase := string(list.Items[i].Status.Phase)
+			if phase == "" {
+				phase = string(crawlv1alpha1.PhasePending)
+			}
+			counts[phase]++
+		}
+		return counts
+	}
+}

config/default/discovery_service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: controller-manager
+    app.kubernetes.io/name: egress-proxies-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: controller-manager-discovery-service
+  namespace: system
+spec:
+  ports:
+  - name: discovery
+    port: 8090
+    protocol: TCP
+    targetPort: discovery
+  selector:
+    control-plane: controller-manager
+    app.kubernetes.io/name: egress-proxies-operator

config/default/kustomization.yaml

@@ -22,6 +22,8 @@ resources:
 #- ../prometheus
 # [METRICS] Expose the controller manager metrics service.
 - metrics_service.yaml
+# Expose the discovery/lease HTTP API inside the cluster.
+- discovery_service.yaml
 
 # Uncomment the patches line if you enable Metrics
 patches:

config/manager/kustomization.yaml

@@ -1,5 +1,6 @@
 resources:
 - manager.yaml
+- providers_config.yaml
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:

config/manager/manager.yaml

@@ -63,12 +63,28 @@ spec:
         args:
           - --leader-elect
           - --health-probe-bind-address=:8081
+          - --providers-config=/etc/proxy-operator/providers.yaml
+        env:
+          # Bearer token for the discovery API. Optional: without the
+          # Secret the API serves unauthenticated (with a loud warning).
+          # Create it with:
+          #   kubectl -n egress-proxies-operator-system create secret \
+          #     generic discovery-token --from-literal=token=<your-token>
+          - name: DISCOVERY_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: discovery-token
+                key: token
+                optional: true
         image: controller:latest
         name: manager
         ports:
         - containerPort: 8081
           name: health
           protocol: TCP
+        - containerPort: 8090
+          name: discovery
+          protocol: TCP
         securityContext:
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
@@ -96,7 +112,13 @@ spec:
           requests:
             cpu: 10m
             memory: 64Mi
-        volumeMounts: []
+        volumeMounts:
-      volumes: []
+        - name: providers-config
+          mountPath: /etc/proxy-operator
+          readOnly: true
+      volumes:
+      - name: providers-config
+        configMap:
+          name: providers-config
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10

config/manager/providers_config.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: providers-config
+  namespace: system
+  labels:
+    app.kubernetes.io/name: egress-proxies-operator
+    app.kubernetes.io/managed-by: kustomize
+data:
+  # Mounted at /etc/proxy-operator/providers.yaml (--providers-config).
+  # The default ships only the kubernetes-pod provider so the operator runs
+  # out of the box; add gcp entries (type: gcp, gcp.project: ...) for real
+  # egress fleets — see config/samples/providers-config.yaml.
+  providers.yaml: |
+    providers:
+      - name: kubernetes
+        type: kubernetes

config/rbac/role.yaml

@@ -4,6 +4,16 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:

config/samples/crawl_v1alpha1_proxy.yaml

@@ -1,9 +0,0 @@
-apiVersion: crawl.example.com/v1alpha1
-kind: Proxy
-metadata:
-  labels:
-    app.kubernetes.io/name: egress-proxies-operator
-    app.kubernetes.io/managed-by: kustomize
-  name: proxy-sample
-spec:
-  # TODO(user): Add fields here

config/samples/kustomization.yaml

@@ -1,4 +1,8 @@
 ## Append samples of your project ##
+# providers-config.yaml is deliberately absent: it is a sample
+# --providers-config file, not a Kubernetes manifest.
 resources:
-- crawl_v1alpha1_proxy.yaml
+- proxy_kubernetes.yaml
+- proxy_gcp.yaml
+- proxy_external.yaml
 # +kubebuilder:scaffold:manifestskustomizesamples

config/samples/providers-config.yaml

@@ -0,0 +1,21 @@
+# Sample --providers-config file (not a Kubernetes manifest). In-cluster
+# this content lives in the providers-config ConfigMap
+# (config/manager/providers_config.yaml); for `make run-dev` a
+# kubernetes-only variant is at hack/providers-dev.yaml.
+#
+# Named provider instances: "gcp-eu" and "gcp-us" are two configs of the
+# same type. spec.provider on a Proxy refers to the name, not the type.
+providers:
+  - name: kubernetes
+    type: kubernetes
+    # kubernetes:
+    #   image: ubuntu/squid:6.6-24.04_edge   # the default
+  - name: gcp-eu
+    type: gcp
+    gcp:
+      project: my-project
+      # network: default            # VPC network name
+      # networkTag: proxy-operator  # firewall tag on created instances
+      # diskSizeGb: 10
+      # auth: Application Default Credentials (workload identity
+      # in-cluster, gcloud ADC locally). No key-file plumbing.

config/samples/proxy_external.yaml

@@ -0,0 +1,15 @@
+# An External proxy: the VM exists outside the operator's control; the
+# operator only tracks and health-checks it through the endpoint. No
+# finalizer, no provider calls, and deleting the CR touches nothing.
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-external-sample
+spec:
+  mode: External
+  endpoint:
+    host: 203.0.113.7
+    port: 3128
+  attributes:
+    geo: eu
+    purpose: crawl

config/samples/proxy_gcp.yaml

@@ -0,0 +1,36 @@
+# A Managed proxy backed by GCP: the operator creates a VM with an
+# ephemeral external IP and installs Squid via cloud-init. Requires a
+# providers-config entry named "gcp-eu" (see providers-config.yaml) and
+# Application Default Credentials with compute.instanceAdmin.v1.
+#
+# All three placement fields are required for GCP; the operator sets the
+# Proxy to Failed with a message naming any missing one.
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-gcp-sample
+spec:
+  mode: Managed
+  provider: gcp-eu
+  placement:
+    zone: europe-west1-b
+    machineType: e2-micro
+    image: projects/debian-cloud/global/images/family/debian-12
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_port 3128
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+  attributes:
+    geo: eu
+    purpose: crawl

config/samples/proxy_kubernetes.yaml

@@ -0,0 +1,14 @@
+# A Managed proxy backed by the kubernetes-pod provider: the operator runs
+# a real Squid pod in this cluster. This is the local-dev/CI sample — pods
+# share the cluster's egress IP, so it exercises the full lifecycle but
+# does not provide a distinct egress path (use the gcp provider for that).
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-kubernetes-sample
+spec:
+  mode: Managed
+  provider: kubernetes
+  attributes:
+    geo: local
+    purpose: crawl

docs/api.md

@@ -0,0 +1,323 @@
+# Discovery API reference
+
+The operator serves an HTTP API (the *discovery API*) that crawler clients
+use to find and lease egress proxies: list healthy proxies filtered by
+attributes, acquire a TTL-based lease on one, release it early, and report
+how a target site treated the proxy. It is implemented in
+[`internal/discovery`](../internal/discovery/) with lease state in
+[`internal/lease`](../internal/lease/); the only Kubernetes interaction is
+reading `Proxy` resources from the manager's cache.
+
+## Base URL
+
+The API listens on `:8090` (`--discovery-addr`) inside the manager pod and
+is exposed by a Service
+([config/default/discovery_service.yaml](../config/default/discovery_service.yaml)).
+
+In-cluster:
+
+```text
+http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090
+```
+
+From a workstation, port-forward:
+
+```sh
+kubectl -n egress-proxies-operator-system port-forward \
+  svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090 &
+```
+
+Set `BASE_URL` to wherever you reach the API; all examples below use it:
+
+```sh
+export BASE_URL=localhost:8090   # via the port-forward above
+# or, from inside the cluster:
+# export BASE_URL=http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090
+```
+
+## Authentication
+
+A single static bearer token, read from the `DISCOVERY_TOKEN` environment
+variable at startup. The shipped Deployment populates it from the
+`discovery-token` Secret (key `token`), which is **optional** — if the
+Secret is absent or the token is empty, the API serves **unauthenticated**
+(the manager logs a loud warning at startup). Create the Secret:
+
+```sh
+kubectl -n egress-proxies-operator-system create secret generic discovery-token \
+  --from-literal=token="$(openssl rand -hex 24)"
+```
+
+Send the token on every request:
+
+```sh
+export TOKEN=<the token>
+curl -s -H "Authorization: Bearer $TOKEN" "$BASE_URL/v1/proxies" | jq
+```
+
+A missing or wrong token gets `401 {"error":"unauthorized",...}`.
+`GET /healthz` is always exempt.
+
+The curl examples below omit the `-H "Authorization: Bearer $TOKEN"` flag
+for brevity — add it to every call when auth is enabled.
+
+## Conventions
+
+- Requests and responses are JSON. Errors share one envelope:
+
+  ```json
+  {"error": "<machine_code>", "message": "<human-readable text>"}
+  ```
+
+- Request bodies are capped at **64 KiB** (larger bodies fail the JSON
+  decode with `400 invalid_body`).
+- Proxies with a deletion timestamp (being finalized) are excluded from
+  every response and never offered for lease.
+
+## Configuration
+
+| Setting | Default | Meaning |
+|---|---|---|
+| `--discovery-addr` | `:8090` | Listen address of the API |
+| `--max-lease-ttl` | `1h` | Maximum `ttlSeconds` a client may request |
+| `--lease-cooldown` | `15m` | Cooldown window applied on `rate_limited`/`banned` reports |
+| `DISCOVERY_TOKEN` (env) | empty | Bearer token; empty disables auth |
+
+The shipped Deployment passes none of these flags, so the defaults apply.
+
+## Endpoints
+
+### `GET /healthz`
+
+Liveness check. Unauthenticated, always `200` with body `ok`.
+
+```sh
+curl -s "$BASE_URL/healthz"
+```
+
+### `GET /v1/proxies` — list proxies
+
+Query parameters (all optional):
+
+| Parameter | Values | Effect |
+|---|---|---|
+| `healthy` | `true` \| `false` | Keep only proxies whose `Healthy` condition matches. Any other value → `400 invalid_query`. |
+| `attr.<key>` | any string | Exact match on `spec.attributes[<key>]`. Repeatable; **all** given pairs must match. |
+
+List everything:
+
+```sh
+curl -s "$BASE_URL/v1/proxies" | jq
+```
+
+List healthy proxies in a given geo:
+
+```sh
+curl -s "$BASE_URL/v1/proxies?healthy=true&attr.geo=eu" | jq
+```
+
+Response — `200`, proxies sorted by `id`, an empty match is `200` with
+`"count": 0` (never `404`):
+
+```json
+{
+  "proxies": [
+    {
+      "id": "default/proxy-kubernetes-sample",
+      "ip": "10.244.1.7",
+      "port": 3128,
+      "attributes": {"geo": "local"},
+      "phase": "Ready",
+      "healthy": true,
+      "latencyMillis": 42,
+      "activeLeases": 1,
+      "maxLeases": 5
+    }
+  ],
+  "count": 1
+}
+```
+
+Proxy object fields (the same shape appears inside lease responses):
+
+| Field | Meaning |
+|---|---|
+| `id` | `namespace/name` of the `Proxy` resource; used as the stable key everywhere |
+| `ip` | Effective host — `spec.endpoint.host` for `External` proxies, `status.ip` for `Managed` (empty until the backing VM/pod is up) |
+| `port` | Effective port (default `3128`) |
+| `attributes` | `spec.attributes` — free-form selection labels (`geo`, `asn`, `purpose`, …); omitted when empty |
+| `phase` | `Pending` \| `Provisioning` \| `Ready` \| `Unhealthy` \| `Deleting` \| `Failed` |
+| `healthy` | `true` iff the `Healthy` condition is `True` (the through-the-proxy health probe passes) |
+| `latencyMillis` | Latency of the last status-affecting health probe |
+| `activeLeases` | Currently active leases on this proxy |
+| `maxLeases` | Lease capacity (default `5`; an explicit `0` means unleasable) |
+
+### `POST /v1/leases` — acquire a lease
+
+Picks a healthy proxy with free capacity matching the selector and grants
+an exclusive-slot, TTL-based lease on it.
+
+Request body (every field optional; `{}` is valid):
+
+```json
+{
+  "selector": {"geo": "eu"},
+  "ttlSeconds": 300,
+  "target": "example.com"
+}
+```
+
+| Field | Default | Meaning |
+|---|---|---|
+| `selector` | none | Attribute equality filter, same semantics as `attr.<key>` above |
+| `ttlSeconds` | `300` (5 min) | Lease lifetime; must be ≤ `--max-lease-ttl` (default 1 h), else `400 invalid_ttl` |
+| `target` | none | The site you intend to crawl; enables per-target cooldowns (see below) |
+
+```sh
+curl -s -XPOST "$BASE_URL/v1/leases" \
+  -d '{"selector":{"geo":"eu"},"ttlSeconds":300,"target":"example.com"}' | jq
+```
+
+Success — `201`:
+
+```json
+{
+  "leaseID": "P3X6HHQTPCM5UTGVGE3B5UPS3A",
+  "proxy": {
+    "id": "default/proxy-eu-1",
+    "ip": "34.88.10.20",
+    "port": 3128,
+    "attributes": {"geo": "eu"},
+    "phase": "Ready",
+    "healthy": true,
+    "latencyMillis": 42,
+    "activeLeases": 1,
+    "maxLeases": 5
+  },
+  "expiresAt": "2026-08-11T22:05:00Z",
+  "ttlSeconds": 300
+}
+```
+
+Use `proxy.ip` and `proxy.port` as an HTTP proxy for the lease's lifetime:
+
+```sh
+curl -x http://34.88.10.20:3128 https://example.com
+```
+
+No match — `409` with diagnostic counts explaining why nothing qualified:
+
+```json
+{
+  "error": "no_match",
+  "message": "no healthy proxy with free capacity matched the selector",
+  "considered": 3,
+  "atCapacity": 1,
+  "inCooldown": 1,
+  "unhealthy": 1
+}
+```
+
+| Count | Meaning |
+|---|---|
+| `considered` | Proxies that matched the selector (before health/capacity checks) |
+| `atCapacity` | Skipped because `activeLeases >= maxLeases` |
+| `inCooldown` | Skipped because of an active cooldown for this target (or a global one) |
+| `unhealthy` | Skipped because the `Healthy` condition is not `True` |
+
+Leases expire on their own — releasing is only needed to free the slot
+early. There is no renew/extend endpoint; acquire a new lease instead.
+
+### `DELETE /v1/leases/{id}` — release early
+
+Frees the lease's capacity slot immediately. Idempotent: always `204`,
+including for unknown or already-expired lease IDs.
+
+```sh
+curl -si -XDELETE "$BASE_URL/v1/leases/P3X6HHQTPCM5UTGVGE3B5UPS3A"
+```
+
+### `POST /v1/leases/{id}/report` — report an outcome
+
+Tell the operator how the target site treated the proxy. This is the
+feedback signal that drives cooldowns.
+
+Request body:
+
+```json
+{"result": "rate_limited", "target": "example.com"}
+```
+
+| Field | Values | Meaning |
+|---|---|---|
+| `result` | `ok` \| `rate_limited` \| `banned` | Anything else → `400 invalid_result` |
+| `target` | optional | Which site produced the result; falls back to the lease's `target`, then to global |
+
+```sh
+curl -si -XPOST "$BASE_URL/v1/leases/P3X6HHQTPCM5UTGVGE3B5UPS3A/report" \
+  -d '{"result":"rate_limited","target":"example.com"}'
+```
+
+Responses: `204` on success, `404 unknown_lease` if the lease ID was never
+issued or has aged out.
+
+Semantics:
+
+- `ok` is a pure acknowledgement — nothing is recorded.
+- `rate_limited` and `banned` currently behave **identically**: both put
+  the proxy in one cooldown window (default 15 min, `--lease-cooldown`)
+  for the resolved target.
+- An expired lease remains reportable for one cooldown window past its
+  TTL, so a late "we got rate-limited" still lands.
+
+## Proxy selection and cooldowns
+
+How `POST /v1/leases` picks among eligible proxies (healthy, matching the
+selector, not being deleted, not at capacity, not in cooldown), in order:
+
+1. fewest `activeLeases` (least-loaded),
+2. lowest `latencyMillis`,
+3. lexicographic `id` (deterministic tie-break).
+
+Cooldowns are keyed by **(proxy, target)**:
+
+- A report **with a target** blocks that proxy only for lease requests
+  naming the **same target**. Other targets — and requests with no
+  target — still get the proxy.
+- A report **without a target**, on a lease that also had no target,
+  creates a **global** cooldown: the proxy is blocked for *all* lease
+  requests until the window passes. Always pass `target` on leases and
+  reports unless you really mean "this proxy is bad for everyone".
+
+## End-to-end example
+
+```sh
+# 1. Acquire a lease for crawling example.com through an EU proxy
+LEASE=$(curl -s -XPOST "$BASE_URL/v1/leases" \
+  -H "Authorization: Bearer $TOKEN" \
+  -d '{"selector":{"geo":"eu"},"ttlSeconds":600,"target":"example.com"}')
+LEASE_ID=$(echo "$LEASE" | jq -r .leaseID)
+PROXY=$(echo "$LEASE" | jq -r '"\(.proxy.ip):\(.proxy.port)"')
+
+# 2. Crawl through the leased proxy
+curl -x "http://$PROXY" https://example.com/some/page
+
+# 3. Got a 429? Report it — example.com-bound leases will avoid this
+#    proxy for the next 15 minutes
+curl -s -XPOST "$BASE_URL/v1/leases/$LEASE_ID/report" \
+  -H "Authorization: Bearer $TOKEN" \
+  -d '{"result":"rate_limited","target":"example.com"}'
+
+# 4. Done early? Release the slot (otherwise the TTL frees it)
+curl -s -XDELETE "$BASE_URL/v1/leases/$LEASE_ID" \
+  -H "Authorization: Bearer $TOKEN"
+```
+
+## Caveats
+
+- **Lease and cooldown state is in-memory and per-process.** An operator
+  restart drops all active leases and cooldowns. Clients must tolerate a
+  granted lease disappearing (a subsequent report returns `404`).
+- **Run a single replica.** The API is served by every manager replica but
+  is not leader-elected, and lease state is not shared between replicas;
+  the shipped Deployment pins `replicas: 1`.

docs/architecture.md

@@ -1,10 +1,20 @@
 # Architecture
 
-> **Status:** the operator is built through Step 5 (health engine) of
+## Components
-> [docs/plans/2026-08-07-1747-proxy-operator.md](plans/2026-08-07-1747-proxy-operator.md).
+
-> This document currently covers the event/reconcile flow; the components
+| Component | Package | Runs as | Leader-elected | Role |
-> table and the Decisions section arrive with Step 10, and the diagrams
+|---|---|---|---|---|
-> below grow as the lease store, discovery API, and orphan GC land.
+| Proxy CRD + helpers | `api/v1alpha1` | types | — | `Proxy` spec/status, CEL validation, defaulting, pure helpers |
+| Reconciler | `internal/controller` | controller | yes (with the manager) | the state machine: provision, replace, delete, represent health |
+| Provider contract | `internal/provider` | library | — | `Provider` interface, error taxonomy, deterministic naming, config, metrics decorator |
+| kubernetes provider | `internal/provider/kubernetes` | library | — | real Squid pods in this cluster (local dev/CI) |
+| gcp provider | `internal/provider/gcp` | library | — | Compute Engine VMs, four API calls, fire-and-forget ops |
+| Health engine | `internal/health` | Runnable | yes | through-the-proxy probes, thresholds, transition events |
+| Lease store | `internal/lease` | Runnable (expiry sweep) | no | in-memory leases + cooldowns, single mutex |
+| Discovery API | `internal/discovery` | Runnable | no | HTTP list/lease/release/report on `:8090` |
+| Orphan GC | `internal/gc` | Runnable | yes | deletes tagged instances whose CR is gone |
+| Metrics | `internal/metrics` | library | — | explicit registration, scrape-time collectors |
+| Composition root | `cmd/main.go` | binary | — | flags, provider registry, wires everything onto one manager |
 
 ## Event flow: cluster events → reconciler functions
 
@@ -117,20 +127,32 @@ reconcileDelete(ctx, p)                          reconcileExternal(ctx, p)
         ──► RequeueAfter: DeletionPoll (poll until gone)
 ```
 
-### 5. What provider calls do back in the cluster (kubernetes pod provider)
+### 5. What provider calls do in the outside world
 
 ```text
+kubernetes pod provider (internal/provider/kubernetes/)
 prov.Create ──► buildPod (pure) ──► client.Create(corev1.Pod)   ─┐ these cause Pod events,
 prov.Get    ──► client.Get(Pod) → phase/IP → InstanceState       │ but the operator does NOT
 prov.Delete ──► client.Delete(Pod, tolerate NotFound)            │ watch Pods — it observes
 prov.ListByTag ─► client.List(Pods by labels, all namespaces)   ─┘ them by polling prov.Get
                                                                     on each RequeueAfter tick
+
+gcp provider (internal/provider/gcp/) — instances.{Insert,Get,Delete,AggregatedList}, nothing else
+prov.Create ──► buildInsertRequest (pure) ──► instances.Insert   ─┐ fire-and-forget:
+                  409 alreadyExists = success (idempotent retry)  │ Operation.Wait is never
+prov.Get    ──► instances.Get → status/NatIP → InstanceState      │ called; readiness is
+                  RUNNING without NatIP = still Provisioning      │ discovered by Get polls,
+prov.Delete ──► instances.Delete (404 = success)                  │ exactly like the pod
+prov.ListByTag ─► AggregatedList(label filter,                   ─┘ provider
+                    ReturnPartialSuccess: true)
+providerID = zones/<zone>/instances/<name> — zone-qualified, so Get/Delete
+stay correct even mid-replacement after a zone edit
 ```
 
-The reconciler never watches provider-side resources (Pods now, GCP VMs
+The reconciler never watches provider-side resources (Pods or GCP VMs).
-later). All instance-state observation is poll-based through the
+All instance-state observation is poll-based through the `Provider`
-`Provider` interface, so the same flow works identically for a cloud API
+interface, so the same flow works identically for a cloud API that has no
-that has no watch mechanism at all.
+watch mechanism at all.
 
 ### 6. Health engine (`internal/health/`) — probes and transitions
 
@@ -183,3 +205,202 @@ Consequence worth knowing: `status.lastHealthCheckTime` is the time of the
 last *status-affecting* probe, not the most recent probe — suppressed
 probes deliberately never write status. True probe recency will live in
 metrics (Step 9).
+
+### 7. Discovery + lease API (`internal/discovery/`, `internal/lease/`)
+
+HTTP-driven, not cluster-event-driven: crawler clients call in; the only
+Kubernetes interaction is reading Proxies from the manager's cache. The
+server is a non-leader-elected Runnable (all replicas would serve, but the
+deployment ships `replicas: 1` because lease state is per-process — an
+operator restart drops all leases and cooldowns, a documented caveat).
+Client-facing reference with request/response schemas and curl examples:
+[api.md](api.md).
+
+```text
+crawler client
+  │  Authorization: Bearer $DISCOVERY_TOKEN   (empty token = auth disabled, loud startup warning)
+  ▼
+Server.handler()      middleware, outermost first        (server.go)
+  recover → request-log → MaxBytesReader(64KiB) → bearer auth (constant-time; /healthz exempt)
+  │
+  ├─ GET  /healthz            ──► 200 ok (unauthenticated)
+  │
+  ├─ GET  /v1/proxies?attr.k=v&healthy=true            (handlers.go)
+  │     Reader.List(Proxies) ── manager cache
+  │     filter: attributes equality + Healthy condition
+  │     + Store.Counts() for activeLeases
+  │     ──► 200 {"proxies":[...], "count":N}   (empty list is 200, not 404)
+  │
+  ├─ POST /v1/leases  {"selector":{...},"ttlSeconds":300,"target":"..."}
+  │     Reader.List → filter selector; unhealthy matches counted, not offered
+  │     Store.Acquire(healthy candidates, target, ttl)   ── one lock: select+insert
+  │     │    selection: fewest active leases, then latency, then name
+  │     ├─ granted ──► 201 {leaseID, proxy:{...}, expiresAt, ttlSeconds}
+  │     └─ ErrNoMatch ──► 409 {"error":"no_match", considered, atCapacity,
+  │                            inCooldown, unhealthy}
+  │
+  ├─ DELETE /v1/leases/{id} ──► Store.Release ──► always 204 (idempotent)
+  │
+  └─ POST /v1/leases/{id}/report {"result":"ok|rate_limited|banned","target":"..."}
+        Store.Report ── rate_limited/banned ⇒ cooldown[{proxy,target}] for
+        │               CooldownWindow (target falls back: report → lease → global)
+        ├─ 204 │ 400 invalid_result │ 404 unknown_lease
+        └─ an expired lease still resolves for CooldownWindow past its TTL —
+           a late report lands exactly when the proxy is being rate-limited
+
+Store.Start(ctx)  ── manager Runnable, NOT leader-elected: sweeps expired
+                     leases + cooldowns; correctness never depends on the
+                     sweep (every read checks ExpiresAt against the clock)
+```
+
+### 8. Orphan GC (`internal/gc/`) — the crash-safety net
+
+Timer-driven, leader-elected (destructive ⇒ single writer). Exists for the
+one gap the reconciler cannot close alone: a crash after a provider Create
+but before the status write that records the instance.
+
+```text
+Sweeper.Start(ctx)   ── refuses to run when the cache is namespace-
+  │                     restricted unless --gc-allow-namespaced is explicit
+  │                     (an incomplete live set would "orphan" live VMs)
+  └─ every Interval (10m; first sweep a full interval after start):
+       sweep(ctx)
+         │ Reader.List(Proxies) → live UID set
+         │   List fails → skip the whole sweep (never guess)
+         │   a CR with deletionTimestamp still counts as LIVE — its
+         │   finalizer owns that deletion; GC racing it double-deletes
+         └ per provider: ListByTag
+             │ error → log, continue with the next provider
+             └ delete only when ALL hold:
+                 has the proxy-operator-uid label   (ownership proof)
+                 older than MinAge (10m)            (not mid-create)
+                 UID matches no existing CR         (truly orphaned)
+               each kill logged loudly with provider, providerID, UID
+```
+
+### 9. Metrics (`internal/metrics/`)
+
+Registered explicitly from `cmd/main.go` (no `init()`; tests use fresh
+registries). Two kinds:
+
+- **Scrape-time collectors** — `proxy_operator_proxies{phase}` and
+  `proxy_operator_leases_active` read the cache / lease store at every
+  scrape; reconcile-incremented gauges inevitably drift and leak series.
+- **Fed vectors** — `healthcheck_duration_seconds{proxy}` and
+  `healthcheck_failures_total{proxy}` observe EVERY probe (status writes
+  are transition-only; metrics carry the high-frequency signal), and the
+  health engine deletes a proxy's series when it prunes its state;
+  `lease_requests_total{outcome}` from the discovery handlers;
+  `provider_requests_total{provider,op,result}` from the
+  `provider.WithMetrics` decorator — the one place `Class()` is called
+  purely for observability.
+
+Each consuming package defines its own small recorder interface
+(`health.ProbeMetrics`, `discovery.LeaseMetrics`, `provider.RequestRecorder`);
+`metrics.Metrics` satisfies all of them structurally, so no package other
+than `cmd/main.go` imports the metrics package.
+
+## Decisions
+
+Judgment calls the spec left open, and deliberate deviations — recorded so
+they read as choices, not accidents. Chronological by build step.
+
+- **Registry takes its constructor map as a parameter** instead of holding
+  a package-level map: avoids the provider⇄registry import cycle and puts
+  the wiring at the composition root, where it is visible.
+- **Mock provider replaced by the kubernetes-pod provider** (user
+  decision, mid-build): a simulated in-memory provider was too far from
+  the real system to build confidence in. Local dev/CI now runs real
+  `ubuntu/squid` pods (Canonical's actively maintained image, verified
+  50M+ pulls, pinned tag) in the operator's own cluster. Trade-off
+  accepted: envtest has no kubelet, so end-to-end proof lives in the kind
+  quickstart, and cluster pods share one egress IP — distinct egress
+  paths remain the GCP provider's job.
+- **`RequeueAfter: RequeueNow` instead of the plan's `Requeue: true`:**
+  `ctrl.Result.Requeue` is deprecated in controller-runtime v0.24; a fifth
+  configurable interval (default 1s) keeps identical semantics and stays
+  shrinkable in tests.
+- **Quota exhaustion is a wait, not a failure:** `ErrQuotaExceeded` sets a
+  condition and requeues slowly (5m) with a nil error — off the backoff
+  curve, out of the error log, and never `phase: Failed`. Only
+  `ErrPermanent` latches Failed, keyed to the generation so a spec edit
+  auto-recovers.
+- **The finalizer path never latches permanent failures:** a permanent
+  error during deletion keeps retrying visibly instead — latching there
+  would wedge the object forever with no path out but manual finalizer
+  surgery.
+- **Health transitions travel reconciler-ward over a channel**
+  (`source.Channel`), not direct status patches: `phase` derives from both
+  provisioning and health, so two status writers would race and flap. One
+  writer of status; the engine owns health *state*, the reconciler its
+  *representation*; write-only-on-transition falls out for free.
+- **Health state seeds from the existing Healthy condition on leader
+  handover** (verdict kept, counters zeroed, first probe jittered), so a
+  healthy fleet doesn't flap to Unknown on restart — but a real
+  transition still needs a full threshold run. A never-probed proxy skips
+  the jitter and probes on the next tick: startup spread matters for
+  restarts, not for a single new proxy.
+- **Latency suppression is `max(20ms, 50%)` + a 60s rate limit, and only
+  while the verdict is healthy.** The spec's bare ">50% change" is
+  undefined at 0 and lets a proxy jittering 40↔61ms write status forever;
+  the healthy-only guard (found by test) stops a below-threshold success
+  streak from emitting latency updates for a proxy still reported
+  unhealthy. Consequence: `status.lastHealthCheckTime` means "last
+  status-affecting probe" — true probe recency is in the metrics.
+- **Deterministic instance names** are `proxy-` + 16 chars of
+  base32(SHA-256(CR UID)): legal for both GCP (`[a-z2-7]` ⊂ `[-a-z0-9]`,
+  22 ≤ 63 chars) and Pod names, 80 bits against birthday collisions at a
+  fleet of tens. The replacement VM therefore has the *same name* as the
+  one being deleted — which is why replacement polls to NotFound before
+  recreating instead of racing a 409.
+- **`banned` and `rate_limited` share one cooldown window:** a second
+  duration knob the spec doesn't ask for; the report's semantic
+  difference is preserved in the API but not the store.
+- **Report targets fall back report → lease → global**, so a client that
+  leased with a target can't accidentally poison the proxy's global pool
+  by omitting the target in its report.
+- **The 409 body's `considered` counts unhealthy matches too** (the store
+  only ever sees healthy candidates): `considered = atCapacity +
+  inCooldown + unhealthy + eligible-but-outranked`, keeping the numbers
+  additive for a human debugging "why no proxy?".
+- **TTLs above `--max-lease-ttl` are a 400, not a silent clamp** — a
+  client asking for a week should find out.
+- **Discovery is not leader-elected and ships `replicas: 1`:** caches
+  start before non-leader-election runnables (verified in
+  controller-runtime's ordering), and a leader-elected server would leave
+  non-leader replicas as broken Service endpoints. One replica because
+  lease state is per-process.
+- **GCP `Create` requires zone, machineType, and image** and fails
+  `ErrPermanent` naming the missing field — inventing machine-type
+  defaults would silently create billable VMs of arbitrary shape.
+- **Unknown GCP instance statuses map to `Stopped`:** the reconciler's
+  answer to Stopped is delete-and-recreate, the always-safe move for
+  cattle when the API grows a new state.
+- **Kubernetes 403s classify as `ErrPermanent`** even though quota
+  exhaustion also surfaces as 403 (indistinguishable from RBAC denial in
+  `apierrors`): not hammering an API server that may never allow the
+  request is the safer default; a real ResourceQuota 403 forgoes the
+  gentler quota backoff. Documented at the classification site.
+- **GC kills log at Info with a `WARNING:` prefix** — logr has no Warn
+  level; the plan's "log at Warn" is met in spirit with provider,
+  providerID, and UID always attached. Same convention as the
+  discovery server's empty-token warning.
+- **GC trusts only provable orphans:** instances without the UID label
+  are never deleted, a CR with a deletionTimestamp still counts as live
+  (its finalizer owns that deletion), and an unreadable Proxy list skips
+  the whole sweep. The namespace guard refuses to sweep a
+  namespace-restricted cache without `--gc-allow-namespaced`.
+- **Cloud-init Secrets must carry `crawl.example.com/cloud-init: "true"`:**
+  the manager caches only labelled Secrets (the operator holds
+  cluster-wide Secret read RBAC — an unrestricted cache would hold every
+  Secret in scope). Unlabelled referenced Secrets are invisible by
+  construction, surfacing as `CloudInitError`.
+- **Events RBAC from the plan is omitted:** nothing wires an
+  EventRecorder in the prototype, and granting verbs nothing uses would
+  be RBAC lint noise. Add the marker together with the recorder if events
+  land later.
+- **logr, not slog, inside controller paths:** the repo convention says
+  `slog`, but `log.FromContext(ctx)` hands controller-runtime's logr
+  logger to everything running under the manager — fighting that would
+  mean two logging systems in one process. Noted as a deviation rather
+  than silently ignored.

docs/demo/create-gcp-proxies.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+# Demo: create N Managed proxies backed by the gcp provider, named
+# proxy-gcp-demo-1 .. proxy-gcp-demo-N, each in a randomly picked EU zone
+# so the fleet gets egress IPs from different locations.
+#
+# Usage:
+#   ./create-gcp-proxies.sh <count>
+#
+# Optional environment:
+#   NAMESPACE     namespace to create the proxies in (default: current context)
+#   GCP_PROVIDER  provider NAME from providers.yaml (default: gcp-eu)
+#   ZONES         space-separated zone list to pick from (default: EU zones below)
+set -euo pipefail
+
+if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
+    echo "usage: $(basename "$0") <count>  (positive integer)" >&2
+    exit 1
+fi
+count="$1"
+provider="${GCP_PROVIDER:-gcp-eu}"
+
+# GCP zones in the EU where e2-micro is generally available. Override with
+# ZONES="zone1 zone2 ..." if your project has quota only in some of them.
+default_zones=(
+    europe-west1-b europe-west1-c europe-west1-d       # Belgium
+    europe-west2-a europe-west2-b europe-west2-c       # London
+    europe-west3-a europe-west3-b europe-west3-c       # Frankfurt
+    europe-west4-a europe-west4-b europe-west4-c       # Netherlands
+    europe-west6-a europe-west6-b europe-west6-c       # Zurich
+    europe-west8-a europe-west8-b europe-west8-c       # Milan
+    europe-west9-a europe-west9-b europe-west9-c       # Paris
+    europe-central2-a europe-central2-b europe-central2-c   # Warsaw
+    europe-north1-a europe-north1-b europe-north1-c    # Finland
+    europe-southwest1-a europe-southwest1-b europe-southwest1-c   # Madrid
+)
+if [ -n "${ZONES:-}" ]; then
+    read -r -a zones <<< "${ZONES}"
+else
+    zones=("${default_zones[@]}")
+fi
+
+if ! command -v kubectl &> /dev/null; then
+    echo "ERROR: kubectl is required but not installed" >&2
+    exit 1
+fi
+
+ns_args=()
+if [ -n "${NAMESPACE:-}" ]; then
+    ns_args=(-n "${NAMESPACE}")
+fi
+
+for i in $(seq 1 "${count}"); do
+    zone="${zones[RANDOM % ${#zones[@]}]}"
+    echo "Creating proxy-gcp-demo-${i} in ${zone} ..."
+    kubectl apply "${ns_args[@]}" -f - << EOF
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-gcp-demo-${i}
+spec:
+  mode: Managed
+  provider: ${provider}              # must match a provider NAME in providers.yaml
+  placement:
+    zone: ${zone}
+    machineType: e2-micro
+    # debian-cloud images have no cloud-init, so spec.cloudInit (passed as
+    # user-data metadata) would be silently ignored there. Ubuntu images do.
+    image: projects/ubuntu-os-cloud/global/images/family/ubuntu-2404-lts-amd64
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      package_update: true
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+  attributes:
+    geo: eu
+    zone: ${zone}
+    purpose: crawl
+EOF
+done
+
+echo ""
+echo "Created ${count} proxies. VMs take a few minutes to provision and pass"
+echo "the health check. Watch them come up with:"
+echo "  kubectl get px ${ns_args[*]:-} -w"

docs/demo/create-kubernetes-proxies.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Demo: create N Managed proxies backed by the kubernetes-pod provider,
+# named proxy-kubernetes-demo-1 .. proxy-kubernetes-demo-N. Pods share the
+# cluster's egress IP — this exercises the full lifecycle, not distinct
+# egress paths (use create-gcp-proxies.sh for that).
+#
+# Usage:
+#   ./create-kubernetes-proxies.sh <count>
+#
+# Optional environment:
+#   NAMESPACE  namespace to create the proxies in (default: current context)
+set -euo pipefail
+
+if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
+    echo "usage: $(basename "$0") <count>  (positive integer)" >&2
+    exit 1
+fi
+count="$1"
+
+if ! command -v kubectl &> /dev/null; then
+    echo "ERROR: kubectl is required but not installed" >&2
+    exit 1
+fi
+
+ns_args=()
+if [ -n "${NAMESPACE:-}" ]; then
+    ns_args=(-n "${NAMESPACE}")
+fi
+
+for i in $(seq 1 "${count}"); do
+    echo "Creating proxy-kubernetes-demo-${i} ..."
+    kubectl apply "${ns_args[@]}" -f - << EOF
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-kubernetes-demo-${i}
+spec:
+  mode: Managed
+  provider: kubernetes
+  attributes:
+    geo: local
+    purpose: crawl
+EOF
+done
+
+echo ""
+echo "Created ${count} proxies. Watch them come up with:"
+echo "  kubectl get px ${ns_args[*]:-} -w"

docs/demo/run-demo.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Demo driver: opens a tmux session with a 2x2 pane grid:
+#
+#   top-left:     show-egress-ips-table.sh in a 10s loop, run inside the
+#                 netshoot pod against the in-cluster discovery Service
+#   top-right:    watch -n3 kubectl get px
+#   bottom-left:  create-kubernetes-proxies.sh <count>
+#   bottom-right: create-gcp-proxies.sh <count>
+#
+# Usage:
+#   ./run-demo.sh
+#
+# Optional environment:
+#   COUNT         proxies each create script makes (default: 4)
+#   SESSION       tmux session name (default: proxy-demo; an existing
+#                 session with this name is killed and recreated)
+#   NETSHOOT_POD  pod to exec into for the egress-IP loop (default: netshoot)
+#   DEMO_DIR      where the demo scripts live (default: this script's dir)
+set -euo pipefail
+
+COUNT="${COUNT:-4}"
+SESSION="${SESSION:-proxy-demo}"
+NETSHOOT_POD="${NETSHOOT_POD:-netshoot}"
+DEMO_DIR="${DEMO_DIR:-$(cd "$(dirname "$0")" && pwd)}"
+BASE_URL="http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090"
+
+for tool in tmux kubectl; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+if ! kubectl get pod "${NETSHOOT_POD}" &> /dev/null; then
+    echo "ERROR: pod ${NETSHOOT_POD} not found — start one with:" >&2
+    echo "  kubectl run netshoot --image=nicolaka/netshoot -- sleep infinity" >&2
+    exit 1
+fi
+
+echo "Copying show-egress-ips-table.sh into pod ${NETSHOOT_POD} ..."
+kubectl cp "${DEMO_DIR}/show-egress-ips-table.sh" "${NETSHOOT_POD}:/tmp/show-egress-ips-table.sh"
+
+if tmux has-session -t "${SESSION}" 2> /dev/null; then
+    echo "Killing existing tmux session ${SESSION}"
+    tmux kill-session -t "${SESSION}"
+fi
+
+# 2x2 grid: after these splits pane indexes are 0 top-left, 1 top-right,
+# 2 bottom-left, 3 bottom-right; tiled layout evens them into quarters.
+tmux new-session -d -s "${SESSION}"
+tmux split-window -h -t "${SESSION}:0"
+tmux split-window -v -t "${SESSION}:0.0"
+tmux split-window -v -t "${SESSION}:0.1"
+tmux select-layout -t "${SESSION}:0" tiled
+
+loop_cmd="BASE_URL=${BASE_URL}; while true; do bash /tmp/show-egress-ips-table.sh \"\$BASE_URL\"; echo; sleep 10; done"
+tmux send-keys -t "${SESSION}:0.0" "kubectl exec -it ${NETSHOOT_POD} -- bash -c '${loop_cmd}'" C-m
+tmux send-keys -t "${SESSION}:0.1" "watch -n3 kubectl get px" C-m
+tmux send-keys -t "${SESSION}:0.2" "bash ${DEMO_DIR}/create-kubernetes-proxies.sh ${COUNT}" C-m
+tmux send-keys -t "${SESSION}:0.3" "bash ${DEMO_DIR}/create-gcp-proxies.sh ${COUNT}" C-m
+
+tmux select-pane -t "${SESSION}:0.2"
+if [ -n "${TMUX:-}" ]; then
+    tmux switch-client -t "${SESSION}"
+else
+    tmux attach-session -t "${SESSION}"
+fi

docs/demo/show-egress-ips-table.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+# Demo: condensed-table variant of show-egress-ips.sh, made to fit a small
+# tmux pane. One line per proxy: which proxy the request goes through, its
+# endpoint, location (zone/geo attribute), and the egress IP the IP-echo
+# site saw — or unhealthy/FAILED.
+#
+# Usage:
+#   ./show-egress-ips-table.sh <BASE_URL>    e.g. ./show-egress-ips-table.sh localhost:8090
+#
+# Optional environment:
+#   TOKEN        bearer token for the discovery API (see docs/api.md)
+#   IP_ECHO_URL  site that returns the caller's IP as JSON with an "ip" field
+#                (default: https://api.ipify.org?format=json)
+set -euo pipefail
+
+if [ $# -ne 1 ]; then
+    echo "usage: $(basename "$0") <BASE_URL>  (e.g. localhost:8090)" >&2
+    exit 1
+fi
+BASE_URL="$1"
+IP_ECHO_URL="${IP_ECHO_URL:-https://api.ipify.org?format=json}"
+
+for tool in curl jq; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+auth_args=()
+if [ -n "${TOKEN:-}" ]; then
+    auth_args=(-H "Authorization: Bearer ${TOKEN}")
+fi
+
+if ! proxies_json=$(curl -sS --fail "${auth_args[@]}" "${BASE_URL}/v1/proxies") \
+    || ! echo "${proxies_json}" | jq -e . > /dev/null 2>&1; then
+    echo "ERROR: could not fetch proxy list from ${BASE_URL}" >&2
+    exit 1
+fi
+
+total=$(echo "${proxies_json}" | jq -r '.count')
+fmt="%-31s %-21s %-21s %s\n"
+
+echo "${total} proxies @ $(date +%H:%M:%S)"
+# shellcheck disable=SC2059
+printf "${fmt}" "PROXY" "ENDPOINT" "LOCATION" "EGRESS-IP"
+
+probed=0
+skipped=0
+failed=0
+while IFS= read -r proxy; do
+    id=$(echo "${proxy}" | jq -r '.id')
+    endpoint=$(echo "${proxy}" | jq -r '"\(.ip):\(.port)"')
+    location=$(echo "${proxy}" | jq -r '.attributes.zone // .attributes.geo // "-"')
+    healthy=$(echo "${proxy}" | jq -r '.healthy')
+
+    if [ "${healthy}" != "true" ]; then
+        # shellcheck disable=SC2059
+        printf "${fmt}" "${id}" "${endpoint}" "${location}" "(unhealthy)"
+        skipped=$((skipped + 1))
+        continue
+    fi
+
+    if response=$(curl -sS --max-time 10 -x "http://${endpoint}" "${IP_ECHO_URL}" 2> /dev/null); then
+        egress=$(echo "${response}" | jq -r '.ip // "?"' 2> /dev/null || echo "?")
+        probed=$((probed + 1))
+    else
+        egress="FAILED"
+        failed=$((failed + 1))
+    fi
+    # shellcheck disable=SC2059
+    printf "${fmt}" "${id}" "${endpoint}" "${location}" "${egress}"
+done < <(echo "${proxies_json}" | jq -c '.proxies[]')
+
+echo "-- ${probed} probed, ${skipped} unhealthy, ${failed} failed --"

docs/demo/show-egress-ips.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Demo: list proxies from the discovery API and show the egress IP each
+# healthy one provides, by calling an IP-echo site through it.
+#
+# Usage:
+#   ./show-egress-ips.sh <BASE_URL>          e.g. ./show-egress-ips.sh localhost:8090
+#
+# Optional environment:
+#   TOKEN        bearer token for the discovery API (see docs/api.md)
+#   IP_ECHO_URL  site that returns the caller's IP as JSON
+#                (default: https://api.ipify.org?format=json)
+set -euo pipefail
+
+if [ $# -ne 1 ]; then
+    echo "usage: $(basename "$0") <BASE_URL>  (e.g. localhost:8090)" >&2
+    exit 1
+fi
+BASE_URL="$1"
+IP_ECHO_URL="${IP_ECHO_URL:-https://api.ipify.org?format=json}"
+
+for tool in curl jq; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+auth_args=()
+if [ -n "${TOKEN:-}" ]; then
+    auth_args=(-H "Authorization: Bearer ${TOKEN}")
+fi
+
+echo "Fetching proxies from ${BASE_URL}/v1/proxies ..."
+if ! proxies_json=$(curl -sS --fail "${auth_args[@]}" "${BASE_URL}/v1/proxies"); then
+    echo "ERROR: could not fetch proxy list from ${BASE_URL}" >&2
+    exit 1
+fi
+if ! echo "${proxies_json}" | jq -e . > /dev/null; then
+    echo "ERROR: response from ${BASE_URL}/v1/proxies is not valid JSON" >&2
+    exit 1
+fi
+
+total=$(echo "${proxies_json}" | jq -r '.count')
+echo "Found ${total} proxies"
+echo ""
+
+probed=0
+skipped=0
+failed=0
+while IFS= read -r proxy; do
+    id=$(echo "${proxy}" | jq -r '.id')
+    ip=$(echo "${proxy}" | jq -r '.ip')
+    port=$(echo "${proxy}" | jq -r '.port')
+    healthy=$(echo "${proxy}" | jq -r '.healthy')
+
+    if [ "${healthy}" != "true" ]; then
+        echo "--- skipping ${id} (unhealthy) ---"
+        echo ""
+        skipped=$((skipped + 1))
+        continue
+    fi
+
+    echo "=== via ${id} — http://${ip}:${port} ==="
+    if response=$(curl -sS --max-time 10 -x "http://${ip}:${port}" "${IP_ECHO_URL}"); then
+        echo "${response}" | jq . 2> /dev/null || echo "${response}"
+        probed=$((probed + 1))
+    else
+        echo "WARNING: request through ${id} failed" >&2
+        failed=$((failed + 1))
+    fi
+    echo ""
+done < <(echo "${proxies_json}" | jq -c '.proxies[]')
+
+echo "Done: ${total} proxies — ${probed} probed, ${skipped} skipped (unhealthy), ${failed} failed"

docs/gcp-in-specific-project.md

@@ -0,0 +1,255 @@
+## Project egress-proxy
+
+```bash
+PROJECT_ID=egress-proxy
+
+# 1. Create the service account
+gcloud iam service-accounts create proxy-operator \
+  --project ${PROJECT_ID} \
+  --display-name "egress-proxies-operator"
+
+# Output: 
+# Created service account [proxy-operator].
+# Service account email: proxy-operator@egress-proxy.iam.gserviceaccount.com
+
+# 2. Grant compute.instanceAdmin.v1 on the project
+gcloud projects add-iam-policy-binding ${PROJECT_ID} \
+  --member "serviceAccount:proxy-operator@${PROJECT_ID}.iam.gserviceaccount.com" \
+  --role roles/compute.instanceAdmin.v1
+
+# Output:
+# ---------
+# Updated IAM policy for project [egress-proxy].
+# bindings:
+# - members:
+#   - serviceAccount:proxy-operator@egress-proxy.iam.gserviceaccount.com
+#   role: roles/compute.instanceAdmin.v1
+# - members:
+#   - serviceAccount:541231138892@cloudservices.gserviceaccount.com
+#   role: roles/compute.instanceGroupManagerServiceAgent
+# - members:
+#   - serviceAccount:service-541231138892@compute-system.iam.gserviceaccount.com
+#   role: roles/compute.serviceAgent
+# - members:
+#   - user:admin@fujultimate.cz
+#   role: roles/owner
+# etag: BwZYuFXko24=
+# version: 1
+
+# 3. Create the JSON key (this is what goes into the Secret)
+SA_KEY_PATH=sa_key.json
+gcloud iam service-accounts keys create $SA_KEY_PATH \
+  --iam-account proxy-operator@${PROJECT_ID}.iam.gserviceaccount.com
+
+# output:
+# created key [fdff85174a8e80bbd684e76c4d9fe28e2f4b2ddf] of type [json] as [sa_key.json] for [proxy-operator@egress-proxy.iam.gserviceaccount.com]
+
+# 4. A **firewall rule**: created VMs get network tag `proxy-operator` (the
+#    default; configurable as `gcp.networkTag`), an ephemeral external IP,
+#    and Squid listening on 3128.
+
+gcloud compute firewall-rules create allow-proxy-operator \
+    --project $PROJECT_ID \
+    --network default \
+    --allow tcp:3128 \
+    --target-tags proxy-operator \
+    --source-ranges 94.230.145.216/32
+```
+
+## Phase 2 - resources in kube
+
+```bash
+SA_KEY_PATH=sa_key.json
+kubectl -n egress-proxies-operator-system create secret generic gcp-credentials \
+  --from-file=key.json=$SA_KEY_PATH
+```
+
+
+## Appendix - full manifests
+
+```bash
+# crawl CR
+kubectl apply -f - <<'EOF'
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-gcp-sample
+spec:
+  mode: Managed
+  provider: gcp-eu                   # must match a provider NAME in providers.yaml
+  placement:
+    zone: europe-west1-b
+    machineType: e2-micro
+    # debian-cloud images have no cloud-init, so spec.cloudInit (passed as
+    # user-data metadata) would be silently ignored there. Ubuntu images do.
+    image: projects/ubuntu-os-cloud/global/images/family/ubuntu-2404-lts-amd64
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      package_update: true
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+  attributes:
+    geo: eu
+    purpose: crawl
+EOF
+
+
+
+# configmap
+kubectl apply -f - <<'EOF'
+apiVersion: v1
+data:
+  providers.yaml: |
+    providers:
+      - name: kubernetes
+        type: kubernetes
+      - name: gcp-eu          # spec.provider on a Proxy refers to this NAME, not the type
+        type: gcp
+        gcp:
+            project: egress-proxy
+            # network: default            # these three default as shown
+            # networkTag: proxy-operator
+            # diskSizeGb: 10
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: egress-proxies-operator
+  name: egress-proxies-operator-providers-config
+  namespace: egress-proxies-operator-system
+EOF
+
+# operator deployment
+kubectl apply -f - <<'EOF'
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    deployment.kubernetes.io/revision: "2"
+  labels:
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: egress-proxies-operator
+    control-plane: controller-manager
+  name: egress-proxies-operator-controller-manager
+  namespace: egress-proxies-operator-system
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: egress-proxies-operator
+      control-plane: controller-manager
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/default-container: manager
+      labels:
+        app.kubernetes.io/name: egress-proxies-operator
+        control-plane: controller-manager
+    spec:
+      containers:
+        - args:
+            - --metrics-bind-address=:8443
+            - --leader-elect
+            - --health-probe-bind-address=:8081
+            - --providers-config=/etc/proxy-operator/providers.yaml
+          command:
+            - /manager
+          env:
+            - name: DISCOVERY_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  key: token
+                  name: discovery-token
+                  optional: true
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: /var/secrets/gcp/key.json
+          image: egress-proxies-operator:dev
+          imagePullPolicy: IfNotPresent
+          livenessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthz
+              port: 8081
+              scheme: HTTP
+            initialDelaySeconds: 15
+            periodSeconds: 20
+            successThreshold: 1
+            timeoutSeconds: 1
+          name: manager
+          ports:
+            - containerPort: 8081
+              name: health
+              protocol: TCP
+            - containerPort: 8090
+              name: discovery
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /readyz
+              port: 8081
+              scheme: HTTP
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          resources:
+            limits:
+              cpu: 500m
+              memory: 128Mi
+            requests:
+              cpu: 10m
+              memory: 64Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: true
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /etc/proxy-operator
+              name: providers-config
+              readOnly: true
+            - mountPath: /var/secrets/gcp
+              name: gcp-credentials
+              readOnly: true
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      serviceAccount: egress-proxies-operator-controller-manager
+      serviceAccountName: egress-proxies-operator-controller-manager
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - configMap:
+            defaultMode: 420
+            name: egress-proxies-operator-providers-config
+          name: providers-config
+        - name: gcp-credentials
+          secret:
+            defaultMode: 420
+            secretName: gcp-credentials
+EOF
+```

docs/gcp-vm-validation.md

@@ -0,0 +1,158 @@
+# Validating real VM creation on GCP
+
+Recipe for wiring the GCP provider into a live cluster and watching a
+`Proxy` CR create a real Compute Engine VM. Angle brackets mark values you
+supply: `<PROJECT_ID>`, `<SA_KEY_PATH>`, `<ZONE>`, `<CLUSTER_EGRESS_IP>`,
+`<REGISTRY_IMAGE>`.
+
+The one important fact up front: **the operator takes no GCP credentials
+through its own config.** The client is built with Application Default
+Credentials (`internal/provider/gcp/gcp.go`, `New()`); there is no
+key-file field in the providers config. The only secret to prepare is a
+service-account JSON key, injected via the standard
+`GOOGLE_APPLICATION_CREDENTIALS` mechanism. On GKE you would use workload
+identity instead and skip the key entirely.
+
+## 1. GCP-side prerequisites (prepared outside the cluster)
+
+1. A project — `<PROJECT_ID>` — with the **Compute Engine API enabled**.
+2. A **service account** with `roles/compute.instanceAdmin.v1` on the
+   project. The operator only calls instances
+   `Insert`/`Get`/`Delete`/`AggregatedList` and does not attach a service
+   account to the VMs it creates, so no `iam.serviceAccountUser` is
+   needed.
+3. A **JSON key** for that service account, saved at `<SA_KEY_PATH>`.
+4. A **firewall rule**: created VMs get network tag `proxy-operator` (the
+   default; configurable as `gcp.networkTag`), an ephemeral external IP,
+   and Squid listening on 3128.
+
+   ```sh
+   gcloud compute firewall-rules create allow-proxy-operator \
+     --project <PROJECT_ID> \
+     --network default \
+     --allow tcp:3128 \
+     --target-tags proxy-operator \
+     --source-ranges <CLUSTER_EGRESS_IP>/32
+   ```
+
+   The source range must cover the cluster's egress IP — the operator's
+   CONNECT health probes originate there, and without the rule the Proxy
+   hangs at `Running`/unhealthy instead of reaching `Ready`. ⚠️ The
+   sample cloud-init configures `http_access allow all`, so on a public
+   IP this is an open proxy — keep the source ranges tight.
+
+## 2. Create the credentials Secret
+
+Namespace is `egress-proxies-operator-system` after kustomize prefixing:
+
+```sh
+kubectl -n egress-proxies-operator-system create secret generic gcp-credentials \
+  --from-file=key.json=<SA_KEY_PATH>
+```
+
+## 3. Add a GCP entry to the providers ConfigMap
+
+Edit `config/manager/providers_config.yaml` (mounted at
+`/etc/proxy-operator/providers.yaml`):
+
+```yaml
+providers:
+  - name: kubernetes
+    type: kubernetes
+  - name: gcp-eu          # spec.provider on a Proxy refers to this NAME, not the type
+    type: gcp
+    gcp:
+      project: <PROJECT_ID>
+      # network: default            # these three default as shown
+      # networkTag: proxy-operator
+      # diskSizeGb: 10
+```
+
+The config is validated fail-fast at startup — a typo shows up
+immediately in the manager log, not on first use.
+
+## 4. Mount the Secret and point ADC at it
+
+In `config/manager/manager.yaml`, add to the manager container:
+
+```yaml
+env:
+  - name: GOOGLE_APPLICATION_CREDENTIALS
+    value: /var/secrets/gcp/key.json
+volumeMounts:
+  - name: gcp-credentials
+    mountPath: /var/secrets/gcp
+    readOnly: true
+volumes:
+  - name: gcp-credentials
+    secret:
+      secretName: gcp-credentials
+```
+
+(`volumeMounts` merges into the existing container list; `volumes` into
+the existing pod-level list.)
+
+## 5. Deploy and create the Proxy
+
+```sh
+make deploy IMG=<REGISTRY_IMAGE>
+```
+
+`config/samples/proxy_gcp.yaml` is usable as-is once `spec.provider`
+matches the name from step 3. All three placement fields are mandatory
+for GCP — a missing one sets the Proxy to `Failed` with a message naming
+it:
+
+```yaml
+spec:
+  mode: Managed
+  provider: gcp-eu
+  placement:
+    zone: <ZONE>                     # e.g. europe-west1-b
+    machineType: e2-micro
+    image: projects/debian-cloud/global/images/family/debian-12
+```
+
+```sh
+kubectl apply -f config/samples/proxy_gcp.yaml
+```
+
+## 6. What you should see
+
+```sh
+kubectl get proxy -w
+```
+
+`Provisioning` → `Running` (VM's external IP published in status) →
+`Ready` (CONNECT health probe succeeded through the public IP). Then:
+
+```sh
+# the VM exists and carries the GC labels
+gcloud compute instances list --project <PROJECT_ID> \
+  --filter 'labels.proxy-operator-managed=yes'
+
+# the proxy actually tunnels — should print the VM's external IP
+curl -x http://<EXTERNAL_IP>:3128 https://ifconfig.me
+```
+
+Cleanup — the finalizer deletes the VM:
+
+```sh
+kubectl delete proxy proxy-gcp-sample
+gcloud compute instances list --project <PROJECT_ID>   # should be empty again
+```
+
+## Gotchas
+
+- **The orphan GC sweeps the whole project**: any VM labeled
+  `proxy-operator-managed=yes` whose UID does not match a live Proxy CR
+  in *this* cluster is deleted once past the age threshold. Do not point
+  two operator installs at the same project, and do not hand-create VMs
+  with that label.
+- **VM creation is fire-and-forget** — the provider never waits on the
+  insert operation; progress is discovered by polling `Get`. A quota
+  error or bad image name surfaces on the Proxy's status/conditions a
+  reconcile later, not synchronously. `kubectl describe proxy` is the
+  place to look when something stalls.
+- **e2-micro costs pennies but is not free everywhere** — remember to
+  delete the CR (or check `gcloud compute instances list`) when done.

docs/plans-executions/2026-08-07-1747-proxy-operator.md

@@ -10,13 +10,13 @@ Pairs with [docs/plans/2026-08-07-1747-proxy-operator.md](../plans/2026-08-07-17
 - [x] Step 3 — Kubernetes pod provider (`internal/provider/kubernetes/`; first built as an in-memory mock, then replaced — see the two Step 3 sections below)
 - [x] Step 4 — Reconciler (`internal/controller/`)
 - [x] Step 5 — Health engine (`internal/health/`)
-- [ ] Step 6 — Lease store (`internal/lease/`)
+- [x] Step 6 — Lease store (`internal/lease/`)
-- [ ] Step 7 — Discovery API (`internal/discovery/`)
+- [x] Step 7 — Discovery API (`internal/discovery/`)
-- [ ] Step 8 — GCP provider (`internal/provider/gcp/`)
+- [x] Step 8 — GCP provider (`internal/provider/gcp/`)
-- [ ] Step 9 — Orphan GC + metrics
+- [x] Step 9 — Orphan GC + metrics
-- [ ] Step 10 — Wiring, config, docs
+- [x] Step 10 — Wiring, config, docs
-- [ ] Step 11 — Tests
+- [x] Step 11 — Tests
-- [ ] Verification (vet/test/kind e2e) + commit, push, open MR
+- [x] Verification (vet/test/kind e2e) + commit, push, open MR
 
 ## Step 0 — Branch and scaffold
 
@@ -694,3 +694,438 @@ engine deliberately knows nothing about conditions except reading one at
 seed time, keeping the state/representation split honest. The
 `hint`-driven `wg.Go` idiom (Go 1.25+) replaced the classic
 `wg.Add/defer wg.Done` in the worker pool.
+
+## Step 6 — Lease store (`internal/lease/`)
+
+Implemented `store.go` per the plan: `Acquire` takes the whole candidate
+set so selection and insertion happen under the one store mutex (no
+overcommit between concurrent requests), selection is a linear scan +
+`slices.SortFunc` on `(activeLeases asc, latency asc, name asc)`,
+`AcquireStats{Considered, AtCapacity, InCooldown}` feeds Step 7's 409
+body, cooldowns live in a `map[{proxy, target}]time.Time` (empty target =
+global pool), and expired leases are retained for `CooldownWindow` past
+their TTL so a late `Report` — arriving exactly when a proxy is being
+rate-limited — still resolves and records its cooldown.
+
+Semantics pinned against the spec (§8) rather than guessed:
+
+- Report results are exactly `ok | rate_limited | banned` (`ParseResult`
+  gives the API layer its 400 check). `rate_limited` and `banned` both
+  record a cooldown for the same window; `ok` records nothing.
+  Distinguishing ban duration from rate-limit duration would be a second
+  knob the spec doesn't ask for — noted for the Decisions section.
+- Cooldown scoping: the global cooldown (empty target) always applies; a
+  target-scoped cooldown additionally blocks acquisitions for that target;
+  acquisitions without a target see only the global pool ("a proxy
+  rate-limited by one site is still fine for everyone else").
+- A `Report` without a target falls back to the lease's own target before
+  falling back to global — so a client that leased with a target doesn't
+  accidentally poison the whole proxy by omitting it in the report.
+
+Design notes:
+
+- **Correctness never depends on the sweep.** Every read path
+  (`Acquire`/`ActiveCount`/`Counts`) compares `ExpiresAt` against the
+  injected clock, so TTL expiry frees capacity immediately even if the
+  background loop hasn't run; the sweep is purely garbage collection. The
+  plan's `ExpireLoop` became `Start(ctx)` + `NeedLeaderElection() false`
+  so the store satisfies `manager.Runnable` directly — Step 10 just
+  `mgr.Add(store)`s it. Not leader-elected because lease state is
+  per-process and must expire wherever the discovery API is serving.
+- The store knows nothing about Proxy objects — `Candidate` carries the
+  opaque key, `MaxLeases`, and latency; the discovery layer does the
+  health/attribute filtering. The spec's `LeaseStore` interface will be
+  defined consumer-side in `internal/discovery` (Step 7), per Go idiom;
+  this package exports only the concrete in-memory `*Store`.
+- Lease IDs come from `crypto/rand.Text()` (Go 1.24+); returned `Lease`
+  values are copies so callers can't mutate store internals.
+
+Tests (94.8% coverage, `-race -count=2` clean): capacity + release
+freeing slots, `MaxLeases=0` unleasable, least-loaded/latency/name
+selection order, target-scoped vs global cooldown scoping, cooldown
+expiry via the injected fake clock, TTL freeing capacity with no sweep,
+report-on-expired-but-retained lease (then `ErrUnknownLease` after
+retention), `ok` recording nothing, idempotent release, `ParseResult`,
+40 concurrent acquires against `MaxLeases=5` granting exactly 5, and the
+`Start` loop sweeping then stopping cleanly on cancel.
+
+```bash
+go test -race -count=2 ./internal/lease/
+make test    # whole repo green, other packages' coverage unchanged
+```
+
+Worth noting: `docs/architecture.md` was not extended this step — the
+lease store is HTTP-driven, not cluster-event-driven, so its diagram
+belongs with the discovery API and lands in Step 7 (banner updated to say
+so).
+
+## Step 7 — Discovery API (`internal/discovery/`)
+
+Implemented `server.go` (Runnable + middleware chain) and `handlers.go`
+(the four endpoints + `proxyView` wire shape) per the plan: stdlib
+`http.ServeMux` method+wildcard routing (no third-party router — see the
+plan clarification commit: this is a stdlib feature since Go 1.22, the
+project stays on the pinned Go 1.26), middleware outermost-first recover →
+request-log → `MaxBytesReader(64KiB)` → constant-time bearer auth with
+`/healthz` exempt, empty `DISCOVERY_TOKEN` serving unauthenticated with a
+loud startup warning, `NeedLeaderElection() = false` with the plan's
+runnable-ordering rationale in the doc comment, and graceful `Shutdown`
+with a 10 s grace on ctx cancel.
+
+The `LeaseStore` interface landed consumer-side in this package (spec §8
+wants handlers swappable to a CRD/Redis store); `internal/lease.*Store`
+satisfies it without modification.
+
+Judgment calls the plan/spec left open:
+
+- **409 arithmetic:** the store only ever sees healthy candidates, so its
+  `Considered` excludes unhealthy matches. The handler counts unhealthy
+  selector-matches itself and reports `considered = healthy + unhealthy`,
+  keeping the plan's example arithmetic (7 = 2+2+3) consistent.
+- **TTL handling:** omitted/zero `ttlSeconds` → 300 s default; negative or
+  above `MaxLeaseTTL` (default 1h, flag in Step 10) → 400 `invalid_ttl`
+  rather than silent clamping — a client asking for a week-long lease
+  should find out, not get an hour quietly.
+- **Grant response includes the fresh `activeLeases`** (the just-granted
+  lease counted), read back via `Store.Counts()` after the acquire.
+- Proxies with a deletionTimestamp are filtered out of both list and
+  candidate selection — a proxy mid-teardown shouldn't be advertised.
+
+Tests (87.3% coverage, `-race -count=2` clean, green on first run):
+httptest over the real handler chain with a fake cache reader and a real
+`lease.Store` — auth on/off/wrong-token/healthz-exempt, list filtering
+(attributes, healthy, combined, empty-is-200), grant shape (201, default
+TTL, lowest-latency pick, RFC3339 expiresAt, activeLeases=1), the full
+409 body arithmetic, invalid TTL/body/result, idempotent 204 release,
+report→cooldown→409 round-trip, 404 on unknown lease, and a real
+`Start` on `127.0.0.1:0` (via the new `BoundAddr()` accessor) serving
+healthz then shutting down cleanly on cancel.
+
+```bash
+go test -race -count=2 ./internal/discovery/
+make test    # whole repo green
+```
+
+Worth noting: `go mod tidy` promoted `github.com/go-logr/logr` from
+indirect to direct (the server holds a `logr.Logger` field). The
+`--discovery-addr`, `--max-lease-ttl` flags and the `DISCOVERY_TOKEN`
+Secret mount arrive with `cmd/main.go` in Step 10. `docs/architecture.md`
+gained §7 covering the whole HTTP path and the store's sweep Runnable.
+
+## Step 8 — GCP provider (`internal/provider/gcp/`)
+
+Implemented per the plan: `gcp.go` (Provider + the flattened `instancesAPI`
+test seam + providerID handling + state mapping), `insert.go` (pure
+`buildInsertRequest` + config defaults), `errors.go` (HTTP-code → taxonomy
+classification). Only the four calls the spec allows — instances Insert /
+Get / Delete / AggregatedList — and `Operation.Wait` is never called:
+Create/Delete return once the operation is submitted, `409 alreadyExists`
+on Insert and `404` on Delete both count as success, which is what makes
+repeat calls after a crash correct.
+
+Dependency added (the plan's environment check pinned it):
+
+```bash
+go get cloud.google.com/go/compute@v1.65.0 google.golang.org/api@latest
+# resolved google.golang.org/api v0.292.0; go mod tidy pulled the auth/gax chain
+```
+
+Key shapes, all straight from the plan:
+
+- **providerID `zones/<zone>/instances/<name>`** — Get/Delete parse the
+  zone out of the ID instead of re-reading `spec.placement.zone`, which is
+  wrong exactly when a zone edit is the replacement being processed.
+- **The seam is not an SDK mirror** — verified the plan's premise against
+  the vendored source before designing around it:
+  `InstancesScopedListPairIterator` has an unexported `nextFunc`, so a
+  fake cannot construct one. The seam flattens `AggregatedList` to
+  `[]*computepb.Instance` and returns operations as just their name.
+- `AggregatedList` sets `ReturnPartialSuccess: true` (one unreachable
+  zone must not fail a GC sweep) and filters by
+  `labels.proxy-operator-managed = true`.
+- `RUNNING` without a `NatIP` maps to `Provisioning` — never publish an
+  empty IP. Unknown/new GCP statuses map to `Stopped`: the reconciler's
+  response is delete-and-recreate, always safe for cattle.
+- Classification: 404→NotFound; 429 and 403-with-
+  `quotaExceeded`/`rateLimitExceeded`→Quota; 400/401/403-other→Permanent;
+  everything else (408, 5xx, network, unknown)→Transient.
+
+Judgment call: `placement.zone`/`machineType`/`image` are all required at
+`Create` — missing values fail as `ErrPermanent` with a message naming
+the empty fields, rather than inventing defaults the spec doesn't define.
+A wrong guess here would silently create billable VMs of an arbitrary
+shape; a Failed condition telling the user what to set is strictly better.
+
+Tests (75.2%, `-race -count=2` clean, green on first run): the plan's
+primary field-by-field `buildInsertRequest` assertion (machine-type URL,
+boot disk, the exact `{External NAT, ONE_TO_ONE_NAT}` access config,
+user-data metadata, GC labels, network tag) plus config overrides and
+no-metadata-without-cloud-init; the full classification table including
+`errors.Is` AND `errors.As` through the multi-unwrap; and fake-seam tests
+for zone-qualified IDs, 409-is-success, permanent-on-bad-placement (no
+API call made), the nine-row state-mapping table, 404 paths, malformed
+providerIDs, and the ListByTag filter/partial-success assertions. The
+uncovered remainder is `New()` (dials real Google with ADC) and the
+`realInstances` adapter — the same deliberately-untested posture as the
+kubernetes provider's `New()`.
+
+Worth noting: gopls suggested replacing `proto.String(x)` with Go 1.26's
+`new(x)` expression; left as `proto.String` — it is the universal
+protobuf-construction idiom and matches every example in the SDK docs.
+Registry wiring (`"gcp": gcp.New`) happens at the composition root in
+Step 10, as designed in Step 2. `docs/architecture.md` §5 now shows both
+providers' call mappings.
+
+## Step 9 — Orphan GC + metrics
+
+Implemented `internal/gc/gc.go` (the `Sweeper` Runnable) and
+`internal/metrics/metrics.go` (explicit-registration metric set), plus the
+`provider.WithMetrics` decorator deferred from Step 2 into
+`internal/provider/metrics.go`, and the observation hooks in the health
+engine and discovery server.
+
+**GC**, per the plan: `NeedLeaderElection() = true` (destructive ⇒ single
+writer), 10 min ticker with the first sweep one full interval after start,
+per-provider `ListByTag` with log-and-continue on provider errors, and a
+kill requires all three of: our UID label present, older than `MinAge`
+(10 min), and the UID matching no existing CR — where a CR with a
+`deletionTimestamp` still counts as live (its finalizer owns that
+deletion; GC racing it would double-delete). Two safety behaviors worth
+naming: a failed Proxy `List` skips the whole sweep (an unreadable live
+set proves nothing orphaned), and the namespace-scope guard makes `Start`
+refuse to run against a namespace-restricted cache unless
+`--gc-allow-namespaced` is explicit, with the flag named in the error.
+One deviation of record: the plan says kills log "at Warn", but logr has
+no Warn level — kills log at Info with a `WARNING:` prefix carrying
+provider, providerID, and UID, same convention as the discovery server's
+empty-token warning.
+
+**Metrics**, per the plan: no `init()` — `Metrics.Register(reg, phases,
+leases)` is called explicitly by the composition root (Step 10), which is
+also what lets every test use a fresh registry (asserted by a test that
+registers two sets on two registries). `proxy_operator_proxies{phase}`
+and `proxy_operator_leases_active` are scrape-time collectors fed by
+closures — a reconcile-incremented gauge drifts and leaks series on
+delete; reading the source of truth at scrape time cannot. The
+probe vectors observe **every** probe (status stays transition-only;
+metrics carry the high-frequency signal), and the health engine calls
+`ForgetProxy` when it prunes a state entry so per-proxy series don't leak.
+`provider_requests_total{provider,op,result}` comes from the
+`WithMetrics` decorator — the one place `Class()` is called purely for
+observability, with results labelled ok / not_found / quota_exceeded /
+permanent / transient.
+
+**Decoupling shape:** each consuming package defines its own small
+recorder interface (`health.ProbeMetrics`, `discovery.LeaseMetrics`,
+`provider.RequestRecorder`); `metrics.Metrics` satisfies all of them
+structurally. Only `cmd/main.go` will import `internal/metrics`.
+
+Tests (`-race -count=2` clean): GC's true-orphan matrix in one sweep
+(live kept, deleting-CR kept, young kept, unlabelled kept, orphan
+deleted), broken-provider isolation, list-failure skips sweep, the
+namespace guard both ways, and the Start loop sweeping then stopping;
+metrics via `prometheus/testutil` — `GatherAndCompare` on the scrape-time
+collectors, ForgetProxy dropping series, outcome/result label counts;
+the decorator's five-way classification table with error passthrough
+asserted. Coverage: gc 86.1%, metrics 95.0%, provider up to 97.1%;
+health/discovery re-ran green with the hooks in place.
+
+```bash
+go test -race -count=2 ./internal/gc/ ./internal/metrics/ ./internal/provider/ ./internal/health/ ./internal/discovery/
+make test    # whole repo green
+```
+
+Worth noting: `prometheus/client_golang` was already in the module via
+controller-runtime's metrics server, so no new dependency — `go mod tidy`
+just promoted it to direct. `docs/architecture.md` gained §8 (GC sweep)
+and §9 (metrics shape).
+
+## Step 10 — Wiring, config, docs
+
+The composition root and everything around it. `cmd/main.go` now: parses
+the plan's flag set (plus `--gc-allow-namespaced` from Step 9), loads the
+provider config first and fails fast, builds the registry with
+`{"kubernetes": kubernetes.New, "gcp": gcp.New}`, wraps every provider in
+`provider.WithMetrics`, then adds the lease store, health engine,
+discovery server, and GC sweeper to one manager and hands the reconciler
+its providers + health snapshotter + events channel. Metrics register on
+controller-runtime's global registry with scrape-time closures (phase
+counts from the cache, active leases summed from `store.Counts()`).
+
+Two cache decisions became concrete here:
+
+- The Secret cache is restricted to Secrets labelled
+  `crawl.example.com/cloud-init=true` (new constant
+  `v1alpha1.LabelCloudInit`) — the operator holds cluster-wide Secret
+  read RBAC, and an unrestricted cache would hold every Secret in scope.
+  Consequence documented in the README: an unlabelled referenced Secret
+  is invisible → `CloudInitError`.
+- `--proxy-namespace` restricts the whole cache via `DefaultNamespaces`
+  and flips the GC sweeper's `NamespaceRestricted` guard.
+
+Manifests: `config/manager/manager.yaml` gained the
+`--providers-config` arg, the optional `DISCOVERY_TOKEN` secretKeyRef
+(`optional: true` — without the Secret the API runs unauthenticated with
+its loud warning), the ConfigMap volume mount, and containerPort 8090;
+new `config/manager/providers_config.yaml` (kubernetes-only default) and
+`config/default/discovery_service.yaml`. RBAC: the pods marker landed in
+the controller RBAC block (cluster-scoped role — the kubernetes
+provider's ListByTag spans namespaces). The plan's events RBAC was
+deliberately omitted: nothing wires an EventRecorder, and unused verbs
+are lint noise — recorded in Decisions.
+
+Samples: `proxy_kubernetes.yaml` / `proxy_gcp.yaml` (with a working
+Squid-installing cloud-init) / `proxy_external.yaml` replace the scaffold
+placeholder; `providers-config.yaml` documents both provider blocks;
+`hack/providers-dev.yaml` + a new `run-dev` Makefile target run locally
+with plain-HTTP metrics:
+
+```bash
+make run-dev
+# go run ./cmd/main.go --providers-config hack/providers-dev.yaml \
+#   --metrics-bind-address :8080 --metrics-secure=false
+```
+
+Docs: README rewritten per the plan (60-second architecture, kind
+quickstart, in-cluster deploy incl. token Secret creation, GCP setup with
+the IAM roles, the two prominent caveats, the no-substitutions version
+pins note). `docs/architecture.md` gained the components table and the
+full Decisions section — the plan's listed decisions plus everything
+accumulated in this log (RequeueNow, quota≠Failed, 409 arithmetic,
+TTL-400-not-clamp, GCP required placement, unknown-status→Stopped,
+banned==rate_limited window, GC logging convention, the Secret label
+contract, events-RBAC omission, logr-not-slog). `CHANGELOG.md` got its
+first entry with a real timestamp.
+
+Verified: `make test` green across the repo (coverage unchanged),
+`bin/kustomize build config/default` and `config/samples` render clean,
+`make build` produces the binary, e2e-tagged build + vet clean. The kind
+end-to-end run is deliberately still ahead — it is the Verification
+step's job, after Step 11 closes the remaining test gaps.
+
+Worth noting: `make run-dev` passes `--metrics-secure=false` because the
+scaffold's secure-serving default requires authn/authz reachability that
+a local process doesn't have; in-cluster deployments keep the secure
+default from the kustomize patch. The `providers` map wrapping happens
+*before* any consumer sees it, so the reconciler and GC only ever hold
+instrumented providers.
+
+## Step 11 — Tests
+
+Most of the plan's Step 11 list was deliberately front-loaded into the
+step that built each component (the action-table suite, computePhase and
+SpecHash tables, lease-store matrix incl. the concurrent `-race` case,
+discovery httptest suite, health threshold/CONNECT tests, GCP
+`buildInsertRequest` + classification, name-derivation tests from
+Step 2). This step closed what remained — the envtest-only coverage —
+and audited the list item by item.
+
+Added to `internal/controller/proxy_controller_test.go`:
+
+- **The CEL cases only a real API server can test** (fake clients run
+  neither CEL nor structural defaulting): six invalid-create rejections
+  (Managed-without-provider, External-with-provider,
+  External-without-endpoint, Managed-with-endpoint, cloudInit
+  both/neither), mode-mutation rejection, provider mutation *and removal*
+  rejection (the `has(self.x)==has(oldSelf.x)` form exists exactly for
+  the removal case), and the `+kubebuilder:default={}` assertion — a
+  Proxy created with no `healthCheck` comes back with every nested
+  default materialized, plus port and maxLeases defaults.
+- **Ready-through-health**: Managed proxy walks to Running (phase still
+  Provisioning — "no health verdict yet must not be Ready"), then a fake
+  `HealthSnapshotter` supplies a healthy snapshot and the phase flips to
+  Ready with latency in status.
+- **Quota + permanent, envtest edition**: quota → condition
+  QuotaExceeded, `RequeueAfter = QuotaRetry`, nil error, phase *not*
+  Failed; then permanent → Failed and the generation latch provably stops
+  further provider calls.
+- **Adopt**: strip the spec-hash annotation off a Running proxy (as an
+  operator upgrade with a changed hash-input struct would), reconcile,
+  and assert the annotation is restored byte-identical, the providerID
+  unchanged, and zero provider deletes.
+
+One repo-wide change: `make test` now runs with `-race` (the plan's
+"everything runs with -race" was previously only true of the manual
+verification runs, not the canonical target):
+
+```make
+go test -race $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+```
+
+Everything green on the first run of the new specs; full suite ~9 s for
+the controller package under race, `-short` still skips envtest in 0.6 s.
+
+Worth noting: the provider-removal CEL test has a subtlety worth keeping —
+removing `provider` alone would also trip the required-iff rule, so the
+test flips mode and adds an endpoint in the same update to isolate the
+immutability rules as the thing that rejects. The plan's remaining
+checklist item is Verification: the throwaway-kind-cluster run of the
+README quickstart, then push + MR.
+
+## Verification — kind end-to-end
+
+Static checks first (`go vet ./...`, `make build`, full `make test` with
+`-race`): all green. Then the real thing, per the spec's §13 "run the kind
+quickstart yourself and fix what breaks" — and two things broke, both now
+fixed.
+
+**Finding 1 — `make run-dev` cannot produce a Ready proxy on kind.** The
+operator on the host provisions the pod fine (Provisioned=True, IP
+published), but the health probe originates on the host, and kind pod IPs
+(10.244.x.x) are not host-routable — every probe fails by construction
+and the proxy latches `Unhealthy`:
+
+```text
+Healthy=False: Get "https://www.gstatic.com/generate_204":
+  proxyconnect tcp: dial tcp 10.244.0.5:3128: connect: connection refused
+```
+
+Everything around the failure worked exactly as designed (thresholds,
+condition, phase, and the finalizer delete ran clean from the host). Fix:
+the README quickstart now deploys the operator **in-cluster**
+(docker-build → kind load → deploy → port-forward 8090), with the
+run-dev limitation documented in both the quickstart and the Development
+section.
+
+**Finding 2 — Squid was OOM-killed at startup in-cluster.** With the
+operator deployed in-cluster the pod crash-looped (`OOMKilled`, empty
+logs). Root cause: squid sizes its file-descriptor tables from
+`RLIMIT_NOFILE`, and containerd under kind sets that effectively
+unlimited (~10^9) — squid allocates gigabytes before it ever listens.
+Fix in the generated config (`internal/provider/kubernetes/pod.go`):
+`max_filedescriptors 1024` (the load-bearing line) plus `cache_mem 16 MB`
+(a crawling forward proxy gains nothing from squid's 256 MB default),
+with a regression assertion added to `pod_test.go`.
+
+**With both fixes, the full pass:**
+
+```bash
+kind create cluster --name proxy-operator-demo
+make install
+make docker-build IMG=egress-proxies-operator:dev
+kind load docker-image egress-proxies-operator:dev --name proxy-operator-demo
+make deploy IMG=egress-proxies-operator:dev
+kubectl apply -f config/samples/proxy_kubernetes.yaml
+# → Ready 10.244.0.9 lat=89ms Provisioned=True Healthy=True   (~30 s)
+kubectl -n egress-proxies-operator-system port-forward svc/...-discovery-service 8090:8090 &
+curl -s 'localhost:8090/v1/proxies?healthy=true'          # count:1, latencyMillis:89
+curl -s -XPOST localhost:8090/v1/leases -d '{"selector":{"geo":"local"},"ttlSeconds":300}'
+#   → 201 {leaseID, proxy(activeLeases:1), expiresAt, ttlSeconds:300}
+curl -XPOST .../report -d '{"result":"rate_limited","target":"example.com"}'   # 204
+curl -XPOST /v1/leases -d '{...,"target":"example.com"}'  # 409 {inCooldown:1} ✓
+curl -XDELETE /v1/leases/<id>   # 204, and 204 again ✓
+kubectl delete -f config/samples/proxy_kubernetes.yaml    # finalizer: pod Terminating, CR gone
+kind delete cluster --name proxy-operator-demo
+```
+
+A real Squid pod went Ready through a real CONNECT probe, a lease was
+held on it, the cooldown machinery answered a 409 with correct
+arithmetic, and the finalizer cleaned up — the plan's success bar, met
+with the actual product.
+
+Worth noting: `make deploy` runs `kustomize edit set image` and mutates
+`config/manager/kustomization.yaml` in the working tree — reverted before
+committing (the repo keeps the pinned stanza). The health probe's ~89 ms
+latency is gstatic-through-squid from a kind pod on this machine;
+metrics-side observations were not separately checked in-cluster (covered
+by unit tests).

docs/plans-executions/2026-08-11-1742-gcp-provider-verbose-logging.md

@@ -0,0 +1,42 @@
+# Execution: Verbose (V-level) logging in the GCP provider
+
+Plan: `docs/plans/2026-08-11-1742-gcp-provider-verbose-logging.md`
+
+- [x] Step 0 — Save and commit the plan
+- [x] Step 1 — V(1)/V(2) logging in `internal/provider/gcp` (gcp.go, errors.go)
+- [x] Step 2 — Tests (verbosity tiers, error detail, cloud-init leak guard)
+- [ ] Step 3 — CHANGELOG entry (after the user confirms it works live)
+
+## Step 1 — logging in the provider
+
+Went as planned: context-carried logger (`logf.FromContext(ctx).WithName("gcp")`),
+V(1) one line per API call, V(2) request/list detail, opNames captured from the
+`instancesAPI` seam instead of being discarded. `logAPIError` lives in
+`errors.go` (next to `classify`, whose imports it shares) rather than `gcp.go`
+as loosely implied by the plan — same package, so no behavioural difference.
+These are the first `.V(n)` calls and the first logging import anywhere under
+`internal/provider/`.
+
+Worth noting: the gopls `errorsastype` suggestion fired on the new
+`errors.As` in `logAPIError` (Go's newer `errors.AsType`); kept `errors.As`
+for consistency with the three existing uses in the same file. Same for the
+`newexpr` (`proto.String` → `new`) suggestions — the codebase consistently
+uses `proto.String`.
+
+## Step 2 — tests
+
+`funcr.New` as the capturing sink, injected via `logr.NewContext`, exactly the
+seam the plan predicted. One deviation: instead of a single
+`TestLogging_verbosity` table, it split into three tests — `_verbosityTiers`
+(table over V=0/1/2, incl. the cloud-init sentinel leak assertion),
+`_apiErrorKeepsHTTPDetail` (403 quotaExceeded keeps `httpStatus`/reason at
+V(1)), and `_treatedAsSuccessPathsAreExplicit` (409-on-create /
+404-on-delete each log their "treated as success" line) — the last two
+exercise fake error wiring that didn't fit the tier table cleanly.
+
+Verified with:
+
+```bash
+go test -race ./internal/provider/gcp/
+go test ./...
+```

docs/plans-executions/2026-08-11-1802-bake-commit-version.md

@@ -0,0 +1,61 @@
+# Execution: Bake the git commit into the operator binary and log it at startup
+
+Plan: `docs/plans/2026-08-11-1802-bake-commit-version.md`
+
+- [x] Step 0 — Save and commit the plan
+- [x] Step 1 — `internal/version` package + tests
+- [x] Step 2 — `cmd/main.go`: `--version` flag + startup log line
+- [x] Step 3 — Makefile `GIT_COMMIT` + `--build-arg` wiring
+- [x] Step 4 — Dockerfile `-ldflags` stamp + OCI revision label
+- [ ] Step 5 — CHANGELOG entry (after the user confirms it works live)
+
+## Steps 1–4
+
+Mostly as planned. One deviation worth recording: the plan claimed
+`build`/`run` targets need no changes because Go's automatic VCS stamp covers
+host builds — that turned out to be only half true. Go skips VCS stamping
+when the build target is a *file argument* rather than a package pattern, and
+the Makefile's `build` target used `go build -o bin/manager cmd/main.go`.
+Verified empirically:
+
+```bash
+go build -o bin/manager cmd/main.go && ./bin/manager --version   # unknown (no vcs settings)
+go build -o bin/manager ./cmd       && ./bin/manager --version   # e4d2a191d0c2-dirty
+```
+
+So `build:` now uses `go build -o bin/manager ./cmd`. `go run` never stamps
+VCS info regardless of invocation form — `make run`/`run-dev` print
+`commit=unknown`, which is acceptable for dev loops (the Dockerfile path uses
+the explicit ldflags stamp and is unaffected; it kept `cmd/main.go`).
+
+The ldflags path was verified independently:
+
+```bash
+go build -ldflags "-X gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version.Commit=deadbeef1234" \
+  -o bin/manager-stamped cmd/main.go
+./bin/manager-stamped --version   # deadbeef1234
+```
+
+Worth noting: `cmd/main.go` imports k8s apimachinery as `runtime`, so the
+stdlib runtime needed an alias (`goruntime "runtime"`) for
+`goruntime.Version()` in the startup line. The `--version` check happens
+right after `flag.Parse()`, before logger and manager setup, so it works
+without a kubeconfig.
+
+## Verification
+
+```bash
+go vet ./... && go test ./...           # all green, incl. new resolve() table tests
+go build -o bin/manager ./cmd && ./bin/manager --version   # e4d2a191d0c2-dirty
+```
+
+The image-level check initially failed (Docker daemon not running); after
+the daemon was started it passed in full:
+
+```bash
+make docker-build IMG=egress-proxies-operator:dev
+docker run --rm egress-proxies-operator:dev --version
+# ae434a7167ec-dirty  (matches git rev-parse --short=12 HEAD + untracked files)
+docker inspect egress-proxies-operator:dev --format '{{index .Config.Labels "org.opencontainers.image.revision"}}'
+# ae434a7167ec-dirty
+```

docs/plans-executions/2026-08-11-1838-gcp-http-wire-logging-v5.md

@@ -0,0 +1,63 @@
+# Execution: GCP HTTP wire logging at V(5)
+
+Plan: `docs/plans/2026-08-11-1838-gcp-http-wire-logging-v5.md`
+
+- [x] Step 1 — `wireLogger` + `option.WithLogger` wiring in `internal/provider/gcp/gcp.go`
+- [x] Step 2 — Tests (`TestWireLogger_gatesAtV5`, `TestWireLogger_infoLandsAtV1`)
+- [x] Step 3 — Live verification at `--zap-log-level=5` (user, on cluster)
+- [x] Step 3b — Post-verification fix: drop auth records, elide huge fields
+- [ ] Step 4 — CHANGELOG entry (after live confirmation of 3b; batch with the
+      two earlier pending entries: GCP V-logging, version stamp)
+
+## Steps 1–2
+
+Went exactly as planned — the whole feature is ~10 lines of production code
+because both halves already existed: the compute SDK logs full HTTP
+request/response records at slog Debug to an injectable logger, and
+`logr.ToSlogHandler` does the slog→logr bridging. The only real design
+content is the level shift (`base.V(1)` + slog-Debug's +4 = V(5)) and the
+startup warning line when V(5) is active (raw payloads include cloud-init
+user-data, which the curated V(2) logging deliberately hides).
+
+Worth noting for future readers:
+
+- `option.WithLogger` **disables** `GOOGLE_SDK_GO_LOGGING_LEVEL` for this
+  client (documented SDK precedence) — `--zap-log-level` is now the only knob
+  for GCP wire logs.
+- The V(5) check in `New` runs once at startup; that is sound because the zap
+  level is fixed by flags at process start.
+- Added `TestWireLogger_infoLandsAtV1` beyond the plan's table — it pins the
+  shift arithmetic from the other side (slog Info → V(1)), so a future logr
+  mapping change would fail loudly.
+
+Verified with:
+
+```bash
+go test -race ./internal/provider/gcp/
+go build ./... && go test ./...
+```
+
+## Step 3b — what live verification exposed, and the fix
+
+Live V(5) output revealed two problems the plan missed:
+
+1. **Security: the injected logger propagates into `cloud.google.com/go/auth`**,
+   which logs its own token exchange (`auth.go:571/576`) — signed JWT
+   assertion in the request, full bearer access token in the response. The
+   plan's "auth token is safe" analysis only covered the compute client's
+   request headers, not the auth library's own records. Fix: `wireLogger`
+   now wraps the handler in a filter that drops every Debug record except
+   the compute client's `"api request"`/`"api response"` (allowlist, so
+   future SDK additions fail closed); Warn/Error still pass through.
+2. **Readability: GCP responses embed multi-KB blobs** (Shielded-VM UEFI
+   dbx databases) that swamp the line. Fix: string fields >1KiB are elided
+   to `[elided N bytes]` by default, recursively through payload
+   maps/arrays. Opt-out via new manager flag
+   `--gcp-wire-log-full-payloads` (threaded through a constructor closure
+   in `cmd/main.go` → `gcp.NewWithWireOptions`; the `registry.Constructor`
+   signature stays unchanged). Chosen by the user: elision on by default,
+   verbatim available on demand. Auth records are dropped in both modes.
+
+The filter/elision logic lives in `internal/provider/gcp/wirelog.go` with
+tests covering: auth-record drop (both modes), elision marker + small-field
+preservation, verbatim mode, and the original V(5) gating.

docs/plans-executions/2026-08-11-1935-gitea-build-workflow.md

@@ -0,0 +1,53 @@
+# Execution: Distilled Gitea Actions image-build workflow
+
+Plan: `docs/plans/2026-08-11-1935-gitea-build-workflow.md`
+
+- [x] Step 1 — Create `.gitea/workflows/build.yaml`
+- [x] Step 2 — Replace CLAUDE.md CI TODO with a CI/CD subsection
+- [x] Step 3 — Push branch + open MR
+- [ ] Step 4 — CHANGELOG entry (after the first successful run is confirmed)
+
+## Steps 1–2 — workflow + CLAUDE.md
+
+The workflow distills the house pattern from 9 sibling projects (survey in the plan)
+plus improvements none of them combine: an immutable `sha-<12>` tag, a lightweight
+test gate, `:latest` moving only on real tag pushes, and a `concurrency` group.
+
+Work happened in a git worktree off `origin/main` so the main checkout (which had
+unrelated uncommitted changes) stayed untouched:
+
+```bash
+git worktree add -b feat/gitea-build-workflow \
+  "$SCRATCH/wt-build" origin/main
+```
+
+Deviation from the plan's assumptions: while planning, `feat/proxy-operator` was
+still unmerged and the plan noted `main` lacked `internal/`/`test/`. By execution
+time `origin/main` had moved (`076bc66..f7000f7` — the proxy-operator MR merged),
+so the branch and its CI gate cover the full operator code.
+
+Verified the workflow parses and the check-gate commands pass on this exact tree
+(ruby stands in for a YAML linter because the system python3 has no `yaml` module):
+
+```bash
+ruby -ryaml -e "YAML.load_file('.gitea/workflows/build.yaml'); puts 'YAML OK'"
+go vet ./... && go build ./... && go test -short ./...   # all packages ok
+```
+
+## Step 3 — push + MR
+
+Branch pushed and MR opened with `tea` (the worktree was then removed and the main
+checkout switched onto the branch so the files are visible locally):
+
+```bash
+tea pr create --title "Add Gitea Actions image-build workflow" \
+  --description "..." --base main --head feat/gitea-build-workflow
+# → https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/pulls/2
+```
+
+Worth noting: the workflow itself cannot run end-to-end until (a) the MR merges
+(it only triggers on tags / manual dispatch, not branch pushes) and (b) the
+`REGISTRY_TOKEN` secret is created in this repo's Gitea settings (PAT with
+`write:package`). First real verification = manual dispatch with tag
+`manual-test`, expecting `manual-test` + `sha-…` in Packages and `:latest`
+untouched.

docs/plans-executions/2026-08-11-2152-discovery-api-docs.md

@@ -0,0 +1,38 @@
+# Execution: Discovery API documentation
+
+Plan: [2026-08-11-2152-discovery-api-docs.md](../plans/2026-08-11-2152-discovery-api-docs.md)
+
+- [x] Step 1 — Write `docs/api.md` full API reference
+- [x] Step 2 — Add pointers in README and architecture.md
+
+## Step 1 — docs/api.md
+
+Wrote the full reference: base URL (in-cluster FQDN + port-forward), bearer
+auth, error envelope, configuration table, all five routes with schemas,
+status codes and curl examples, the selection/cooldown semantics section,
+an end-to-end curl walkthrough, and the in-memory/single-replica caveats.
+All facts were taken from the code, not from memory of prior docs.
+
+Worth noting: the doc explicitly calls out two things no earlier doc
+stated for clients — that a report **without** a target on a targetless
+lease creates a *global* cooldown (blocking the proxy for everyone), and
+that `rate_limited` and `banned` currently behave identically. Both came
+straight from `internal/lease/store.go` and are easy to trip over.
+
+## Step 2 — Pointers + verification
+
+Added one-line links to the new doc in README's quickstart (above the curl
+block) and in `docs/architecture.md` §7. Verified the documented behavior
+against the tree rather than trusting the write-up:
+
+```sh
+go build ./... && go test -short ./internal/discovery/ ./internal/lease/
+```
+
+Both pass; a grep of `server_test.go` confirmed every documented status
+code and error code (`invalid_ttl`, `invalid_query`, `invalid_result`,
+`no_match`, `unknown_lease`, 201/204/401/404/409) is asserted by tests.
+
+Worth noting: CHANGELOG entry deliberately deferred until the user
+confirms the docs read well, per the CHANGELOG convention's
+"once the user confirms it works" clause.

docs/plans-executions/2026-08-11-2220-demo-scripts.md

@@ -0,0 +1,55 @@
+# Execution: Demo scripts
+
+Plan: [2026-08-11-2220-demo-scripts.md](../plans/2026-08-11-2220-demo-scripts.md)
+
+- [x] Step 0 — Branch `feat/demo-scripts` + plan commit
+- [x] Step 1 — `docs/demo/show-egress-ips.sh`
+- [x] Extra (added iteratively, not in the original plan) — proxy-creation scripts
+
+## Step 0 + Step 1
+
+Branched off `main`, committed the plan alone, then wrote
+`docs/demo/show-egress-ips.sh`: takes `BASE_URL` as its argument, lists
+`/v1/proxies` (bearer auth via optional `TOKEN` env), probes each healthy
+proxy with `curl -x http://ip:port` against an IP-echo site
+(`IP_ECHO_URL`, default ipify JSON), banners which proxy each request goes
+through, skips unhealthy ones, and ends with a probed/skipped/failed
+summary. Per user request the script was left uncommitted for iteration
+and no verification beyond `bash -n` was run.
+
+## Extra — create-kubernetes-proxies.sh, create-gcp-proxies.sh
+
+Added on the same branch before the first commit:
+
+- `create-kubernetes-proxies.sh <count>` — creates
+  `proxy-kubernetes-demo-1..N` with the kubernetes provider, spec taken
+  from `config/samples/proxy_kubernetes.yaml`, applied via
+  `kubectl apply -f -` heredocs. Optional `NAMESPACE` env.
+- `create-gcp-proxies.sh <count>` — creates `proxy-gcp-demo-1..N` from the
+  user-supplied gcp-eu manifest (e2-micro, Ubuntu 24.04, Squid
+  cloud-init), each with a zone picked randomly from a hardcoded list of
+  27 EU zones so the fleet gets egress IPs from different locations.
+  Env overrides: `ZONES`, `GCP_PROVIDER` (default `gcp-eu`), `NAMESPACE`.
+
+## Extra — run-demo.sh, show-egress-ips-table.sh
+
+Second round of iterative additions:
+
+- `run-demo.sh` — tmux demo driver: 2x2 tiled grid with the egress-IP
+  table looping every 10s inside a netshoot pod (script `kubectl cp`'d
+  into the pod, `BASE_URL` set to the in-cluster Service FQDN),
+  `watch -n3 kubectl get px`, and both create scripts auto-running with
+  `COUNT` proxies each (default 4). Env knobs: `COUNT`, `SESSION`,
+  `NETSHOOT_POD`, `DEMO_DIR` (defaults to the script's own dir).
+- `show-egress-ips-table.sh` — condensed one-line-per-proxy variant of
+  `show-egress-ips.sh` sized for a tmux pane: PROXY / ENDPOINT /
+  LOCATION (zone→geo attribute fallback) / EGRESS-IP columns, with
+  `(unhealthy)` and `FAILED` inline instead of verbose output. The
+  verbose script stays for standalone use; the demo driver uses the
+  table variant.
+
+Worth noting: beyond the user's sample manifest, the gcp script also
+writes the picked zone into `attributes.zone`, so the discovery API
+exposes each proxy's location and leases can select on it. The zone list
+is static — if a project lacks quota in some region, `ZONES` narrows the
+pool; nothing validates zones against the live project.

docs/plans-executions/2026-09-04-1404-kvm-provider.md

@@ -0,0 +1,285 @@
+# Execution: KVM/libvirt provider
+
+Plan: [2026-09-04-1404-kvm-provider.md](../plans/2026-09-04-1404-kvm-provider.md)
+
+- [x] Step 0 — Save plan + branch `feat/kvm-provider`
+- [x] Step 1 — Config (`KVMConfig` & friends, `validate()` case, tests)
+- [x] Step 2 — Pure builders (`domainxml.go`, `volume.go`, `seed.go`, `id.go`, `metadata.go`)
+- [x] Step 3 — Transport + errors (`conn.go`, `errors.go`)
+- [x] Step 3b — Image cache (`image.go`)
+- [x] Step 4 — Provider methods (`kvm.go`) over a fake API
+- [ ] Step 5 — Wiring, manifests, samples, docs, CHANGELOG
+- [ ] Step 6 — Integration test against a real hypervisor
+- [ ] Step 7 — Manual validation + MR
+
+## Step 0 — Save plan + branch
+
+Plan committed alone on `feat/proxy-operator`, then branched:
+
+```bash
+git checkout -b feat/kvm-provider
+```
+
+## Step 1 — Config
+
+The new structs (`KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage`) and
+`validateKVM` went into their own file, `internal/provider/config_kvm.go`,
+rather than being appended to `config.go` as the plan said: the KVM block has
+far more fields and rules than the other two providers, and keeping it
+separate leaves `config.go` readable. Tests likewise live in
+`config_kvm_test.go` instead of extending the table in `config_test.go`; the
+invalid-config table covers every required field, duplicate host names, image
+URL scheme, sha256 shape, and the disk format enum.
+
+While adding the `kvm` case to `validate()`, the pairwise "type is X but a Y
+block is set" checks (which would have needed six branches with three types)
+were replaced by a `setBlocks()` helper that lists the non-nil blocks and
+rejects any that don't match `type`. The error strings are unchanged, so the
+existing GCP/kubernetes tests still pass.
+
+Dependencies were fetched up front so the API signatures in the plan could be
+checked with `go doc`; they stay `// indirect` in `go.mod` until Step 2 imports
+them:
+
+```bash
+go get github.com/digitalocean/go-libvirt@latest github.com/kdomanski/iso9660@v0.4.0
+# go-libvirt resolved to v0.0.0-20260814190004-1a83157e1858 (no tagged release)
+# side effect: golang.org/x/crypto 0.54.0 → 0.56.0, x/mod, x/text, x/tools bumped
+```
+
+Worth noting: `go doc` showed that go-libvirt's `New(net.Conn)` is deprecated
+in favour of `NewWithDialer(socket.Dialer)`, where `Dialer` is a one-method
+`Dial() (net.Conn, error)` interface. Step 3 should wrap the SSH client's unix
+forward in that interface instead of using `New`; the plan's transport design
+is otherwise unaffected.
+
+## Step 2 — Pure builders
+
+`internal/provider/kvm/` now has the five builders from the plan plus
+`kvm.go` holding the package doc and `withDefaults` (disk size 10 GiB, the
+default vendor-data, `/var/run/libvirt/libvirt-sock`, pool `default`, image
+format `qcow2`). Everything is a pure function over `encoding/xml` structs or
+strings, unit-tested by unmarshalling the output back and checking fields.
+
+Two design choices differ from the plan text:
+
+- **Domain disks reference pool volumes, not host paths.** The plan had
+  `buildDomainXML` take the overlay and seed paths (needing two
+  `StorageVolGetPath` round-trips per Create). The domain XML now uses
+  `<disk type="volume"><source pool=... volume=.../>`, so only the *base*
+  image path is ever looked up (the overlay's `<backingStore>` needs a real
+  path). Fewer RPCs and the builder's inputs are all names.
+- **Capacities are always `unit="bytes"`.** libvirt's `G` means GiB but
+  `GB` means 10^9; spelling the byte count out removes the ambiguity from
+  both the overlay and the seed/base volumes.
+
+The metadata element is written with a default namespace
+(`<instance xmlns="…/kvm">`) because `encoding/xml` cannot emit prefixes on
+marshal. libvirt may hand it back prefixed; `parseInstanceMetadata` is
+tested against both spellings since `encoding/xml` matches on the namespace
+URI. Whether libvirt accepts the default-namespace form on `DomainDefineXML`
+is confirmed in Step 6 against a real host, not here.
+
+Checked before relying on `github.com/kdomanski/iso9660` for the seed: its
+filename mangling lowercases and keeps `-`, so `user-data`/`meta-data`/
+`vendor-data` come out as `user-data;1` etc. in the directory record, which
+the Linux isofs driver (and therefore cloud-init's NoCloud mount) presents as
+`user-data`. The seed test opens the ISO with the same library's reader and
+asserts label `cidata`, all three files, and byte-identical output for
+identical inputs. `NewWriter()` stages files in a temp directory, so
+`Cleanup()` is deferred.
+
+`go mod tidy` ran as part of this step; it promoted `kdomanski/iso9660` to a
+direct dependency and dropped `go-libvirt` again because nothing imports it
+yet — Step 3 brings it back.
+
+Worth noting: `withDefaults` copies the `Images` map and `Hosts` slice
+instead of mutating the config in place, so the registry's shared
+`ProviderConfig` stays untouched. The default vendor-data installs
+`qemu-guest-agent` via `packages:` + `runcmd`, which means the first boot
+needs network access to the distro mirrors; a Proxy on an isolated bridge
+would still work (ARP discovery), just without the agent.
+
+## Step 3 — Transport + errors
+
+`conn.go` holds the `libvirtAPI` seam (17 methods, signatures as in the
+plan), a per-host connection pool, and the SSH dialer; `errors.go` holds
+`classify`, `isLibvirtError`, `hasCode` and `logAPIError`. Both are
+unit-tested; the whole package still runs in about a second under `-race`.
+
+The one thing the plan did not anticipate: **go-libvirt RPCs take no
+context and never time out**. A hung hypervisor would pin a reconcile worker
+forever. So the pool exposes `call(ctx, host, fn)` rather than handing out
+raw clients: `fn` runs in a goroutine, and if `ctx` ends first the pool
+closes the SSH connection out from under it (which makes the blocked RPC
+return) and reports `ctx.Err()`. The same helper (`awaitOrCancel`) bounds
+`lv.Connect()` during dial. Two more liveness measures for the same reason:
+the SSH handshake gets a deadline (ctx's, or 30 s), and each connection runs
+an OpenSSH `keepalive@openssh.com` loop every 15 s that closes the client
+when a request errors or goes unanswered for one interval.
+
+Connection-drop policy in `call`: any error that is *not* a `libvirt.Error`
+(so io.EOF, "connection closed", SSH failures) drops the cached connection
+and the next call redials; a daemon-reported `libvirt.Error` keeps it. Dials
+are serialised per host (concurrent Creates on one host share a dial) but
+hosts don't block each other.
+
+As Step 1 noted, the dialer uses `libvirt.NewWithDialer` with a
+`socket.Dialer` that wraps `ssh.Client.Dial("unix", socket)` rather than the
+deprecated `New(net.Conn)`.
+
+Tests: the pool is exercised through a fake dialer (reuse, redial after
+`Disconnected`, drop on transport error, keep on libvirt error, cancel
+releases a blocked call, concurrent callers share one dial, hosts
+independent). The SSH layer is tested against an in-process
+`x/crypto/ssh` server that serves `direct-streamlocal@openssh.com` channels
+by dialing the requested unix path (an echo socket standing in for
+`libvirt-sock`), with keys generated per test and a real `known_hosts` file
+written via `knownhosts.Line`. Covered: forward round-trip and the socket
+path the server saw, unknown host key rejected with `knownhosts.KeyError`,
+`insecureIgnoreHostKey`, wrong client key, cancelled ctx, missing key or
+known_hosts file, and keepalive both closing a stalled client and leaving a
+healthy one alone. The libvirt handshake (`Connect()`) itself is *not*
+covered here — faking libvirt's XDR RPC just for that is not worth it; the
+Step 6 integration test covers it against a real host.
+
+`go mod tidy` promoted `go-libvirt` and `golang.org/x/crypto` to direct
+dependencies.
+
+Worth noting: `classify` maps `ErrOperationInvalid` (e.g. destroying a
+domain that is already off) to Transient; Delete in Step 4 must check that
+code itself via `hasCode` and treat it as success rather than relying on
+the classifier. Nothing maps to `ErrQuotaExceeded` — libvirt has no quota
+concept; an out-of-space pool surfaces as a Transient `ErrInternalError`.
+
+## Step 3b — Image cache
+
+`image.go` follows the plan closely: `resolveImage` maps `placement.image`
+to a configured key or a direct http(s) URL (else Permanent);
+`imageCache.ensure` checks base volume + marker (+ marker content when a
+checksum is configured), tears down and re-fetches when anything is off,
+streams the HTTP body through `StorageVolUpload` with a `sha256` tee,
+refreshes the pool, then writes the marker last. A per-`(host, volume)`
+mutex serialises concurrent callers; the pool itself is the only cache
+state.
+
+Errors from this file are pre-tagged with the taxonomy sentinel via
+`fmt.Errorf("%w: …", provider.ErrPermanent, …)` and `classify` now honours
+such tags before looking at libvirt codes. That keeps one `wrapErr` path in
+the provider instead of special-casing image errors.
+
+`fake_test.go` landed with this step rather than Step 4: an in-memory
+`libvirtAPI` with pools, volumes (content, format, capacity, backing path),
+domains (state, metadata, per-source interface addresses), per-method error
+injection (`fail` always / `failOnce`) and an `uploadHook` to break a stream
+mid-way. It models the libvirt error codes the provider branches on
+(`ErrStorageVolExist`, `ErrNoStorageVol`, `ErrNoDomain`,
+`ErrOperationInvalid`, `ErrNoDomainMetadata`, `ErrAgentUnresponsive`).
+
+Tests cover every case in the plan plus: no checksum configured (marker is
+still written with the computed digest; second call needs no marker read),
+403/500 alongside 404/503, missing pool → Permanent, cancelled ctx →
+Transient with nothing created, and different hosts fetching independently.
+
+One test-harness lesson worth keeping: `httptest`'s server drops
+`Content-Length` and switches to chunked encoding for bodies larger than
+its 2 KiB write buffer, so the fake mirror sets the header explicitly.
+Real distro mirrors send it; a server that doesn't is reported as
+Permanent with a message naming the header.
+
+Worth noting: the base volume is created with `capacity = Content-Length`
+and the *image's* format (`qcow2`), and the raw bytes are uploaded over it.
+That relies on libvirt's upload writing the file verbatim (which is how
+`virsh vol-upload` populates volumes); the following `StoragePoolRefresh`
+makes libvirt re-read the real virtual size from the qcow2 header. Step 6
+confirms this on a real host. Also, the whole download runs inside one
+`pool.call`, so cancellation aborts the HTTP body (ctx-bound) and the
+upload stream fails; the half-written base volume is deleted on the spot
+and, failing that, on the next attempt via the missing marker.
+
+## Step 4 — Provider methods
+
+`kvm.go` now holds `Provider`, `New` (loads the SSH key and known_hosts
+up front via `newSSHAuth`, so a bad path fails at startup), the
+`newWithDialer` seam, and `Create`/`Get`/`Delete`/`ListByTag`. Each
+method resolves its host from config, then runs the whole libvirt
+conversation inside one `hostPool.call`, so a cancelled reconcile drops
+the connection and the blocked RPC returns. Errors pass through `classify`
+in one `wrapErr`, as the Step 3b note intended.
+
+Behaviour that goes beyond the plan text, or that a future reader should
+know before touching this file:
+
+- **Create looks up the domain first.** If it exists, only the running
+  check runs and the volumes are never touched. If not, nothing can be
+  using the volumes yet: the overlay is reused when it exists (its
+  creation is atomic; `ErrStorageVolExist` → reuse) but the **seed is
+  always deleted and rebuilt**. The plan said "exists → skip upload"; the
+  reviewer pointed out that a reconcile cancelled mid-upload leaves a
+  truncated seed that the compensating delete cannot remove, because
+  `hostPool.call` has already closed the connection the delete would go
+  over. Skipping such a seed would boot the VM without cloud-init and no
+  retry would ever repair it. Rebuilding costs one ~40 KiB upload per
+  attempt before the domain exists, nothing afterwards. The table test
+  breaks Create after each step and re-runs it, and a separate test
+  starts from a leftover truncated seed.
+- **Get swallows address-source errors.** `DomainInterfaceAddresses` from
+  ARP returns nothing until the guest has sent traffic and the agent
+  source errors with `ErrAgentUnresponsive` until `qemu-guest-agent` is
+  up, so both are logged at V(2) and mean "no IP yet" → `Provisioning`.
+  Loopback, link-local and unspecified addresses are skipped when
+  choosing the first IPv4 (the agent reports `lo` too).
+- **State mapping:** `running`/`blocked` are active; `nostate` →
+  Provisioning (the reconciler answers Stopped with delete-and-recreate,
+  too drastic for a state libvirt could not determine); everything else
+  (paused, pmsuspended, shutdown, shutoff, crashed) → Stopped.
+- **Delete treats a missing pool as "nothing to clean"** rather than
+  Permanent: failing there would pin the Proxy's finalizer on a config
+  edit that renamed the pool. Domain removal is destroy (`ErrOperationInvalid`
+  on a stopped domain is fine) → undefine with managed-save, snapshot
+  metadata and nvram flags → overlay and seed deleted by name.
+- **ListByTag** reads each domain's metadata once (passed into the same
+  `inspect` that Get uses, so no second `DomainGetMetadata` round-trip)
+  and skips domains without the operator's element. A domain that cannot
+  be read at all — vanished mid-sweep, corrupt metadata, state call
+  failing — is *skipped*, not fatal: GC never deletes what it does not
+  see, so leaving one out is safe, whereas one bad domain blocking GC for
+  every other one (the first version's behaviour, caught in review) is
+  not. An unreachable host still fails the whole sweep.
+  `ConnectListAllDomains` is called with flags 0 (active and inactive).
+- A host named in a providerID but no longer in `kvm.hosts` is Permanent
+  for Get and Delete: the operator cannot reach it, and retrying forever
+  would hide the config problem.
+
+Tests run the provider over a `hostDialer` that maps host names to
+`fakeAPI` instances (an unmapped host is "unreachable"). The image is
+pre-seeded in the pool for most tests; one test runs the full first-use
+download through `httptest` and confirms a second Proxy on the same host
+does not download again.
+
+Found while running the package: `TestBuildSeedISO_deterministic` from
+Step 2 was wrong. `iso9660.ImageWriter` stamps `time.Now()` into the
+volume descriptor, so two builds in different seconds differ by six
+timestamp bytes; the test had only passed because both builds landed in
+the same second. Byte identity is nothing the provider depends on (an
+existing seed is never re-uploaded), so the test now compares label,
+files and size, and the comment on `buildSeedISO` no longer claims
+identical output.
+
+The `go-operator-reviewer` agent ran on the first version of this step
+and found the two bugs above (truncated seed, wedged GC sweep) plus the
+double metadata read and the `nostate` mapping; all were fixed before
+committing. One finding it raised is deliberately **not** fixed here
+because it lives in Step 3b's `image.go` and changes the volume naming
+the plan chose: re-fetching a base image (checksum edited in config, or
+upstream content changed) deletes `base-<key>.<fmt>` while existing
+overlays still back onto it, breaking those VMs' disks. Content-addressed
+base names would avoid it. Flagged for the user; until decided, Step 5's
+docs must warn that an image's `sha256`/`url` must not change while
+Proxies using it exist.
+
+Worth noting: `go test -cover` reports 88.2 % for the package; the
+uncovered lines are the real SSH/libvirt dial path and log-only
+branches. `go vet` and `-race` (three runs) are clean. Wiring into
+`cmd/main.go` is Step 5, so `New` is not yet reachable from a running
+operator.

docs/plans/2026-08-07-1747-proxy-operator.md

@@ -391,7 +391,10 @@ pods would refuse connections while still being Service endpoints. The 1-replica
 constraint comes from lease state being per-process, which the spec already accepts —
 both facts go in the README caveats.
 
-stdlib `http.ServeMux` with Go 1.22 method+wildcard patterns:
+stdlib `http.ServeMux` using its method+wildcard patterns (`"GET /path"`,
+`"/{id}"` + `r.PathValue`) — a stdlib feature available since Go 1.22, used
+here so no third-party router is needed; the project itself stays on the
+pinned Go 1.26:
 `GET /v1/proxies`, `POST /v1/leases`, `DELETE /v1/leases/{id}`,
 `POST /v1/leases/{id}/report`, plus unauthenticated `GET /healthz`.
 Middleware outermost-first: recover → request-log → `MaxBytesReader(64KiB)` → bearer

docs/plans/2026-08-11-1742-gcp-provider-verbose-logging.md

@@ -0,0 +1,87 @@
+# Plan: Verbose (V-level) logging in the GCP provider
+
+**Created:** 2026-08-11 17:42
+
+## Context
+
+Debugging GCP provisioning is currently blind: the operator has zero `.V(n)` calls
+anywhere, so `--zap-log-level=debug` (or any numeric level) reveals nothing about
+what the GCP provider is doing — which API calls it makes, with what parameters,
+and what came back. The goal: with debug/V-level logging enabled, see the details
+of every GCP Compute API call (Insert/Get/Delete/AggregatedList) including a
+summary of the response; with default `info` level, the provider stays as quiet
+as today.
+
+## Approach (the "how")
+
+**Logger source — context-carried, not injected.** Provider methods all take
+`ctx`, and the reconciler already builds a per-request logger
+(`logf.FromContext(ctx)` in `internal/controller/proxy_controller.go:117`) that
+carries the proxy's name/namespace. The GCP provider will do
+`log := logf.FromContext(ctx).WithName("gcp").WithValues("provider", p.name)` at
+the top of each public method. Zero wiring changes (no registry/constructor/struct
+changes), and every provider log line automatically inherits the reconcile
+context (which Proxy triggered it). Calls from the GC sweeper inherit its
+`orphan-gc` logger name the same way.
+
+**Verbosity scheme** (logr convention: `.Info()` = V(0), `debug` flag = V(1)):
+
+- **V(1)** — one line per GCP API call, after it returns: operation, identifying
+  params, outcome. Examples:
+  - `Create`: `"GCP insert instance"` with `zone`, `name`, `machineType`,
+    `image`, `opName` (currently discarded at gcp.go:117 — capture it, it's the
+    only handle for correlating with GCP's operation log), plus a line for the
+    409-already-exists path.
+  - `Get`: `"GCP get instance"` with `zone`, `name`, `status`, mapped `state`, `ip`.
+  - `Delete`: `"GCP delete instance"` with `zone`, `name`, `opName`, and the
+    404-treated-as-success path.
+  - `ListByTag`: `"GCP aggregated list"` with `filter`, `count`.
+  - Error paths at V(1) too: log the raw classification (HTTP status / reason
+    from `googleapi.Error`) before it's wrapped, since the wrapped error the
+    reconciler sees is coarser.
+- **V(2)** — request/response detail: full curated insert-request summary
+  (network, networkTag, diskSizeGB, port, labels, `cloudInitBytes` = `len`),
+  per-instance lines in `ListByTag` (id, state, uid, age).
+
+**Curated fields, never raw proto dumps.** `CreateRequest.CloudInit` is resolved
+user-data possibly sourced from a Secret, and it lands in the insert request's
+metadata — so logging the request proto wholesale would leak it. Log named safe
+fields only; for cloud-init, log only its byte length. This is a hard rule, and
+a test asserts it.
+
+**Where the calls live: the `Provider` methods in
+`internal/provider/gcp/gcp.go`** (Create/Get/Delete/ListByTag), not in
+`realInstances` (deliberately untested by design, gcp.go:86) and not an HTTP
+round-tripper (would log auth headers/user-data, unredactable). The
+`instancesAPI` fake seam (`newWithAPI`, gcp.go:96) keeps everything testable.
+To surface `opName`, change `Provider.Create`/`Delete` to capture the string
+their `instancesAPI` calls already return instead of discarding it.
+
+## Files to change
+
+- `internal/provider/gcp/gcp.go` — add `logf` import; V(1)/V(2) logging in
+  `Create`, `Get`, `Delete`, `ListByTag`; capture opNames. Only file with
+  production changes.
+- `internal/provider/gcp/gcp_test.go` — new table-driven test
+  `TestLogging_verbosity` (name TBD per house `Test<Function>_<scenario>`
+  style): inject a capturing logger via `logf.IntoContext(ctx, funcr.New(...))`
+  (`github.com/go-logr/logr/funcr`, logr already a direct dep), assert:
+  - at V(1): expected message + keys per operation (incl. opName),
+  - at V(0): nothing logged,
+  - **cloud-init content never appears in any log output** (grep the captured
+    lines for a sentinel string placed in `CloudInit`).
+- No changes to `provider.Provider` interface, registry, `cmd/main.go`,
+  manifests, or the kubernetes provider (it can copy this pattern later).
+
+## Verification
+
+```bash
+go test -race ./internal/provider/gcp/...
+go build ./...
+```
+
+Optional live check: run the manager with `--zap-log-level=2` against the GCP
+project and confirm insert/get lines appear during a Proxy reconcile, and that
+`--zap-log-level=info` stays quiet.
+
+Also append a CHANGELOG.md entry per house convention once confirmed working.

docs/plans/2026-08-11-1802-bake-commit-version.md

@@ -0,0 +1,75 @@
+# Plan: Bake the git commit into the operator binary and log it at startup
+
+**Created:** 2026-08-11 18:02
+
+## Context
+
+There is no versioning yet, and the image tag (`egress-proxies-operator:dev`,
+`imagePullPolicy: IfNotPresent`) says nothing about what code is actually
+running. The user wants the commit hash baked into the image at build time,
+and — the key requirement — the binary itself must know it and print it into
+the log stream during initialization, so `kubectl logs | head` answers "which
+version is running". Docker builds cannot use Go's automatic VCS stamp because
+`.dockerignore` excludes `.git` (correctly — re-including it would bust layer
+caching), so the hash must travel git → Makefile → `--build-arg` → `-ldflags -X`.
+
+## Implementation
+
+**1. New package `internal/version`** (`version.go` + `version_test.go`):
+
+- `var Commit string` — stamped at link time via
+  `-ldflags "-X gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version.Commit=<hash>"`.
+- `func Resolve() string` — returns `Commit` if non-empty; otherwise falls
+  back to `debug.ReadBuildInfo()` VCS settings (`vcs.revision` truncated to
+  12 chars, `-dirty` suffix when `vcs.modified=true`), so plain host builds
+  (`make build`, `make run`, `go run`) are stamped for free since `.git` is
+  present there; `"unknown"` when neither source is available (e.g. `go test`).
+- Internal `resolve(ldflagsCommit string, readBuildInfo func() (*debug.BuildInfo, bool)) string`
+  so the fallback logic is table-testable with a fake build-info func
+  (house style: stdlib testing, `t.Parallel()`, subtests,
+  `Test<Function>_<scenario>` names).
+
+**2. `cmd/main.go`**:
+
+- Add `--version` bool flag; immediately after the existing `flag.Parse()`
+  (main.go:132), if set: print `version.Resolve()` to stdout and exit 0
+  (before logger/manager setup).
+- Right after `ctrl.SetLogger(...)` (main.go:134):
+  `setupLog.Info("Starting egress-proxies-operator", "commit", version.Resolve(), "goVersion", runtime.Version())`
+  — first line of every run, plain V(0) so it appears at any log level.
+
+**3. `Makefile`**:
+
+- Near the other variables:
+  `GIT_COMMIT ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)$(shell test -z "$$(git status --porcelain 2>/dev/null)" || echo -dirty)`
+  (`git status --porcelain` catches staged and untracked changes, which
+  `git diff --quiet` misses).
+- `docker-build`: add `--build-arg GIT_COMMIT=$(GIT_COMMIT)`.
+- `docker-buildx`: add the same `--build-arg` to the `buildx build` line.
+- `build`/`run`/`run-dev` stay untouched — the ReadBuildInfo fallback covers them.
+
+**4. `Dockerfile`**:
+
+- `ARG GIT_COMMIT=unknown` in the builder stage; extend the existing
+  `go build` with `-ldflags "-X <module>/internal/version.Commit=${GIT_COMMIT}"`.
+- Re-declare `ARG GIT_COMMIT` in the distroless stage and add
+  `LABEL org.opencontainers.image.revision="${GIT_COMMIT}"` so the hash is
+  also visible via `docker inspect` without running the binary.
+
+No changes to deploy manifests, providers, or the reconciler. CHANGELOG entry
+after the user confirms it works (house convention).
+
+## Verification
+
+```bash
+go test ./internal/version/            # resolve() table tests
+go build ./... && go test ./...        # nothing else broke
+go run ./cmd/main.go --version         # host build: VCS-stamped hash (+ -dirty), exit 0
+make docker-build IMG=egress-proxies-operator:dev
+docker run --rm egress-proxies-operator:dev --version          # prints the baked commit
+docker inspect egress-proxies-operator:dev \
+  --format '{{index .Config.Labels "org.opencontainers.image.revision"}}'
+```
+
+Live check: redeploy on the cluster and confirm the first log line carries
+`"commit"`.

docs/plans/2026-08-11-1838-gcp-http-wire-logging-v5.md

@@ -0,0 +1,85 @@
+# Plan: GCP HTTP wire logging at V(5)
+
+**Created:** 2026-08-11 18:38
+
+## Context
+
+The GCP provider logs curated call summaries at V(1)/V(2), but when debugging
+against the real API the user wants ground truth: the actual HTTP requests and
+responses ("gory details") — visible at `--zap-log-level=5`, in the same log
+stream as everything else. The compute SDK already produces exactly this:
+`cloud.google.com/go/compute@v1.65.0/apiv1/helpers.go:60,70` logs
+`"api request"`/`"api response"` (method, URL, headers, full JSON payloads,
+lazily via `internallog.HTTPRequest/HTTPResponse`) to an injectable
+`*slog.Logger` at slog Debug level. We inject one bridged to the operator's
+zap sink, level-shifted so those Debug records surface only at V(5).
+
+Level scheme after this change: V(1) call outcomes, V(2) curated detail,
+V(5) raw HTTP traffic. V(3)/V(4) reserved.
+
+## Mechanism (verified in module sources)
+
+- `option.WithLogger(*slog.Logger)` exists in `google.golang.org/api@v0.292.0`
+  (option.go:529) and **takes precedence over `GOOGLE_SDK_GO_LOGGING_LEVEL`**
+  — after this change, V(5) is the single knob for this client; document that.
+- `logr.ToSlogHandler` (go-logr/logr v1.4.3, already a direct dep) maps
+  slog Debug → logr V(4), plus the base logger's V-bias. logr's own docs
+  (sloghandler.go:180-184): `slog.New(ToSlogHandler(logrV2)).Debug()` ≈ V(6).
+  So a base of `.V(1)` lands Debug at exactly V(5).
+- Gating is cheap: the slog handler's `Enabled()` consults the zap sink, so
+  below level 5 the SDK's lazy `LogValuer`s are never evaluated.
+
+## Implementation
+
+**`internal/provider/gcp/gcp.go`** (only production file):
+
+1. New pure function:
+   ```go
+   // wireLogger returns the slog logger handed to the SDK: its Debug-level
+   // "api request"/"api response" records (slog Debug = +4 on the logr
+   // scale) land at V(5) on top of the base's V(1) shift.
+   func wireLogger(base logr.Logger) *slog.Logger {
+       return slog.New(logr.ToSlogHandler(base.V(1)))
+   }
+   ```
+2. In `New` (gcp.go:96): pass it to the client —
+   `compute.NewInstancesRESTClient(ctx, option.WithLogger(wireLogger(logf.Log.WithName("gcp").WithName("http"))))`.
+   Base is the process-root `logf.Log` (client is built once at startup;
+   `ctrl.SetLogger` runs before `registry.Build` in cmd/main.go, so it
+   resolves to the real zap logger).
+3. One-time notice in `New`: if `logf.Log.V(5).Enabled()`, log at Info:
+   `"GCP HTTP wire logging active — request payloads include cloud-init user-data"`
+   (the secret-leak warning our curated V(2) logging exists to avoid; at V(5)
+   the user has explicitly opted into raw payloads).
+4. New imports: `log/slog`, `google.golang.org/api/option` (module already in
+   go.mod as a direct dep; `option` package is a first-time import in the repo).
+
+**`internal/provider/gcp/gcp_test.go`**:
+
+- `TestWireLogger_gatesAtV5`: table over funcr sink verbosities
+  (`funcr.Options{Verbosity: N}`, pattern already used by `captureContext`):
+  at 5 a `Debug("api request", ...)` through `wireLogger` emits (message and
+  attrs present); at 4 it emits nothing; an `Info` record through the same
+  logger lands at V(1) (sanity-check of the shift).
+- `New` itself stays untested by design (dials real Google endpoints —
+  existing convention, gcp.go:86-87).
+
+No changes to manifests, Makefile, other providers, or the reconciler.
+CHANGELOG entry after the user confirms it works (house convention) — this
+plus the two earlier pending entries (GCP V-logging, version stamp).
+
+## Verification
+
+```bash
+go test -race ./internal/provider/gcp/
+go build ./... && go test ./...
+```
+
+Live (the real proof, needs the cluster):
+
+```bash
+# rebuild + load image, set --zap-log-level=5, restart, then:
+kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager -f \
+  | grep -m2 'api request\|api response'   # full URL/headers/payload visible
+# and at --zap-log-level=2: the same grep stays silent while V(2) lines still appear
+```

docs/plans/2026-08-11-1935-gitea-build-workflow.md

@@ -0,0 +1,135 @@
+# Plan: Distilled Gitea Actions image-build workflow
+**Created:** 2026-08-11 19:35
+
+## Context
+
+This repo (`egress-proxies-operator`) has a Dockerfile, a Makefile with `docker-build`/`docker-push` targets, and a Gitea remote — but no CI workflow (CLAUDE.md flags this as a TODO). A survey of all projects under `/Users/jan.novak/srv` found 9 image-build workflows, all variations of one lineage: trigger on `workflow_dispatch` + tag push, `docker login` to `gitea.home.hrajfrisbee.cz` with `secrets.REGISTRY_TOKEN`, raw `docker build`/`docker push`, `runs-on: ubuntu-latest`, `permissions: {contents: read, packages: write}`.
+
+The best individual ideas are scattered:
+- **aviso_v2**: quality-gate job before build; computes `sha-<short>` as a second immutable tag via `$GITHUB_OUTPUT`.
+- **gateway-helper-operator** (closest sibling — same kubebuilder shape): passes build args (`GIT_COMMIT` etc.), tags `:latest` alongside the version tag.
+- **psmf-data-sync test.yaml**: `actions/setup-go@v5` with `go-version-file: go.mod` + module cache (proven to work on the act_runner).
+
+Goal: distill these into one `build.yaml` for this repo. User decisions: triggers = **tags + manual dispatch only** (house convention, no builds from main); build tool = **raw docker CLI** (the runner bind-mounts docker.sock, so this just works); **lightweight test gate** (`go vet` + `go build` + `go test -short`, no envtest download); **amd64 only**.
+
+## The workflow
+
+Create `.gitea/workflows/build.yaml`:
+
+```yaml
+name: Build and Push
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Image tag'
+        required: true
+        default: 'latest'
+  push:
+    tags:
+      - '*'
+
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Vet
+        run: go vet ./...
+      - name: Build
+        run: go build ./...
+      - name: Test (short)
+        run: go test -short ./...
+
+  build:
+    needs: check
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Compute image tags
+        id: meta
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            TAG="${{ inputs.tag }}"
+          else
+            TAG="${{ github.ref_name }}"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "sha=sha-$(echo '${{ github.sha }}' | cut -c1-12)" >> "$GITHUB_OUTPUT"
+
+      - name: Login to Gitea registry
+        run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login -u ${{ github.actor }} --password-stdin gitea.home.hrajfrisbee.cz
+
+      - name: Build and push
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker build \
+            --build-arg GIT_COMMIT=$(echo '${{ github.sha }}' | cut -c1-12) \
+            --label org.opencontainers.image.source=https://gitea.home.hrajfrisbee.cz/${{ github.repository }} \
+            --label org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
+            -t "$IMAGE:${{ steps.meta.outputs.tag }}" \
+            -t "$IMAGE:${{ steps.meta.outputs.sha }}" \
+            .
+          docker push "$IMAGE:${{ steps.meta.outputs.tag }}"
+          docker push "$IMAGE:${{ steps.meta.outputs.sha }}"
+
+      - name: Push latest (tag builds only)
+        if: github.event_name == 'push'
+        run: |
+          IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}
+          docker tag "$IMAGE:${{ steps.meta.outputs.tag }}" "$IMAGE:latest"
+          docker push "$IMAGE:latest"
+```
+
+### What's distilled vs. improved over the existing workflows
+
+Distilled (house patterns kept as-is): triggers, `REGISTRY_TOKEN` + `github.actor` login, image name `gitea.home.hrajfrisbee.cz/${{ github.repository }}`, `ubuntu-latest`, raw docker CLI, `permissions` block.
+
+Improvements none of the existing workflows have all of:
+1. **`sha-<12>` immutable tag** alongside the human tag (aviso_v2 had this; nobody else) — lets deployments pin exactly what was built.
+2. **Test gate** (aviso_v2 had one; the Go projects don't) — lightweight variant per user choice; uses `go-version-file: go.mod` so the Go version never drifts from the module.
+3. **`GIT_COMMIT` build arg** matches the Makefile/Dockerfile contract — the binary's `internal/version.Commit` and the `org.opencontainers.image.revision` label get the real commit (12-char, same width as the Makefile's `git rev-parse --short=12`; no `-dirty` needed since CI checkouts are clean).
+4. **`:latest` only on real tag pushes**, not manual dispatch — gateway-helper pushed `latest` unconditionally, which lets an ad-hoc dispatch of an old ref clobber `latest`.
+5. **`concurrency` group** — cancels a superseded run of the same ref (none of the 20 surveyed workflows have this).
+6. **OCI `source`/`created` labels** added at build time (revision label already comes from the Dockerfile).
+
+## Files
+
+- **Create** `.gitea/workflows/build.yaml` — content above.
+- **Update** `CLAUDE.md` — replace the `TODO: no .gitea/workflows/ CI pipeline exists yet` note in the Git Commits section with a short CI/CD subsection describing the workflow (triggers, secret, tags produced).
+- **Update** `CHANGELOG.md` — new top entry (after user confirms it works, per convention; timestamp via `date "+%Y-%m-%d %H:%M %Z"`).
+- Copy this plan to `docs/plans/YYYY-MM-DD-HHMM-gitea-build-workflow.md` (timestamp via `date "+%Y-%m-%d-%H%M"`) and commit it first, per CLAUDE.md ordering rule.
+
+## Branch & MR
+
+House convention: feature → own branch + MR. Dockerfile and `cmd/` already exist on `main`, so:
+
+1. `git checkout -b feat/gitea-build-workflow origin/main` (do not touch the current `feat/proxy-operator` branch's uncommitted `.claude/settings.json` change — leave it be).
+2. Commit plan file, then the workflow + CLAUDE.md update (with `Co-Authored-By: Claude <noreply@anthropic.com>`).
+3. `git push -u origin feat/gitea-build-workflow`, open MR with `tea pr create --base main --head feat/gitea-build-workflow`. Do not merge.
+
+Note: `main` has no `internal/`/`test/` dirs yet (those are on `feat/proxy-operator`), which is fine — the workflow only fires on tags/dispatch, and by then the operator branch will be merged. `go build ./...` / `go test -short ./...` work on both branch states.
+
+## Prerequisite (user action)
+
+`REGISTRY_TOKEN` secret must exist in this repo's Gitea settings (Settings → Actions → Secrets): a personal access token with `write:package` scope — same as every other project uses. Flag this in the MR description.
+
+## Verification
+
+The workflow doesn't trigger on branch pushes, so end-to-end verification happens after merge:
+1. Local sanity: `docker build --build-arg GIT_COMMIT=test -t scratch-check .` (confirms the build args/labels line is valid) — or at minimum a YAML parse check.
+2. After the MR merges: run the workflow manually via Gitea UI (Actions → Build and Push → Run workflow, tag `manual-test`), confirm both `manual-test` and `sha-…` tags appear under Packages, and that `:latest` was NOT updated.
+3. Then push a real version tag (e.g. `v0.1.0`) and confirm `v0.1.0`, `sha-…`, and `latest` all appear.

docs/plans/2026-08-11-2152-discovery-api-docs.md

@@ -0,0 +1,58 @@
+# Plan: Discovery API documentation (docs/api.md)
+**Created:** 2026-08-11 21:52
+
+
+## Context
+
+The operator serves an HTTP discovery/lease API on `:8090` ([internal/discovery/](internal/discovery/)) that crawler clients use to list proxies, acquire TTL leases, release them, and report rate-limiting. There is no dedicated API reference today: README has four quickstart curls (no auth header, no schemas), and `docs/architecture.md` §7 has an ASCII route map. The user wants full documentation with curl examples for every feature.
+
+**Decisions made with user:** doc lives in a new `docs/api.md`; commit straight to `main` (no MR).
+
+## Deliverable
+
+### 1. New file `docs/api.md` — full API reference
+
+Content (all facts verified against code during planning):
+
+- **Overview & base URL** — what the API is; in-cluster FQDN `http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090` (Service: `config/default/discovery_service.yaml`); local access via `kubectl port-forward svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090`.
+- **Authentication** — static bearer token from `DISCOVERY_TOKEN` env var (populated from the optional `discovery-token` Secret, key `token`; `config/manager/manager.yaml`). Empty token ⇒ auth disabled with startup warning. Curl: `-H "Authorization: Bearer $TOKEN"` on every example. `/healthz` always exempt.
+- **Conventions** — JSON everywhere; error envelope `{"error":"<code>","message":"<text>"}`; request bodies capped at 64 KiB; proxies with a deletion timestamp are excluded from all responses.
+- **Configuration table** — `--discovery-addr` (default `:8090`), `--max-lease-ttl` (default 1h), `--lease-cooldown` (default 15m), `DISCOVERY_TOKEN`. Note the shipped Deployment passes none of these flags, so defaults apply.
+- **Endpoints**, each with request/response schema, status codes, and a copy-pasteable curl example:
+  - `GET /healthz` — liveness, unauthenticated.
+  - `GET /v1/proxies` — filters `healthy=true|false` (else 400 `invalid_query`) and repeatable `attr.<key>=<value>` (verbatim equality on `spec.attributes`, all pairs must match). Response `{"proxies":[proxyView...],"count":N}` sorted by id. Full `proxyView` field table: `id` (ns/name), `ip`, `port`, `attributes`, `phase` (Pending/Provisioning/Ready/Unhealthy/Deleting/Failed), `healthy` (condition `Healthy` == True), `latencyMillis`, `activeLeases`, `maxLeases` (default 5; explicit 0 = unleasable).
+  - `POST /v1/leases` — body `{selector, ttlSeconds, target}` all optional; TTL defaults 5m, capped at max-lease-ttl (else 400 `invalid_ttl`). 201 `{leaseID, proxy, expiresAt, ttlSeconds}`; 409 `no_match` with `considered/atCapacity/inCooldown/unhealthy` counts (documented meanings).
+  - `DELETE /v1/leases/{id}` — early release; always 204, idempotent.
+  - `POST /v1/leases/{id}/report` — body `{result: ok|rate_limited|banned, target}`; 204, 404 `unknown_lease`, 400 `invalid_result`. `ok` is a pure ack; `rate_limited` and `banned` behave identically today (both start one cooldown window).
+- **Selection & cooldown semantics** (short section — this is the non-obvious part clients need):
+  - Selection order: fewest active leases → lowest latency → lexicographic id; deterministic; only healthy, non-deleting proxies with free capacity are candidates.
+  - Cooldown: 15m default (`--lease-cooldown`), keyed `{proxy, target}`. Report with a target blocks only leases requesting that target; report without a target (and lease without one) creates a **global** cooldown blocking all acquisitions of that proxy — call this footgun out explicitly.
+  - Expired leases stay reportable for one cooldown window past TTL.
+- **End-to-end workflow example** — numbered curl walkthrough: acquire → use `proxy.ip:port` as HTTP proxy (`curl -x`) → report `rate_limited` on 429 → release. Using a `jq`-extracted `leaseID`.
+- **Caveats** — lease/cooldown state is in-memory and per-process: single replica only, operator restart drops all leases and cooldowns.
+
+### 2. Pointers to the new doc (small edits)
+
+- `README.md`: one-line link near the quickstart curl section ("full reference: docs/api.md").
+- `docs/architecture.md` §7: one-line link to `docs/api.md` as the detailed reference.
+
+### 3. Housekeeping per CLAUDE.md
+
+- First action post-approval: copy this plan to `docs/plans/<timestamp>-discovery-api-docs.md` (timestamp from `date "+%Y-%m-%d-%H%M"`), commit it alone.
+- Then write the docs, commit to `main` with `Co-Authored-By: Claude <noreply@anthropic.com>` trailer, push.
+- Append execution summary + status checklist to `docs/plans-executions/<same-timestamp>-discovery-api-docs.md` in the docs commit.
+- Add `CHANGELOG.md` entry (timestamp via `date "+%Y-%m-%d %H:%M %Z"`) once the user confirms.
+
+## Key source files (facts source of truth)
+
+- [internal/discovery/handlers.go](internal/discovery/handlers.go), [internal/discovery/server.go](internal/discovery/server.go) — routes, schemas, status codes, auth, limits.
+- [internal/lease/store.go](internal/lease/store.go) — selection order, cooldown/retention, stats.
+- [api/v1alpha1/proxy_types.go](api/v1alpha1/proxy_types.go), [api/v1alpha1/helpers.go](api/v1alpha1/helpers.go) — defaults (port 3128, maxLeases 5), phases, conditions.
+- [cmd/main.go](cmd/main.go) — flags/env defaults.
+- [config/default/discovery_service.yaml](config/default/discovery_service.yaml), [config/manager/manager.yaml](config/manager/manager.yaml) — service DNS, token secret.
+
+## Verification
+
+- Cross-check every documented status code / field name against `internal/discovery/server_test.go` expectations.
+- Sanity-run `go build ./... && go test -short ./internal/discovery/ ./internal/lease/` (no code changes expected — confirms docs match current behavior, not a stale tree).
+- Optionally lint the curl JSON bodies by piping each through `jq .` locally.

docs/plans/2026-08-11-2220-demo-scripts.md

@@ -0,0 +1,33 @@
+# Plan: Demo script — egress IP check through each healthy proxy
+**Created:** 2026-08-11 22:20
+
+
+## Context
+
+First of a planned series of demo scripts under `docs/demo/`. This one showcases the discovery API end to end without leases: list proxies from `$BASE_URL/v1/proxies`, and for each **healthy** one, call an IP-echo site through it (`curl -x`) to show the egress IP that proxy provides — clearly labeling which proxy each request goes through. The user explicitly asked to start by switching to a new branch. The script will be iterated on: **write it but do not commit it** — the user wants to add things to it before anything is committed.
+
+## Steps
+
+### Step 0 — Branch
+
+Create `feat/demo-scripts` off `main`. Commit only the plan copy (`docs/plans/<timestamp>-demo-scripts.md`, timestamp via `date "+%Y-%m-%d-%H%M"`) per CLAUDE.md — nothing else gets committed this round.
+
+### Step 1 — `docs/demo/show-egress-ips.sh` (new file, executable, left uncommitted)
+
+Style: match [.devcontainer/post-install.sh](.devcontainer/post-install.sh) — `#!/bin/bash`, `set -euo pipefail`, `ERROR:`/`WARNING:` messages, `${VAR}` braces.
+
+Behavior:
+
+1. **Args/env:** `BASE_URL` is `$1` (required; missing → usage text + exit 1, e.g. `usage: show-egress-ips.sh <BASE_URL>  (e.g. localhost:8090)`). Optional env: `TOKEN` (bearer token, same name docs/api.md uses; when set, send `Authorization: Bearer $TOKEN`), `IP_ECHO_URL` (default `https://api.ipify.org?format=json` — returns `{"ip":"..."}`).
+2. **Dependency check:** `command -v curl`, `command -v jq` → `ERROR` + exit 1 if missing.
+3. **Fetch** `"$BASE_URL/v1/proxies"` once (no server-side `healthy` filter — fetch all so unhealthy ones can be shown as skipped, which makes the demo more informative). Fail with a clear error if curl or JSON parsing fails.
+4. **Iterate** proxies with `jq -c '.proxies[]'`; for each, extract `id`, `ip`, `port`, `healthy`:
+   - unhealthy → print `--- skipping <id> (unhealthy) ---`
+   - healthy → print a clear banner naming the proxy before the request, e.g. `=== via <id> — http://<ip>:<port> ===`, then `curl -sS --max-time 10 -x "http://${ip}:${port}" "$IP_ECHO_URL"`; print the JSON response. A failed probe prints `WARNING: request through <id> failed` and continues (guard so `set -e` doesn't kill the loop).
+5. Finish with a one-line summary: N proxies, M probed, K skipped/failed.
+
+Reference for API shapes: [docs/api.md](docs/api.md) (`proxies[].id/ip/port/healthy`; `curl -x http://ip:port` usage is already documented there and in README).
+
+### Deliberately deferred (user will iterate on the script first)
+
+- No commit of the script, no push beyond the plan commit, no MR, no execution summary, no CHANGELOG — all wait until the user says the script (or script set) is ready.

docs/plans/2026-09-04-1404-kvm-provider.md

@@ -0,0 +1,313 @@
+# Plan: KVM/libvirt provider
+
+**Created:** 2026-09-04 14:04
+
+## Context
+
+The operator provisions egress proxies through named provider instances (today
+`gcp` and `kubernetes`). The user has several KVM hypervisors on the home
+network and wants proxy VMs created there, using stock Debian/Ubuntu cloud
+images so the VM stays a normal machine they can customise through their own
+cloud-init user-data. Longer term the operator may become a generic "run a VM
+or container with a set of scripts" tool; the KVM provider should not
+foreclose that.
+
+The provider contract (`internal/provider/provider.go`: `Create`/`Get`/`Delete`/
+`ListByTag`, keyed by `NameFromUID`, cloud-init passed verbatim, GC-by-tag) fits
+libvirt with **no CRD or reconciler changes**. All work is a new provider
+package plus config/wiring/docs, following the GCP provider's shape.
+
+Decisions taken with the user (2026-09-04):
+
+- **Transport:** SSH to each hypervisor, forwarding the libvirt unix socket
+  (`ssh.Client.Dial("unix", "/var/run/libvirt/libvirt-sock")` → `libvirt.New(conn)`).
+  Pure Go, no cgo, no libvirt libs in the image. Verified: `x/crypto/ssh`
+  `Client.Dial` has a `"unix"` case (`dialStreamLocal`), go-libvirt `New(net.Conn)`.
+- **Topology:** one provider instance can span several hosts from day one
+  (config has a `hosts` list; providerID encodes the host), so adding hosts
+  later is a config edit with no providerID migration. With one host configured
+  `placement.zone` may be omitted.
+- **Networking:** VM NIC on a host bridge to the LAN, IP from the router's
+  DHCP. IP discovery via `DomainInterfaceAddresses` (ARP source first, then
+  guest agent). Provider-injected `vendor-data` installs `qemu-guest-agent`
+  so agent discovery works on stock images without touching user-data.
+- **Base image:** a stock Debian/Ubuntu cloud image (qcow2, cloud-init
+  preinstalled) that **the operator fetches itself** on first use per host:
+  `placement.image` names an entry in the provider's `images:` map (url +
+  optional sha256) or is an `https://` URL directly. The provider streams the
+  download through libvirt's `StorageVolUpload` into the host's pool, so
+  nothing is pre-placed and the operator needs no filesystem access on the
+  host. Per-VM disk is a qcow2 overlay with the cached base as backing store.
+- **Seed:** NoCloud ISO (`user-data`, `meta-data`, `vendor-data`) built in
+  pure Go with `github.com/kdomanski/iso9660` (`WriteTo(w, "cidata")`) and
+  streamed to the pool with `StorageVolUpload(vol, io.Reader, ...)`. Nothing
+  to install on the hypervisors.
+
+## Design
+
+### Config (`internal/provider/config.go`)
+
+```yaml
+- name: kvm-home
+  type: kvm
+  kvm:
+    sshKeyFile: /etc/proxy-operator/kvm/id_ed25519        # required
+    knownHostsFile: /etc/proxy-operator/kvm/known_hosts   # required unless insecureIgnoreHostKey
+    insecureIgnoreHostKey: false
+    diskSizeGb: 10                    # overlay capacity; cloud-init growpart expands root fs
+    vendorData: |                     # optional override; default installs qemu-guest-agent
+      #cloud-config
+      ...
+    images:                           # optional; placement.image names a key (or is an https URL)
+      debian-13:
+        url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+        sha256: "…"                   # optional; verified after download, re-fetch when it changes
+        format: qcow2                 # default qcow2
+      ubuntu-24.04:
+        url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+    machineTypes:                     # required, ≥1; placement.machineType names a key
+      small:  {vcpus: 1, memoryMiB: 1024}
+      medium: {vcpus: 2, memoryMiB: 2048}
+    hosts:                            # required, ≥1, unique names
+      - name: nas1                    # placement.zone value
+        address: nas1.home.lan:22     # required
+        user: virt                    # required; must be in the libvirt group
+        socket: /var/run/libvirt/libvirt-sock   # default
+        pool: default                 # default "default"
+        bridge: br0                   # required
+```
+
+Add `KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage` structs; a `"kvm"`
+case in `validate()` (cross-block checks like the others, required fields
+above, unique host names, image URLs must be `http(s)`, sha256 if set must be
+64 hex chars). Tests in `config_test.go` extend the existing table.
+
+### Placement mapping
+
+| Placement field | KVM meaning |
+|---|---|
+| `zone` | host name from `hosts`; empty → the single configured host, else `ErrPermanent` |
+| `machineType` | key in `machineTypes`; unknown → `ErrPermanent` |
+| `image` | key in `images:`, or an `https://` URL; anything else → `ErrPermanent`. Fetched into the host's pool on first use (see *Image cache*) |
+| `region` | ignored |
+
+### providerID
+
+`hosts/<host>/domains/<name>` with `formatProviderID`/`parseProviderID`
+(same self-contained style as GCP's `zones/<zone>/instances/<name>`).
+
+### GC contract without labels
+
+Domain XML gets a `<metadata>` child in a private namespace, e.g.
+`xmlns="https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm"`,
+carrying `<managed>true</managed><uid>…</uid><createdAt>RFC3339</createdAt>`.
+`Get` reads it via `DomainGetMetadata(dom, DomainMetadataElement, uri)`;
+`ListByTag` does `ConnectListAllDomains` on every host and keeps domains where
+that call succeeds and `managed == "true"`. `CreatedAt` comes from the metadata
+(libvirt has no creation time). A host that is unreachable makes `ListByTag`
+return an error (GC skips the sweep that round rather than acting on a partial view).
+
+### Connection management (`internal/provider/kvm/conn.go`)
+
+Per-host lazily dialed `*ssh.Client` + `*libvirt.Libvirt`, cached under a
+mutex, re-dialed when `Disconnected()` is closed or a call fails with a
+transport error. SSH auth: private key from `sshKeyFile`; host key check via
+`knownhosts.New(knownHostsFile)` unless `insecureIgnoreHostKey`. Dial timeout
+from `ctx`. The provider calls libvirt through a narrow interface:
+
+```go
+type libvirtAPI interface {
+    DomainLookupByName(name string) (libvirt.Domain, error)
+    DomainDefineXML(xml string) (libvirt.Domain, error)
+    DomainCreate(d libvirt.Domain) error
+    DomainDestroy(d libvirt.Domain) error
+    DomainUndefineFlags(d libvirt.Domain, f libvirt.DomainUndefineFlagsValues) error
+    DomainGetState(d libvirt.Domain, flags uint32) (int32, int32, error)
+    DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, f libvirt.DomainModificationImpact) (string, error)
+    DomainInterfaceAddresses(d libvirt.Domain, src uint32, flags uint32) ([]libvirt.DomainInterface, error)
+    ConnectListAllDomains(need int32, f libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error)
+    StoragePoolLookupByName(name string) (libvirt.StoragePool, error)
+    StorageVolLookupByName(p libvirt.StoragePool, name string) (libvirt.StorageVol, error)
+    StorageVolGetPath(v libvirt.StorageVol) (string, error)
+    StorageVolCreateXML(p libvirt.StoragePool, xml string, f libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error)
+    StorageVolUpload(v libvirt.StorageVol, r io.Reader, off, n uint64, f libvirt.StorageVolUploadFlags) error
+    StorageVolDownload(v libvirt.StorageVol, w io.Writer, off, n uint64, f libvirt.StorageVolDownloadFlags) error
+    StorageVolDelete(v libvirt.StorageVol, f libvirt.StorageVolDeleteFlags) error
+    StoragePoolRefresh(p libvirt.StoragePool, flags uint32) error
+}
+```
+
+(All signatures verified against go-libvirt `main` on 2026-09-04.) A
+`hostDialer func(ctx, KVMHost) (libvirtAPI, io.Closer, error)` is the test seam;
+`newWithDialer(cfg, dialer)` mirrors GCP's `newWithAPI`.
+
+### Image cache (`internal/provider/kvm/image.go`)
+
+`ensureImage(ctx, host, api, img) (basePath string, err error)` makes the base
+image present in the host's pool, once per host per image:
+
+- Volume names: `base-<key>.<format>` for an `images:` key, or
+  `base-<sha256(url)[:16]>.<format>` for a URL given directly in
+  `placement.image`. Marker volume `<base>.sha256` (raw, 64 bytes) holds the
+  digest of the verified upload.
+- **Ready check:** base volume exists *and* marker exists *and* (no `sha256`
+  configured, or marker content == configured `sha256`). Read the marker with
+  `StorageVolDownload`. Ready → return `StorageVolGetPath(base)`.
+- **Not ready:** delete both volumes if present (partial upload from a crash,
+  or a changed checksum), then `GET url` with the ctx, require a
+  `Content-Length` (`ErrPermanent` otherwise), `StorageVolCreateXML` the base
+  with `<capacity unit="bytes">Content-Length</capacity>` and the configured
+  format, `StorageVolUpload(base, io.TeeReader(body, sha256), 0, 0, 0)` (length
+  0 = until EOF, so nothing is buffered in the operator), then compare the
+  digest with the configured one (mismatch → delete base, `ErrPermanent`),
+  `StoragePoolRefresh` so libvirt re-reads the real qcow2 capacity, create the
+  marker volume and upload the digest last.
+- HTTP 4xx → `ErrPermanent`; 5xx, network, or upload transport errors →
+  `ErrTransient` (partial volumes are cleaned on the next attempt).
+- A per-`(host, image)` `sync.Mutex` map serialises concurrent Creates for the
+  same image on the same host; the second caller finds it ready.
+- **Judgment call:** the fetch runs synchronously inside `Create`, so the first
+  Proxy for a given image on a given host holds a reconcile worker for the
+  download (LAN speed, a few hundred MB, once). This keeps `Create` idempotent
+  and needs no background state. Async fetch with a `Provisioning`-style
+  progress status is a possible later improvement, noted in the docs.
+
+### Create (idempotent, fire-and-forget after the image is cached)
+
+1. Resolve host, machine type, pool, and image source (`images:` key or URL;
+   else `ErrPermanent`). `ensureImage` → base path.
+2. Overlay volume `<name>.qcow2`: `StorageVolCreateXML` with
+   `<capacity unit="G">diskSizeGb</capacity>`, `<target><format type="qcow2"/></target>`,
+   `<backingStore><path>base</path><format type="qcow2"/></backingStore>`.
+   `ErrStorageVolExist` → reuse.
+3. Seed volume `<name>-seed.iso`: build ISO in memory (`user-data` = req.CloudInit,
+   `meta-data` = `instance-id: <name>\nlocal-hostname: <name>\n`, `vendor-data` =
+   cfg or default), `StorageVolCreateXML` with exact capacity + `<format type="raw"/>`,
+   then `StorageVolUpload`. Exists → skip upload (content is deterministic per spec hash;
+   spec changes recreate the whole instance anyway).
+4. `DomainLookupByName(name)`; if `ErrNoDomain` → `DomainDefineXML(buildDomainXML(...))`.
+5. `DomainGetState`; if not running → `DomainCreate`. Return providerID.
+
+`buildDomainXML` is a pure function over `encoding/xml` structs: `type="kvm"`,
+q35 machine, `host-passthrough` CPU, vcpus/memory from the profile, virtio disk
+(overlay) + sata cdrom (seed), `interface type="bridge"` with virtio model,
+`org.qemu.guest_agent.0` virtio channel, serial console, virtio rng, no
+graphics, and the `<metadata>` block. Unit-tested field by field like
+`buildInsertRequest`.
+
+### Get
+
+`parseProviderID` → host → `DomainLookupByName` (`ErrNoDomain` → `ErrNotFound`)
+→ `DomainGetState` + `DomainGetMetadata` + `DomainInterfaceAddresses`
+(ARP, then agent; first IPv4 wins; agent errors while the agent isn't up yet
+are swallowed). Mapping:
+
+| libvirt state | IP | Instance.State |
+|---|---|---|
+| running | yes | Running |
+| running | no | Provisioning |
+| shutoff / crashed / paused / pmsuspended | – | Stopped |
+| no domain | – | `ErrNotFound` |
+
+### Delete (idempotent)
+
+Destroy if running (`ErrOperationInvalid` on a stopped domain is fine) →
+`DomainUndefineFlags(managed-save | snapshots-metadata | nvram)` → delete
+`<name>.qcow2` and `<name>-seed.iso` (`ErrNoStorageVol` fine). Volumes are
+deleted by name even when the domain is already gone, so a crash between
+volume creation and define still gets cleaned up on the next reconcile.
+Known gap (documented, not solved now): volumes orphaned with no domain *and*
+no CR are invisible to `ListByTag`.
+
+### Errors (`internal/provider/kvm/errors.go`)
+
+`classify(err)` on `libvirt.Error.Code`: `ErrNoDomain`, `ErrNoStorageVol` →
+`ErrNotFound`; `ErrNoStoragePool`, `ErrXMLError`, `ErrConfigUnsupported`,
+`ErrNoSupport`, `ErrAuthFailed`, `ErrInvalidArg` → `ErrPermanent`; SSH dial /
+host-key / transport errors and everything else → `ErrTransient`. Image
+download: HTTP 4xx, missing Content-Length, checksum mismatch → `ErrPermanent`;
+5xx and network errors → `ErrTransient`. Quota is not detectable from libvirt;
+left as transient (revisit with a capacity check later).
+
+### Wiring & manifests
+
+- `cmd/main.go`: add `"kvm": kvm.New` to the builtins map.
+- `config/manager/manager.yaml`: mount Secret `kvm-ssh` (optional) at
+  `/etc/proxy-operator/kvm` (keys `id_ed25519`, `known_hosts`); document the
+  `kubectl create secret generic kvm-ssh --from-file=...` command.
+- `config/samples/providers-config.yaml`: add the `kvm-home` block with the
+  `images:` map (Debian 13 and Ubuntu 24.04 official cloud-image URLs);
+  `config/samples/proxy_kvm.yaml`: Proxy with `placement: {zone: nas1,
+  machineType: small, image: debian-13}` and the same squid cloud-init as the
+  GCP sample plus `ssh_authorized_keys`.
+- `hack/providers-dev.yaml` stays kubernetes-only (needs no secrets).
+- Deps: `github.com/digitalocean/go-libvirt` (latest pseudo-version),
+  `github.com/kdomanski/iso9660 v0.4.0`, promote `golang.org/x/crypto` to direct.
+
+### Docs
+
+- New `docs/kvm-provider.md`: hypervisor prerequisites (user in `libvirt`
+  group, a bridge to the LAN, a storage pool with enough space), the config
+  reference including `images:`, how the image cache works (`base-*` and
+  `*.sha256` volumes in the pool, when a re-download happens, how to evict
+  with `virsh vol-delete`), the placement table, how to log into a VM
+  (`ssh_authorized_keys` in user-data, `virsh console` on the host), and the
+  known gaps (no quota detection, orphan volumes without a domain,
+  synchronous first-time image fetch).
+- Update the provider tables in `README.md`, `docs/architecture.md` §5,
+  `docs/testing.md` (integration test section).
+
+### Towards the generic runner (not built now, just kept open)
+
+Provider knows nothing about squid. Future hooks: `vendor-data` for injecting an
+agent, `QEMUDomainAgentCommand` for reading exit status/output, and a future
+`Placement`-independent "job" mode in the CRD. None of this changes the KVM
+provider's shape.
+
+## Steps
+
+0. **Save plan + branch.** Copy this file to `docs/plans/<ts>-kvm-provider.md`,
+   commit alone, then `git checkout -b feat/kvm-provider` off `feat/proxy-operator`
+   (the working main for this project).
+1. **Config.** `KVMConfig` & friends, `validate()` case, table tests.
+2. **Pure builders.** `internal/provider/kvm/{domainxml.go,volume.go,seed.go,id.go,metadata.go}`
+   + tests: domain XML fields, volume XML, seed contents (unpack the ISO with
+   the same library's reader and assert the three files + label `cidata`),
+   providerID round-trip, metadata marshal/parse.
+3. **Transport + errors.** `conn.go` (SSH dial, host-key policy, per-host cache,
+   reconnect), `errors.go` + classification table test. SSH dialer tested
+   against an in-process `x/crypto/ssh` server that serves a unix socket
+   forward (cheap, keeps the dial code covered).
+3b. **Image cache.** `image.go` + `image_test.go`: `httptest.Server` serving a
+   small fake image over the `fakeAPI` (which records uploaded bytes);
+   cases: fresh fetch, already ready, marker missing (partial upload → redo),
+   checksum changed → redo, checksum mismatch → Permanent + cleanup, 404 →
+   Permanent, 503 → Transient, two concurrent callers → one download.
+4. **Provider methods.** `kvm.go`: `New`, `newWithDialer`, `Create/Get/Delete/ListByTag`
+   over `libvirtAPI`; `kvm_test.go` with a hand-written `fakeAPI` (domains and
+   volumes in maps, per-call error injection) covering idempotent re-Create
+   after each partial step, state/IP mapping, delete-when-gone, ListByTag
+   across two hosts and with one host down.
+5. **Wiring, manifests, samples, docs, CHANGELOG.**
+6. **Integration test** (`//go:build integration`, `kvm_integration_test.go`):
+   skipped unless `KVM_TEST_HOST`, `KVM_TEST_USER`, `KVM_TEST_SSH_KEY`,
+   `KVM_TEST_BRIDGE`, `KVM_TEST_IMAGE_URL` are set; fetches the image (or finds
+   it cached), creates, waits for Running+IP, deletes, asserts per-VM volumes
+   gone and the base image still cached. Deviation from the testcontainers convention,
+   called out in `docs/testing.md`: libvirt+KVM can't run inside a container
+   without `/dev/kvm`, so this targets a real hypervisor.
+7. **Manual validation** on one real hypervisor, results in
+   `docs/plans-executions/` (like `docs/gcp-vm-validation.md`), then MR via `tea`.
+
+## Verification
+
+- `make test` green (`-race`, envtest suite untouched but must still pass).
+- `go test -short ./internal/provider/...` covers builders, fake-API provider, errors, config.
+- Integration: `KVM_TEST_HOST=nas1.home.lan … go test -tags=integration -run TestKVM ./internal/provider/kvm/`.
+- End to end: deploy with the `kvm-ssh` Secret and `kvm-home` config against a
+  host with no cached image, apply `config/samples/proxy_kvm.yaml`, watch the
+  operator log the image fetch and `virsh vol-list` show `base-debian-13.qcow2`
+  plus its `.sha256` marker, watch `kubectl get px -w` reach `Ready` with a LAN
+  IP, apply a second Proxy and confirm no second download, confirm the VM shows up in `GET /v1/proxies` and works with
+  `hack/` demo egress-IP check, delete the Proxy and confirm `virsh list --all`
+  and `virsh vol-list default` are clean.

docs/reviews/2026-08-10-1134-proxy-operator-pr-review.md

@@ -0,0 +1,131 @@
+# PR review findings: feat/proxy-operator
+
+**Created:** 2026-08-10 11:34
+**Scope:** `origin/main...feat/proxy-operator` (merge-base 076bc66, 25 commits, ~80 files)
+**Reviewers:** `go-operator-reviewer` + `operator-reviewer` agents; findings consolidated, most severe first. Check off items as they're processed.
+
+Both reviewers rated the core reconcile architecture sound: single status writer with one
+deferred patch, finalizer added before any provider call, Get-before-RemoveFinalizer on
+delete, CEL immutability rules correctly split to avoid the oldSelf-on-CREATE trap,
+leader-election gating on destructive runnables, GC tombstone rules (MinAge, UID-less
+instances never deleted).
+
+## Merge-blockers
+
+- [ ] **Discovery leases proxies with an empty IP** — found independently by both reviewers.
+  `internal/discovery/handlers.go:151`, `internal/controller/proxy_controller.go:226`
+  During instance replacement (and the Get→NotFound recovery path) the reconciler clears
+  `status.ip` but only the create branch removes the `Healthy` condition, and the health
+  engine prunes state for empty-host proxies so nothing refreshes it. For the whole
+  delete→recreate window (minutes on GCP), `isHealthy` still returns true and
+  `handleAcquireLease` grants `201 Created` with `"ip": ""`, burning a `MaxLeases` slot.
+  **Fix:** add `EffectiveHost() != ""` to `isHealthy` (covers list + acquire), and
+  remove/downgrade `Healthy` wherever `status.IP` is cleared.
+
+- [ ] **Orphan GC deletes other installations' fleets in a shared GCP project.**
+  `internal/provider/gcp/insert.go:57`, `internal/gc/gc.go:93`
+  Instances are tagged only `proxy-operator-managed=true` + CR UID; the sweeper deletes any
+  tagged instance whose UID isn't in *its own cluster's* Proxy list. Two clusters sharing a
+  GCP project delete each other's VMs every GC interval in a permanent loop.
+  **Fix:** add an installation-identity label (cluster/deployment ID) set by both providers
+  and filtered on in `ListByTag`.
+
+- [ ] **Permanent-error latch wedges proxies on failures that aren't spec-caused.**
+  `internal/controller/proxy_controller.go:126`
+  Latch keys on `observedGeneration == generation`, but two failure inputs live outside the
+  spec: an unconfigured provider (config fix + restart doesn't bump generation, and
+  `spec.provider` is CEL-immutable → stuck `Failed` short of deleting the CR) and resolved
+  Secret content (Secret fix enqueues a reconcile that short-circuits at the latch before
+  re-resolving cloud-init).
+  **Fix:** latch should also consider current spec-hash / provider availability.
+
+## Worth fixing
+
+- [ ] **Deletion-path failures invisible in status** — flagged by both reviewers.
+  `internal/controller/proxy_controller.go:319`, `:266`
+  `deletionFailure` swallows `ErrQuotaExceeded` (nil error, no status write); unconfigured
+  provider returns a bare error forever. A Proxy wedged in `Deleting` shows nothing in
+  `kubectl describe`. Stage `setProvisioned(p, False, ReasonDeleting, ...)` before returning.
+  Also: `Delete` is resubmitted on every `DeletionPoll` pass, churning GCP quota — a state
+  check on the `Get` result would avoid it.
+
+- [ ] **Lost providerID on `setSpecHash` conflict.**
+  `internal/controller/proxy_controller.go:161`
+  On Update conflict the function returns before `p.Status.ProviderID = id`, so the deferred
+  patch persists an empty providerID for a just-created instance. Self-heals via GC.
+  **Fix:** set `p.Status.ProviderID = id` before returning the error (one line).
+
+- [ ] **Terminating pods still report `StateRunning`.**
+  `internal/provider/kubernetes/kubernetes.go:151`
+  A pod with a deletionTimestamp keeps `phase=Running` + `PodIP` while terminating, so drift
+  reconcile republishes `Provisioned=True` and discovery keeps leasing a dying pod.
+  **Fix:** map non-zero `pod.DeletionTimestamp` to `StateTerminated` in `instanceFromPod`.
+
+- [ ] **Stale-cache spec-hash race deletes the freshly created replacement instance.**
+  `internal/controller/proxy_controller.go:176`
+  Instance name derives from CR UID, so old and new instances share a providerID. A reconcile
+  served a cached object from before a just-completed replacement re-enters `replaceInstance`
+  and deletes the *new* healthy instance. Converges, but destroys a good instance.
+  **Fix:** re-read uncached before the destructive branch, or compare `inst.CreatedAt`
+  against the annotation-update time.
+
+- [ ] **`observedGeneration` written before the generation is actually processed.**
+  `internal/controller/status.go:114`
+  Set unconditionally in `patchStatusIfChanged`, including on the finalizer-add pass and
+  `resolveCloudInit` failures — misleads kstatus-style tooling. Set it only once the state
+  machine has genuinely evaluated the spec.
+
+- [ ] **No event filtering on the Proxy watch.**
+  `internal/controller/proxy_controller.go:402`
+  Every self-inflicted status patch triggers a follow-up reconcile with an extra cloud `Get`,
+  roughly doubling provider read traffic. Caution: a plain `GenerationChangedPredicate`
+  breaks the finalizer flow (relies on its own Update event to re-enter) — needs a
+  status-only/resourceVersion-only filter or an explicit requeue in the finalizer pass.
+
+- [ ] **Unlabelled cloud-init Secrets produce a misleading NotFound with endless backoff.**
+  `internal/controller/proxy_controller.go:344`, `cmd/main.go:210`
+  The label-restricted cache turns "exists but missing `crawl.example.com/cloud-init=true`"
+  into `CloudInitError: not found`. Mention the label requirement in the condition message,
+  or read via uncached `APIReader` and validate the label explicitly.
+
+- [ ] **RBAC over-grant.**
+  `config/rbac/role.yaml:25`
+  `create;delete` on `proxies` is scaffold residue (controller never creates/deletes CRs);
+  cluster-wide `pods create/delete` and `secrets get/list/watch` apply even when only the GCP
+  provider is configured — pod rules belong in an optional kustomize component.
+
+## Simplifications
+
+- [ ] **Delete `internal/provider/registry`** — 14 lines of logic, one caller
+  (`cmd/main.go:148`); fold `Build`/`Constructor` into the composition root. Also fixes the
+  two-sources-of-truth problem: `internal/provider/config.go:84` hardcodes
+  `"kubernetes"`/`"gcp"` while `registry.Build` dispatches through a caller-supplied map —
+  validate against the constructor map instead. Net −1 package, −44 lines, −92 test lines.
+
+- [ ] **Collapse `LeaseStore` interface to `*lease.Store`.**
+  `internal/discovery/server.go:28`
+  Single implementation and not a test seam (tests wire the real `lease.NewStore`).
+  Keep `HealthSnapshotter` and `instancesAPI` — those are genuine seams.
+
+- [ ] **Replace metrics nil-guards with no-op defaults.**
+  `internal/health/engine.go:68`, `internal/discovery/server.go:38`,
+  `internal/provider/metrics.go:8`
+  Keep the interfaces (legit "no prometheus in domain packages" rationale) but default the
+  fields to a no-op impl — `provider.WithMetrics` already dereferences unconditionally, so
+  the guards are inconsistent anyway.
+
+## Nice-to-have
+
+- [ ] Add an `OwnerReference` to provider pods (`internal/provider/kubernetes/pod.go:24`) —
+  free cascading deletion if the finalizer is ever bypassed; `CreateRequest` already carries
+  Namespace/ProxyName/UID.
+- [ ] `Close()` the GCP `*compute.InstancesClient` (`internal/provider/gcp/gcp.go:89`) —
+  harmless today, a leak the moment providers are rebuilt on config reload.
+- [ ] Fix `.golangci` config: it references a missing `logcheck` plugin, so the linter only
+  runs with the project config disabled.
+- [ ] External-mode endpoint edits don't reset health-engine counters
+  (`internal/health/engine.go:206` keys on name+UID): flipping `endpoint.host` keeps the old
+  host's `Healthy=True` for `failureThreshold × interval`. Arguably a replacement, not a flap.
+- [ ] `init()` funcs at `api/v1alpha1/proxy_types.go:353` and `cmd/main.go:67` conflict with
+  the repo's "no `init()`" convention; kubebuilder-idiomatic, but scheme registration could
+  use the scaffold's `SchemeBuilder.Register` at package var scope.

docs/testing.md

@@ -0,0 +1,123 @@
+# Testing
+
+What tests exist, how to run them, and where the boundaries of the
+automated suites are. The philosophy throughout: **test against the most
+real double available** — a real envtest API server over a fake client, a
+real CONNECT-capable proxy stub over a mocked HTTP client, the real lease
+store under the discovery handlers — and leave the gaps that only real
+infrastructure can close to the kind verification run, documented at the
+bottom.
+
+## Running
+
+```sh
+make test              # THE canonical run: codegen + fmt + vet + envtest setup,
+                       # then `go test -race` on every package, with coverage
+go test -short ./...   # skip the envtest suite (runs in <1s per package)
+go tool cover -html=cover.out   # browse coverage from the last make test
+```
+
+Targeted runs:
+
+```sh
+go test -race ./internal/lease/                  # one package
+go test -race -run TestAcquire ./internal/lease/ # one test (or a prefix)
+go test -race -count=2 ./internal/health/        # flake-shaking: run twice
+```
+
+**Gotcha — plain `go test ./...` fails the controller package** with an
+error like `/usr/local/kubebuilder/bin/etcd: no such file`. That is not a
+code problem: the envtest suite needs `KUBEBUILDER_ASSETS` pointing at the
+API-server/etcd binaries, which only `make test` sets up (via
+`setup-envtest`). Either use `make test`, or export it once:
+
+```sh
+KUBEBUILDER_ASSETS="$PWD/bin/k8s/<version>-<os>-<arch>" go test -race ./...
+```
+
+Everything runs with `-race` — `make test` enforces it; keep the flag on
+manual runs too.
+
+## The suites, package by package
+
+| Package | Files | What is covered | Test double / technique |
+|---|---|---|---|
+| `api/v1alpha1` | `helpers_test.go` | `EffectivePort/EffectiveHost/HealthCheckOrDefault/MaxLeasesOrDefault` | pure table-driven; CEL rules are **not** unit-testable — see envtest below |
+| `internal/controller` | `reconcile_test.go` | every row of the reconciler's action table: create, poll, publish IP, replace, adopt, NotFound recovery, delete/finalizer, quota/permanent/transient errors, cloud-init resolution | `controller-runtime/pkg/client/fake` + an in-test `stubProvider` |
+| | `status_test.go` | `computePhase` truth table (10 rows) | pure |
+| | `spechash_test.go` | hash stability, nil/empty normalization, sensitivity to every replacement-triggering field | pure |
+| | `health_test.go` | Healthy-condition representation: Ready/Unhealthy phases, no-verdict, stale-verdict cleanup on replacement, nil snapshotter | fake client + `fakeSnapshotter` |
+| | `proxy_controller_test.go` (envtest, ginkgo) | full lifecycles against a **real API server**: provision→Running, spec-change replacement, finalizer deletion, External tracking, Ready-through-health, quota-vs-permanent, adoption; **all CEL rules** (six invalid creates, mode/provider immutability incl. removal) and the `default={}` materialization | envtest apiserver + stub provider; the only place CEL and structural defaulting actually execute |
+| `internal/provider` | `name_test.go` | deterministic naming: idempotency, `^proxy-[a-z2-7]{16}$`, 10k-UID distinctness | pure |
+| | `errors_test.go` | taxonomy: `Class()` mapping, `errors.Is` **and** `errors.As` through the multi-unwrap | pure |
+| | `config_test.go` | providers-config parsing + fail-fast validation | pure |
+| | `metrics_test.go` | `WithMetrics` decorator: classified result labels, error passthrough | fake recorder + static provider |
+| `internal/provider/kubernetes` | `kubernetes_test.go`, `pod_test.go` | Create/Get/Delete/ListByTag over real Pod objects; pure `buildPod`/`squidConf` incl. the `max_filedescriptors` OOM-regression assertion | `client/fake` (real `corev1.Pod`s, no kubelet) |
+| `internal/provider/gcp` | `insert_test.go`, `errors_test.go`, `gcp_test.go` | field-by-field `buildInsertRequest`; HTTP-code classification table; zone-qualified IDs, 409-is-success, 9-row state mapping, ListByTag filter + `ReturnPartialSuccess` | pure builder needs **no fake**; the rest uses the flattened `instancesAPI` seam |
+| `internal/health` | `probe_test.go` | probes through a **real CONNECT-capable proxy stub** to a real TLS server: tunnel success, refused CONNECT, wrong status, dead proxy, plain-http forward | `httptest` + hijacked bidirectional tunnel |
+| | `engine_test.go` | thresholds, first-verdict, latency suppression matrix, dropped-event retry, stale-probe UID guard, tick scheduling/seeding/pruning, `Start` end-to-end | fake `client.Reader` + injected `probeFn` |
+| `internal/lease` | `store_test.go` | capacity, `MaxLeases=0`, all three selection tie-breaks, target-scoped vs global cooldowns, expiry via fake clock, report-on-retained-lease, idempotent release, **40 concurrent acquires never exceeding capacity** | injectable clock; the concurrency case is why `-race` matters |
+| `internal/discovery` | `server_test.go` | auth matrix (incl. `/healthz` exemption), list filtering, grant shape, full 409 arithmetic, invalid TTL/body/result, idempotent release, report→cooldown→409 round trip, `Start`/shutdown | `httptest` over the real handler chain, fake cache reader, **real** `lease.Store` |
+| `internal/gc` | `gc_test.go` | the true-orphan matrix (live/deleting/young/unlabelled all kept), broken-provider isolation, list-failure skips sweep, namespace guard, sweep loop | fake reader + canned-list provider |
+| `internal/metrics` | `metrics_test.go` | scrape-time collectors (`GatherAndCompare`), per-proxy series cleanup via `ForgetProxy`, label counts, fresh-registry-per-test property | `prometheus/client_golang/testutil` |
+
+Conventions (from `CLAUDE.md`): stdlib `testing`, table-driven by default,
+`t.Parallel()` where safe, tests next to source, ginkgo only in the
+envtest suite the scaffold generated.
+
+## What the automated suites deliberately do NOT cover
+
+- **`New()` constructors that dial real infrastructure**: the kubernetes
+  provider's `New` (connects to whatever your kubeconfig points at), the
+  GCP provider's `New` and its `realInstances` SDK adapter (ADC + real
+  Google endpoints). Both are thin; both are exercised by the kind run /
+  real deployments. Their 0% coverage is by design — do not "fix" it.
+- **CEL and structural defaulting under the fake client**: the fake
+  client runs neither. Every unit test that relies on defaults calls the
+  `*OrDefault` helpers; every CEL rule is asserted in the envtest suite
+  instead.
+- **A container actually starting and serving**: envtest has no kubelet,
+  so a Pod created there sits Pending forever. Whether Squid really comes
+  up and tunnels CONNECT is provable only on a real cluster — that is the
+  kind verification's job, and it is exactly what caught the Squid OOM
+  bug (below).
+- **Live GCP**: no test talks to Google. The seam boundary
+  (`buildInsertRequest` + classification) is tested exhaustively instead.
+
+## The scaffolded `make test-e2e` suite
+
+`test/e2e/` is the kubebuilder-generated smoke suite (build image → kind
+cluster → deploy → assert the manager pod runs and serves metrics). It
+compiles only under `-tags=e2e`, manages its own kind cluster
+(`make test-e2e` / `make cleanup-test-e2e`), and has been kept compiling
+(`go vet -tags=e2e ./...` is part of the routine) but is **not part of
+`make test` and was not used for the release verification** — the manual
+kind run below covers strictly more. Treat it as scaffold to grow into if
+CI wants an automated cluster smoke test.
+
+## The kind verification run (the real end-to-end)
+
+The README quickstart **is** the e2e test, run by hand before the MR:
+deploy the operator in-cluster on a throwaway kind cluster, watch a real
+Squid pod reach `Ready` via a real CONNECT probe, then exercise the whole
+lease lifecycle through the discovery API (list → lease → report
+`rate_limited` → same-target lease answered 409 `inCooldown:1` → release
+204 twice) and delete the CR to watch the finalizer remove the pod.
+
+Worth knowing about that run (full detail in
+[plans-executions/2026-08-07-1747-proxy-operator.md](plans-executions/2026-08-07-1747-proxy-operator.md),
+"Verification" section): it was not a clean pass-through. It caught two
+real bugs the automated suites structurally could not —
+
+1. `make run-dev` on a laptop can never produce a `Ready`
+   kubernetes-provider proxy: health probes originate on the host, which
+   cannot route to kind's pod network. The quickstart was rewritten to
+   deploy in-cluster.
+2. Squid was OOM-killed at startup: it sizes file-descriptor tables from
+   `RLIMIT_NOFILE`, which containerd under kind sets effectively
+   unlimited. Fixed with `max_filedescriptors 1024` in the generated
+   config, plus a regression assertion in `pod_test.go`.
+
+That is the pattern to keep: when the kind run finds something, the fix
+lands **with a unit-level regression test**, so the manual run stays a
+discovery tool rather than a recurring gate.

go.mod

@@ -3,8 +3,16 @@ module gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator
 go 1.26.0
 
 require (
+	cloud.google.com/go/compute v1.65.0
+	github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858
+	github.com/go-logr/logr v1.4.3
+	github.com/kdomanski/iso9660 v0.4.0
 	github.com/onsi/ginkgo/v2 v2.27.4
 	github.com/onsi/gomega v1.39.0
+	github.com/prometheus/client_golang v1.23.2
+	golang.org/x/crypto v0.56.0
+	google.golang.org/api v0.292.0
+	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
 	k8s.io/api v0.36.0
 	k8s.io/apimachinery v0.36.0
 	k8s.io/client-go v0.36.0
@@ -13,7 +21,10 @@ require (
 )
 
 require (
-	cel.dev/expr v0.25.1 // indirect
+	cel.dev/expr v0.25.2 // indirect
+	cloud.google.com/go/auth v0.22.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -26,7 +37,6 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
-	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
@@ -37,17 +47,20 @@ require (
 	github.com/google/gnostic-models v0.7.0 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect
+	github.com/googleapis/gax-go/v2 v2.23.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.19.2 // indirect
@@ -56,33 +69,33 @@ require (
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
-	go.opentelemetry.io/otel v1.41.0 // indirect
+	go.opentelemetry.io/otel v1.44.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.41.0 // indirect
+	go.opentelemetry.io/otel/metric v1.44.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.40.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.44.0 // indirect
-	go.opentelemetry.io/otel/trace v1.41.0 // indirect
+	go.opentelemetry.io/otel/trace v1.44.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
-	golang.org/x/mod v0.32.0 // indirect
+	golang.org/x/mod v0.38.0 // indirect
-	golang.org/x/net v0.49.0 // indirect
+	golang.org/x/net v0.57.0 // indirect
-	golang.org/x/oauth2 v0.34.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
-	golang.org/x/sys v0.40.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
-	golang.org/x/term v0.39.0 // indirect
+	golang.org/x/term v0.45.0 // indirect
-	golang.org/x/text v0.33.0 // indirect
+	golang.org/x/text v0.41.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
-	golang.org/x/tools v0.41.0 // indirect
+	golang.org/x/tools v0.48.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect
-	google.golang.org/grpc v1.79.3 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
-	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	google.golang.org/grpc v1.83.0 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -1,5 +1,15 @@
-cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
+cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs=
-cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
+cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
+cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
+cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
+cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ=
+cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s=
+cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
+cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
+cloud.google.com/go/compute v1.65.0 h1:K0a3NRvazE7sZn5qswwI6BtlaZv1fgR5wFop5LZCLz8=
+cloud.google.com/go/compute v1.65.0/go.mod h1:vFq+Ztj9Rzhc8zf1t6hGp/6NdrEVG1GakkyVRQPRgKc=
+cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
+cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
@@ -18,6 +28,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858 h1:8xCFt73OddCD5WXxoYidJBWp3bJMC+CpE7Zwf5tcwk8=
+github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858/go.mod h1:qb0Ofa71d3oXARQf633h2tNaeBxLsVxuDp+jcsVO2+4=
 github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
 github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
@@ -68,8 +80,14 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
+github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.3.19 h1:mMOE7DN2+p76/EdIrmAy9B9bH+yC4563vmnJ34QR8i4=
+github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k=
+github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE=
+github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -80,6 +98,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE
 github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg=
+github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -154,22 +174,22 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
-go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
+go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
-go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
+go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
-go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
+go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
-go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
+go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
-go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
+go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
-go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
+go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
-go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
+go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
-go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
+go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
-go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
+go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
-go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
+go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
 go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
 go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -182,36 +202,42 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
 go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
+golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=
 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
-golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
+golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
-golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
+golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
-golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
-golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
-golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
-golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
-golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
-golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
-golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
-golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
+golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
-golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
+golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
-golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
+golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
-golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
-gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
-google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
+google.golang.org/api v0.292.0 h1:Ewiwo/GTtiaPZSNAZQUcWLh8AYDEoPmIXyJfeoTSMHU=
-google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
+google.golang.org/api v0.292.0/go.mod h1:07kjmMnFGm2RQuCza2EZM/5N68G/fVvFb1xKjWqoFA0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ=
+google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
-google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
+google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU=
-google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
+google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

hack/providers-dev.yaml

@@ -0,0 +1,6 @@
+# Providers config for `make run-dev`: local development against the
+# current kubeconfig context (e.g. a kind cluster). Only the
+# kubernetes-pod provider — no cloud credentials needed.
+providers:
+  - name: kubernetes
+    type: kubernetes

internal/controller/proxy_controller.go

@@ -82,6 +82,9 @@ type ProxyReconciler struct {
 // +kubebuilder:rbac:groups=crawl.example.com,resources=proxies/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=crawl.example.com,resources=proxies/finalizers,verbs=update
 // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
+// The pod verbs are for the kubernetes-pod provider; cluster-scoped, since
+// its ListByTag enumerates the operator's Pods across all namespaces.
+// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;delete
 
 // Reconcile fetches the Proxy named by req into p (r.Get fills the struct
 // through the pointer), dispatches to the delete/external/managed state

internal/controller/proxy_controller_test.go

@@ -28,6 +28,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
 )
 
@@ -233,6 +234,105 @@ var _ = Describe("Proxy controller", func() {
 		Expect(apierrors.IsNotFound(err)).To(BeTrue(), "proxy should be fully deleted")
 	})
 
+	It("reaches Ready once the health engine has a verdict", func() {
+		const name = "e2e-ready"
+		stub := &stubProvider{createID: "inst-rdy"}
+		r := newEnvtestReconciler(stub)
+		DeferCleanup(func() { cleanup(r, stub, name) })
+
+		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec:       managedSpec(),
+		})).To(Succeed())
+		_, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		stub.getInst = &provider.Instance{ID: "inst-rdy", IP: "10.3.3.3", State: provider.StateRunning}
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseProvisioning),
+			"no health verdict yet — must not be Ready")
+
+		By("supplying a healthy snapshot")
+		r.Health = fakeSnapshotter{ok: true, snap: health.Snapshot{
+			Healthy: true, Latency: 21 * time.Millisecond, LastProbe: time.Now(),
+		}}
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		p := fetch(name)
+		Expect(p.Status.Phase).To(Equal(crawlv1alpha1.PhaseReady))
+		Expect(p.Status.LatencyMillis).To(Equal(int64(21)))
+	})
+
+	It("treats quota exhaustion as a wait and a permanent error as Failed", func() {
+		const name = "e2e-errors"
+		stub := &stubProvider{
+			createErr: provider.Wrap(provider.ErrQuotaExceeded, "create", "stub", "", nil),
+		}
+		r := newEnvtestReconciler(stub)
+		DeferCleanup(func() { cleanup(r, stub, name) })
+
+		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec:       managedSpec(),
+		})).To(Succeed())
+		_, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("quota: condition set, slow requeue, phase NOT Failed")
+		res, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred(), "quota must not count as an error (stays off the backoff curve)")
+		Expect(res.RequeueAfter).To(Equal(r.QuotaRetry))
+		p := fetch(name)
+		cond := apimeta.FindStatusCondition(p.Status.Conditions, crawlv1alpha1.ConditionProvisioned)
+		Expect(cond.Reason).To(Equal(ReasonQuotaExceeded))
+		Expect(p.Status.Phase).NotTo(Equal(crawlv1alpha1.PhaseFailed))
+
+		By("permanent: phase Failed and no further provider calls")
+		stub.createErr = provider.Wrap(provider.ErrPermanent, "create", "stub", "", nil)
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetch(name).Status.Phase).To(Equal(crawlv1alpha1.PhaseFailed))
+		callsAfterLatch := stub.createCalls
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(stub.createCalls).To(Equal(callsAfterLatch), "the latch must stop provider calls")
+	})
+
+	It("adopts an instance when the spec-hash annotation is stripped", func() {
+		const name = "e2e-adopt"
+		stub := &stubProvider{createID: "inst-adopt"}
+		r := newEnvtestReconciler(stub)
+		DeferCleanup(func() { cleanup(r, stub, name) })
+
+		Expect(k8sClient.Create(ctx, &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec:       managedSpec(),
+		})).To(Succeed())
+		_, err := envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		stub.getInst = &provider.Instance{ID: "inst-adopt", IP: "10.4.4.4", State: provider.StateRunning}
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		originalHash := fetch(name).Annotations[crawlv1alpha1.AnnotationSpecHash]
+		Expect(originalHash).NotTo(BeEmpty())
+
+		By("stripping the annotation, as an operator-version upgrade with a changed hash input would")
+		p := fetch(name)
+		delete(p.Annotations, crawlv1alpha1.AnnotationSpecHash)
+		Expect(k8sClient.Update(ctx, p)).To(Succeed())
+
+		_, err = envReconcile(r, name)
+		Expect(err).NotTo(HaveOccurred())
+		p = fetch(name)
+		Expect(p.Annotations[crawlv1alpha1.AnnotationSpecHash]).To(Equal(originalHash), "hash must be restored")
+		Expect(p.Status.ProviderID).To(Equal("inst-adopt"), "adoption must keep the instance")
+		Expect(stub.deleteCalls).To(BeZero(), "adoption must never replace")
+	})
+
 	It("tracks an External proxy without touching providers", func() {
 		const name = "e2e-external"
 		stub := &stubProvider{}
@@ -264,3 +364,157 @@ var _ = Describe("Proxy controller", func() {
 		Expect(apierrors.IsNotFound(err)).To(BeTrue())
 	})
 })
+
+// These specs assert the CRD's CEL rules and structural defaulting against
+// the real envtest API server — the fake client runs neither, which is the
+// documented caveat on the action-table unit tests.
+var _ = Describe("Proxy CRD validation (CEL)", func() {
+	const ns = "default"
+
+	managed := func(name string) *crawlv1alpha1.Proxy {
+		return &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec: crawlv1alpha1.ProxySpec{
+				Mode:     crawlv1alpha1.ModeManaged,
+				Provider: "stub",
+			},
+		}
+	}
+	external := func(name string) *crawlv1alpha1.Proxy {
+		return &crawlv1alpha1.Proxy{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Spec: crawlv1alpha1.ProxySpec{
+				Mode:     crawlv1alpha1.ModeExternal,
+				Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
+			},
+		}
+	}
+	mustCreate := func(p *crawlv1alpha1.Proxy) {
+		GinkgoHelper()
+		Expect(k8sClient.Create(ctx, p)).To(Succeed())
+		DeferCleanup(func() { _ = k8sClient.Delete(ctx, p) })
+	}
+
+	It("rejects invalid creates", func() {
+		invalid := []struct {
+			about string
+			spec  crawlv1alpha1.ProxySpec
+			want  string
+		}{
+			{
+				about: "Managed without provider",
+				spec:  crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged},
+				want:  "provider is required when mode is Managed",
+			},
+			{
+				about: "External with provider",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeExternal, Provider: "stub",
+					Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
+				},
+				want: "provider must not be set when mode is External",
+			},
+			{
+				about: "External without endpoint",
+				spec:  crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeExternal},
+				want:  "endpoint is required when mode is External",
+			},
+			{
+				about: "Managed with endpoint",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
+					Endpoint: &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"},
+				},
+				want: "endpoint must not be set when mode is Managed",
+			},
+			{
+				about: "cloudInit with both inline and secretRef",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
+					CloudInit: &crawlv1alpha1.CloudInitSpec{
+						Inline:    "#cloud-config",
+						SecretRef: &crawlv1alpha1.SecretKeySelector{Name: "s"},
+					},
+				},
+				want: "exactly one of inline or secretRef",
+			},
+			{
+				about: "cloudInit with neither inline nor secretRef",
+				spec: crawlv1alpha1.ProxySpec{
+					Mode: crawlv1alpha1.ModeManaged, Provider: "stub",
+					CloudInit: &crawlv1alpha1.CloudInitSpec{},
+				},
+				want: "exactly one of inline or secretRef",
+			},
+		}
+		for _, tc := range invalid {
+			p := &crawlv1alpha1.Proxy{
+				ObjectMeta: metav1.ObjectMeta{Name: "cel-invalid", Namespace: ns},
+				Spec:       tc.spec,
+			}
+			err := k8sClient.Create(ctx, p)
+			Expect(err).To(HaveOccurred(), tc.about)
+			Expect(err.Error()).To(ContainSubstring(tc.want), tc.about)
+		}
+	})
+
+	It("rejects mode mutation", func() {
+		p := external("cel-mode-immutable")
+		mustCreate(p)
+		p.Spec = crawlv1alpha1.ProxySpec{Mode: crawlv1alpha1.ModeManaged, Provider: "stub"}
+		err := k8sClient.Update(ctx, p)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("mode is immutable"))
+	})
+
+	It("rejects provider mutation and removal", func() {
+		p := managed("cel-provider-immutable")
+		mustCreate(p)
+
+		p.Spec.Provider = "other"
+		err := k8sClient.Update(ctx, p)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("provider is immutable"))
+
+		// Removal must also be rejected — the has()==has() form exists
+		// exactly because a field-level rule would not fire on absence.
+		// (Dropping provider alone would also trip the required-iff rule,
+		// so flip mode too and check the immutability rules win.)
+		fresh := fetchProxy(ns, "cel-provider-immutable")
+		fresh.Spec.Provider = ""
+		fresh.Spec.Endpoint = &crawlv1alpha1.EndpointSpec{Host: "203.0.113.9"}
+		fresh.Spec.Mode = crawlv1alpha1.ModeExternal
+		err = k8sClient.Update(ctx, fresh)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("immutable"))
+	})
+
+	It("materializes every nested healthCheck default when healthCheck is omitted", func() {
+		p := managed("cel-defaults")
+		mustCreate(p)
+
+		got := fetchProxy(ns, "cel-defaults")
+		// The +kubebuilder:default={} assertion: structural defaulting only
+		// descends into values that exist, so without it a nil healthCheck
+		// would get none of these.
+		hc := got.Spec.HealthCheck
+		Expect(hc).NotTo(BeNil())
+		Expect(hc.ProbeURL).To(Equal(crawlv1alpha1.DefaultProbeURL))
+		Expect(hc.IntervalSeconds).To(Equal(crawlv1alpha1.DefaultHealthCheckIntervalSeconds))
+		Expect(hc.TimeoutSeconds).To(Equal(crawlv1alpha1.DefaultHealthCheckTimeoutSeconds))
+		Expect(hc.FailureThreshold).To(Equal(crawlv1alpha1.DefaultFailureThreshold))
+		Expect(hc.SuccessThreshold).To(Equal(crawlv1alpha1.DefaultSuccessThreshold))
+		Expect(hc.ExpectedStatusCodes).To(Equal(crawlv1alpha1.DefaultExpectedStatusCodes))
+
+		Expect(got.Spec.Port).To(Equal(crawlv1alpha1.DefaultPort))
+		Expect(got.Spec.MaxLeases).NotTo(BeNil())
+		Expect(*got.Spec.MaxLeases).To(Equal(crawlv1alpha1.DefaultMaxLeases))
+	})
+})
+
+func fetchProxy(ns, name string) *crawlv1alpha1.Proxy {
+	GinkgoHelper()
+	p := &crawlv1alpha1.Proxy{}
+	Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, p)).To(Succeed())
+	return p
+}

internal/discovery/handlers.go

@@ -0,0 +1,241 @@
+package discovery
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"slices"
+	"strings"
+	"time"
+
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+)
+
+// proxyView is the wire shape of one proxy in list and lease responses.
+type proxyView struct {
+	ID            string            `json:"id"` // namespace/name
+	IP            string            `json:"ip"`
+	Port          int32             `json:"port"`
+	Attributes    map[string]string `json:"attributes,omitempty"`
+	Phase         string            `json:"phase"`
+	Healthy       bool              `json:"healthy"`
+	LatencyMillis int64             `json:"latencyMillis"`
+	ActiveLeases  int               `json:"activeLeases"`
+	MaxLeases     int32             `json:"maxLeases"`
+}
+
+func viewOf(p *crawlv1alpha1.Proxy, activeLeases int) proxyView {
+	return proxyView{
+		ID:            client.ObjectKeyFromObject(p).String(),
+		IP:            p.EffectiveHost(),
+		Port:          p.EffectivePort(),
+		Attributes:    p.Spec.Attributes,
+		Phase:         string(p.Status.Phase),
+		Healthy:       isHealthy(p),
+		LatencyMillis: p.Status.LatencyMillis,
+		ActiveLeases:  activeLeases,
+		MaxLeases:     p.MaxLeasesOrDefault(),
+	}
+}
+
+func isHealthy(p *crawlv1alpha1.Proxy) bool {
+	return apimeta.IsStatusConditionTrue(p.Status.Conditions, crawlv1alpha1.ConditionHealthy)
+}
+
+// matchesAttributes is spec.attributes equality: every selector pair must
+// be present verbatim.
+func matchesAttributes(p *crawlv1alpha1.Proxy, selector map[string]string) bool {
+	for k, v := range selector {
+		if p.Spec.Attributes[k] != v {
+			return false
+		}
+	}
+	return true
+}
+
+// GET /v1/proxies?attr.<key>=<value>&healthy=true|false
+func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
+	selector := map[string]string{}
+	var healthyFilter *bool
+	for key, values := range r.URL.Query() {
+		switch {
+		case key == "healthy":
+			switch values[0] {
+			case "true":
+				healthyFilter = ptr(true)
+			case "false":
+				healthyFilter = ptr(false)
+			default:
+				writeError(w, http.StatusBadRequest, "invalid_query",
+					fmt.Sprintf("healthy must be true or false, got %q", values[0]))
+				return
+			}
+		case strings.HasPrefix(key, "attr."):
+			selector[strings.TrimPrefix(key, "attr.")] = values[0]
+		}
+	}
+
+	var list crawlv1alpha1.ProxyList
+	if err := s.Reader.List(r.Context(), &list); err != nil {
+		s.log.Error(err, "listing proxies")
+		writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
+		return
+	}
+	counts := s.Store.Counts()
+
+	views := []proxyView{}
+	for i := range list.Items {
+		p := &list.Items[i]
+		if !p.DeletionTimestamp.IsZero() || !matchesAttributes(p, selector) {
+			continue
+		}
+		v := viewOf(p, counts[client.ObjectKeyFromObject(p).String()])
+		if healthyFilter != nil && v.Healthy != *healthyFilter {
+			continue
+		}
+		views = append(views, v)
+	}
+	slices.SortFunc(views, func(a, b proxyView) int { return strings.Compare(a.ID, b.ID) })
+
+	writeJSON(w, http.StatusOK, map[string]any{"proxies": views, "count": len(views)})
+}
+
+type leaseRequest struct {
+	Selector   map[string]string `json:"selector"`
+	TTLSeconds int64             `json:"ttlSeconds"`
+	Target     string            `json:"target"`
+}
+
+type leaseResponse struct {
+	LeaseID    string    `json:"leaseID"`
+	Proxy      proxyView `json:"proxy"`
+	ExpiresAt  time.Time `json:"expiresAt"`
+	TTLSeconds int64     `json:"ttlSeconds"`
+}
+
+// POST /v1/leases
+func (s *Server) handleAcquireLease(w http.ResponseWriter, r *http.Request) {
+	var req leaseRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid_body", err.Error())
+		return
+	}
+
+	ttl := time.Duration(req.TTLSeconds) * time.Second
+	if req.TTLSeconds == 0 {
+		ttl = defaultTTL
+	}
+	if ttl < 0 || ttl > s.MaxLeaseTTL {
+		writeError(w, http.StatusBadRequest, "invalid_ttl",
+			fmt.Sprintf("ttlSeconds must be between 1 and %d", int64(s.MaxLeaseTTL.Seconds())))
+		return
+	}
+
+	var list crawlv1alpha1.ProxyList
+	if err := s.Reader.List(r.Context(), &list); err != nil {
+		s.log.Error(err, "listing proxies for lease")
+		writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
+		return
+	}
+
+	// The store gets only healthy, live candidates; unhealthy matches are
+	// counted here because the store never sees them.
+	var candidates []lease.Candidate
+	byKey := map[string]*crawlv1alpha1.Proxy{}
+	unhealthy := 0
+	for i := range list.Items {
+		p := &list.Items[i]
+		if !p.DeletionTimestamp.IsZero() || !matchesAttributes(p, req.Selector) {
+			continue
+		}
+		if !isHealthy(p) {
+			unhealthy++
+			continue
+		}
+		key := client.ObjectKeyFromObject(p).String()
+		byKey[key] = p
+		candidates = append(candidates, lease.Candidate{
+			Proxy:     key,
+			MaxLeases: p.MaxLeasesOrDefault(),
+			Latency:   time.Duration(p.Status.LatencyMillis) * time.Millisecond,
+		})
+	}
+
+	granted, stats, err := s.Store.Acquire(r.Context(), lease.AcquireRequest{
+		Candidates: candidates,
+		Target:     req.Target,
+		TTL:        ttl,
+	})
+	if errors.Is(err, lease.ErrNoMatch) {
+		if s.Metrics != nil {
+			s.Metrics.LeaseRequest("no_match")
+		}
+		writeJSON(w, http.StatusConflict, map[string]any{
+			"error":      "no_match",
+			"message":    "no healthy proxy with free capacity matched the selector",
+			"considered": stats.Considered + unhealthy,
+			"atCapacity": stats.AtCapacity,
+			"inCooldown": stats.InCooldown,
+			"unhealthy":  unhealthy,
+		})
+		return
+	}
+	if err != nil {
+		writeError(w, http.StatusBadRequest, "invalid_request", err.Error())
+		return
+	}
+
+	if s.Metrics != nil {
+		s.Metrics.LeaseRequest("granted")
+	}
+	writeJSON(w, http.StatusCreated, leaseResponse{
+		LeaseID:    granted.ID,
+		Proxy:      viewOf(byKey[granted.Proxy], s.Store.Counts()[granted.Proxy]),
+		ExpiresAt:  granted.ExpiresAt,
+		TTLSeconds: int64(ttl.Seconds()),
+	})
+}
+
+// DELETE /v1/leases/{id} — early release, always 204: releasing an unknown
+// or already expired lease is not an error.
+func (s *Server) handleReleaseLease(w http.ResponseWriter, r *http.Request) {
+	s.Store.Release(r.Context(), r.PathValue("id"))
+	w.WriteHeader(http.StatusNoContent)
+}
+
+type reportRequest struct {
+	Result string `json:"result"`
+	Target string `json:"target"`
+}
+
+// POST /v1/leases/{id}/report
+func (s *Server) handleReportLease(w http.ResponseWriter, r *http.Request) {
+	var req reportRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid_body", err.Error())
+		return
+	}
+	result, ok := lease.ParseResult(req.Result)
+	if !ok {
+		writeError(w, http.StatusBadRequest, "invalid_result",
+			fmt.Sprintf("result must be one of ok, rate_limited, banned; got %q", req.Result))
+		return
+	}
+	if err := s.Store.Report(r.Context(), r.PathValue("id"), result, req.Target); err != nil {
+		if errors.Is(err, lease.ErrUnknownLease) {
+			writeError(w, http.StatusNotFound, "unknown_lease", "no such lease")
+			return
+		}
+		s.log.Error(err, "reporting lease", "leaseID", r.PathValue("id"))
+		writeError(w, http.StatusInternalServerError, "internal", "report failed")
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func ptr[T any](v T) *T { return &v }

internal/discovery/server.go

@@ -0,0 +1,234 @@
+// Package discovery implements the HTTP API crawler clients use to find
+// and lease proxies: list healthy proxies filtered by attributes, acquire a
+// TTL-based lease, release it early, and report how a target treated the
+// proxy. Reads go through the manager's informer cache; lease state lives
+// in the injected store.
+package discovery
+
+import (
+	"context"
+	"crypto/subtle"
+	"encoding/json"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/go-logr/logr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+)
+
+// LeaseStore is what the handlers need from a lease backend. Defined here,
+// consumer-side, so a CRD- or Redis-backed store can replace the in-memory
+// one (which internal/lease's *Store satisfies) without touching handlers.
+type LeaseStore interface {
+	Acquire(ctx context.Context, req lease.AcquireRequest) (*lease.Lease, lease.AcquireStats, error)
+	Release(ctx context.Context, id string)
+	Report(ctx context.Context, id string, result lease.Result, target string) error
+	Counts() map[string]int
+}
+
+// LeaseMetrics counts lease acquisitions by outcome. Implemented by
+// internal/metrics; defined here so this package carries no metrics
+// dependency.
+type LeaseMetrics interface {
+	LeaseRequest(outcome string)
+}
+
+const (
+	defaultAddr     = ":8090"
+	defaultTTL      = 5 * time.Minute
+	defaultMaxTTL   = time.Hour
+	maxBodyBytes    = 64 << 10
+	shutdownGrace   = 10 * time.Second
+	readHeadTimeout = 5 * time.Second
+)
+
+// Server serves the discovery API as a manager Runnable.
+type Server struct {
+	// Reader lists Proxies from the manager's cache.
+	Reader client.Reader
+	// Store is the lease backend.
+	Store LeaseStore
+	// Addr is the listen address (default ":8090"; --discovery-addr).
+	Addr string
+	// Token is the static bearer token from DISCOVERY_TOKEN. Empty
+	// disables auth — allowed for the prototype, but loudly warned about
+	// at startup, because in-cluster that is a silent security hole.
+	Token string
+	// MaxLeaseTTL caps requested lease TTLs (default 1h; --max-lease-ttl).
+	MaxLeaseTTL time.Duration
+	// Metrics, when non-nil, counts lease requests by outcome.
+	Metrics LeaseMetrics
+
+	log logr.Logger
+
+	mu        sync.Mutex
+	boundAddr string
+}
+
+// NeedLeaderElection is false, and the deployment ships replicas: 1.
+// Verified against controller-runtime's runnable ordering: caches start and
+// sync before non-leader-election runnables, so cache reads here are safe.
+// If this were leader-elected, non-leader replicas would refuse connections
+// while still being Service endpoints. The 1-replica constraint comes from
+// lease state being per-process — both facts are README caveats.
+func (s *Server) NeedLeaderElection() bool { return false }
+
+// BoundAddr returns the actual listen address once Start has bound it —
+// meaningful when Addr uses port 0 (tests).
+func (s *Server) BoundAddr() string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.boundAddr
+}
+
+// Start listens and serves until ctx ends, then shuts down gracefully with
+// a 10-second grace period.
+func (s *Server) Start(ctx context.Context) error {
+	if s.Addr == "" {
+		s.Addr = defaultAddr
+	}
+	if s.MaxLeaseTTL == 0 {
+		s.MaxLeaseTTL = defaultMaxTTL
+	}
+	s.log = logf.FromContext(ctx).WithName("discovery")
+	if s.Token == "" {
+		s.log.Info("WARNING: DISCOVERY_TOKEN is empty — the discovery API is served without authentication")
+	}
+
+	ln, err := net.Listen("tcp", s.Addr)
+	if err != nil {
+		return err
+	}
+	s.mu.Lock()
+	s.boundAddr = ln.Addr().String()
+	s.mu.Unlock()
+
+	srv := &http.Server{
+		Handler:           s.handler(),
+		ReadHeaderTimeout: readHeadTimeout,
+		ReadTimeout:       10 * time.Second,
+		WriteTimeout:      10 * time.Second,
+		IdleTimeout:       60 * time.Second,
+	}
+
+	errCh := make(chan error, 1)
+	go func() { errCh <- srv.Serve(ln) }()
+	s.log.Info("discovery API listening", "addr", s.boundAddr)
+
+	select {
+	case err := <-errCh:
+		return err
+	case <-ctx.Done():
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownGrace)
+		defer cancel()
+		if err := srv.Shutdown(shutdownCtx); err != nil {
+			return err
+		}
+		<-errCh // always http.ErrServerClosed after a clean Shutdown
+		return nil
+	}
+}
+
+// handler assembles the mux and the middleware chain, outermost first:
+// recover → request-log → body-size cap → bearer auth.
+func (s *Server) handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok\n"))
+	})
+	mux.HandleFunc("GET /v1/proxies", s.handleListProxies)
+	mux.HandleFunc("POST /v1/leases", s.handleAcquireLease)
+	mux.HandleFunc("DELETE /v1/leases/{id}", s.handleReleaseLease)
+	mux.HandleFunc("POST /v1/leases/{id}/report", s.handleReportLease)
+
+	var h http.Handler = mux
+	h = s.authMiddleware(h)
+	h = maxBytesMiddleware(h)
+	h = s.logMiddleware(h)
+	h = s.recoverMiddleware(h)
+	return h
+}
+
+func (s *Server) recoverMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if p := recover(); p != nil {
+				s.log.Error(nil, "panic in discovery handler", "panic", p, "path", r.URL.Path)
+				writeError(w, http.StatusInternalServerError, "internal", "internal server error")
+			}
+		}()
+		next.ServeHTTP(w, r)
+	})
+}
+
+// statusRecorder captures the response code for the request log.
+type statusRecorder struct {
+	http.ResponseWriter
+	status int
+}
+
+func (r *statusRecorder) WriteHeader(code int) {
+	r.status = code
+	r.ResponseWriter.WriteHeader(code)
+}
+
+func (s *Server) logMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/healthz" {
+			next.ServeHTTP(w, r) // probes are noise
+			return
+		}
+		rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
+		start := time.Now()
+		next.ServeHTTP(rec, r)
+		s.log.Info("request",
+			"method", r.Method, "path", r.URL.Path,
+			"status", rec.status, "duration", time.Since(start).String())
+	})
+}
+
+func maxBytesMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
+		next.ServeHTTP(w, r)
+	})
+}
+
+func (s *Server) authMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if s.Token == "" || r.URL.Path == "/healthz" {
+			next.ServeHTTP(w, r)
+			return
+		}
+		token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if !ok || subtle.ConstantTimeCompare([]byte(token), []byte(s.Token)) != 1 {
+			writeError(w, http.StatusUnauthorized, "unauthorized", "missing or invalid bearer token")
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+// errorBody is the shared error shape:
+// {"error":"<machine_code>","message":"<human>"}.
+type errorBody struct {
+	Error   string `json:"error"`
+	Message string `json:"message"`
+}
+
+func writeError(w http.ResponseWriter, status int, code, message string) {
+	writeJSON(w, status, errorBody{Error: code, Message: message})
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(v)
+}

internal/discovery/server_test.go

@@ -0,0 +1,386 @@
+package discovery
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+)
+
+func testProxy(name string, attrs map[string]string, healthy bool, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy {
+	p := &crawlv1alpha1.Proxy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name, Namespace: "default", UID: types.UID("uid-" + name),
+		},
+		Spec: crawlv1alpha1.ProxySpec{
+			Mode:       crawlv1alpha1.ModeExternal,
+			Endpoint:   &crawlv1alpha1.EndpointSpec{Host: "10.0.0.1", Port: 3128},
+			Attributes: attrs,
+		},
+	}
+	p.Status.IP = "10.0.0.1"
+	p.Status.Phase = crawlv1alpha1.PhaseReady
+	status := metav1.ConditionFalse
+	if healthy {
+		status = metav1.ConditionTrue
+	}
+	p.Status.Conditions = []metav1.Condition{{
+		Type: crawlv1alpha1.ConditionHealthy, Status: status,
+		Reason: "Probing", LastTransitionTime: metav1.Now(),
+	}}
+	for _, m := range mut {
+		m(p)
+	}
+	return p
+}
+
+func withMaxLeases(n int32) func(*crawlv1alpha1.Proxy) {
+	return func(p *crawlv1alpha1.Proxy) { p.Spec.MaxLeases = &n }
+}
+
+func withLatency(ms int64) func(*crawlv1alpha1.Proxy) {
+	return func(p *crawlv1alpha1.Proxy) { p.Status.LatencyMillis = ms }
+}
+
+// newTestServer wires the handler chain to a fake cache reader and a real
+// lease store, served over httptest.
+func newTestServer(t *testing.T, token string, proxies ...*crawlv1alpha1.Proxy) (*httptest.Server, *Server) {
+	t.Helper()
+	s := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("scheme: %v", err)
+	}
+	builder := fake.NewClientBuilder().WithScheme(s)
+	for _, p := range proxies {
+		builder = builder.WithObjects(p)
+	}
+	srv := &Server{
+		Reader:      builder.Build(),
+		Store:       lease.NewStore(15 * time.Minute),
+		Token:       token,
+		MaxLeaseTTL: time.Hour,
+	}
+	ts := httptest.NewServer(srv.handler())
+	t.Cleanup(ts.Close)
+	return ts, srv
+}
+
+type response struct {
+	status int
+	body   map[string]any
+}
+
+func do(t *testing.T, ts *httptest.Server, method, path, token string, body any) response {
+	t.Helper()
+	var reader io.Reader
+	if body != nil {
+		if s, ok := body.(string); ok {
+			reader = bytes.NewBufferString(s)
+		} else {
+			b, err := json.Marshal(body)
+			if err != nil {
+				t.Fatalf("marshaling request body: %v", err)
+			}
+			reader = bytes.NewBuffer(b)
+		}
+	}
+	req, err := http.NewRequestWithContext(context.Background(), method, ts.URL+path, reader)
+	if err != nil {
+		t.Fatalf("building request: %v", err)
+	}
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
+	}
+	resp, err := ts.Client().Do(req)
+	if err != nil {
+		t.Fatalf("%s %s: %v", method, path, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	out := response{status: resp.StatusCode}
+	raw, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("reading response: %v", err)
+	}
+	if len(raw) > 0 && resp.Header.Get("Content-Type") == "application/json" {
+		if err := json.Unmarshal(raw, &out.body); err != nil {
+			t.Fatalf("decoding response %q: %v", raw, err)
+		}
+	}
+	return out
+}
+
+func TestAuth(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "sekrit", testProxy("p1", nil, true))
+
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "", nil); got.status != http.StatusUnauthorized {
+		t.Errorf("no token: status %d, want 401", got.status)
+	}
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "wrong", nil); got.status != http.StatusUnauthorized {
+		t.Errorf("wrong token: status %d, want 401", got.status)
+	}
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "sekrit", nil); got.status != http.StatusOK {
+		t.Errorf("correct token: status %d, want 200", got.status)
+	}
+	if got := do(t, ts, http.MethodGet, "/healthz", "", nil); got.status != http.StatusOK {
+		t.Errorf("healthz without token: status %d, want 200 (always unauthenticated)", got.status)
+	}
+}
+
+func TestAuth_disabledWithEmptyToken(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
+	if got := do(t, ts, http.MethodGet, "/v1/proxies", "", nil); got.status != http.StatusOK {
+		t.Errorf("status %d, want 200 with auth disabled", got.status)
+	}
+}
+
+func TestListProxies(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "",
+		testProxy("eu-healthy", map[string]string{"geo": "eu", "purpose": "crawl"}, true),
+		testProxy("eu-sick", map[string]string{"geo": "eu"}, false),
+		testProxy("us-healthy", map[string]string{"geo": "us"}, true),
+	)
+
+	tests := []struct {
+		name      string
+		query     string
+		wantCount int
+		wantFirst string
+	}{
+		{name: "no filter returns everything", query: "", wantCount: 3, wantFirst: "default/eu-healthy"},
+		{name: "healthy filter", query: "?healthy=true", wantCount: 2},
+		{name: "unhealthy filter", query: "?healthy=false", wantCount: 1, wantFirst: "default/eu-sick"},
+		{name: "attribute filter", query: "?attr.geo=eu", wantCount: 2},
+		{name: "attribute and health combined", query: "?attr.geo=eu&healthy=true", wantCount: 1, wantFirst: "default/eu-healthy"},
+		{name: "two attributes must both match", query: "?attr.geo=eu&attr.purpose=crawl", wantCount: 1, wantFirst: "default/eu-healthy"},
+		{name: "no matches is 200 with count 0", query: "?attr.geo=mars", wantCount: 0},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got := do(t, ts, http.MethodGet, "/v1/proxies"+tc.query, "", nil)
+			if got.status != http.StatusOK {
+				t.Fatalf("status %d, want 200", got.status)
+			}
+			count := int(got.body["count"].(float64))
+			proxies := got.body["proxies"].([]any)
+			if count != tc.wantCount || len(proxies) != tc.wantCount {
+				t.Fatalf("count = %d (len %d), want %d", count, len(proxies), tc.wantCount)
+			}
+			if tc.wantFirst != "" {
+				first := proxies[0].(map[string]any)
+				if first["id"] != tc.wantFirst {
+					t.Errorf("first id = %v, want %s", first["id"], tc.wantFirst)
+				}
+			}
+		})
+	}
+
+	t.Run("invalid healthy value is 400", func(t *testing.T) {
+		t.Parallel()
+		if got := do(t, ts, http.MethodGet, "/v1/proxies?healthy=maybe", "", nil); got.status != http.StatusBadRequest {
+			t.Errorf("status %d, want 400", got.status)
+		}
+	})
+}
+
+func TestAcquireLease_grantShape(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "",
+		testProxy("eu1", map[string]string{"geo": "eu"}, true, withLatency(30)),
+		testProxy("eu2", map[string]string{"geo": "eu"}, true, withLatency(10)),
+	)
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
+		"selector": map[string]string{"geo": "eu"},
+	})
+	if got.status != http.StatusCreated {
+		t.Fatalf("status %d (%v), want 201", got.status, got.body)
+	}
+	if got.body["leaseID"] == "" || got.body["leaseID"] == nil {
+		t.Error("empty leaseID")
+	}
+	if got.body["ttlSeconds"].(float64) != 300 {
+		t.Errorf("ttlSeconds = %v, want the 300 default", got.body["ttlSeconds"])
+	}
+	proxy := got.body["proxy"].(map[string]any)
+	if proxy["id"] != "default/eu2" {
+		t.Errorf("granted %v, want default/eu2 (lower latency at equal load)", proxy["id"])
+	}
+	if proxy["activeLeases"].(float64) != 1 {
+		t.Errorf("activeLeases = %v, want 1 (this grant included)", proxy["activeLeases"])
+	}
+	if _, err := time.Parse(time.RFC3339, got.body["expiresAt"].(string)); err != nil {
+		t.Errorf("expiresAt %v is not RFC3339: %v", got.body["expiresAt"], err)
+	}
+}
+
+func TestAcquireLease_noMatchBody(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "",
+		testProxy("eu-tiny", map[string]string{"geo": "eu"}, true, withMaxLeases(1)),
+		testProxy("eu-sick", map[string]string{"geo": "eu"}, false),
+	)
+
+	body := map[string]any{"selector": map[string]string{"geo": "eu"}}
+	if got := do(t, ts, http.MethodPost, "/v1/leases", "", body); got.status != http.StatusCreated {
+		t.Fatalf("first acquire: status %d, want 201", got.status)
+	}
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", body)
+	if got.status != http.StatusConflict {
+		t.Fatalf("second acquire: status %d, want 409", got.status)
+	}
+	want := map[string]float64{"considered": 2, "atCapacity": 1, "inCooldown": 0, "unhealthy": 1}
+	for k, v := range want {
+		if got.body[k].(float64) != v {
+			t.Errorf("%s = %v, want %v (body %v)", k, got.body[k], v, got.body)
+		}
+	}
+	if got.body["error"] != "no_match" {
+		t.Errorf("error = %v, want no_match", got.body["error"])
+	}
+}
+
+func TestAcquireLease_badRequests(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
+
+	tests := []struct {
+		name     string
+		body     any
+		wantCode string
+	}{
+		{name: "ttl above the cap", body: map[string]any{"ttlSeconds": 999999}, wantCode: "invalid_ttl"},
+		{name: "negative ttl", body: map[string]any{"ttlSeconds": -5}, wantCode: "invalid_ttl"},
+		{name: "malformed json", body: "{not json", wantCode: "invalid_body"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got := do(t, ts, http.MethodPost, "/v1/leases", "", tc.body)
+			if got.status != http.StatusBadRequest || got.body["error"] != tc.wantCode {
+				t.Errorf("= %d/%v, want 400/%s", got.status, got.body["error"], tc.wantCode)
+			}
+		})
+	}
+}
+
+func TestReleaseLease_alwaysNoContent(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", nil, true))
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{})
+	if got.status != http.StatusCreated {
+		t.Fatalf("acquire: status %d, want 201", got.status)
+	}
+	id := got.body["leaseID"].(string)
+
+	for _, path := range []string{"/v1/leases/" + id, "/v1/leases/" + id, "/v1/leases/never-existed"} {
+		if got := do(t, ts, http.MethodDelete, path, "", nil); got.status != http.StatusNoContent {
+			t.Errorf("DELETE %s: status %d, want 204", path, got.status)
+		}
+	}
+}
+
+func TestReportLease(t *testing.T) {
+	t.Parallel()
+	ts, _ := newTestServer(t, "", testProxy("p1", map[string]string{"geo": "eu"}, true))
+
+	got := do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
+		"selector": map[string]string{"geo": "eu"}, "target": "example.com",
+	})
+	if got.status != http.StatusCreated {
+		t.Fatalf("acquire: status %d, want 201", got.status)
+	}
+	id := got.body["leaseID"].(string)
+	reportPath := fmt.Sprintf("/v1/leases/%s/report", id)
+
+	if got := do(t, ts, http.MethodPost, reportPath, "", map[string]any{"result": "rate_limited", "target": "example.com"}); got.status != http.StatusNoContent {
+		t.Fatalf("report: status %d, want 204", got.status)
+	}
+
+	// The cooldown from the report now blocks same-target acquisition.
+	got = do(t, ts, http.MethodPost, "/v1/leases", "", map[string]any{
+		"selector": map[string]string{"geo": "eu"}, "target": "example.com",
+	})
+	if got.status != http.StatusConflict || got.body["inCooldown"].(float64) != 1 {
+		t.Errorf("post-report acquire = %d/%v, want 409 with inCooldown 1", got.status, got.body)
+	}
+
+	t.Run("invalid result value", func(t *testing.T) {
+		got := do(t, ts, http.MethodPost, reportPath, "", map[string]any{"result": "throttled"})
+		if got.status != http.StatusBadRequest || got.body["error"] != "invalid_result" {
+			t.Errorf("= %d/%v, want 400/invalid_result", got.status, got.body["error"])
+		}
+	})
+	t.Run("unknown lease", func(t *testing.T) {
+		got := do(t, ts, http.MethodPost, "/v1/leases/never-existed/report", "", map[string]any{"result": "ok"})
+		if got.status != http.StatusNotFound || got.body["error"] != "unknown_lease" {
+			t.Errorf("= %d/%v, want 404/unknown_lease", got.status, got.body["error"])
+		}
+	})
+}
+
+func TestStart_servesAndShutsDown(t *testing.T) {
+	t.Parallel()
+	s := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("scheme: %v", err)
+	}
+	srv := &Server{
+		Reader: fake.NewClientBuilder().WithScheme(s).Build(),
+		Store:  lease.NewStore(time.Minute),
+		Addr:   "127.0.0.1:0",
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- srv.Start(ctx) }()
+
+	var addr string
+	deadline := time.After(5 * time.Second)
+	for addr == "" {
+		select {
+		case <-deadline:
+			t.Fatal("server never bound")
+		case <-time.After(5 * time.Millisecond):
+			addr = srv.BoundAddr()
+		}
+	}
+
+	resp, err := http.Get("http://" + addr + "/healthz")
+	if err != nil {
+		t.Fatalf("healthz: %v", err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("healthz status %d, want 200", resp.StatusCode)
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start returned %v, want nil after graceful shutdown", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}

internal/gc/gc.go

@@ -0,0 +1,125 @@
+// Package gc implements orphan garbage collection: a periodic sweep that
+// deletes provider instances tagged by this operator whose owning Proxy CR
+// no longer exists — the safety net for crashes between a provider Create
+// and the status write that records it.
+package gc
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// Sweeper is the manager Runnable running the sweep loop.
+type Sweeper struct {
+	// Reader lists Proxies from the manager's cache to establish the live
+	// UID set.
+	Reader client.Reader
+	// Providers are the configured backends; each is swept independently.
+	Providers map[string]provider.Provider
+
+	// Interval between sweeps (default 10m). The first sweep runs one full
+	// interval after start, not immediately — right after startup the
+	// cache is coldest and an in-flight create is most likely.
+	Interval time.Duration
+	// MinAge exempts young instances (default 10m): an instance mid-create
+	// may not have its status write landed yet; deleting it would race the
+	// reconciler.
+	MinAge time.Duration
+
+	// NamespaceRestricted must be set when the manager cache is limited to
+	// one namespace. Then the live-UID set is incomplete, and a sweep
+	// would delete VMs owned by Proxies the cache cannot see — so Start
+	// refuses unless AllowNamespaced (--gc-allow-namespaced) is explicit.
+	NamespaceRestricted bool
+	AllowNamespaced     bool
+
+	now func() time.Time
+}
+
+// NeedLeaderElection is true: the sweep is destructive and must have a
+// single writer.
+func (s *Sweeper) NeedLeaderElection() bool { return true }
+
+// Start runs the sweep loop until ctx ends.
+func (s *Sweeper) Start(ctx context.Context) error {
+	if s.NamespaceRestricted && !s.AllowNamespaced {
+		return errors.New(
+			"orphan GC refuses to run against a namespace-restricted cache: proxies outside the namespace " +
+				"would count as orphans and their instances would be deleted; pass --gc-allow-namespaced to override")
+	}
+	if s.Interval == 0 {
+		s.Interval = 10 * time.Minute
+	}
+	if s.MinAge == 0 {
+		s.MinAge = 10 * time.Minute
+	}
+	if s.now == nil {
+		s.now = time.Now
+	}
+
+	ticker := time.NewTicker(s.Interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			s.sweep(ctx)
+		}
+	}
+}
+
+// sweep deletes tagged instances whose UID matches no existing Proxy CR.
+// A CR with a deletionTimestamp still counts as live: its finalizer owns
+// that deletion, and GC racing it would double-delete. A UID becomes
+// orphan-eligible only once the object is fully gone.
+func (s *Sweeper) sweep(ctx context.Context) {
+	log := logf.FromContext(ctx).WithName("orphan-gc")
+
+	var list crawlv1alpha1.ProxyList
+	if err := s.Reader.List(ctx, &list); err != nil {
+		// Without the live set nothing can be proven orphaned; skip the
+		// whole sweep rather than guess.
+		log.Error(err, "listing proxies; skipping this sweep")
+		return
+	}
+	live := make(map[string]bool, len(list.Items))
+	for i := range list.Items {
+		live[string(list.Items[i].UID)] = true
+	}
+
+	for name, prov := range s.Providers {
+		instances, err := prov.ListByTag(ctx)
+		if err != nil {
+			// One broken provider must not abort the sweep for the rest.
+			log.Error(err, "listing instances; skipping this provider", "provider", name)
+			continue
+		}
+		for _, inst := range instances {
+			switch {
+			case inst.UID == "":
+				// Managed label without a UID label shouldn't exist for
+				// anything this operator created; without ownership proof,
+				// never delete.
+				continue
+			case live[inst.UID]:
+				continue
+			case s.now().Sub(inst.CreatedAt) < s.MinAge:
+				continue
+			}
+			log.Info("WARNING: deleting orphaned instance",
+				"provider", name, "providerID", inst.ID, "uid", inst.UID)
+			if err := prov.Delete(ctx, inst.ID); err != nil {
+				log.Error(err, "deleting orphaned instance",
+					"provider", name, "providerID", inst.ID, "uid", inst.UID)
+			}
+		}
+	}
+}

internal/gc/gc_test.go

@@ -0,0 +1,214 @@
+package gc
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// listProvider serves a canned instance list and records deletions.
+type listProvider struct {
+	mu        sync.Mutex
+	instances []provider.Instance
+	listErr   error
+	deleted   []string
+}
+
+func (l *listProvider) Create(context.Context, provider.CreateRequest) (string, error) {
+	return "", errors.New("not used")
+}
+
+func (l *listProvider) Get(context.Context, string) (*provider.Instance, error) {
+	return nil, provider.ErrNotFound
+}
+
+func (l *listProvider) Delete(_ context.Context, id string) error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.deleted = append(l.deleted, id)
+	return nil
+}
+
+func (l *listProvider) ListByTag(context.Context) ([]provider.Instance, error) {
+	return l.instances, l.listErr
+}
+
+func (l *listProvider) deletedIDs() []string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return append([]string(nil), l.deleted...)
+}
+
+func proxyWithUID(name, uid string, mut ...func(*crawlv1alpha1.Proxy)) *crawlv1alpha1.Proxy {
+	p := &crawlv1alpha1.Proxy{
+		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", UID: types.UID(uid)},
+		Spec: crawlv1alpha1.ProxySpec{
+			Mode:     crawlv1alpha1.ModeManaged,
+			Provider: "stub",
+		},
+	}
+	for _, m := range mut {
+		m(p)
+	}
+	return p
+}
+
+func newReader(t *testing.T, objs ...client.Object) client.Reader {
+	t.Helper()
+	s := runtime.NewScheme()
+	if err := crawlv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("scheme: %v", err)
+	}
+	return fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build()
+}
+
+func oldInstance(id, uid string) provider.Instance {
+	return provider.Instance{ID: id, UID: uid, State: provider.StateRunning,
+		CreatedAt: time.Now().Add(-time.Hour)}
+}
+
+func newSweeper(reader client.Reader, providers map[string]provider.Provider) *Sweeper {
+	return &Sweeper{
+		Reader:    reader,
+		Providers: providers,
+		Interval:  10 * time.Minute,
+		MinAge:    10 * time.Minute,
+		now:       time.Now,
+	}
+}
+
+func TestSweep_deletesOnlyTrueOrphans(t *testing.T) {
+	t.Parallel()
+
+	deletingCR := proxyWithUID("deleting", "uid-deleting", func(p *crawlv1alpha1.Proxy) {
+		now := metav1.Now()
+		p.DeletionTimestamp = &now
+		p.Finalizers = []string{crawlv1alpha1.FinalizerName}
+	})
+	prov := &listProvider{instances: []provider.Instance{
+		oldInstance("inst-live", "uid-live"),
+		oldInstance("inst-orphan", "uid-orphan"),
+		oldInstance("inst-deleting", "uid-deleting"),
+		{ID: "inst-young", UID: "uid-young-orphan", State: provider.StateRunning,
+			CreatedAt: time.Now().Add(-time.Minute)},
+		oldInstance("inst-unlabelled", ""),
+	}}
+	s := newSweeper(
+		newReader(t, proxyWithUID("live", "uid-live"), deletingCR),
+		map[string]provider.Provider{"stub": prov},
+	)
+
+	s.sweep(context.Background())
+
+	got := prov.deletedIDs()
+	if len(got) != 1 || got[0] != "inst-orphan" {
+		t.Errorf("deleted %v, want exactly [inst-orphan]:\n"+
+			"live CR's instance must stay; a deleting CR still owns its instance (finalizer, not GC);\n"+
+			"young instances may be mid-create; unlabelled instances have no ownership proof", got)
+	}
+}
+
+func TestSweep_providerErrorDoesNotAbortOthers(t *testing.T) {
+	t.Parallel()
+	broken := &listProvider{listErr: errors.New("cloud is down")}
+	working := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
+	s := newSweeper(newReader(t), map[string]provider.Provider{
+		"broken":  broken,
+		"working": working,
+	})
+
+	s.sweep(context.Background())
+
+	if got := working.deletedIDs(); len(got) != 1 {
+		t.Errorf("working provider deletions = %v, want the orphan despite the broken provider", got)
+	}
+}
+
+// errReader fails every List: without the live set nothing can be proven
+// orphaned, so the sweep must delete nothing.
+type errReader struct{ client.Reader }
+
+func (errReader) List(context.Context, client.ObjectList, ...client.ListOption) error {
+	return errors.New("cache broken")
+}
+
+func TestSweep_listFailureSkipsSweep(t *testing.T) {
+	t.Parallel()
+	prov := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
+	s := newSweeper(errReader{}, map[string]provider.Provider{"stub": prov})
+
+	s.sweep(context.Background())
+
+	if got := prov.deletedIDs(); len(got) != 0 {
+		t.Errorf("deleted %v with an unreadable live set, want nothing", got)
+	}
+}
+
+func TestStart_namespaceGuard(t *testing.T) {
+	t.Parallel()
+
+	s := newSweeper(newReader(t), nil)
+	s.NamespaceRestricted = true
+	err := s.Start(context.Background())
+	if err == nil || !strings.Contains(err.Error(), "--gc-allow-namespaced") {
+		t.Errorf("Start with restricted cache = %v, want refusal naming the override flag", err)
+	}
+
+	s2 := newSweeper(newReader(t), nil)
+	s2.NamespaceRestricted = true
+	s2.AllowNamespaced = true
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- s2.Start(ctx) }()
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start with override = %v, want it to run until cancel", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}
+
+func TestStart_sweepsOnIntervalAndStops(t *testing.T) {
+	t.Parallel()
+	prov := &listProvider{instances: []provider.Instance{oldInstance("inst-orphan", "uid-orphan")}}
+	s := newSweeper(newReader(t), map[string]provider.Provider{"stub": prov})
+	s.Interval = 5 * time.Millisecond
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- s.Start(ctx) }()
+
+	deadline := time.After(5 * time.Second)
+	for len(prov.deletedIDs()) == 0 {
+		select {
+		case <-deadline:
+			t.Fatal("no sweep ran")
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start = %v, want nil", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}

internal/health/engine.go

@@ -62,6 +62,14 @@ type probeJob struct {
 	interval time.Duration
 }
 
+// ProbeMetrics receives every probe result and the retirement of a
+// proxy's series. Implemented by internal/metrics; defined here so this
+// package carries no metrics dependency.
+type ProbeMetrics interface {
+	ObserveProbe(proxy string, latency time.Duration, success bool)
+	ForgetProxy(proxy string)
+}
+
 // Engine runs the probe scheduler and worker pool as a manager Runnable. It
 // never writes Proxy status itself — keeping the reconciler the single
 // status writer — and instead emits a GenericEvent per status-affecting
@@ -87,6 +95,10 @@ type Engine struct {
 	// ProbeTLSConfig overrides TLS verification for https probe URLs; nil
 	// means system roots. Needed for private CAs (and tests).
 	ProbeTLSConfig *tls.Config
+	// Metrics, when non-nil, is fed on every probe — the status writes are
+	// transition-only by design, so metrics are where high-frequency
+	// signal (true probe recency, every latency sample) lives.
+	Metrics ProbeMetrics
 
 	probeFn func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult
 
@@ -220,6 +232,11 @@ func (e *Engine) tick(ctx context.Context, now time.Time, jobs chan<- probeJob)
 	for key := range e.states {
 		if _, ok := probeable[key]; !ok {
 			delete(e.states, key)
+			if e.Metrics != nil {
+				// Retire the per-proxy series with the state, or series
+				// for deleted proxies leak forever.
+				e.Metrics.ForgetProxy(key.String())
+			}
 		}
 	}
 }
@@ -252,6 +269,10 @@ func newState(p *crawlv1alpha1.Proxy, now time.Time, interval time.Duration) *st
 // threshold-crossing flip, or a material latency change (beyond
 // max(LatencyFloor, 50% of reported) and rate-limited by MinReportInterval).
 func (e *Engine) record(job probeJob, res probeResult, now time.Time) {
+	if e.Metrics != nil {
+		e.Metrics.ObserveProbe(job.key.String(), res.latency, res.ok)
+	}
+
 	e.mu.Lock()
 	defer e.mu.Unlock()
 

internal/lease/store.go

@@ -0,0 +1,319 @@
+// Package lease implements the in-memory lease store behind the discovery
+// API: TTL-based proxy assignment with server-side usage tracking and
+// per-(proxy, target) cooldowns. Accepted prototype limitation, documented
+// in the README: state is per-process, so an operator restart drops all
+// leases and cooldowns — clients must tolerate a lease vanishing (their
+// requests still work; they just re-lease).
+package lease
+
+import (
+	"cmp"
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Result is a client's report of how a leased proxy behaved against a
+// target. ResultRateLimited and ResultBanned record a cooldown; ResultOK is
+// an acknowledgement and records nothing.
+type Result string
+
+const (
+	ResultOK          Result = "ok"
+	ResultRateLimited Result = "rate_limited"
+	ResultBanned      Result = "banned"
+)
+
+// ParseResult maps a wire value to a Result; ok is false for anything
+// unknown, which the API layer turns into a 400.
+func ParseResult(s string) (Result, bool) {
+	switch r := Result(s); r {
+	case ResultOK, ResultRateLimited, ResultBanned:
+		return r, true
+	default:
+		return "", false
+	}
+}
+
+var (
+	// ErrNoMatch means no candidate could take a lease; AcquireStats says
+	// why, and the API layer turns both into the 409 body.
+	ErrNoMatch = errors.New("lease: no candidate available")
+	// ErrUnknownLease means the lease ID does not resolve (404). Reports on
+	// recently expired leases do NOT hit this — see the retention note on
+	// Store.
+	ErrUnknownLease = errors.New("lease: unknown lease id")
+)
+
+// Candidate is one leasable proxy as seen by the caller at selection time.
+// The store itself knows nothing about Proxy objects — the discovery layer
+// filters for health/attributes and passes what selection needs.
+type Candidate struct {
+	// Proxy is the opaque proxy key ("namespace/name").
+	Proxy string
+	// MaxLeases caps concurrent leases; 0 means unleasable.
+	MaxLeases int32
+	// Latency is the proxy's last reported latency, used as the tie-break.
+	Latency time.Duration
+}
+
+// Lease is a granted assignment. Values returned by the store are copies;
+// mutating them does not affect the store.
+type Lease struct {
+	ID        string
+	Proxy     string
+	Target    string
+	ExpiresAt time.Time
+}
+
+// AcquireRequest carries the candidate set and lease parameters. Acquire
+// deliberately takes the whole candidate set, not a pre-chosen proxy:
+// selection and insertion must happen under one lock, or two concurrent
+// requests both see "3 of 5 used" and overcommit.
+type AcquireRequest struct {
+	Candidates []Candidate
+	// Target scopes the cooldown check; empty means the global pool.
+	Target string
+	TTL    time.Duration
+}
+
+// AcquireStats explains an ErrNoMatch (and is returned on success too):
+// every candidate is either leased, at capacity, or in cooldown.
+type AcquireStats struct {
+	Considered int
+	AtCapacity int
+	InCooldown int
+}
+
+type cooldownKey struct{ proxy, target string }
+
+// Store is the in-memory lease store. One mutex guards everything: at tens
+// of proxies and human-rate QPS, sharding would be premature complexity.
+//
+// Retention: an expired lease is kept for CooldownWindow past its TTL so a
+// Report arriving just after expiry still resolves — which matters most
+// exactly when a proxy is being rate-limited. Acquire and the counts ignore
+// retained leases; only the sweep finally drops them.
+type Store struct {
+	// CooldownWindow is how long a reported proxy/target pair is excluded
+	// from selection (default 15m; --lease-cooldown in Step 10).
+	CooldownWindow time.Duration
+	// SweepInterval is how often the expiry sweep runs (default 30s).
+	SweepInterval time.Duration
+
+	now func() time.Time
+
+	mu        sync.Mutex
+	byID      map[string]*Lease
+	byProxy   map[string]map[string]*Lease
+	cooldowns map[cooldownKey]time.Time
+}
+
+// NewStore returns a ready Store. A non-positive cooldownWindow selects the
+// 15-minute default.
+func NewStore(cooldownWindow time.Duration) *Store {
+	if cooldownWindow <= 0 {
+		cooldownWindow = 15 * time.Minute
+	}
+	return &Store{
+		CooldownWindow: cooldownWindow,
+		SweepInterval:  30 * time.Second,
+		now:            time.Now,
+		byID:           map[string]*Lease{},
+		byProxy:        map[string]map[string]*Lease{},
+		cooldowns:      map[cooldownKey]time.Time{},
+	}
+}
+
+// Acquire selects the least-loaded eligible candidate (ties: lowest
+// latency, then name, so selection is deterministic and testable) and
+// grants a lease on it.
+func (s *Store) Acquire(_ context.Context, req AcquireRequest) (*Lease, AcquireStats, error) {
+	stats := AcquireStats{Considered: len(req.Candidates)}
+	if req.TTL <= 0 {
+		return nil, stats, fmt.Errorf("lease: non-positive TTL %v", req.TTL)
+	}
+	now := s.now()
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	type eligible struct {
+		cand   Candidate
+		active int
+	}
+	var elig []eligible
+	for _, c := range req.Candidates {
+		if s.inCooldownLocked(c.Proxy, req.Target, now) {
+			stats.InCooldown++
+			continue
+		}
+		active := s.activeCountLocked(c.Proxy, now)
+		if int32(active) >= c.MaxLeases {
+			stats.AtCapacity++
+			continue
+		}
+		elig = append(elig, eligible{cand: c, active: active})
+	}
+	if len(elig) == 0 {
+		return nil, stats, ErrNoMatch
+	}
+
+	slices.SortFunc(elig, func(a, b eligible) int {
+		if c := cmp.Compare(a.active, b.active); c != 0 {
+			return c
+		}
+		if c := cmp.Compare(a.cand.Latency, b.cand.Latency); c != 0 {
+			return c
+		}
+		return strings.Compare(a.cand.Proxy, b.cand.Proxy)
+	})
+
+	l := &Lease{
+		ID:        rand.Text(),
+		Proxy:     elig[0].cand.Proxy,
+		Target:    req.Target,
+		ExpiresAt: now.Add(req.TTL),
+	}
+	s.byID[l.ID] = l
+	if s.byProxy[l.Proxy] == nil {
+		s.byProxy[l.Proxy] = map[string]*Lease{}
+	}
+	s.byProxy[l.Proxy][l.ID] = l
+
+	granted := *l
+	return &granted, stats, nil
+}
+
+// Release drops a lease early. Idempotent: releasing an unknown or already
+// expired lease is a no-op, so the API's DELETE can always answer 204.
+func (s *Store) Release(_ context.Context, id string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.dropLocked(id)
+}
+
+// Report records the outcome of using a lease. Rate-limited and banned
+// results put the (proxy, target) pair in cooldown — target taken from the
+// report, falling back to the lease's own target, falling back to the
+// global pool. Reports on recently expired leases still resolve (see the
+// retention note on Store).
+func (s *Store) Report(_ context.Context, id string, result Result, target string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	l, ok := s.byID[id]
+	if !ok {
+		return ErrUnknownLease
+	}
+	if result == ResultOK {
+		return nil
+	}
+	if target == "" {
+		target = l.Target
+	}
+	s.cooldowns[cooldownKey{proxy: l.Proxy, target: target}] = s.now().Add(s.CooldownWindow)
+	return nil
+}
+
+// ActiveCount returns the number of unexpired leases held on one proxy.
+func (s *Store) ActiveCount(proxy string) int {
+	now := s.now()
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.activeCountLocked(proxy, now)
+}
+
+// Counts returns the active-lease count per proxy, for the discovery list
+// endpoint and the metrics collector. Proxies with no active leases are
+// absent from the map.
+func (s *Store) Counts() map[string]int {
+	now := s.now()
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	counts := make(map[string]int, len(s.byProxy))
+	for proxy := range s.byProxy {
+		if n := s.activeCountLocked(proxy, now); n > 0 {
+			counts[proxy] = n
+		}
+	}
+	return counts
+}
+
+// Start runs the expiry sweep until ctx ends; it satisfies
+// manager.Runnable so cmd/main.go can mgr.Add the store directly.
+func (s *Store) Start(ctx context.Context) error {
+	ticker := time.NewTicker(s.SweepInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			s.sweep(s.now())
+		}
+	}
+}
+
+// NeedLeaderElection is false: lease state is per-process and the discovery
+// API serves wherever this process runs, so the sweep must run there too.
+func (s *Store) NeedLeaderElection() bool { return false }
+
+// sweep drops leases past their retention window and elapsed cooldowns.
+// Correctness never depends on sweep timing — every read path checks
+// expiry against the clock — so this is purely garbage collection.
+func (s *Store) sweep(now time.Time) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for id, l := range s.byID {
+		if now.After(l.ExpiresAt.Add(s.CooldownWindow)) {
+			s.dropLocked(id)
+		}
+	}
+	for k, until := range s.cooldowns {
+		if now.After(until) {
+			delete(s.cooldowns, k)
+		}
+	}
+}
+
+func (s *Store) dropLocked(id string) {
+	l, ok := s.byID[id]
+	if !ok {
+		return
+	}
+	delete(s.byID, id)
+	delete(s.byProxy[l.Proxy], id)
+	if len(s.byProxy[l.Proxy]) == 0 {
+		delete(s.byProxy, l.Proxy)
+	}
+}
+
+func (s *Store) activeCountLocked(proxy string, now time.Time) int {
+	n := 0
+	for _, l := range s.byProxy[proxy] {
+		if now.Before(l.ExpiresAt) {
+			n++
+		}
+	}
+	return n
+}
+
+// inCooldownLocked: the global cooldown (empty target) always applies; a
+// target-scoped cooldown additionally applies to acquisitions for that
+// target. An acquisition without a target sees only the global pool — a
+// proxy rate-limited by one site is still fine for everyone else.
+func (s *Store) inCooldownLocked(proxy, target string, now time.Time) bool {
+	if until, ok := s.cooldowns[cooldownKey{proxy: proxy}]; ok && now.Before(until) {
+		return true
+	}
+	if target == "" {
+		return false
+	}
+	until, ok := s.cooldowns[cooldownKey{proxy: proxy, target: target}]
+	return ok && now.Before(until)
+}

internal/lease/store_test.go

@@ -0,0 +1,365 @@
+package lease
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+)
+
+// fakeClock is an injectable, manually advanced clock.
+type fakeClock struct {
+	mu  sync.Mutex
+	cur time.Time
+}
+
+func newFakeClock() *fakeClock {
+	return &fakeClock{cur: time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC)}
+}
+
+func (c *fakeClock) Now() time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.cur
+}
+
+func (c *fakeClock) Advance(d time.Duration) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.cur = c.cur.Add(d)
+}
+
+func newTestStore() (*Store, *fakeClock) {
+	s := NewStore(15 * time.Minute)
+	clock := newFakeClock()
+	s.now = clock.Now
+	return s, clock
+}
+
+func candidate(proxy string, maxLeases int32, latency time.Duration) Candidate {
+	return Candidate{Proxy: proxy, MaxLeases: maxLeases, Latency: latency}
+}
+
+func mustAcquire(t *testing.T, s *Store, req AcquireRequest) *Lease {
+	t.Helper()
+	l, _, err := s.Acquire(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Acquire: %v", err)
+	}
+	return l
+}
+
+func TestAcquire_capacity(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 2, 0)}, TTL: time.Minute}
+
+	l1 := mustAcquire(t, s, req)
+	l2 := mustAcquire(t, s, req)
+	if l1.ID == l2.ID {
+		t.Fatal("two leases share an ID")
+	}
+	if got := s.ActiveCount("ns/p1"); got != 2 {
+		t.Fatalf("ActiveCount = %d, want 2", got)
+	}
+
+	_, stats, err := s.Acquire(context.Background(), req)
+	if !errors.Is(err, ErrNoMatch) {
+		t.Fatalf("third acquire error = %v, want ErrNoMatch", err)
+	}
+	want := AcquireStats{Considered: 1, AtCapacity: 1}
+	if stats != want {
+		t.Errorf("stats = %+v, want %+v", stats, want)
+	}
+
+	// Early release frees the slot again.
+	s.Release(context.Background(), l1.ID)
+	mustAcquire(t, s, req)
+}
+
+func TestAcquire_maxLeasesZeroIsUnleasable(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+		Candidates: []Candidate{candidate("ns/p1", 0, 0)},
+		TTL:        time.Minute,
+	})
+	if !errors.Is(err, ErrNoMatch) {
+		t.Fatalf("err = %v, want ErrNoMatch", err)
+	}
+	if stats.AtCapacity != 1 {
+		t.Errorf("stats = %+v, want the unleasable proxy counted AtCapacity", stats)
+	}
+}
+
+func TestAcquire_selectionOrder(t *testing.T) {
+	t.Parallel()
+
+	t.Run("least loaded wins", func(t *testing.T) {
+		t.Parallel()
+		s, _ := newTestStore()
+		mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{candidate("ns/a", 5, 10*time.Millisecond)}, TTL: time.Minute,
+		})
+		l := mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{
+				candidate("ns/a", 5, 10*time.Millisecond), // 1 active, lower latency
+				candidate("ns/b", 5, 90*time.Millisecond), // 0 active
+			},
+			TTL: time.Minute,
+		})
+		if l.Proxy != "ns/b" {
+			t.Errorf("chose %s, want the least-loaded ns/b", l.Proxy)
+		}
+	})
+
+	t.Run("latency breaks the load tie", func(t *testing.T) {
+		t.Parallel()
+		s, _ := newTestStore()
+		l := mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{
+				candidate("ns/a", 5, 90*time.Millisecond),
+				candidate("ns/b", 5, 10*time.Millisecond),
+			},
+			TTL: time.Minute,
+		})
+		if l.Proxy != "ns/b" {
+			t.Errorf("chose %s, want the lower-latency ns/b", l.Proxy)
+		}
+	})
+
+	t.Run("name breaks a full tie deterministically", func(t *testing.T) {
+		t.Parallel()
+		s, _ := newTestStore()
+		l := mustAcquire(t, s, AcquireRequest{
+			Candidates: []Candidate{
+				candidate("ns/b", 5, 10*time.Millisecond),
+				candidate("ns/a", 5, 10*time.Millisecond),
+			},
+			TTL: time.Minute,
+		})
+		if l.Proxy != "ns/a" {
+			t.Errorf("chose %s, want ns/a (lexicographic tie-break)", l.Proxy)
+		}
+	})
+}
+
+func TestAcquire_cooldownScoping(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, "example.com"); err != nil {
+		t.Fatalf("Report: %v", err)
+	}
+
+	// Same target: excluded.
+	_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+		Candidates: cands, Target: "example.com", TTL: time.Minute,
+	})
+	if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
+		t.Errorf("same-target acquire = (%v, %+v), want ErrNoMatch with InCooldown=1", err, stats)
+	}
+
+	// Different target: fine.
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "other.org", TTL: time.Minute})
+
+	// No target (global pool): a target-scoped cooldown does not apply.
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+}
+
+func TestAcquire_globalCooldownBlocksEverything(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	// A lease without a target, reported banned without a target: the
+	// cooldown lands on the global pool.
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+	if err := s.Report(context.Background(), l.ID, ResultBanned, ""); err != nil {
+		t.Fatalf("Report: %v", err)
+	}
+
+	for _, target := range []string{"", "example.com"} {
+		_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+			Candidates: cands, Target: target, TTL: time.Minute,
+		})
+		if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
+			t.Errorf("acquire(target=%q) = (%v, %+v), want global cooldown to block", target, err, stats)
+		}
+	}
+}
+
+func TestAcquire_cooldownExpires(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); err != nil {
+		t.Fatalf("Report: %v", err)
+	}
+	if _, _, err := s.Acquire(context.Background(), AcquireRequest{Candidates: cands, TTL: time.Minute}); !errors.Is(err, ErrNoMatch) {
+		t.Fatal("expected cooldown to block immediately after the report")
+	}
+
+	clock.Advance(15*time.Minute + time.Second)
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, TTL: time.Minute})
+}
+
+func TestExpiry_freesCapacityWithoutSweep(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 1, 0)}, TTL: time.Minute}
+
+	mustAcquire(t, s, req)
+	if _, _, err := s.Acquire(context.Background(), req); !errors.Is(err, ErrNoMatch) {
+		t.Fatal("capacity 1 not enforced")
+	}
+
+	clock.Advance(2 * time.Minute)
+	// No sweep has run; expiry must still free capacity and zero the counts.
+	if got := s.ActiveCount("ns/p1"); got != 0 {
+		t.Fatalf("ActiveCount after TTL = %d, want 0", got)
+	}
+	if counts := s.Counts(); len(counts) != 0 {
+		t.Fatalf("Counts after TTL = %v, want empty", counts)
+	}
+	mustAcquire(t, s, req)
+}
+
+func TestReport_expiredButRetainedLease(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+
+	// TTL lapses; the report arrives late — exactly when the proxy is being
+	// rate-limited, which is when the cooldown matters most.
+	clock.Advance(5 * time.Minute)
+	s.sweep(clock.Now())
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); err != nil {
+		t.Fatalf("Report on an expired-but-retained lease: %v", err)
+	}
+	// The cooldown fell back to the lease's own target.
+	_, stats, err := s.Acquire(context.Background(), AcquireRequest{
+		Candidates: cands, Target: "example.com", TTL: time.Minute,
+	})
+	if !errors.Is(err, ErrNoMatch) || stats.InCooldown != 1 {
+		t.Errorf("acquire = (%v, %+v), want cooldown from the late report", err, stats)
+	}
+
+	// Past the retention window the sweep finally drops it.
+	clock.Advance(15 * time.Minute)
+	s.sweep(clock.Now())
+	if err := s.Report(context.Background(), l.ID, ResultRateLimited, ""); !errors.Is(err, ErrUnknownLease) {
+		t.Errorf("Report after retention = %v, want ErrUnknownLease", err)
+	}
+}
+
+func TestReport_okRecordsNothing(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	cands := []Candidate{candidate("ns/p1", 5, 0)}
+	l := mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+
+	if err := s.Report(context.Background(), l.ID, ResultOK, "example.com"); err != nil {
+		t.Fatalf("Report(ok): %v", err)
+	}
+	mustAcquire(t, s, AcquireRequest{Candidates: cands, Target: "example.com", TTL: time.Minute})
+}
+
+func TestRelease_isIdempotent(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	l := mustAcquire(t, s, AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 1, 0)}, TTL: time.Minute})
+
+	s.Release(context.Background(), l.ID)
+	s.Release(context.Background(), l.ID)
+	s.Release(context.Background(), "never-existed")
+	if got := s.ActiveCount("ns/p1"); got != 0 {
+		t.Errorf("ActiveCount = %d, want 0", got)
+	}
+}
+
+func TestParseResult(t *testing.T) {
+	t.Parallel()
+	for _, valid := range []string{"ok", "rate_limited", "banned"} {
+		if _, ok := ParseResult(valid); !ok {
+			t.Errorf("ParseResult(%q) rejected a valid value", valid)
+		}
+	}
+	for _, invalid := range []string{"", "OK", "throttled", "rate-limited"} {
+		if _, ok := ParseResult(invalid); ok {
+			t.Errorf("ParseResult(%q) accepted an invalid value", invalid)
+		}
+	}
+}
+
+func TestAcquire_concurrentNeverOvercommits(t *testing.T) {
+	t.Parallel()
+	s, _ := newTestStore()
+	req := AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 5, 0)}, TTL: time.Minute}
+
+	const attempts = 40
+	var wg sync.WaitGroup
+	granted := make(chan *Lease, attempts)
+	for range attempts {
+		wg.Go(func() {
+			if l, _, err := s.Acquire(context.Background(), req); err == nil {
+				granted <- l
+			}
+		})
+	}
+	wg.Wait()
+	close(granted)
+
+	var n int
+	for range granted {
+		n++
+	}
+	if n != 5 {
+		t.Errorf("%d of %d concurrent acquires granted, want exactly MaxLeases=5", n, attempts)
+	}
+	if got := s.ActiveCount("ns/p1"); got != 5 {
+		t.Errorf("ActiveCount = %d, want 5", got)
+	}
+}
+
+func TestStart_sweepsAndStops(t *testing.T) {
+	t.Parallel()
+	s, clock := newTestStore()
+	s.SweepInterval = time.Millisecond
+
+	l := mustAcquire(t, s, AcquireRequest{Candidates: []Candidate{candidate("ns/p1", 5, 0)}, TTL: time.Minute})
+	clock.Advance(20 * time.Minute) // past TTL + retention
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- s.Start(ctx) }()
+
+	deadline := time.After(5 * time.Second)
+	for {
+		if err := s.Report(context.Background(), l.ID, ResultOK, ""); errors.Is(err, ErrUnknownLease) {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("sweep never dropped the lease")
+		case <-time.After(5 * time.Millisecond):
+		}
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("Start returned %v, want nil", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not stop on cancel")
+	}
+}

internal/metrics/metrics.go

@@ -0,0 +1,119 @@
+// Package metrics defines the operator's Prometheus metrics. Nothing here
+// registers itself — no init(), per house rules — the composition root
+// calls Register explicitly, which also lets every test use a fresh
+// registry.
+package metrics
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metrics holds the vector metrics the operator's components feed. The
+// consuming packages (health, discovery, provider) each define their own
+// small recorder interface; *Metrics satisfies all of them structurally,
+// so none of them import this package's prometheus surface.
+type Metrics struct {
+	healthcheckDuration *prometheus.HistogramVec
+	healthcheckFailures *prometheus.CounterVec
+	leaseRequests       *prometheus.CounterVec
+	providerRequests    *prometheus.CounterVec
+}
+
+// New builds the metric set, unregistered.
+func New() *Metrics {
+	return &Metrics{
+		healthcheckDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "proxy_operator_healthcheck_duration_seconds",
+			Help:    "Duration of through-the-proxy health probes.",
+			Buckets: prometheus.DefBuckets,
+		}, []string{"proxy"}),
+		healthcheckFailures: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_healthcheck_failures_total",
+			Help: "Failed health probes.",
+		}, []string{"proxy"}),
+		leaseRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_lease_requests_total",
+			Help: "Lease acquisition requests by outcome.",
+		}, []string{"outcome"}),
+		providerRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "proxy_operator_provider_requests_total",
+			Help: "Provider API calls by operation and classified result.",
+		}, []string{"provider", "op", "result"}),
+	}
+}
+
+// Register registers the vectors plus the two scrape-time collectors.
+// proxyPhases and activeLeases are read at every scrape: gauges derived
+// from reconcile-time increments inevitably drift and leak series on
+// delete; reading the source of truth cannot.
+func (m *Metrics) Register(reg prometheus.Registerer, proxyPhases func() map[string]int, activeLeases func() int) error {
+	collectors := []prometheus.Collector{
+		m.healthcheckDuration,
+		m.healthcheckFailures,
+		m.leaseRequests,
+		m.providerRequests,
+		&constCollector{
+			desc: prometheus.NewDesc("proxy_operator_proxies",
+				"Proxy objects by phase.", []string{"phase"}, nil),
+			read: proxyPhases,
+		},
+		&constCollector{
+			desc: prometheus.NewDesc("proxy_operator_leases_active",
+				"Currently active leases.", nil, nil),
+			read: func() map[string]int { return map[string]int{"": activeLeases()} },
+		},
+	}
+	for _, c := range collectors {
+		if err := reg.Register(c); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ObserveProbe records one health probe. Called on every probe — metrics
+// are the home for high-frequency signal that must never touch status.
+func (m *Metrics) ObserveProbe(proxy string, latency time.Duration, success bool) {
+	m.healthcheckDuration.WithLabelValues(proxy).Observe(latency.Seconds())
+	if !success {
+		m.healthcheckFailures.WithLabelValues(proxy).Inc()
+	}
+}
+
+// ForgetProxy drops the per-proxy series when the health engine prunes its
+// state — without this, series for deleted proxies leak forever.
+func (m *Metrics) ForgetProxy(proxy string) {
+	m.healthcheckDuration.DeleteLabelValues(proxy)
+	m.healthcheckFailures.DeleteLabelValues(proxy)
+}
+
+// LeaseRequest records a lease acquisition outcome ("granted"|"no_match").
+func (m *Metrics) LeaseRequest(outcome string) {
+	m.leaseRequests.WithLabelValues(outcome).Inc()
+}
+
+// ProviderRequest records one provider API call with its classified result.
+func (m *Metrics) ProviderRequest(provider, op, result string) {
+	m.providerRequests.WithLabelValues(provider, op, result).Inc()
+}
+
+// constCollector reads a label→value map at scrape time and emits one
+// gauge sample per entry. An empty-string label key means "no labels".
+type constCollector struct {
+	desc *prometheus.Desc
+	read func() map[string]int
+}
+
+func (c *constCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.desc }
+
+func (c *constCollector) Collect(ch chan<- prometheus.Metric) {
+	for label, value := range c.read() {
+		if label == "" {
+			ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(value))
+			continue
+		}
+		ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(value), label)
+	}
+}

internal/metrics/metrics_test.go

@@ -0,0 +1,113 @@
+package metrics
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+// register wires a fresh registry — the reason Register exists instead of
+// init()-time self-registration.
+func register(t *testing.T, m *Metrics, phases map[string]int, active int) *prometheus.Registry {
+	t.Helper()
+	reg := prometheus.NewRegistry()
+	err := m.Register(reg,
+		func() map[string]int { return phases },
+		func() int { return active },
+	)
+	if err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+	return reg
+}
+
+func TestRegister_scrapeTimeCollectors(t *testing.T) {
+	t.Parallel()
+	m := New()
+	reg := register(t, m, map[string]int{"Ready": 3, "Provisioning": 1}, 7)
+
+	expected := `
+# HELP proxy_operator_leases_active Currently active leases.
+# TYPE proxy_operator_leases_active gauge
+proxy_operator_leases_active 7
+# HELP proxy_operator_proxies Proxy objects by phase.
+# TYPE proxy_operator_proxies gauge
+proxy_operator_proxies{phase="Provisioning"} 1
+proxy_operator_proxies{phase="Ready"} 3
+`
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(expected),
+		"proxy_operator_proxies", "proxy_operator_leases_active"); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestObserveProbe_andForget(t *testing.T) {
+	t.Parallel()
+	m := New()
+	reg := register(t, m, nil, 0)
+
+	m.ObserveProbe("default/p1", 30*time.Millisecond, true)
+	m.ObserveProbe("default/p1", 40*time.Millisecond, false)
+	m.ObserveProbe("default/p2", 10*time.Millisecond, true)
+
+	if got := testutil.CollectAndCount(m.healthcheckDuration); got != 2 {
+		t.Errorf("duration series = %d, want 2 (one per proxy)", got)
+	}
+	if got := testutil.ToFloat64(m.healthcheckFailures.WithLabelValues("default/p1")); got != 1 {
+		t.Errorf("p1 failures = %v, want 1 (only the failed probe)", got)
+	}
+
+	m.ForgetProxy("default/p1")
+	if got := testutil.CollectAndCount(m.healthcheckDuration); got != 1 {
+		t.Errorf("duration series after ForgetProxy = %d, want 1 — series must not leak", got)
+	}
+	if got := testutil.CollectAndCount(m.healthcheckFailures); got != 0 {
+		t.Errorf("failure series after ForgetProxy = %d, want 0", got)
+	}
+	_ = reg
+}
+
+func TestLeaseRequest(t *testing.T) {
+	t.Parallel()
+	m := New()
+	register(t, m, nil, 0)
+
+	m.LeaseRequest("granted")
+	m.LeaseRequest("granted")
+	m.LeaseRequest("no_match")
+
+	if got := testutil.ToFloat64(m.leaseRequests.WithLabelValues("granted")); got != 2 {
+		t.Errorf("granted = %v, want 2", got)
+	}
+	if got := testutil.ToFloat64(m.leaseRequests.WithLabelValues("no_match")); got != 1 {
+		t.Errorf("no_match = %v, want 1", got)
+	}
+}
+
+func TestProviderRequest(t *testing.T) {
+	t.Parallel()
+	m := New()
+	register(t, m, nil, 0)
+
+	m.ProviderRequest("gcp-eu", "create", "ok")
+	m.ProviderRequest("gcp-eu", "create", "quota_exceeded")
+
+	if got := testutil.ToFloat64(m.providerRequests.WithLabelValues("gcp-eu", "create", "ok")); got != 1 {
+		t.Errorf("ok = %v, want 1", got)
+	}
+	if got := testutil.ToFloat64(m.providerRequests.WithLabelValues("gcp-eu", "create", "quota_exceeded")); got != 1 {
+		t.Errorf("quota_exceeded = %v, want 1", got)
+	}
+}
+
+func TestRegister_freshRegistryPerTest(t *testing.T) {
+	t.Parallel()
+	// Registering the same metric set on two registries must both succeed —
+	// the property init()-style global registration would break.
+	m1, m2 := New(), New()
+	register(t, m1, nil, 0)
+	register(t, m2, nil, 0)
+}

internal/provider/config.go

@@ -21,6 +21,23 @@ type ProviderConfig struct {
 	Type       string            `json:"type"`
 	Kubernetes *KubernetesConfig `json:"kubernetes,omitempty"`
 	GCP        *GCPConfig        `json:"gcp,omitempty"`
+	KVM        *KVMConfig        `json:"kvm,omitempty"`
+}
+
+// setBlocks names the type-specific blocks that are non-nil, so validate can
+// reject a block that doesn't match Type without a pairwise check per type.
+func (p ProviderConfig) setBlocks() []string {
+	var out []string
+	if p.Kubernetes != nil {
+		out = append(out, "kubernetes")
+	}
+	if p.GCP != nil {
+		out = append(out, "gcp")
+	}
+	if p.KVM != nil {
+		out = append(out, "kvm")
+	}
+	return out
 }
 
 // KubernetesConfig configures the kubernetes-pod provider, which creates
@@ -81,22 +98,28 @@ func (c *Config) validate() error {
 		}
 		seen[p.Name] = true
 
+		prefix := fmt.Sprintf("providers[%d] %q", i, p.Name)
 		switch p.Type {
-		case "kubernetes":
+		case "kubernetes", "gcp", "kvm":
-			if p.GCP != nil {
-				return fmt.Errorf("providers[%d] %q: type is kubernetes but a gcp block is set", i, p.Name)
-			}
-		case "gcp":
-			if p.Kubernetes != nil {
-				return fmt.Errorf("providers[%d] %q: type is gcp but a kubernetes block is set", i, p.Name)
-			}
-			if p.GCP == nil || p.GCP.Project == "" {
-				return fmt.Errorf("providers[%d] %q: gcp.project is required", i, p.Name)
-			}
 		case "":
-			return fmt.Errorf("providers[%d] %q: type is required", i, p.Name)
+			return fmt.Errorf("%s: type is required", prefix)
 		default:
-			return fmt.Errorf("providers[%d] %q: unknown provider type %q", i, p.Name, p.Type)
+			return fmt.Errorf("%s: unknown provider type %q", prefix, p.Type)
+		}
+		for _, block := range p.setBlocks() {
+			if block != p.Type {
+				return fmt.Errorf("%s: type is %s but a %s block is set", prefix, p.Type, block)
+			}
+		}
+		switch p.Type {
+		case "gcp":
+			if p.GCP == nil || p.GCP.Project == "" {
+				return fmt.Errorf("%s: gcp.project is required", prefix)
+			}
+		case "kvm":
+			if err := validateKVM(prefix, p.KVM); err != nil {
+				return err
+			}
 		}
 	}
 	return nil

internal/provider/config_kvm.go

@@ -0,0 +1,138 @@
+package provider
+
+import (
+	"fmt"
+	"net/url"
+	"regexp"
+)
+
+// KVMConfig configures a kvm provider instance: one or more libvirt/KVM
+// hypervisors reached over SSH (the libvirt unix socket is forwarded through
+// the SSH connection, so hosts need no libvirtd TCP/TLS listener).
+// placement.zone on a Proxy selects a host by name; with a single host it may
+// be omitted.
+type KVMConfig struct {
+	// SSHKeyFile is the path to the private key used for every host. Required.
+	SSHKeyFile string `json:"sshKeyFile"`
+	// KnownHostsFile is an OpenSSH known_hosts file used to verify host keys.
+	// Required unless InsecureIgnoreHostKey is set.
+	KnownHostsFile string `json:"knownHostsFile,omitempty"`
+	// InsecureIgnoreHostKey skips host-key verification entirely.
+	InsecureIgnoreHostKey bool `json:"insecureIgnoreHostKey,omitempty"`
+	// DiskSizeGB is the capacity of each VM's overlay disk. Default 10. The
+	// guest's cloud-init growpart expands the root filesystem into it.
+	DiskSizeGB int64 `json:"diskSizeGb,omitempty"`
+	// VendorData is cloud-init vendor-data placed on every seed next to the
+	// Proxy's own user-data. Default: installs and starts qemu-guest-agent so
+	// the provider can read the VM's IP even before it shows up in ARP.
+	VendorData string `json:"vendorData,omitempty"`
+	// Images are named base images; placement.image names a key here (or is
+	// an http(s) URL directly). The provider downloads each image into a
+	// host's storage pool on first use and caches it there.
+	Images map[string]KVMImage `json:"images,omitempty"`
+	// MachineTypes are named size profiles; placement.machineType names a
+	// key here. Required, at least one.
+	MachineTypes map[string]KVMMachineType `json:"machineTypes"`
+	// Hosts are the hypervisors. Required, at least one, unique names.
+	Hosts []KVMHost `json:"hosts"`
+}
+
+// KVMImage is one downloadable base image (a distro cloud image with
+// cloud-init preinstalled, e.g. Debian's generic qcow2).
+type KVMImage struct {
+	// URL is where to fetch the image from. Required, http or https.
+	URL string `json:"url"`
+	// SHA256 is the expected hex digest of the file. Optional; when set it is
+	// verified after download and a later change triggers a re-download.
+	SHA256 string `json:"sha256,omitempty"`
+	// Format is the image's disk format, "qcow2" (default) or "raw".
+	Format string `json:"format,omitempty"`
+}
+
+// KVMMachineType is a VM size profile.
+type KVMMachineType struct {
+	VCPUs     int32 `json:"vcpus"`
+	MemoryMiB int64 `json:"memoryMiB"`
+}
+
+// KVMHost is one hypervisor.
+type KVMHost struct {
+	// Name is the placement.zone value that selects this host. Required.
+	Name string `json:"name"`
+	// Address is the SSH endpoint, host:port. Required.
+	Address string `json:"address"`
+	// User is the SSH user; it must be allowed to use the libvirt socket
+	// (typically a member of the libvirt group). Required.
+	User string `json:"user"`
+	// Socket is the libvirt unix socket path on the host.
+	// Default /var/run/libvirt/libvirt-sock.
+	Socket string `json:"socket,omitempty"`
+	// Pool is the libvirt storage pool for base images, overlay disks and
+	// seed ISOs. Default "default".
+	Pool string `json:"pool,omitempty"`
+	// Bridge is the host bridge the VM's NIC attaches to; it must lead to a
+	// network the cluster can reach and that hands out DHCP leases. Required.
+	Bridge string `json:"bridge"`
+}
+
+var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
+
+func validateKVM(prefix string, c *KVMConfig) error {
+	if c == nil {
+		return fmt.Errorf("%s: kvm block is required", prefix)
+	}
+	if c.SSHKeyFile == "" {
+		return fmt.Errorf("%s: kvm.sshKeyFile is required", prefix)
+	}
+	if c.KnownHostsFile == "" && !c.InsecureIgnoreHostKey {
+		return fmt.Errorf("%s: kvm.knownHostsFile is required unless kvm.insecureIgnoreHostKey is true", prefix)
+	}
+	if c.DiskSizeGB < 0 {
+		return fmt.Errorf("%s: kvm.diskSizeGb must not be negative", prefix)
+	}
+	for name, img := range c.Images {
+		p := fmt.Sprintf("%s: kvm.images[%q]", prefix, name)
+		u, err := url.Parse(img.URL)
+		if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
+			return fmt.Errorf("%s.url must be an http(s) URL (got %q)", p, img.URL)
+		}
+		if img.SHA256 != "" && !sha256Hex.MatchString(img.SHA256) {
+			return fmt.Errorf("%s.sha256 must be 64 hex characters", p)
+		}
+		if img.Format != "" && img.Format != "qcow2" && img.Format != "raw" {
+			return fmt.Errorf("%s.format must be qcow2 or raw (got %q)", p, img.Format)
+		}
+	}
+	if len(c.MachineTypes) == 0 {
+		return fmt.Errorf("%s: kvm.machineTypes must define at least one profile", prefix)
+	}
+	for name, mt := range c.MachineTypes {
+		if mt.VCPUs <= 0 || mt.MemoryMiB <= 0 {
+			return fmt.Errorf("%s: kvm.machineTypes[%q] needs vcpus > 0 and memoryMiB > 0", prefix, name)
+		}
+	}
+	if len(c.Hosts) == 0 {
+		return fmt.Errorf("%s: kvm.hosts must list at least one host", prefix)
+	}
+	seen := make(map[string]bool, len(c.Hosts))
+	for i, h := range c.Hosts {
+		p := fmt.Sprintf("%s: kvm.hosts[%d]", prefix, i)
+		if h.Name == "" {
+			return fmt.Errorf("%s: name is required", p)
+		}
+		if seen[h.Name] {
+			return fmt.Errorf("%s: duplicate host name %q", p, h.Name)
+		}
+		seen[h.Name] = true
+		if h.Address == "" {
+			return fmt.Errorf("%s %q: address is required", p, h.Name)
+		}
+		if h.User == "" {
+			return fmt.Errorf("%s %q: user is required", p, h.Name)
+		}
+		if h.Bridge == "" {
+			return fmt.Errorf("%s %q: bridge is required", p, h.Name)
+		}
+	}
+	return nil
+}

internal/provider/config_kvm_test.go

@@ -0,0 +1,208 @@
+package provider
+
+import (
+	"strings"
+	"testing"
+)
+
+const validKVMYAML = `
+providers:
+  - name: kvm-home
+    type: kvm
+    kvm:
+      sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+      knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+      diskSizeGb: 20
+      images:
+        debian-13:
+          url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+          sha256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+        ubuntu:
+          url: http://mirror.local/noble.img
+          format: raw
+      machineTypes:
+        small: {vcpus: 1, memoryMiB: 1024}
+        medium: {vcpus: 2, memoryMiB: 2048}
+      hosts:
+        - name: nas1
+          address: nas1.home.lan:22
+          user: virt
+          bridge: br0
+        - name: lab2
+          address: 10.0.0.2:2222
+          user: root
+          socket: /run/libvirt/virtqemud-sock
+          pool: fast
+          bridge: br1
+`
+
+func TestLoadConfig_kvmValid(t *testing.T) {
+	t.Parallel()
+	cfg, err := LoadConfig([]byte(validKVMYAML))
+	if err != nil {
+		t.Fatalf("LoadConfig() error = %v, want nil", err)
+	}
+	k := cfg.Providers[0].KVM
+	if k == nil {
+		t.Fatal("providers[0].kvm = nil")
+	}
+	if k.SSHKeyFile != "/etc/proxy-operator/kvm/id_ed25519" || k.DiskSizeGB != 20 {
+		t.Errorf("kvm = %+v, want sshKeyFile and diskSizeGb parsed", k)
+	}
+	if got := k.Images["debian-13"].SHA256; !strings.HasPrefix(got, "0123") {
+		t.Errorf("images[debian-13].sha256 = %q", got)
+	}
+	if k.Images["ubuntu"].Format != "raw" {
+		t.Errorf("images[ubuntu].format = %q, want raw", k.Images["ubuntu"].Format)
+	}
+	if mt := k.MachineTypes["medium"]; mt.VCPUs != 2 || mt.MemoryMiB != 2048 {
+		t.Errorf("machineTypes[medium] = %+v, want {2 2048}", mt)
+	}
+	if len(k.Hosts) != 2 || k.Hosts[1].Socket != "/run/libvirt/virtqemud-sock" || k.Hosts[1].Pool != "fast" {
+		t.Errorf("hosts = %+v, want two hosts with lab2's socket and pool parsed", k.Hosts)
+	}
+}
+
+func TestLoadConfig_kvmInsecureHostKeyNeedsNoKnownHosts(t *testing.T) {
+	t.Parallel()
+	yaml := strings.Replace(validKVMYAML,
+		"knownHostsFile: /etc/proxy-operator/kvm/known_hosts",
+		"insecureIgnoreHostKey: true", 1)
+	if _, err := LoadConfig([]byte(yaml)); err != nil {
+		t.Fatalf("LoadConfig() error = %v, want nil", err)
+	}
+}
+
+func TestLoadConfig_kvmInvalid(t *testing.T) {
+	t.Parallel()
+	minimal := func(kvmBody string) string {
+		return "providers:\n  - name: kvm-home\n    type: kvm\n    kvm:\n" + indent(kvmBody, 6)
+	}
+	hostsAndTypes := `
+machineTypes:
+  small: {vcpus: 1, memoryMiB: 1024}
+hosts:
+  - {name: nas1, address: nas1:22, user: virt, bridge: br0}
+`
+	tests := []struct {
+		name       string
+		yaml       string
+		wantErrSub string
+	}{
+		{
+			name:       "missing kvm block",
+			yaml:       "providers:\n  - name: kvm-home\n    type: kvm\n",
+			wantErrSub: "kvm block is required",
+		},
+		{
+			name:       "missing sshKeyFile",
+			yaml:       minimal("knownHostsFile: /kh\n" + hostsAndTypes),
+			wantErrSub: "kvm.sshKeyFile is required",
+		},
+		{
+			name:       "missing knownHostsFile without insecure flag",
+			yaml:       minimal("sshKeyFile: /key\n" + hostsAndTypes),
+			wantErrSub: "kvm.knownHostsFile is required unless",
+		},
+		{
+			name:       "negative disk size",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\ndiskSizeGb: -1\n" + hostsAndTypes),
+			wantErrSub: "diskSizeGb must not be negative",
+		},
+		{
+			name:       "no machine types",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nhosts:\n  - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
+			wantErrSub: "machineTypes must define at least one",
+		},
+		{
+			name:       "machine type with zero memory",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1}\nhosts:\n  - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `machineTypes["small"] needs vcpus > 0 and memoryMiB > 0`,
+		},
+		{
+			name:       "no hosts",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\n"),
+			wantErrSub: "kvm.hosts must list at least one host",
+		},
+		{
+			name:       "duplicate host name",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, user: virt, bridge: br0}\n  - {name: nas1, address: b:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `duplicate host name "nas1"`,
+		},
+		{
+			name:       "host missing address",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, user: virt, bridge: br0}\n"),
+			wantErrSub: "address is required",
+		},
+		{
+			name:       "host missing user",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, bridge: br0}\n"),
+			wantErrSub: "user is required",
+		},
+		{
+			name:       "host missing bridge",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, user: virt}\n"),
+			wantErrSub: "bridge is required",
+		},
+		{
+			name:       "image with non-http url",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: file:///tmp/x.qcow2}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].url must be an http(s) URL`,
+		},
+		{
+			name:       "image with bad sha256",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.qcow2, sha256: abc}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].sha256 must be 64 hex characters`,
+		},
+		{
+			name:       "image with unknown format",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.vmdk, format: vmdk}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].format must be qcow2 or raw`,
+		},
+		{
+			name: "kvm type with gcp block",
+			yaml: `
+providers:
+  - name: p1
+    type: kvm
+    gcp:
+      project: my-project`,
+			wantErrSub: "type is kvm but a gcp block is set",
+		},
+		{
+			name: "gcp type with kvm block",
+			yaml: `
+providers:
+  - name: p1
+    type: gcp
+    gcp:
+      project: my-project
+    kvm:
+      sshKeyFile: /key`,
+			wantErrSub: "type is gcp but a kvm block is set",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			_, err := LoadConfig([]byte(tc.yaml))
+			if err == nil {
+				t.Fatalf("LoadConfig() error = nil, want error containing %q", tc.wantErrSub)
+			}
+			if !strings.Contains(err.Error(), tc.wantErrSub) {
+				t.Errorf("LoadConfig() error = %q, want substring %q", err.Error(), tc.wantErrSub)
+			}
+		})
+	}
+}
+
+func indent(s string, n int) string {
+	pad := strings.Repeat(" ", n)
+	lines := strings.Split(strings.TrimLeft(s, "\n"), "\n")
+	for i, l := range lines {
+		if l != "" {
+			lines[i] = pad + l
+		}
+	}
+	return strings.Join(lines, "\n")
+}

internal/provider/gcp/errors.go

@@ -0,0 +1,81 @@
+package gcp
+
+import (
+	"errors"
+	"net/http"
+	"slices"
+
+	"github.com/go-logr/logr"
+	"google.golang.org/api/googleapi"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// classify maps a GCP API error onto the provider taxonomy:
+// 404 → NotFound; 429 and quota-flavored 403s → QuotaExceeded;
+// 400/401/other 403s → Permanent; 408/5xx and anything unrecognized
+// (network errors, context cancellation) → Transient, because retrying is
+// always safer than latching Failed on an error nobody taught this
+// function to recognize.
+func classify(err error) error {
+	var gerr *googleapi.Error
+	if !errors.As(err, &gerr) {
+		return provider.ErrTransient
+	}
+	switch {
+	case gerr.Code == http.StatusNotFound:
+		return provider.ErrNotFound
+	case gerr.Code == http.StatusTooManyRequests:
+		return provider.ErrQuotaExceeded
+	case gerr.Code == http.StatusForbidden && hasReason(gerr, "quotaExceeded", "rateLimitExceeded"):
+		return provider.ErrQuotaExceeded
+	case gerr.Code == http.StatusBadRequest,
+		gerr.Code == http.StatusUnauthorized,
+		gerr.Code == http.StatusForbidden:
+		return provider.ErrPermanent
+	default:
+		return provider.ErrTransient
+	}
+}
+
+func (p *Provider) wrapErr(op, id string, err error) error {
+	return provider.Wrap(classify(err), op, p.name, id, err)
+}
+
+// logAPIError records the raw googleapi error shape (HTTP status, reasons)
+// at V(1) — classify collapses it onto the coarser provider taxonomy, so
+// this line is the only place the original status survives.
+func logAPIError(log logr.Logger, op string, err error) {
+	if !log.V(1).Enabled() {
+		return
+	}
+	kv := []any{"op", op, "error", err.Error()}
+	var gerr *googleapi.Error
+	if errors.As(err, &gerr) {
+		reasons := make([]string, 0, len(gerr.Errors))
+		for _, item := range gerr.Errors {
+			reasons = append(reasons, item.Reason)
+		}
+		kv = append(kv, "httpStatus", gerr.Code, "reasons", reasons)
+	}
+	log.V(1).Info("GCP API call failed", kv...)
+}
+
+func hasReason(gerr *googleapi.Error, reasons ...string) bool {
+	for _, item := range gerr.Errors {
+		if slices.Contains(reasons, item.Reason) {
+			return true
+		}
+	}
+	return false
+}
+
+func isAlreadyExists(err error) bool {
+	var gerr *googleapi.Error
+	return errors.As(err, &gerr) && gerr.Code == http.StatusConflict
+}
+
+func isNotFound(err error) bool {
+	var gerr *googleapi.Error
+	return errors.As(err, &gerr) && gerr.Code == http.StatusNotFound
+}

internal/provider/gcp/errors_test.go

@@ -0,0 +1,71 @@
+package gcp
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"google.golang.org/api/googleapi"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func gerr(code int, reasons ...string) error {
+	e := &googleapi.Error{Code: code, Message: "boom"}
+	for _, r := range reasons {
+		e.Errors = append(e.Errors, googleapi.ErrorItem{Reason: r})
+	}
+	return e
+}
+
+func TestClassify(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		err  error
+		want error
+	}{
+		{name: "404 is NotFound", err: gerr(404), want: provider.ErrNotFound},
+		{name: "429 is Quota", err: gerr(429), want: provider.ErrQuotaExceeded},
+		{name: "403 quotaExceeded is Quota", err: gerr(403, "quotaExceeded"), want: provider.ErrQuotaExceeded},
+		{name: "403 rateLimitExceeded is Quota", err: gerr(403, "rateLimitExceeded"), want: provider.ErrQuotaExceeded},
+		{name: "403 plain is Permanent", err: gerr(403, "forbidden"), want: provider.ErrPermanent},
+		{name: "400 is Permanent", err: gerr(400), want: provider.ErrPermanent},
+		{name: "401 is Permanent", err: gerr(401), want: provider.ErrPermanent},
+		{name: "408 is Transient", err: gerr(408), want: provider.ErrTransient},
+		{name: "500 is Transient", err: gerr(500), want: provider.ErrTransient},
+		{name: "503 is Transient", err: gerr(503), want: provider.ErrTransient},
+		{name: "409 is Transient (alreadyExists is handled before classify)", err: gerr(409), want: provider.ErrTransient},
+		{name: "plain network error is Transient", err: errors.New("connection reset"), want: provider.ErrTransient},
+		{name: "wrapped googleapi error still classifies", err: fmt.Errorf("calling api: %w", gerr(404)), want: provider.ErrNotFound},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := classify(tc.err); got != tc.want {
+				t.Errorf("classify() = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// The wrapped error must satisfy both halves of the taxonomy contract:
+// errors.Is against the sentinel AND errors.As back to the SDK error.
+func TestWrapErr_isAndAsBothWork(t *testing.T) {
+	t.Parallel()
+	p := &Provider{name: "gcp-eu"}
+	wrapped := p.wrapErr("get", "zones/z/instances/i", gerr(404))
+
+	if !errors.Is(wrapped, provider.ErrNotFound) {
+		t.Error("errors.Is(wrapped, ErrNotFound) = false")
+	}
+	var ge *googleapi.Error
+	if !errors.As(wrapped, &ge) || ge.Code != 404 {
+		t.Error("errors.As back to *googleapi.Error failed")
+	}
+	if provider.Class(wrapped) != provider.ErrNotFound {
+		t.Errorf("Class() = %v, want ErrNotFound", provider.Class(wrapped))
+	}
+}

internal/provider/gcp/gcp.go

@@ -0,0 +1,307 @@
+// Package gcp implements the provider contract on GCP Compute Engine via
+// the modern Cloud Client Library (cloud.google.com/go/compute/apiv1),
+// deliberately restricted to four calls: instances.Insert, Get, Delete,
+// AggregatedList. Operations are fire-and-forget — Operation.Wait is never
+// called; Create/Delete return as soon as the operation is submitted and
+// the reconciler discovers progress by polling Get.
+package gcp
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	compute "cloud.google.com/go/compute/apiv1"
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"github.com/go-logr/logr"
+	"google.golang.org/api/iterator"
+	"google.golang.org/api/option"
+	"google.golang.org/protobuf/proto"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// instancesAPI is the test seam. It deliberately does not mirror the SDK:
+// the SDK's InstancesScopedListPairIterator has an unexported nextFunc, so
+// a fake cannot construct one — the seam flattens AggregatedList to a
+// slice, and returns operations as just their name (the only thing this
+// provider ever uses, since it never waits on them).
+type instancesAPI interface {
+	Insert(ctx context.Context, req *computepb.InsertInstanceRequest) (opName string, err error)
+	Get(ctx context.Context, req *computepb.GetInstanceRequest) (*computepb.Instance, error)
+	Delete(ctx context.Context, req *computepb.DeleteInstanceRequest) (opName string, err error)
+	AggregatedList(ctx context.Context, req *computepb.AggregatedListInstancesRequest) ([]*computepb.Instance, error)
+}
+
+// realInstances adapts *compute.InstancesClient to the seam.
+type realInstances struct {
+	client *compute.InstancesClient
+}
+
+func (r *realInstances) Insert(ctx context.Context, req *computepb.InsertInstanceRequest) (string, error) {
+	op, err := r.client.Insert(ctx, req)
+	if err != nil {
+		return "", err
+	}
+	return op.Name(), nil
+}
+
+func (r *realInstances) Get(ctx context.Context, req *computepb.GetInstanceRequest) (*computepb.Instance, error) {
+	return r.client.Get(ctx, req)
+}
+
+func (r *realInstances) Delete(ctx context.Context, req *computepb.DeleteInstanceRequest) (string, error) {
+	op, err := r.client.Delete(ctx, req)
+	if err != nil {
+		return "", err
+	}
+	return op.Name(), nil
+}
+
+func (r *realInstances) AggregatedList(ctx context.Context, req *computepb.AggregatedListInstancesRequest) ([]*computepb.Instance, error) {
+	it := r.client.AggregatedList(ctx, req)
+	var out []*computepb.Instance
+	for {
+		pair, err := it.Next()
+		if err == iterator.Done {
+			return out, nil
+		}
+		if err != nil {
+			return nil, err
+		}
+		if pair.Value != nil {
+			out = append(out, pair.Value.Instances...)
+		}
+	}
+}
+
+// Provider implements provider.Provider on GCP Compute Engine.
+type Provider struct {
+	name string
+	cfg  provider.GCPConfig
+	api  instancesAPI
+}
+
+// New builds a Provider using Application Default Credentials (workload
+// identity in-cluster, gcloud ADC locally — no key-file plumbing).
+// Deliberately untested: it dials real Google endpoints; everything below
+// it is exercised through newWithAPI.
+func New(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+	return NewWithWireOptions(ctx, pc, WireLogOptions{})
+}
+
+// NewWithWireOptions is New with explicit control over the V(5) wire
+// logging; the injected wire logger surfaces the SDK's HTTP
+// request/response records at V(5). Note option.WithLogger overrides the
+// SDK's own GOOGLE_SDK_GO_LOGGING_LEVEL env var, so --zap-log-level is
+// the only knob.
+func NewWithWireOptions(ctx context.Context, pc provider.ProviderConfig, opts WireLogOptions) (provider.Provider, error) {
+	base := logf.Log.WithName("gcp").WithName("http")
+	if base.V(5).Enabled() {
+		logf.Log.WithName("gcp").Info(
+			"GCP HTTP wire logging active — request payloads include cloud-init user-data",
+			"fullPayloads", opts.FullPayloads)
+	}
+	client, err := compute.NewInstancesRESTClient(ctx, option.WithLogger(wireLogger(base, opts)))
+	if err != nil {
+		return nil, fmt.Errorf("creating GCP instances client: %w", err)
+	}
+	return newWithAPI(pc, &realInstances{client: client}), nil
+}
+
+func newWithAPI(pc provider.ProviderConfig, api instancesAPI) *Provider {
+	cfg := provider.GCPConfig{}
+	if pc.GCP != nil {
+		cfg = *pc.GCP
+	}
+	return &Provider{name: pc.Name, cfg: withDefaults(cfg), api: api}
+}
+
+// logger derives the request-scoped logger from ctx, so provider lines
+// inherit the reconcile context (which Proxy triggered the call).
+func (p *Provider) logger(ctx context.Context) logr.Logger {
+	return logf.FromContext(ctx).WithName("gcp").WithValues("provider", p.name)
+}
+
+// Create submits the insert and returns immediately with the
+// zone-qualified providerID. A 409 alreadyExists is success — the
+// deterministic instance name means a repeat call after a crash found the
+// VM it already created, which is exactly the idempotency the contract
+// demands.
+func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
+	pl := req.Placement
+	if pl.Zone == "" || pl.MachineType == "" || pl.Image == "" {
+		return "", provider.Wrap(provider.ErrPermanent, "create", p.name, "", fmt.Errorf(
+			"gcp requires placement.zone, placement.machineType and placement.image (got zone=%q machineType=%q image=%q)",
+			pl.Zone, pl.MachineType, pl.Image))
+	}
+	log := p.logger(ctx)
+	id := formatProviderID(pl.Zone, req.Name)
+	insertReq := buildInsertRequest(p.cfg, req)
+	// Curated fields only: the request proto embeds the cloud-init
+	// user-data, which may be Secret-sourced and must never reach logs.
+	log.V(2).Info("GCP insert request built",
+		"zone", pl.Zone, "name", req.Name,
+		"network", p.cfg.Network, "networkTag", p.cfg.NetworkTag,
+		"diskSizeGb", p.cfg.DiskSizeGB, "port", req.Port,
+		"labels", insertReq.GetInstanceResource().GetLabels(),
+		"cloudInitBytes", len(req.CloudInit))
+	opName, err := p.api.Insert(ctx, insertReq)
+	switch {
+	case err == nil:
+		log.V(1).Info("GCP instance insert submitted",
+			"zone", pl.Zone, "name", req.Name,
+			"machineType", pl.MachineType, "image", pl.Image, "opName", opName)
+	case isAlreadyExists(err):
+		log.V(1).Info("GCP instance already exists, insert treated as success",
+			"zone", pl.Zone, "name", req.Name)
+	default:
+		logAPIError(log, "create", err)
+		return "", p.wrapErr("create", id, err)
+	}
+	return id, nil
+}
+
+// Get returns the instance state. The providerID carries its own zone, so
+// this stays correct even mid-replacement after a zone edit — re-reading
+// spec.placement.zone would look up the wrong zone exactly then.
+func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instance, error) {
+	zone, name, err := parseProviderID(providerID)
+	if err != nil {
+		return nil, provider.Wrap(provider.ErrPermanent, "get", p.name, providerID, err)
+	}
+	log := p.logger(ctx)
+	inst, err := p.api.Get(ctx, &computepb.GetInstanceRequest{
+		Project:  p.cfg.Project,
+		Zone:     zone,
+		Instance: name,
+	})
+	if err != nil {
+		logAPIError(log, "get", err)
+		return nil, p.wrapErr("get", providerID, err)
+	}
+	out := toInstance(inst, zone)
+	log.V(1).Info("GCP instance fetched",
+		"zone", zone, "name", name,
+		"status", inst.GetStatus(), "state", out.State, "ip", out.IP)
+	return out, nil
+}
+
+// Delete submits the delete and returns; deleting an instance that is
+// already gone is success.
+func (p *Provider) Delete(ctx context.Context, providerID string) error {
+	zone, name, err := parseProviderID(providerID)
+	if err != nil {
+		return provider.Wrap(provider.ErrPermanent, "delete", p.name, providerID, err)
+	}
+	log := p.logger(ctx)
+	opName, err := p.api.Delete(ctx, &computepb.DeleteInstanceRequest{
+		Project:  p.cfg.Project,
+		Zone:     zone,
+		Instance: name,
+	})
+	switch {
+	case err == nil:
+		log.V(1).Info("GCP instance delete submitted",
+			"zone", zone, "name", name, "opName", opName)
+	case isNotFound(err):
+		log.V(1).Info("GCP instance already gone, delete treated as success",
+			"zone", zone, "name", name)
+	default:
+		logAPIError(log, "delete", err)
+		return p.wrapErr("delete", providerID, err)
+	}
+	return nil
+}
+
+// ListByTag sweeps every zone for instances carrying the GC labels.
+// ReturnPartialSuccess matters: without it one unreachable zone fails the
+// entire GC sweep.
+func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
+	log := p.logger(ctx)
+	filter := fmt.Sprintf("labels.%s = %s", provider.LabelManaged, provider.LabelManagedYes)
+	instances, err := p.api.AggregatedList(ctx, &computepb.AggregatedListInstancesRequest{
+		Project:              p.cfg.Project,
+		Filter:               proto.String(filter),
+		ReturnPartialSuccess: proto.Bool(true),
+	})
+	if err != nil {
+		logAPIError(log, "list", err)
+		return nil, p.wrapErr("list", "", err)
+	}
+	log.V(1).Info("GCP instances listed", "filter", filter, "count", len(instances))
+	out := make([]provider.Instance, 0, len(instances))
+	for _, inst := range instances {
+		conv := toInstance(inst, lastPathSegment(inst.GetZone()))
+		out = append(out, *conv)
+		log.V(2).Info("GCP listed instance",
+			"id", conv.ID, "state", conv.State, "uid", conv.UID, "createdAt", conv.CreatedAt)
+	}
+	return out, nil
+}
+
+func toInstance(inst *computepb.Instance, zone string) *provider.Instance {
+	var ip string
+	if nics := inst.GetNetworkInterfaces(); len(nics) > 0 {
+		if acs := nics[0].GetAccessConfigs(); len(acs) > 0 {
+			ip = acs[0].GetNatIP()
+		}
+	}
+	// CreationTimestamp is RFC3339; a parse failure leaves the zero time,
+	// which orphan GC treats as "old" — safe, since a malformed timestamp
+	// never protects a candidate from collection forever.
+	created, _ := time.Parse(time.RFC3339, inst.GetCreationTimestamp())
+	return &provider.Instance{
+		ID:        formatProviderID(zone, inst.GetName()),
+		IP:        ip,
+		State:     mapState(inst.GetStatus(), ip),
+		UID:       inst.GetLabels()[provider.LabelUID],
+		CreatedAt: created,
+	}
+}
+
+// mapState collapses GCP instance statuses onto the provider states. A
+// RUNNING instance without a NatIP maps to Provisioning — an empty IP must
+// never be published as Running. Anything unrecognized maps to Stopped:
+// the reconciler's answer to Stopped is delete-and-recreate, which is
+// always safe for cattle.
+func mapState(status, ip string) provider.InstanceState {
+	switch status {
+	case "PROVISIONING", "STAGING", "REPAIRING":
+		return provider.StateProvisioning
+	case "RUNNING":
+		if ip == "" {
+			return provider.StateProvisioning
+		}
+		return provider.StateRunning
+	case "STOPPING", "STOPPED", "SUSPENDING", "SUSPENDED":
+		return provider.StateStopped
+	case "TERMINATED":
+		return provider.StateTerminated
+	default:
+		return provider.StateStopped
+	}
+}
+
+func formatProviderID(zone, name string) string {
+	return fmt.Sprintf("zones/%s/instances/%s", zone, name)
+}
+
+func parseProviderID(id string) (zone, name string, err error) {
+	parts := strings.Split(id, "/")
+	if len(parts) != 4 || parts[0] != "zones" || parts[2] != "instances" || parts[1] == "" || parts[3] == "" {
+		return "", "", fmt.Errorf("malformed gcp providerID %q, want zones/<zone>/instances/<name>", id)
+	}
+	return parts[1], parts[3], nil
+}
+
+// lastPathSegment extracts the zone name from the URL-style
+// ".../zones/europe-west1-b" the API returns on instances.
+func lastPathSegment(url string) string {
+	if i := strings.LastIndexByte(url, '/'); i >= 0 {
+		return url[i+1:]
+	}
+	return url
+}

internal/provider/gcp/gcp_test.go

@@ -0,0 +1,564 @@
+package gcp
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"strings"
+	"testing"
+	"time"
+
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"github.com/go-logr/logr"
+	"github.com/go-logr/logr/funcr"
+	"google.golang.org/protobuf/proto"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// fakeAPI implements the instancesAPI seam.
+type fakeAPI struct {
+	insertReq *computepb.InsertInstanceRequest
+	insertErr error
+
+	getReq  *computepb.GetInstanceRequest
+	getInst *computepb.Instance
+	getErr  error
+
+	deleteReq *computepb.DeleteInstanceRequest
+	deleteErr error
+
+	listReq   *computepb.AggregatedListInstancesRequest
+	listInsts []*computepb.Instance
+	listErr   error
+}
+
+func (f *fakeAPI) Insert(_ context.Context, req *computepb.InsertInstanceRequest) (string, error) {
+	f.insertReq = req
+	return "op-insert", f.insertErr
+}
+
+func (f *fakeAPI) Get(_ context.Context, req *computepb.GetInstanceRequest) (*computepb.Instance, error) {
+	f.getReq = req
+	return f.getInst, f.getErr
+}
+
+func (f *fakeAPI) Delete(_ context.Context, req *computepb.DeleteInstanceRequest) (string, error) {
+	f.deleteReq = req
+	return "op-delete", f.deleteErr
+}
+
+func (f *fakeAPI) AggregatedList(_ context.Context, req *computepb.AggregatedListInstancesRequest) ([]*computepb.Instance, error) {
+	f.listReq = req
+	return f.listInsts, f.listErr
+}
+
+func newTestProvider(api *fakeAPI) *Provider {
+	return newWithAPI(provider.ProviderConfig{
+		Name: "gcp-eu",
+		Type: "gcp",
+		GCP:  &provider.GCPConfig{Project: "my-project"},
+	}, api)
+}
+
+func TestCreate_returnsZoneQualifiedID(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{}
+	p := newTestProvider(api)
+
+	id, err := p.Create(context.Background(), testCreateRequest())
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if want := "zones/europe-west1-b/instances/proxy-abc123def456ghij"; id != want {
+		t.Errorf("providerID = %s, want %s", id, want)
+	}
+	if api.insertReq.Project != "my-project" || api.insertReq.Zone != "europe-west1-b" {
+		t.Errorf("insert sent to %s/%s, want my-project/europe-west1-b", api.insertReq.Project, api.insertReq.Zone)
+	}
+}
+
+func TestCreate_alreadyExistsIsSuccess(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{insertErr: gerr(409)}
+	p := newTestProvider(api)
+
+	id, err := p.Create(context.Background(), testCreateRequest())
+	if err != nil {
+		t.Fatalf("Create after crash (409): %v — alreadyExists must be success", err)
+	}
+	if want := "zones/europe-west1-b/instances/proxy-abc123def456ghij"; id != want {
+		t.Errorf("providerID = %s, want %s", id, want)
+	}
+}
+
+func TestCreate_incompletePlacementIsPermanent(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{}
+	p := newTestProvider(api)
+	req := testCreateRequest()
+	req.Placement.MachineType = ""
+
+	_, err := p.Create(context.Background(), req)
+	if provider.Class(err) != provider.ErrPermanent {
+		t.Errorf("Class = %v, want ErrPermanent for missing placement", provider.Class(err))
+	}
+	if api.insertReq != nil {
+		t.Error("Insert was called despite invalid placement")
+	}
+}
+
+func TestCreate_quotaErrorClassified(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(&fakeAPI{insertErr: gerr(403, "quotaExceeded")})
+	_, err := p.Create(context.Background(), testCreateRequest())
+	if provider.Class(err) != provider.ErrQuotaExceeded {
+		t.Errorf("Class = %v, want ErrQuotaExceeded", provider.Class(err))
+	}
+}
+
+func TestGet_stateMapping(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name      string
+		status    string
+		natIP     string
+		wantState provider.InstanceState
+		wantIP    string
+	}{
+		{name: "provisioning", status: "PROVISIONING", wantState: provider.StateProvisioning},
+		{name: "staging", status: "STAGING", wantState: provider.StateProvisioning},
+		{name: "repairing", status: "REPAIRING", wantState: provider.StateProvisioning},
+		{name: "running without NatIP stays provisioning", status: "RUNNING", wantState: provider.StateProvisioning},
+		{name: "running with NatIP", status: "RUNNING", natIP: "34.1.2.3", wantState: provider.StateRunning, wantIP: "34.1.2.3"},
+		{name: "stopped", status: "STOPPED", wantState: provider.StateStopped},
+		{name: "suspended", status: "SUSPENDED", wantState: provider.StateStopped},
+		{name: "terminated", status: "TERMINATED", wantState: provider.StateTerminated},
+		{name: "unknown status maps to stopped for recreation", status: "SOMETHING_NEW", wantState: provider.StateStopped},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			inst := &computepb.Instance{
+				Name:              proto.String("proxy-abc"),
+				Status:            proto.String(tc.status),
+				CreationTimestamp: proto.String("2026-08-09T10:00:00+02:00"),
+				Labels: map[string]string{
+					provider.LabelUID: "uid-1",
+				},
+			}
+			if tc.natIP != "" {
+				inst.NetworkInterfaces = []*computepb.NetworkInterface{{
+					AccessConfigs: []*computepb.AccessConfig{{NatIP: proto.String(tc.natIP)}},
+				}}
+			}
+			p := newTestProvider(&fakeAPI{getInst: inst})
+
+			got, err := p.Get(context.Background(), "zones/europe-west1-b/instances/proxy-abc")
+			if err != nil {
+				t.Fatalf("Get: %v", err)
+			}
+			if got.State != tc.wantState || got.IP != tc.wantIP {
+				t.Errorf("state/ip = %s/%q, want %s/%q", got.State, got.IP, tc.wantState, tc.wantIP)
+			}
+			if got.UID != "uid-1" {
+				t.Errorf("UID = %q, want uid-1 (from the GC label)", got.UID)
+			}
+			if got.ID != "zones/europe-west1-b/instances/proxy-abc" {
+				t.Errorf("ID = %s, want the zone-qualified providerID", got.ID)
+			}
+			if got.CreatedAt.IsZero() {
+				t.Error("CreatedAt not parsed from creationTimestamp")
+			}
+		})
+	}
+}
+
+func TestGet_notFound(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(&fakeAPI{getErr: gerr(404)})
+	_, err := p.Get(context.Background(), "zones/z/instances/gone")
+	if !errors.Is(err, provider.ErrNotFound) {
+		t.Errorf("err = %v, want ErrNotFound", err)
+	}
+}
+
+func TestGet_malformedProviderID(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(&fakeAPI{})
+	for _, id := range []string{"", "proxy-abc", "zones//instances/x", "zones/z/instances/", "z/zone/i/name"} {
+		if _, err := p.Get(context.Background(), id); provider.Class(err) != provider.ErrPermanent {
+			t.Errorf("Get(%q): Class = %v, want ErrPermanent", id, provider.Class(err))
+		}
+	}
+}
+
+func TestDelete_notFoundIsSuccess(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{deleteErr: gerr(404)}
+	p := newTestProvider(api)
+	if err := p.Delete(context.Background(), "zones/z/instances/gone"); err != nil {
+		t.Errorf("Delete of missing instance: %v, want nil", err)
+	}
+}
+
+func TestDelete_sendsParsedZoneAndName(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{}
+	p := newTestProvider(api)
+	if err := p.Delete(context.Background(), "zones/us-east1-c/instances/proxy-xyz"); err != nil {
+		t.Fatalf("Delete: %v", err)
+	}
+	if api.deleteReq.Zone != "us-east1-c" || api.deleteReq.Instance != "proxy-xyz" {
+		t.Errorf("delete sent %s/%s, want us-east1-c/proxy-xyz", api.deleteReq.Zone, api.deleteReq.Instance)
+	}
+}
+
+func TestListByTag(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{listInsts: []*computepb.Instance{{
+		Name:              proto.String("proxy-old"),
+		Status:            proto.String("RUNNING"),
+		Zone:              proto.String("https://www.googleapis.com/compute/v1/projects/my-project/zones/europe-west1-b"),
+		CreationTimestamp: proto.String(time.Now().Format(time.RFC3339)),
+		Labels: map[string]string{
+			provider.LabelManaged: provider.LabelManagedYes,
+			provider.LabelUID:     "uid-orphan",
+		},
+		NetworkInterfaces: []*computepb.NetworkInterface{{
+			AccessConfigs: []*computepb.AccessConfig{{NatIP: proto.String("34.9.9.9")}},
+		}},
+	}}}
+	p := newTestProvider(api)
+
+	got, err := p.ListByTag(context.Background())
+	if err != nil {
+		t.Fatalf("ListByTag: %v", err)
+	}
+
+	if want := "labels.proxy-operator-managed = true"; api.listReq.GetFilter() != want {
+		t.Errorf("filter = %q, want %q", api.listReq.GetFilter(), want)
+	}
+	if !api.listReq.GetReturnPartialSuccess() {
+		t.Error("ReturnPartialSuccess not set — one unreachable zone would fail the whole GC sweep")
+	}
+
+	if len(got) != 1 {
+		t.Fatalf("instances = %d, want 1", len(got))
+	}
+	if got[0].ID != "zones/europe-west1-b/instances/proxy-old" {
+		t.Errorf("ID = %s, want the zone parsed out of the URL-style zone field", got[0].ID)
+	}
+	if got[0].UID != "uid-orphan" || got[0].State != provider.StateRunning {
+		t.Errorf("instance = %+v, want uid-orphan/Running", got[0])
+	}
+}
+
+func TestListByTag_errorPropagates(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(&fakeAPI{listErr: gerr(500)})
+	_, err := p.ListByTag(context.Background())
+	if provider.Class(err) != provider.ErrTransient {
+		t.Errorf("Class = %v, want ErrTransient", provider.Class(err))
+	}
+}
+
+func TestParseProviderID_roundTrip(t *testing.T) {
+	t.Parallel()
+	id := formatProviderID("europe-west1-b", "proxy-abc")
+	zone, name, err := parseProviderID(id)
+	if err != nil || zone != "europe-west1-b" || name != "proxy-abc" {
+		t.Errorf("round trip = %s/%s (%v), want europe-west1-b/proxy-abc", zone, name, err)
+	}
+}
+
+// captureContext returns a ctx carrying a funcr logger that records every
+// emitted line, capped at the given verbosity — the test stand-in for
+// --zap-log-level=<verbosity>.
+func captureContext(verbosity int) (context.Context, *[]string) {
+	lines := &[]string{}
+	log := funcr.New(func(prefix, args string) {
+		*lines = append(*lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: verbosity})
+	return logr.NewContext(context.Background(), log), lines
+}
+
+func runningInstance() *computepb.Instance {
+	return &computepb.Instance{
+		Name:              proto.String("proxy-abc123def456ghij"),
+		Status:            proto.String("RUNNING"),
+		Zone:              proto.String("https://www.googleapis.com/compute/v1/projects/my-project/zones/europe-west1-b"),
+		CreationTimestamp: proto.String("2026-08-09T10:00:00+02:00"),
+		Labels: map[string]string{
+			provider.LabelManaged: provider.LabelManagedYes,
+			provider.LabelUID:     "uid-1",
+		},
+		NetworkInterfaces: []*computepb.NetworkInterface{{
+			AccessConfigs: []*computepb.AccessConfig{{NatIP: proto.String("34.1.2.3")}},
+		}},
+	}
+}
+
+func runAllOps(t *testing.T, ctx context.Context, req provider.CreateRequest) {
+	t.Helper()
+	inst := runningInstance()
+	p := newTestProvider(&fakeAPI{getInst: inst, listInsts: []*computepb.Instance{inst}})
+	if _, err := p.Create(ctx, req); err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if _, err := p.Get(ctx, "zones/europe-west1-b/instances/proxy-abc123def456ghij"); err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	if err := p.Delete(ctx, "zones/europe-west1-b/instances/proxy-abc123def456ghij"); err != nil {
+		t.Fatalf("Delete: %v", err)
+	}
+	if _, err := p.ListByTag(ctx); err != nil {
+		t.Fatalf("ListByTag: %v", err)
+	}
+}
+
+func TestLogging_verbosityTiers(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		verbosity   int
+		wantLines   []string
+		absentLines []string
+	}{
+		{
+			name:      "v0 stays silent",
+			verbosity: 0,
+			absentLines: []string{
+				"GCP instance insert submitted",
+				"GCP instance fetched",
+				"GCP instance delete submitted",
+				"GCP instances listed",
+			},
+		},
+		{
+			name:      "v1 logs one line per API call",
+			verbosity: 1,
+			wantLines: []string{
+				`"msg"="GCP instance insert submitted"`,
+				`"opName"="op-insert"`,
+				`"msg"="GCP instance fetched"`,
+				`"status"="RUNNING"`,
+				`"msg"="GCP instance delete submitted"`,
+				`"opName"="op-delete"`,
+				`"msg"="GCP instances listed"`,
+				`"provider"="gcp-eu"`,
+			},
+			absentLines: []string{
+				"GCP insert request built",
+				"GCP listed instance",
+			},
+		},
+		{
+			name:      "v2 adds request and per-instance detail",
+			verbosity: 2,
+			wantLines: []string{
+				`"msg"="GCP insert request built"`,
+				`"cloudInitBytes"=`,
+				`"msg"="GCP listed instance"`,
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			ctx, lines := captureContext(tc.verbosity)
+			req := testCreateRequest()
+			const sentinel = "SENTINEL-cloud-init-must-never-be-logged"
+			req.CloudInit = sentinel
+
+			runAllOps(t, ctx, req)
+
+			joined := strings.Join(*lines, "\n")
+			if tc.verbosity == 0 && len(*lines) != 0 {
+				t.Errorf("verbosity 0 logged %d lines:\n%s", len(*lines), joined)
+			}
+			for _, want := range tc.wantLines {
+				if !strings.Contains(joined, want) {
+					t.Errorf("output missing %q:\n%s", want, joined)
+				}
+			}
+			for _, absent := range tc.absentLines {
+				if strings.Contains(joined, absent) {
+					t.Errorf("output unexpectedly contains %q:\n%s", absent, joined)
+				}
+			}
+			if strings.Contains(joined, sentinel) {
+				t.Errorf("cloud-init content leaked into logs:\n%s", joined)
+			}
+		})
+	}
+}
+
+func TestWireLogger_gatesAtV5(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name      string
+		verbosity int
+		wantDebug bool
+	}{
+		{name: "v5 shows wire records", verbosity: 5, wantDebug: true},
+		{name: "v4 hides wire records", verbosity: 4, wantDebug: false},
+		{name: "v2 hides wire records", verbosity: 2, wantDebug: false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			lines := &[]string{}
+			base := funcr.New(func(prefix, args string) {
+				*lines = append(*lines, prefix+" "+args)
+			}, funcr.Options{Verbosity: tc.verbosity})
+
+			slogger := wireLogger(base, WireLogOptions{})
+			slogger.Debug("api request", "rpcName", "Insert")
+
+			joined := strings.Join(*lines, "\n")
+			if got := strings.Contains(joined, "api request"); got != tc.wantDebug {
+				t.Errorf("Debug record visible = %v, want %v; output:\n%s", got, tc.wantDebug, joined)
+			}
+			if tc.wantDebug && !strings.Contains(joined, "rpcName") {
+				t.Errorf("wire record lost its attrs:\n%s", joined)
+			}
+		})
+	}
+}
+
+func TestWireLogger_infoLandsAtV1(t *testing.T) {
+	t.Parallel()
+	lines := &[]string{}
+	base := funcr.New(func(prefix, args string) {
+		*lines = append(*lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: 1})
+
+	wireLogger(base, WireLogOptions{}).Info("hello")
+
+	if joined := strings.Join(*lines, "\n"); !strings.Contains(joined, "hello") {
+		t.Errorf("slog Info should land at V(1) and be visible at verbosity 1; output:\n%s", joined)
+	}
+}
+
+func captureWireLogger(verbosity int, opts WireLogOptions) (*slog.Logger, *[]string) {
+	lines := &[]string{}
+	base := funcr.New(func(prefix, args string) {
+		*lines = append(*lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: verbosity})
+	return wireLogger(base, opts), lines
+}
+
+func TestWireLogger_dropsNonAPIDebugRecords(t *testing.T) {
+	t.Parallel()
+	slogger, lines := captureWireLogger(9, WireLogOptions{})
+
+	const secret = "assertion=eyJhbGciOiJSUzI1NiJ9.SECRET"
+	slogger.Debug("2LO token request", "request", map[string]any{"payload": secret})
+	slogger.Debug("2LO token response", "response", map[string]any{"payload": "ya29.SECRET-TOKEN"})
+
+	if len(*lines) != 0 {
+		t.Errorf("auth token-exchange records must be dropped; got:\n%s", strings.Join(*lines, "\n"))
+	}
+
+	slogger.Warn("credential refresh failed")
+	if joined := strings.Join(*lines, "\n"); !strings.Contains(joined, "credential refresh failed") {
+		t.Errorf("non-debug SDK records should pass through; output:\n%s", joined)
+	}
+}
+
+func TestWireLogger_elidesLargeFields(t *testing.T) {
+	t.Parallel()
+	slogger, lines := captureWireLogger(9, WireLogOptions{})
+
+	huge := strings.Repeat("x", 4096)
+	slogger.Debug("api response", "response", map[string]any{
+		"status": "200",
+		"payload": map[string]any{
+			"name":  "proxy-abc",
+			"disks": []any{map[string]any{"content": huge}},
+		},
+	})
+
+	joined := strings.Join(*lines, "\n")
+	if strings.Contains(joined, huge[:64]) {
+		t.Errorf("large field not elided:\n%.500s", joined)
+	}
+	if !strings.Contains(joined, "[elided 4096 bytes]") {
+		t.Errorf("elision marker missing:\n%s", joined)
+	}
+	for _, keep := range []string{"proxy-abc", "200", "api response"} {
+		if !strings.Contains(joined, keep) {
+			t.Errorf("small field %q lost during elision:\n%s", keep, joined)
+		}
+	}
+}
+
+func TestWireLogger_fullPayloadsDisablesElision(t *testing.T) {
+	t.Parallel()
+	slogger, lines := captureWireLogger(9, WireLogOptions{FullPayloads: true})
+
+	huge := strings.Repeat("y", 4096)
+	slogger.Debug("api response", "response", map[string]any{"payload": huge})
+
+	joined := strings.Join(*lines, "\n")
+	if !strings.Contains(joined, huge) {
+		t.Errorf("FullPayloads should keep fields verbatim:\n%.200s", joined)
+	}
+
+	slogger.Debug("2LO token response", "response", "ya29.SECRET")
+	if joined := strings.Join(*lines, "\n"); strings.Contains(joined, "ya29.SECRET") {
+		t.Error("auth records must be dropped even with FullPayloads")
+	}
+}
+
+func TestLogging_apiErrorKeepsHTTPDetail(t *testing.T) {
+	t.Parallel()
+	ctx, lines := captureContext(1)
+	p := newTestProvider(&fakeAPI{insertErr: gerr(403, "quotaExceeded")})
+
+	if _, err := p.Create(ctx, testCreateRequest()); err == nil {
+		t.Fatal("Create: want error")
+	}
+
+	joined := strings.Join(*lines, "\n")
+	for _, want := range []string{
+		`"msg"="GCP API call failed"`,
+		`"httpStatus"=403`,
+		`"quotaExceeded"`,
+		`"op"="create"`,
+	} {
+		if !strings.Contains(joined, want) {
+			t.Errorf("output missing %q:\n%s", want, joined)
+		}
+	}
+}
+
+func TestLogging_treatedAsSuccessPathsAreExplicit(t *testing.T) {
+	t.Parallel()
+	ctx, lines := captureContext(1)
+	p := newTestProvider(&fakeAPI{insertErr: gerr(409), deleteErr: gerr(404)})
+
+	if _, err := p.Create(ctx, testCreateRequest()); err != nil {
+		t.Fatalf("Create with 409: %v", err)
+	}
+	if err := p.Delete(ctx, "zones/z/instances/gone"); err != nil {
+		t.Fatalf("Delete with 404: %v", err)
+	}
+
+	joined := strings.Join(*lines, "\n")
+	for _, want := range []string{
+		`"msg"="GCP instance already exists, insert treated as success"`,
+		`"msg"="GCP instance already gone, delete treated as success"`,
+	} {
+		if !strings.Contains(joined, want) {
+			t.Errorf("output missing %q:\n%s", want, joined)
+		}
+	}
+}

internal/provider/gcp/insert.go

@@ -0,0 +1,74 @@
+package gcp
+
+import (
+	"fmt"
+
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"google.golang.org/protobuf/proto"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+const (
+	defaultNetwork    = "default"
+	defaultNetworkTag = "proxy-operator"
+	defaultDiskSizeGB = 10
+	userDataKey       = "user-data"
+)
+
+func withDefaults(cfg provider.GCPConfig) provider.GCPConfig {
+	if cfg.Network == "" {
+		cfg.Network = defaultNetwork
+	}
+	if cfg.NetworkTag == "" {
+		cfg.NetworkTag = defaultNetworkTag
+	}
+	if cfg.DiskSizeGB == 0 {
+		cfg.DiskSizeGB = defaultDiskSizeGB
+	}
+	return cfg
+}
+
+// buildInsertRequest is pure so the field-by-field unit test needs no fake
+// at all — the plan's primary test for this provider.
+func buildInsertRequest(cfg provider.GCPConfig, req provider.CreateRequest) *computepb.InsertInstanceRequest {
+	inst := &computepb.Instance{
+		Name:        proto.String(req.Name),
+		MachineType: proto.String(fmt.Sprintf("zones/%s/machineTypes/%s", req.Placement.Zone, req.Placement.MachineType)),
+		Disks: []*computepb.AttachedDisk{{
+			Boot:       proto.Bool(true),
+			AutoDelete: proto.Bool(true),
+			InitializeParams: &computepb.AttachedDiskInitializeParams{
+				SourceImage: proto.String(req.Placement.Image),
+				DiskSizeGb:  proto.Int64(cfg.DiskSizeGB),
+			},
+		}},
+		NetworkInterfaces: []*computepb.NetworkInterface{{
+			Network: proto.String("global/networks/" + cfg.Network),
+			// An ephemeral external IP: exactly this pair, per the API's
+			// contract for one-to-one NAT.
+			AccessConfigs: []*computepb.AccessConfig{{
+				Name: proto.String("External NAT"),
+				Type: proto.String("ONE_TO_ONE_NAT"),
+			}},
+		}},
+		// The GC contract: every resource this operator creates carries
+		// these two labels, and orphan GC relies on both.
+		Labels: map[string]string{
+			provider.LabelManaged: provider.LabelManagedYes,
+			provider.LabelUID:     req.UID,
+		},
+		Tags: &computepb.Tags{Items: []string{cfg.NetworkTag}},
+	}
+	if req.CloudInit != "" {
+		inst.Metadata = &computepb.Metadata{Items: []*computepb.Items{{
+			Key:   proto.String(userDataKey),
+			Value: proto.String(req.CloudInit),
+		}}}
+	}
+	return &computepb.InsertInstanceRequest{
+		Project:          cfg.Project,
+		Zone:             req.Placement.Zone,
+		InstanceResource: inst,
+	}
+}

internal/provider/gcp/insert_test.go

@@ -0,0 +1,127 @@
+package gcp
+
+import (
+	"testing"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func testCreateRequest() provider.CreateRequest {
+	return provider.CreateRequest{
+		Name:      "proxy-abc123def456ghij",
+		UID:       "11111111-2222-3333-4444-555555555555",
+		Namespace: "default",
+		ProxyName: "eu-proxy-1",
+		Placement: provider.Placement{
+			Zone:        "europe-west1-b",
+			MachineType: "e2-micro",
+			Image:       "projects/debian-cloud/global/images/family/debian-12",
+		},
+		CloudInit: "#cloud-config\npackages: [squid]",
+		Port:      3128,
+	}
+}
+
+func TestBuildInsertRequest_fieldByField(t *testing.T) {
+	t.Parallel()
+	cfg := withDefaults(provider.GCPConfig{Project: "my-project"})
+	req := buildInsertRequest(cfg, testCreateRequest())
+
+	if req.Project != "my-project" || req.Zone != "europe-west1-b" {
+		t.Errorf("project/zone = %s/%s, want my-project/europe-west1-b", req.Project, req.Zone)
+	}
+	inst := req.InstanceResource
+	if inst.GetName() != "proxy-abc123def456ghij" {
+		t.Errorf("name = %s", inst.GetName())
+	}
+	if got, want := inst.GetMachineType(), "zones/europe-west1-b/machineTypes/e2-micro"; got != want {
+		t.Errorf("machineType = %s, want %s", got, want)
+	}
+
+	if len(inst.GetDisks()) != 1 {
+		t.Fatalf("disks = %d, want 1", len(inst.GetDisks()))
+	}
+	disk := inst.GetDisks()[0]
+	if !disk.GetBoot() || !disk.GetAutoDelete() {
+		t.Errorf("boot/autoDelete = %v/%v, want true/true", disk.GetBoot(), disk.GetAutoDelete())
+	}
+	if got, want := disk.GetInitializeParams().GetSourceImage(), "projects/debian-cloud/global/images/family/debian-12"; got != want {
+		t.Errorf("sourceImage = %s, want %s", got, want)
+	}
+	if disk.GetInitializeParams().GetDiskSizeGb() != 10 {
+		t.Errorf("diskSizeGb = %d, want the 10 default", disk.GetInitializeParams().GetDiskSizeGb())
+	}
+
+	if len(inst.GetNetworkInterfaces()) != 1 {
+		t.Fatalf("networkInterfaces = %d, want 1", len(inst.GetNetworkInterfaces()))
+	}
+	nic := inst.GetNetworkInterfaces()[0]
+	if got, want := nic.GetNetwork(), "global/networks/default"; got != want {
+		t.Errorf("network = %s, want %s", got, want)
+	}
+	if len(nic.GetAccessConfigs()) != 1 {
+		t.Fatalf("accessConfigs = %d, want 1", len(nic.GetAccessConfigs()))
+	}
+	ac := nic.GetAccessConfigs()[0]
+	if ac.GetName() != "External NAT" || ac.GetType() != "ONE_TO_ONE_NAT" {
+		t.Errorf("accessConfig = %s/%s, want External NAT/ONE_TO_ONE_NAT", ac.GetName(), ac.GetType())
+	}
+
+	wantLabels := map[string]string{
+		provider.LabelManaged: provider.LabelManagedYes,
+		provider.LabelUID:     "11111111-2222-3333-4444-555555555555",
+	}
+	labels := inst.GetLabels()
+	if len(labels) != len(wantLabels) {
+		t.Errorf("labels = %v, want %v", labels, wantLabels)
+	}
+	for k, v := range wantLabels {
+		if labels[k] != v {
+			t.Errorf("label %s = %q, want %q", k, labels[k], v)
+		}
+	}
+
+	if tags := inst.GetTags().GetItems(); len(tags) != 1 || tags[0] != "proxy-operator" {
+		t.Errorf("tags = %v, want [proxy-operator]", tags)
+	}
+
+	items := inst.GetMetadata().GetItems()
+	if len(items) != 1 || items[0].GetKey() != "user-data" {
+		t.Fatalf("metadata items = %v, want one user-data entry", items)
+	}
+	if items[0].GetValue() != "#cloud-config\npackages: [squid]" {
+		t.Errorf("user-data = %q, want the resolved cloud-init", items[0].GetValue())
+	}
+}
+
+func TestBuildInsertRequest_configOverrides(t *testing.T) {
+	t.Parallel()
+	cfg := withDefaults(provider.GCPConfig{
+		Project:    "my-project",
+		Network:    "crawl-vpc",
+		NetworkTag: "crawl-egress",
+		DiskSizeGB: 42,
+	})
+	req := buildInsertRequest(cfg, testCreateRequest())
+	inst := req.InstanceResource
+
+	if got, want := inst.GetNetworkInterfaces()[0].GetNetwork(), "global/networks/crawl-vpc"; got != want {
+		t.Errorf("network = %s, want %s", got, want)
+	}
+	if tags := inst.GetTags().GetItems(); len(tags) != 1 || tags[0] != "crawl-egress" {
+		t.Errorf("tags = %v, want [crawl-egress]", tags)
+	}
+	if got := inst.GetDisks()[0].GetInitializeParams().GetDiskSizeGb(); got != 42 {
+		t.Errorf("diskSizeGb = %d, want 42", got)
+	}
+}
+
+func TestBuildInsertRequest_noCloudInitMeansNoMetadata(t *testing.T) {
+	t.Parallel()
+	req := testCreateRequest()
+	req.CloudInit = ""
+	built := buildInsertRequest(withDefaults(provider.GCPConfig{Project: "p"}), req)
+	if built.InstanceResource.GetMetadata() != nil {
+		t.Errorf("metadata = %v, want none without cloud-init", built.InstanceResource.GetMetadata())
+	}
+}

internal/provider/gcp/wirelog.go

@@ -0,0 +1,117 @@
+package gcp
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"github.com/go-logr/logr"
+)
+
+// wireLogMaxFieldBytes is the elision threshold for string fields in wire
+// payloads: GCP responses embed multi-KB blobs (Shielded-VM UEFI dbx
+// databases, licenses) that swamp the log line without diagnostic value.
+const wireLogMaxFieldBytes = 1024
+
+// WireLogOptions controls the V(5) HTTP wire logging of the GCP SDK.
+type WireLogOptions struct {
+	// FullPayloads disables field elision and logs payloads verbatim.
+	FullPayloads bool
+}
+
+// wireLogger returns the slog logger handed to the SDK: its Debug-level
+// "api request"/"api response" records (slog Debug = +4 on the logr
+// scale) land at V(5) on top of the base's V(1) shift.
+//
+// Debug records other than the compute client's api request/response are
+// dropped entirely: the same logger propagates into the auth library,
+// whose token-exchange records contain the signed JWT assertion and the
+// bearer access token. Warnings and errors pass through.
+func wireLogger(base logr.Logger, opts WireLogOptions) *slog.Logger {
+	return slog.New(&wireFilterHandler{
+		inner:        logr.ToSlogHandler(base.V(1)),
+		fullPayloads: opts.FullPayloads,
+	})
+}
+
+type wireFilterHandler struct {
+	inner        slog.Handler
+	fullPayloads bool
+}
+
+func (h *wireFilterHandler) Enabled(ctx context.Context, level slog.Level) bool {
+	return h.inner.Enabled(ctx, level)
+}
+
+func (h *wireFilterHandler) Handle(ctx context.Context, rec slog.Record) error {
+	if rec.Level <= slog.LevelDebug && rec.Message != "api request" && rec.Message != "api response" {
+		return nil
+	}
+	if h.fullPayloads {
+		return h.inner.Handle(ctx, rec)
+	}
+	elided := slog.NewRecord(rec.Time, rec.Level, rec.Message, rec.PC)
+	rec.Attrs(func(a slog.Attr) bool {
+		elided.AddAttrs(slog.Attr{Key: a.Key, Value: elideValue(a.Value)})
+		return true
+	})
+	return h.inner.Handle(ctx, elided)
+}
+
+func (h *wireFilterHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
+	return &wireFilterHandler{inner: h.inner.WithAttrs(attrs), fullPayloads: h.fullPayloads}
+}
+
+func (h *wireFilterHandler) WithGroup(name string) slog.Handler {
+	return &wireFilterHandler{inner: h.inner.WithGroup(name), fullPayloads: h.fullPayloads}
+}
+
+func elideValue(v slog.Value) slog.Value {
+	v = v.Resolve()
+	switch v.Kind() {
+	case slog.KindString:
+		if s := v.String(); len(s) > wireLogMaxFieldBytes {
+			return slog.StringValue(elisionMarker(len(s)))
+		}
+		return v
+	case slog.KindGroup:
+		attrs := v.Group()
+		out := make([]slog.Attr, 0, len(attrs))
+		for _, a := range attrs {
+			out = append(out, slog.Attr{Key: a.Key, Value: elideValue(a.Value)})
+		}
+		return slog.GroupValue(out...)
+	case slog.KindAny:
+		return slog.AnyValue(elideAny(v.Any()))
+	default:
+		return v
+	}
+}
+
+func elideAny(v any) any {
+	switch t := v.(type) {
+	case string:
+		if len(t) > wireLogMaxFieldBytes {
+			return elisionMarker(len(t))
+		}
+		return t
+	case map[string]any:
+		out := make(map[string]any, len(t))
+		for k, val := range t {
+			out[k] = elideAny(val)
+		}
+		return out
+	case []any:
+		out := make([]any, len(t))
+		for i, val := range t {
+			out[i] = elideAny(val)
+		}
+		return out
+	default:
+		return v
+	}
+}
+
+func elisionMarker(size int) string {
+	return fmt.Sprintf("[elided %d bytes]", size)
+}

internal/provider/kubernetes/pod.go

@@ -58,10 +58,18 @@ func buildPod(image string, req provider.CreateRequest) *corev1.Pod {
 // there, never interpreted. via/forwarded_for are turned off so the proxy
 // doesn't leak the Pod's identity to the origin.
 func squidConf(port int32) string {
+	// max_filedescriptors is load-bearing in containers: squid sizes its FD
+	// tables from RLIMIT_NOFILE at startup, and containerd commonly sets
+	// that to effectively unlimited (kind: ~10^9) — squid then allocates
+	// gigabytes and is OOM-killed before it ever listens. cache_mem is
+	// trimmed because a forwarding proxy for crawling gains nothing from
+	// squid's 256 MB default cache.
 	return fmt.Sprintf(`http_port %d
 acl all src 0.0.0.0/0
 http_access allow all
 via off
 forwarded_for off
+max_filedescriptors 1024
+cache_mem 16 MB
 `, port)
 }

internal/provider/kubernetes/pod_test.go

@@ -72,7 +72,10 @@ func TestBuildPod_usesRequestPort(t *testing.T) {
 func TestSquidConf_permissive(t *testing.T) {
 	t.Parallel()
 	conf := squidConf(3128)
-	for _, want := range []string{"http_port 3128", "http_access allow all", "via off", "forwarded_for off"} {
+	// max_filedescriptors guards against squid sizing its FD tables from a
+	// container's effectively-unlimited RLIMIT_NOFILE and getting OOM-killed
+	// at startup — found by the kind verification run, must not regress.
+	for _, want := range []string{"http_port 3128", "http_access allow all", "via off", "forwarded_for off", "max_filedescriptors 1024"} {
 		if !strings.Contains(conf, want) {
 			t.Errorf("squidConf() = %q, want it to contain %q", conf, want)
 		}

internal/provider/kvm/conn.go

@@ -0,0 +1,301 @@
+package kvm
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"sync"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+const (
+	defaultHandshakeTimeout = 30 * time.Second
+	keepaliveInterval       = 15 * time.Second
+)
+
+// libvirtAPI is the slice of *libvirt.Libvirt the provider uses, and the
+// test seam: the fake in kvm_test.go implements exactly this.
+type libvirtAPI interface {
+	DomainLookupByName(name string) (libvirt.Domain, error)
+	DomainDefineXML(xml string) (libvirt.Domain, error)
+	DomainCreate(d libvirt.Domain) error
+	DomainDestroy(d libvirt.Domain) error
+	DomainUndefineFlags(d libvirt.Domain, flags libvirt.DomainUndefineFlagsValues) error
+	DomainGetState(d libvirt.Domain, flags uint32) (state int32, reason int32, err error)
+	DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, flags libvirt.DomainModificationImpact) (string, error)
+	DomainInterfaceAddresses(d libvirt.Domain, source uint32, flags uint32) ([]libvirt.DomainInterface, error)
+	ConnectListAllDomains(needResults int32, flags libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error)
+	StoragePoolLookupByName(name string) (libvirt.StoragePool, error)
+	StoragePoolRefresh(pool libvirt.StoragePool, flags uint32) error
+	StorageVolLookupByName(pool libvirt.StoragePool, name string) (libvirt.StorageVol, error)
+	StorageVolGetPath(vol libvirt.StorageVol) (string, error)
+	StorageVolCreateXML(pool libvirt.StoragePool, xml string, flags libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error)
+	StorageVolUpload(vol libvirt.StorageVol, r io.Reader, offset, length uint64, flags libvirt.StorageVolUploadFlags) error
+	StorageVolDownload(vol libvirt.StorageVol, w io.Writer, offset, length uint64, flags libvirt.StorageVolDownloadFlags) error
+	StorageVolDelete(vol libvirt.StorageVol, flags libvirt.StorageVolDeleteFlags) error
+}
+
+// hostClient is one live connection to a hypervisor.
+type hostClient struct {
+	api libvirtAPI
+	// done is closed once the connection is known to be lost. A nil channel
+	// never fires, which suits fakes.
+	done  <-chan struct{}
+	close func()
+}
+
+func (c *hostClient) alive() bool {
+	select {
+	case <-c.done:
+		return false
+	default:
+		return true
+	}
+}
+
+// hostDialer opens a connection to one host. The SSH implementation is the
+// real one; tests inject their own.
+type hostDialer func(ctx context.Context, host provider.KVMHost) (*hostClient, error)
+
+// hostPool caches one connection per host, dialing lazily and redialing
+// after the connection drops. Dials for different hosts don't block each
+// other; concurrent callers for the same host share a single dial.
+type hostPool struct {
+	dial hostDialer
+
+	mu      sync.Mutex
+	entries map[string]*hostEntry
+}
+
+type hostEntry struct {
+	mu     sync.Mutex
+	client *hostClient
+}
+
+func newHostPool(dial hostDialer) *hostPool {
+	return &hostPool{dial: dial, entries: make(map[string]*hostEntry)}
+}
+
+func (p *hostPool) entry(name string) *hostEntry {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	e, ok := p.entries[name]
+	if !ok {
+		e = &hostEntry{}
+		p.entries[name] = e
+	}
+	return e
+}
+
+func (p *hostPool) get(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
+	e := p.entry(host.Name)
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.client != nil {
+		if e.client.alive() {
+			return e.client, nil
+		}
+		e.client.close()
+		e.client = nil
+	}
+	c, err := p.dial(ctx, host)
+	if err != nil {
+		return nil, fmt.Errorf("connecting to host %s (%s): %w", host.Name, host.Address, err)
+	}
+	e.client = c
+	return c, nil
+}
+
+// drop closes and forgets c if it is still the cached client for host, so a
+// connection that was already replaced by a redial is left alone.
+func (p *hostPool) drop(host string, c *hostClient) {
+	e := p.entry(host)
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.client == c {
+		e.client = nil
+	}
+	c.close()
+}
+
+// call runs fn against host's connection. go-libvirt RPCs take no context
+// and block until the daemon answers, so when ctx ends first the connection
+// is closed — which unblocks fn with an error — and ctx.Err() is returned.
+// A transport error (anything that is not a libvirt.Error) also drops the
+// cached connection so the next call redials instead of reusing a dead
+// link.
+func (p *hostPool) call(ctx context.Context, host provider.KVMHost, fn func(api libvirtAPI) error) error {
+	c, err := p.get(ctx, host)
+	if err != nil {
+		return err
+	}
+	err = awaitOrCancel(ctx, func() error { return fn(c.api) }, func() { p.drop(host.Name, c) })
+	if err != nil && !isLibvirtError(err) {
+		p.drop(host.Name, c)
+	}
+	return err
+}
+
+// awaitOrCancel runs fn in the background and returns its result, unless ctx
+// ends first: then abort is invoked (it must make fn return eventually, e.g.
+// by closing the connection fn is blocked on) and ctx.Err() is returned. The
+// abandoned fn's result is discarded.
+func awaitOrCancel(ctx context.Context, fn func() error, abort func()) error {
+	errc := make(chan error, 1)
+	go func() { errc <- fn() }()
+	select {
+	case err := <-errc:
+		return err
+	case <-ctx.Done():
+		abort()
+		return ctx.Err()
+	}
+}
+
+// sshAuth holds the pieces of the SSH client config shared by every host:
+// the private key and the host-key policy, both loaded once at startup so a
+// missing or unreadable file fails the operator fast rather than on the
+// first Create.
+type sshAuth struct {
+	signer  ssh.Signer
+	hostKey ssh.HostKeyCallback
+}
+
+func newSSHAuth(cfg provider.KVMConfig) (*sshAuth, error) {
+	key, err := os.ReadFile(cfg.SSHKeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("reading kvm.sshKeyFile: %w", err)
+	}
+	signer, err := ssh.ParsePrivateKey(key)
+	if err != nil {
+		return nil, fmt.Errorf("parsing kvm.sshKeyFile: %w", err)
+	}
+	hostKey := ssh.InsecureIgnoreHostKey() //nolint:gosec // opt-in via config, documented as insecure
+	if !cfg.InsecureIgnoreHostKey {
+		hostKey, err = knownhosts.New(cfg.KnownHostsFile)
+		if err != nil {
+			return nil, fmt.Errorf("loading kvm.knownHostsFile: %w", err)
+		}
+	}
+	return &sshAuth{signer: signer, hostKey: hostKey}, nil
+}
+
+// dialer returns the production hostDialer: SSH to the host, forward its
+// libvirt unix socket through the connection, and open a libvirt session
+// over that.
+func (a *sshAuth) dialer() hostDialer {
+	return func(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
+		client, err := a.dialSSH(ctx, host)
+		if err != nil {
+			return nil, err
+		}
+		hc, err := connectLibvirt(ctx, client, host.Socket)
+		if err != nil {
+			client.Close()
+			return nil, err
+		}
+		return hc, nil
+	}
+}
+
+// dialSSH establishes the SSH connection. The handshake is bounded by ctx's
+// deadline, or defaultHandshakeTimeout when ctx has none, since a
+// half-open hypervisor must not pin a reconcile worker indefinitely.
+func (a *sshAuth) dialSSH(ctx context.Context, host provider.KVMHost) (*ssh.Client, error) {
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		deadline = time.Now().Add(defaultHandshakeTimeout)
+	}
+	var d net.Dialer
+	nc, err := d.DialContext(ctx, "tcp", host.Address)
+	if err != nil {
+		return nil, fmt.Errorf("ssh dial: %w", err)
+	}
+	_ = nc.SetDeadline(deadline)
+	conn, chans, reqs, err := ssh.NewClientConn(nc, host.Address, &ssh.ClientConfig{
+		User:            host.User,
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(a.signer)},
+		HostKeyCallback: a.hostKey,
+	})
+	if err != nil {
+		nc.Close()
+		return nil, fmt.Errorf("ssh handshake: %w", err)
+	}
+	_ = nc.SetDeadline(time.Time{})
+	return ssh.NewClient(conn, chans, reqs), nil
+}
+
+// unixForward adapts an SSH client to go-libvirt's socket.Dialer: each Dial
+// opens a direct-streamlocal channel to the libvirt socket on the host.
+type unixForward struct {
+	client *ssh.Client
+	path   string
+}
+
+func (u unixForward) Dial() (net.Conn, error) {
+	return u.client.Dial("unix", u.path)
+}
+
+// connectLibvirt opens the libvirt session over the SSH client and starts a
+// keepalive loop so a dead link is noticed within roughly two intervals
+// rather than at the TCP stack's leisure. Closing the returned client tears
+// down the SSH connection, which in turn ends the libvirt session.
+func connectLibvirt(ctx context.Context, client *ssh.Client, socket string) (*hostClient, error) {
+	lv := libvirt.NewWithDialer(unixForward{client: client, path: socket})
+	if err := awaitOrCancel(ctx, lv.Connect, func() { client.Close() }); err != nil {
+		return nil, fmt.Errorf("libvirt connect via %s: %w", socket, err)
+	}
+	stop := make(chan struct{})
+	var once sync.Once
+	go keepalive(client, keepaliveInterval, stop)
+	return &hostClient{
+		api:  lv,
+		done: lv.Disconnected(),
+		close: func() {
+			once.Do(func() {
+				close(stop)
+				client.Close()
+			})
+		},
+	}, nil
+}
+
+// keepalive sends OpenSSH keepalive requests until stop closes; a failed or
+// unanswered request closes the client, which surfaces as Disconnected on
+// the libvirt side and makes the pool redial.
+func keepalive(client *ssh.Client, interval time.Duration, stop <-chan struct{}) {
+	t := time.NewTicker(interval)
+	defer t.Stop()
+	for {
+		select {
+		case <-stop:
+			return
+		case <-t.C:
+		}
+		replied := make(chan error, 1)
+		go func() {
+			_, _, err := client.SendRequest("keepalive@openssh.com", true, nil)
+			replied <- err
+		}()
+		select {
+		case <-stop:
+			return
+		case err := <-replied:
+			if err != nil {
+				client.Close()
+				return
+			}
+		case <-time.After(interval):
+			client.Close()
+			return
+		}
+	}
+}

internal/provider/kvm/conn_test.go

@@ -0,0 +1,569 @@
+package kvm
+
+import (
+	"bytes"
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/pem"
+	"errors"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// --- hostPool over a fake dialer ---
+
+type fakeClient struct {
+	done   chan struct{}
+	closed atomic.Bool
+}
+
+func (c *fakeClient) close() {
+	if c.closed.CompareAndSwap(false, true) {
+		close(c.done)
+	}
+}
+
+type fakeDialer struct {
+	mu      sync.Mutex
+	dials   int
+	err     error
+	delay   time.Duration
+	clients []*fakeClient
+}
+
+func (d *fakeDialer) dial(ctx context.Context, _ provider.KVMHost) (*hostClient, error) {
+	if d.delay > 0 {
+		select {
+		case <-time.After(d.delay):
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.dials++
+	if d.err != nil {
+		return nil, d.err
+	}
+	fc := &fakeClient{done: make(chan struct{})}
+	d.clients = append(d.clients, fc)
+	return &hostClient{api: nil, done: fc.done, close: fc.close}, nil
+}
+
+func (d *fakeDialer) dialCount() int {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.dials
+}
+
+var testHost = provider.KVMHost{Name: "nas1", Address: "nas1:22", User: "virt", Socket: defaultSocket, Pool: "default", Bridge: "br0"}
+
+func noop(libvirtAPI) error { return nil }
+
+func TestHostPool_call_reusesConnection(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	for range 3 {
+		if err := p.call(context.Background(), testHost, noop); err != nil {
+			t.Fatalf("call() error = %v", err)
+		}
+	}
+	if d.dialCount() != 1 {
+		t.Errorf("dials = %d, want 1", d.dialCount())
+	}
+}
+
+func TestHostPool_call_redialsAfterDisconnect(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	if err := p.call(context.Background(), testHost, noop); err != nil {
+		t.Fatalf("call() error = %v", err)
+	}
+	d.clients[0].close()
+	if err := p.call(context.Background(), testHost, noop); err != nil {
+		t.Fatalf("second call() error = %v", err)
+	}
+	if d.dialCount() != 2 {
+		t.Errorf("dials = %d, want 2 after the first connection dropped", d.dialCount())
+	}
+}
+
+func TestHostPool_call_transportErrorDropsConnection(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	err := p.call(context.Background(), testHost, func(libvirtAPI) error { return io.ErrUnexpectedEOF })
+	if !errors.Is(err, io.ErrUnexpectedEOF) {
+		t.Fatalf("call() error = %v, want the transport error passed through", err)
+	}
+	if !d.clients[0].closed.Load() {
+		t.Error("connection not closed after a transport error")
+	}
+	if err := p.call(context.Background(), testHost, noop); err != nil {
+		t.Fatalf("call() after transport error = %v", err)
+	}
+	if d.dialCount() != 2 {
+		t.Errorf("dials = %d, want 2", d.dialCount())
+	}
+}
+
+func TestHostPool_call_libvirtErrorKeepsConnection(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	err := p.call(context.Background(), testHost, func(libvirtAPI) error { return lvErr(libvirt.ErrNoDomain) })
+	if !hasCode(err, libvirt.ErrNoDomain) {
+		t.Fatalf("call() error = %v, want the libvirt error passed through", err)
+	}
+	if err := p.call(context.Background(), testHost, noop); err != nil {
+		t.Fatalf("call() error = %v", err)
+	}
+	if d.dialCount() != 1 {
+		t.Errorf("dials = %d, want 1: a daemon-reported error must not cost the connection", d.dialCount())
+	}
+	if d.clients[0].closed.Load() {
+		t.Error("connection closed after a libvirt error")
+	}
+}
+
+func TestHostPool_call_dialErrorPropagates(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{err: errors.New("ssh: handshake failed")}
+	p := newHostPool(d.dial)
+	err := p.call(context.Background(), testHost, noop)
+	if err == nil || !strings.Contains(err.Error(), "nas1") || !strings.Contains(err.Error(), "handshake failed") {
+		t.Errorf("call() error = %v, want the dial error naming the host", err)
+	}
+}
+
+func TestHostPool_call_cancelAbandonsBlockedCall(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	ctx, cancel := context.WithCancel(context.Background())
+	released := make(chan struct{})
+	go func() {
+		time.Sleep(20 * time.Millisecond)
+		cancel()
+	}()
+	err := p.call(ctx, testHost, func(libvirtAPI) error {
+		// Simulates an RPC stuck on a hung hypervisor: it only returns once
+		// the pool closes the connection out from under it.
+		<-d.clients[0].done
+		close(released)
+		return io.EOF
+	})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("call() error = %v, want context.Canceled", err)
+	}
+	select {
+	case <-released:
+	case <-time.After(time.Second):
+		t.Fatal("blocked call was not released by closing the connection")
+	}
+	if err := p.call(context.Background(), testHost, noop); err != nil {
+		t.Fatalf("call() after cancel = %v", err)
+	}
+	if d.dialCount() != 2 {
+		t.Errorf("dials = %d, want 2", d.dialCount())
+	}
+}
+
+func TestHostPool_call_concurrentCallersShareOneDial(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{delay: 30 * time.Millisecond}
+	p := newHostPool(d.dial)
+	var wg sync.WaitGroup
+	for range 8 {
+		wg.Go(func() {
+			if err := p.call(context.Background(), testHost, noop); err != nil {
+				t.Errorf("call() error = %v", err)
+			}
+		})
+	}
+	wg.Wait()
+	if d.dialCount() != 1 {
+		t.Errorf("dials = %d, want 1", d.dialCount())
+	}
+}
+
+func TestHostPool_call_hostsAreIndependent(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	other := testHost
+	other.Name = "lab2"
+	for _, h := range []provider.KVMHost{testHost, other, testHost, other} {
+		if err := p.call(context.Background(), h, noop); err != nil {
+			t.Fatalf("call(%s) error = %v", h.Name, err)
+		}
+	}
+	if d.dialCount() != 2 {
+		t.Errorf("dials = %d, want one per host", d.dialCount())
+	}
+}
+
+// --- SSH dial against an in-process server ---
+
+func newKeyPair(t *testing.T) (ssh.Signer, []byte) {
+	t.Helper()
+	_, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	signer, err := ssh.NewSignerFromKey(priv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	block, err := ssh.MarshalPrivateKey(priv, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return signer, pem.EncodeToMemory(block)
+}
+
+// testSSHServer accepts one public key and serves
+// direct-streamlocal@openssh.com channels by dialing the requested unix
+// path, which is exactly what OpenSSH does for ssh.Client.Dial("unix").
+type testSSHServer struct {
+	addr    string
+	hostKey ssh.PublicKey
+
+	mu    sync.Mutex
+	paths []string
+}
+
+func startSSHServer(t *testing.T, clientKey ssh.PublicKey, answerGlobalRequests bool) *testSSHServer {
+	t.Helper()
+	hostSigner, _ := newKeyPair(t)
+	cfg := &ssh.ServerConfig{
+		PublicKeyCallback: func(_ ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
+			if bytes.Equal(key.Marshal(), clientKey.Marshal()) {
+				return nil, nil
+			}
+			return nil, errors.New("unknown public key")
+		},
+	}
+	cfg.AddHostKey(hostSigner)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { ln.Close() })
+	s := &testSSHServer{addr: ln.Addr().String(), hostKey: hostSigner.PublicKey()}
+	go func() {
+		for {
+			nc, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			go s.serve(nc, cfg, answerGlobalRequests)
+		}
+	}()
+	return s
+}
+
+func (s *testSSHServer) serve(nc net.Conn, cfg *ssh.ServerConfig, answerGlobalRequests bool) {
+	_, chans, reqs, err := ssh.NewServerConn(nc, cfg)
+	if err != nil {
+		return
+	}
+	if answerGlobalRequests {
+		go ssh.DiscardRequests(reqs)
+	} else {
+		go func() {
+			for range reqs { //nolint:revive // deliberately never replies, to stall keepalives
+			}
+		}()
+	}
+	for newCh := range chans {
+		if newCh.ChannelType() != "direct-streamlocal@openssh.com" {
+			_ = newCh.Reject(ssh.UnknownChannelType, "test server only forwards unix sockets")
+			continue
+		}
+		var payload struct {
+			Path      string
+			Reserved0 string
+			Reserved1 uint32
+		}
+		if err := ssh.Unmarshal(newCh.ExtraData(), &payload); err != nil {
+			_ = newCh.Reject(ssh.ConnectionFailed, err.Error())
+			continue
+		}
+		s.mu.Lock()
+		s.paths = append(s.paths, payload.Path)
+		s.mu.Unlock()
+		target, err := net.Dial("unix", payload.Path)
+		if err != nil {
+			_ = newCh.Reject(ssh.ConnectionFailed, err.Error())
+			continue
+		}
+		ch, chReqs, err := newCh.Accept()
+		if err != nil {
+			target.Close()
+			continue
+		}
+		go ssh.DiscardRequests(chReqs)
+		go func() { _, _ = io.Copy(ch, target); ch.Close() }()
+		go func() { _, _ = io.Copy(target, ch); target.Close() }()
+	}
+}
+
+func (s *testSSHServer) requestedPaths() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.paths...)
+}
+
+// startEchoSocket serves a unix socket that echoes everything back, standing
+// in for libvirt-sock.
+func startEchoSocket(t *testing.T) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "libvirt-sock")
+	ln, err := net.Listen("unix", path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { ln.Close() })
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			go func() { _, _ = io.Copy(c, c); c.Close() }()
+		}
+	}()
+	return path
+}
+
+type sshFixture struct {
+	server  *testSSHServer
+	cfg     provider.KVMConfig
+	host    provider.KVMHost
+	keyFile string
+}
+
+func newSSHFixture(t *testing.T, answerGlobalRequests bool) *sshFixture {
+	t.Helper()
+	clientSigner, clientPEM := newKeyPair(t)
+	server := startSSHServer(t, clientSigner.PublicKey(), answerGlobalRequests)
+	dir := t.TempDir()
+	keyFile := filepath.Join(dir, "id_ed25519")
+	if err := os.WriteFile(keyFile, clientPEM, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	khFile := filepath.Join(dir, "known_hosts")
+	line := knownhosts.Line([]string{server.addr}, server.hostKey)
+	if err := os.WriteFile(khFile, []byte(line+"\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	return &sshFixture{
+		server:  server,
+		keyFile: keyFile,
+		cfg:     provider.KVMConfig{SSHKeyFile: keyFile, KnownHostsFile: khFile},
+		host:    provider.KVMHost{Name: "nas1", Address: server.addr, User: "virt", Socket: startEchoSocket(t)},
+	}
+}
+
+func TestSSHAuth_dialSSH_forwardsUnixSocket(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	auth, err := newSSHAuth(f.cfg)
+	if err != nil {
+		t.Fatalf("newSSHAuth() error = %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	client, err := auth.dialSSH(ctx, f.host)
+	if err != nil {
+		t.Fatalf("dialSSH() error = %v", err)
+	}
+	defer client.Close()
+
+	conn, err := unixForward{client: client, path: f.host.Socket}.Dial()
+	if err != nil {
+		t.Fatalf("unixForward.Dial() error = %v", err)
+	}
+	defer conn.Close()
+	if _, err := conn.Write([]byte("ping")); err != nil {
+		t.Fatal(err)
+	}
+	buf := make([]byte, 4)
+	if _, err := io.ReadFull(conn, buf); err != nil || string(buf) != "ping" {
+		t.Fatalf("echo through the forward = %q, %v; want ping", buf, err)
+	}
+	if paths := f.server.requestedPaths(); len(paths) != 1 || paths[0] != f.host.Socket {
+		t.Errorf("server saw forward requests %v, want [%s]", paths, f.host.Socket)
+	}
+}
+
+func TestSSHAuth_dialSSH_rejectsUnknownHostKey(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	impostor, _ := newKeyPair(t)
+	line := knownhosts.Line([]string{f.server.addr}, impostor.PublicKey())
+	if err := os.WriteFile(f.cfg.KnownHostsFile, []byte(line+"\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	auth, err := newSSHAuth(f.cfg)
+	if err != nil {
+		t.Fatalf("newSSHAuth() error = %v", err)
+	}
+	client, err := auth.dialSSH(context.Background(), f.host)
+	if err == nil {
+		client.Close()
+		t.Fatal("dialSSH() succeeded against a host whose key is not in known_hosts")
+	}
+	var keyErr *knownhosts.KeyError
+	if !errors.As(err, &keyErr) {
+		t.Errorf("dialSSH() error = %v, want a knownhosts.KeyError", err)
+	}
+}
+
+func TestSSHAuth_dialSSH_insecureIgnoreHostKey(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	f.cfg.KnownHostsFile = ""
+	f.cfg.InsecureIgnoreHostKey = true
+	auth, err := newSSHAuth(f.cfg)
+	if err != nil {
+		t.Fatalf("newSSHAuth() error = %v", err)
+	}
+	client, err := auth.dialSSH(context.Background(), f.host)
+	if err != nil {
+		t.Fatalf("dialSSH() error = %v", err)
+	}
+	client.Close()
+}
+
+func TestSSHAuth_dialSSH_rejectsWrongClientKey(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	_, otherPEM := newKeyPair(t)
+	if err := os.WriteFile(f.keyFile, otherPEM, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	auth, err := newSSHAuth(f.cfg)
+	if err != nil {
+		t.Fatalf("newSSHAuth() error = %v", err)
+	}
+	client, err := auth.dialSSH(context.Background(), f.host)
+	if err == nil {
+		client.Close()
+		t.Fatal("dialSSH() succeeded with a key the server does not know")
+	}
+	if !strings.Contains(err.Error(), "ssh handshake") {
+		t.Errorf("dialSSH() error = %v, want an auth failure from the handshake", err)
+	}
+}
+
+func TestSSHAuth_dialSSH_cancelledContext(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	auth, err := newSSHAuth(f.cfg)
+	if err != nil {
+		t.Fatalf("newSSHAuth() error = %v", err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if client, err := auth.dialSSH(ctx, f.host); err == nil {
+		client.Close()
+		t.Fatal("dialSSH() succeeded with an already-cancelled context")
+	}
+}
+
+func TestNewSSHAuth_missingFiles(t *testing.T) {
+	t.Parallel()
+	_, keyPEM := newKeyPair(t)
+	keyFile := filepath.Join(t.TempDir(), "key")
+	if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	tests := []struct {
+		name    string
+		cfg     provider.KVMConfig
+		wantSub string
+	}{
+		{"key file missing", provider.KVMConfig{SSHKeyFile: "/nonexistent/key", InsecureIgnoreHostKey: true}, "sshKeyFile"},
+		{"key file garbage", provider.KVMConfig{SSHKeyFile: keyFile + ".bad", InsecureIgnoreHostKey: true}, "sshKeyFile"},
+		{"known_hosts missing", provider.KVMConfig{SSHKeyFile: keyFile, KnownHostsFile: "/nonexistent/kh"}, "knownHostsFile"},
+	}
+	if err := os.WriteFile(keyFile+".bad", []byte("not a key"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			_, err := newSSHAuth(tc.cfg)
+			if err == nil || !strings.Contains(err.Error(), tc.wantSub) {
+				t.Errorf("newSSHAuth() error = %v, want one mentioning %s", err, tc.wantSub)
+			}
+		})
+	}
+}
+
+func TestKeepalive_closesClientWhenServerStalls(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, false)
+	auth, err := newSSHAuth(f.cfg)
+	if err != nil {
+		t.Fatalf("newSSHAuth() error = %v", err)
+	}
+	client, err := auth.dialSSH(context.Background(), f.host)
+	if err != nil {
+		t.Fatalf("dialSSH() error = %v", err)
+	}
+	defer client.Close()
+	stop := make(chan struct{})
+	defer close(stop)
+	go keepalive(client, 20*time.Millisecond, stop)
+
+	closed := make(chan struct{})
+	go func() { _ = client.Wait(); close(closed) }()
+	select {
+	case <-closed:
+	case <-time.After(2 * time.Second):
+		t.Fatal("client still open after keepalives went unanswered")
+	}
+}
+
+func TestKeepalive_keepsHealthyClientOpen(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	auth, err := newSSHAuth(f.cfg)
+	if err != nil {
+		t.Fatalf("newSSHAuth() error = %v", err)
+	}
+	client, err := auth.dialSSH(context.Background(), f.host)
+	if err != nil {
+		t.Fatalf("dialSSH() error = %v", err)
+	}
+	defer client.Close()
+	stop := make(chan struct{})
+	defer close(stop)
+	go keepalive(client, 10*time.Millisecond, stop)
+	time.Sleep(60 * time.Millisecond)
+	fwd := unixForward{client: client, path: f.host.Socket}
+	conn, err := fwd.Dial()
+	if err != nil {
+		t.Fatalf("forward after several keepalives failed: %v", err)
+	}
+	conn.Close()
+}

internal/provider/kvm/domainxml.go

@@ -0,0 +1,224 @@
+package kvm
+
+import (
+	"encoding/xml"
+	"fmt"
+	"time"
+)
+
+const guestAgentChannel = "org.qemu.guest_agent.0"
+
+// domainParams is everything buildDomainXML needs, already resolved from the
+// config and placement so the builder stays a pure function.
+type domainParams struct {
+	Name      string
+	UID       string
+	CreatedAt time.Time
+	VCPUs     int32
+	MemoryMiB int64
+	Pool      string
+	Bridge    string
+	DiskVol   string
+	SeedVol   string
+}
+
+// buildDomainXML renders a q35 KVM guest: host-passthrough CPU, a virtio
+// overlay disk and a SATA cdrom for the seed (both referenced as pool
+// volumes so no host paths are needed), one virtio NIC on the host bridge,
+// a serial console, the guest-agent channel, a virtio RNG, no graphics, and
+// the operator's metadata element.
+func buildDomainXML(p domainParams) (string, error) {
+	d := domainXML{
+		Type:     "kvm",
+		Name:     p.Name,
+		Metadata: domainMetadataXML{Instance: newInstanceMetadata(p.UID, p.CreatedAt)},
+		Memory:   sizeXML{Unit: "MiB", Value: p.MemoryMiB},
+		VCPU:     p.VCPUs,
+		OS: osXML{
+			Type: osTypeXML{Arch: "x86_64", Machine: "q35", Value: "hvm"},
+			Boot: bootXML{Dev: "hd"},
+		},
+		Features:   featuresXML{ACPI: &struct{}{}, APIC: &struct{}{}},
+		CPU:        cpuXML{Mode: "host-passthrough"},
+		Clock:      clockXML{Offset: "utc"},
+		OnPoweroff: "destroy",
+		OnReboot:   "restart",
+		OnCrash:    "destroy",
+		Devices: devicesXML{
+			Disks: []diskXML{
+				{
+					Type:   "volume",
+					Device: "disk",
+					Driver: driverXML{Name: "qemu", Type: formatQcow2},
+					Source: diskSourceXML{Pool: p.Pool, Volume: p.DiskVol},
+					Target: targetXML{Dev: "vda", Bus: "virtio"},
+				},
+				{
+					Type:     "volume",
+					Device:   "cdrom",
+					Driver:   driverXML{Name: "qemu", Type: formatRaw},
+					Source:   diskSourceXML{Pool: p.Pool, Volume: p.SeedVol},
+					Target:   targetXML{Dev: "sda", Bus: "sata"},
+					ReadOnly: &struct{}{},
+				},
+			},
+			Interfaces: []interfaceXML{{
+				Type:   "bridge",
+				Source: ifaceSourceXML{Bridge: p.Bridge},
+				Model:  modelXML{Type: "virtio"},
+			}},
+			Serials:  []serialXML{{Type: "pty", Target: portXML{Port: 0}}},
+			Consoles: []consoleXML{{Type: "pty", Target: consoleTargetXML{Type: "serial", Port: 0}}},
+			Channels: []channelXML{{
+				Type:   "unix",
+				Target: channelTargetXML{Type: "virtio", Name: guestAgentChannel},
+			}},
+			RNG: rngXML{Model: "virtio", Backend: rngBackendXML{Model: "random", Value: "/dev/urandom"}},
+		},
+	}
+	out, err := xml.Marshal(d)
+	if err != nil {
+		return "", fmt.Errorf("marshalling domain xml: %w", err)
+	}
+	return string(out), nil
+}
+
+type domainXML struct {
+	XMLName    xml.Name          `xml:"domain"`
+	Type       string            `xml:"type,attr"`
+	Name       string            `xml:"name"`
+	Metadata   domainMetadataXML `xml:"metadata"`
+	Memory     sizeXML           `xml:"memory"`
+	VCPU       int32             `xml:"vcpu"`
+	OS         osXML             `xml:"os"`
+	Features   featuresXML       `xml:"features"`
+	CPU        cpuXML            `xml:"cpu"`
+	Clock      clockXML          `xml:"clock"`
+	OnPoweroff string            `xml:"on_poweroff"`
+	OnReboot   string            `xml:"on_reboot"`
+	OnCrash    string            `xml:"on_crash"`
+	Devices    devicesXML        `xml:"devices"`
+}
+
+type domainMetadataXML struct {
+	Instance instanceMetadata
+}
+
+type sizeXML struct {
+	Unit  string `xml:"unit,attr"`
+	Value int64  `xml:",chardata"`
+}
+
+type osXML struct {
+	Type osTypeXML `xml:"type"`
+	Boot bootXML   `xml:"boot"`
+}
+
+type osTypeXML struct {
+	Arch    string `xml:"arch,attr"`
+	Machine string `xml:"machine,attr"`
+	Value   string `xml:",chardata"`
+}
+
+type bootXML struct {
+	Dev string `xml:"dev,attr"`
+}
+
+type featuresXML struct {
+	ACPI *struct{} `xml:"acpi"`
+	APIC *struct{} `xml:"apic"`
+}
+
+type cpuXML struct {
+	Mode string `xml:"mode,attr"`
+}
+
+type clockXML struct {
+	Offset string `xml:"offset,attr"`
+}
+
+type devicesXML struct {
+	Disks      []diskXML      `xml:"disk"`
+	Interfaces []interfaceXML `xml:"interface"`
+	Serials    []serialXML    `xml:"serial"`
+	Consoles   []consoleXML   `xml:"console"`
+	Channels   []channelXML   `xml:"channel"`
+	RNG        rngXML         `xml:"rng"`
+}
+
+type diskXML struct {
+	Type     string        `xml:"type,attr"`
+	Device   string        `xml:"device,attr"`
+	Driver   driverXML     `xml:"driver"`
+	Source   diskSourceXML `xml:"source"`
+	Target   targetXML     `xml:"target"`
+	ReadOnly *struct{}     `xml:"readonly"`
+}
+
+type driverXML struct {
+	Name string `xml:"name,attr"`
+	Type string `xml:"type,attr"`
+}
+
+type diskSourceXML struct {
+	Pool   string `xml:"pool,attr"`
+	Volume string `xml:"volume,attr"`
+}
+
+type targetXML struct {
+	Dev string `xml:"dev,attr"`
+	Bus string `xml:"bus,attr"`
+}
+
+type interfaceXML struct {
+	Type   string         `xml:"type,attr"`
+	Source ifaceSourceXML `xml:"source"`
+	Model  modelXML       `xml:"model"`
+}
+
+type ifaceSourceXML struct {
+	Bridge string `xml:"bridge,attr"`
+}
+
+type modelXML struct {
+	Type string `xml:"type,attr"`
+}
+
+type serialXML struct {
+	Type   string  `xml:"type,attr"`
+	Target portXML `xml:"target"`
+}
+
+type portXML struct {
+	Port int `xml:"port,attr"`
+}
+
+type consoleXML struct {
+	Type   string           `xml:"type,attr"`
+	Target consoleTargetXML `xml:"target"`
+}
+
+type consoleTargetXML struct {
+	Type string `xml:"type,attr"`
+	Port int    `xml:"port,attr"`
+}
+
+type channelXML struct {
+	Type   string           `xml:"type,attr"`
+	Target channelTargetXML `xml:"target"`
+}
+
+type channelTargetXML struct {
+	Type string `xml:"type,attr"`
+	Name string `xml:"name,attr"`
+}
+
+type rngXML struct {
+	Model   string        `xml:"model,attr"`
+	Backend rngBackendXML `xml:"backend"`
+}
+
+type rngBackendXML struct {
+	Model string `xml:"model,attr"`
+	Value string `xml:",chardata"`
+}

internal/provider/kvm/domainxml_test.go

@@ -0,0 +1,113 @@
+package kvm
+
+import (
+	"encoding/xml"
+	"strings"
+	"testing"
+	"time"
+)
+
+func testDomainParams() domainParams {
+	return domainParams{
+		Name:      "proxy-abc123def456ghij",
+		UID:       "11111111-2222-3333-4444-555555555555",
+		CreatedAt: time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC),
+		VCPUs:     2,
+		MemoryMiB: 2048,
+		Pool:      "default",
+		Bridge:    "br0",
+		DiskVol:   "proxy-abc123def456ghij.qcow2",
+		SeedVol:   "proxy-abc123def456ghij-seed.iso",
+	}
+}
+
+func TestBuildDomainXML_fieldByField(t *testing.T) {
+	t.Parallel()
+	raw, err := buildDomainXML(testDomainParams())
+	if err != nil {
+		t.Fatalf("buildDomainXML() error = %v", err)
+	}
+	var d domainXML
+	if err := xml.Unmarshal([]byte(raw), &d); err != nil {
+		t.Fatalf("output is not well-formed XML: %v\n%s", err, raw)
+	}
+
+	if d.Type != "kvm" || d.Name != "proxy-abc123def456ghij" {
+		t.Errorf("type/name = %s/%s", d.Type, d.Name)
+	}
+	if d.VCPU != 2 || d.Memory.Value != 2048 || d.Memory.Unit != "MiB" {
+		t.Errorf("vcpu/memory = %d/%d %s, want 2/2048 MiB", d.VCPU, d.Memory.Value, d.Memory.Unit)
+	}
+	if d.OS.Type.Value != "hvm" || d.OS.Type.Arch != "x86_64" || d.OS.Type.Machine != "q35" || d.OS.Boot.Dev != "hd" {
+		t.Errorf("os = %+v", d.OS)
+	}
+	if d.Features.ACPI == nil || d.Features.APIC == nil {
+		t.Errorf("features = %+v, want acpi and apic", d.Features)
+	}
+	if d.CPU.Mode != "host-passthrough" {
+		t.Errorf("cpu mode = %q", d.CPU.Mode)
+	}
+	if d.OnPoweroff != "destroy" || d.OnReboot != "restart" || d.OnCrash != "destroy" {
+		t.Errorf("lifecycle = %s/%s/%s", d.OnPoweroff, d.OnReboot, d.OnCrash)
+	}
+
+	m := d.Metadata.Instance
+	if !m.managed() || m.UID != "11111111-2222-3333-4444-555555555555" || m.CreatedAt != "2026-09-04T12:00:00Z" {
+		t.Errorf("metadata = %+v", m)
+	}
+	if !strings.Contains(raw, `xmlns="`+metadataURI+`"`) {
+		t.Errorf("metadata element is not namespaced; libvirt rejects that:\n%s", raw)
+	}
+
+	if len(d.Devices.Disks) != 2 {
+		t.Fatalf("disks = %d, want 2", len(d.Devices.Disks))
+	}
+	disk, seed := d.Devices.Disks[0], d.Devices.Disks[1]
+	if disk.Type != "volume" || disk.Device != "disk" || disk.Driver.Type != "qcow2" ||
+		disk.Source.Pool != "default" || disk.Source.Volume != "proxy-abc123def456ghij.qcow2" ||
+		disk.Target.Dev != "vda" || disk.Target.Bus != "virtio" || disk.ReadOnly != nil {
+		t.Errorf("overlay disk = %+v", disk)
+	}
+	if seed.Type != "volume" || seed.Device != "cdrom" || seed.Driver.Type != "raw" ||
+		seed.Source.Pool != "default" || seed.Source.Volume != "proxy-abc123def456ghij-seed.iso" ||
+		seed.Target.Bus != "sata" || seed.ReadOnly == nil {
+		t.Errorf("seed cdrom = %+v", seed)
+	}
+
+	if len(d.Devices.Interfaces) != 1 {
+		t.Fatalf("interfaces = %d, want 1", len(d.Devices.Interfaces))
+	}
+	if nic := d.Devices.Interfaces[0]; nic.Type != "bridge" || nic.Source.Bridge != "br0" || nic.Model.Type != "virtio" {
+		t.Errorf("interface = %+v", nic)
+	}
+
+	if len(d.Devices.Channels) != 1 || d.Devices.Channels[0].Target.Name != guestAgentChannel || d.Devices.Channels[0].Target.Type != "virtio" {
+		t.Errorf("channels = %+v, want one virtio %s channel", d.Devices.Channels, guestAgentChannel)
+	}
+	if len(d.Devices.Serials) != 1 || len(d.Devices.Consoles) != 1 || d.Devices.Consoles[0].Target.Type != "serial" {
+		t.Errorf("serial/console = %+v/%+v", d.Devices.Serials, d.Devices.Consoles)
+	}
+	if d.Devices.RNG.Model != "virtio" || d.Devices.RNG.Backend.Value != "/dev/urandom" {
+		t.Errorf("rng = %+v", d.Devices.RNG)
+	}
+	if strings.Contains(raw, "<graphics") {
+		t.Errorf("domain has a graphics device; headless VMs must not:\n%s", raw)
+	}
+}
+
+func TestBuildDomainXML_escapesUserValues(t *testing.T) {
+	t.Parallel()
+	p := testDomainParams()
+	p.Bridge = `br0" onload="x`
+	raw, err := buildDomainXML(p)
+	if err != nil {
+		t.Fatalf("buildDomainXML() error = %v", err)
+	}
+	var d domainXML
+	if err := xml.Unmarshal([]byte(raw), &d); err != nil {
+		t.Fatalf("output is not well-formed XML: %v", err)
+	}
+	if d.Devices.Interfaces[0].Source.Bridge != p.Bridge {
+		t.Errorf("bridge = %q, want the raw value back", d.Devices.Interfaces[0].Source.Bridge)
+	}
+}

internal/provider/kvm/errors.go

@@ -0,0 +1,72 @@
+package kvm
+
+import (
+	"errors"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// classify maps an error onto the provider taxonomy. Errors the package
+// already tagged with a sentinel (image download failures, placement
+// errors) keep their class. For libvirt errors: missing domain or volume →
+// NotFound; anything the host will keep rejecting (bad XML, unsupported
+// config, denied access, missing pool) → Permanent; every other libvirt
+// code, plus SSH/transport failures and cancellation, → Transient, because
+// retrying is safer than latching Failed on an error nobody taught this
+// function to recognise. libvirt exposes no quota signal, so nothing maps
+// to QuotaExceeded.
+func classify(err error) error {
+	for _, class := range []error{provider.ErrNotFound, provider.ErrQuotaExceeded, provider.ErrPermanent, provider.ErrTransient} {
+		if errors.Is(err, class) {
+			return class
+		}
+	}
+	var lerr libvirt.Error
+	if !errors.As(err, &lerr) {
+		return provider.ErrTransient
+	}
+	switch libvirt.ErrorNumber(lerr.Code) {
+	case libvirt.ErrNoDomain, libvirt.ErrNoStorageVol:
+		return provider.ErrNotFound
+	case libvirt.ErrNoStoragePool,
+		libvirt.ErrXMLError, libvirt.ErrXMLDetail, libvirt.ErrXMLInvalidSchema,
+		libvirt.ErrConfigUnsupported, libvirt.ErrNoSupport,
+		libvirt.ErrOperationUnsupported, libvirt.ErrArgumentUnsupported,
+		libvirt.ErrInvalidArg,
+		libvirt.ErrAuthFailed, libvirt.ErrAccessDenied, libvirt.ErrOperationDenied:
+		return provider.ErrPermanent
+	default:
+		return provider.ErrTransient
+	}
+}
+
+// isLibvirtError reports whether err came back from the libvirt daemon (as
+// opposed to the SSH or RPC transport), which is what decides whether the
+// cached connection is still trustworthy.
+func isLibvirtError(err error) bool {
+	var lerr libvirt.Error
+	return errors.As(err, &lerr)
+}
+
+func hasCode(err error, code libvirt.ErrorNumber) bool {
+	var lerr libvirt.Error
+	return errors.As(err, &lerr) && libvirt.ErrorNumber(lerr.Code) == code
+}
+
+// logAPIError records the raw libvirt code and message at V(1); classify
+// collapses them onto the coarser taxonomy, so this is the only place the
+// original detail survives.
+func logAPIError(log logr.Logger, op string, err error) {
+	if !log.V(1).Enabled() {
+		return
+	}
+	kv := []any{"op", op, "error", err.Error()}
+	var lerr libvirt.Error
+	if errors.As(err, &lerr) {
+		kv = append(kv, "libvirtCode", lerr.Code, "libvirtMessage", lerr.Message)
+	}
+	log.V(1).Info("libvirt call failed", kv...)
+}

internal/provider/kvm/errors_test.go

@@ -0,0 +1,69 @@
+package kvm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"testing"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func lvErr(code libvirt.ErrorNumber) error {
+	return libvirt.Error{Code: uint32(code), Message: code.String()}
+}
+
+func TestClassify(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		err  error
+		want error
+	}{
+		{"no domain", lvErr(libvirt.ErrNoDomain), provider.ErrNotFound},
+		{"no storage vol", lvErr(libvirt.ErrNoStorageVol), provider.ErrNotFound},
+		{"no storage pool", lvErr(libvirt.ErrNoStoragePool), provider.ErrPermanent},
+		{"xml error", lvErr(libvirt.ErrXMLError), provider.ErrPermanent},
+		{"xml detail", lvErr(libvirt.ErrXMLDetail), provider.ErrPermanent},
+		{"config unsupported", lvErr(libvirt.ErrConfigUnsupported), provider.ErrPermanent},
+		{"no support", lvErr(libvirt.ErrNoSupport), provider.ErrPermanent},
+		{"invalid arg", lvErr(libvirt.ErrInvalidArg), provider.ErrPermanent},
+		{"auth failed", lvErr(libvirt.ErrAuthFailed), provider.ErrPermanent},
+		{"access denied", lvErr(libvirt.ErrAccessDenied), provider.ErrPermanent},
+		{"operation invalid", lvErr(libvirt.ErrOperationInvalid), provider.ErrTransient},
+		{"operation failed", lvErr(libvirt.ErrOperationFailed), provider.ErrTransient},
+		{"internal error", lvErr(libvirt.ErrInternalError), provider.ErrTransient},
+		{"storage vol exists", lvErr(libvirt.ErrStorageVolExist), provider.ErrTransient},
+		{"wrapped libvirt error", fmt.Errorf("defining: %w", lvErr(libvirt.ErrNoDomain)), provider.ErrNotFound},
+		{"pre-tagged permanent", fmt.Errorf("%w: image returned HTTP 404", provider.ErrPermanent), provider.ErrPermanent},
+		{"pre-tagged transient wrapping libvirt not-found", fmt.Errorf("%w: %w", provider.ErrTransient, lvErr(libvirt.ErrNoDomain)), provider.ErrTransient},
+		{"transport eof", io.EOF, provider.ErrTransient},
+		{"context cancelled", context.Canceled, provider.ErrTransient},
+		{"plain error", errors.New("ssh: handshake failed"), provider.ErrTransient},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := classify(tc.err); !errors.Is(got, tc.want) {
+				t.Errorf("classify(%v) = %v, want %v", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestHasCode(t *testing.T) {
+	t.Parallel()
+	err := fmt.Errorf("lookup: %w", lvErr(libvirt.ErrNoDomain))
+	if !hasCode(err, libvirt.ErrNoDomain) {
+		t.Error("hasCode(ErrNoDomain) = false through a wrap")
+	}
+	if hasCode(err, libvirt.ErrNoStorageVol) {
+		t.Error("hasCode(ErrNoStorageVol) = true for an ErrNoDomain error")
+	}
+	if hasCode(io.EOF, libvirt.ErrNoDomain) || isLibvirtError(io.EOF) {
+		t.Error("io.EOF classified as a libvirt error")
+	}
+}

internal/provider/kvm/fake_test.go

@@ -0,0 +1,408 @@
+package kvm
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"slices"
+	"sync"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+)
+
+// fakeAPI is an in-memory libvirtAPI: pools of volumes and a set of
+// domains, with per-method error injection. It models just enough libvirt
+// semantics for the provider's decisions to be observable — ErrStorageVolExist
+// on duplicate create, ErrNoDomain/ErrNoStorageVol on missing lookups,
+// ErrOperationInvalid on destroying a stopped domain, and so on.
+type fakeAPI struct {
+	mu       sync.Mutex
+	poolPath string
+	pools    map[string]map[string]*fakeVol
+	domains  map[string]*fakeDomain
+	calls    []string
+	fail     map[string]error
+	failOnce map[string]error
+	// uploadHook, when set, sees every StorageVolUpload before the fake
+	// reads the stream; it can return an error to simulate a broken upload.
+	uploadHook func(vol string, r io.Reader) error
+}
+
+type fakeVol struct {
+	name     string
+	format   string
+	capacity int64
+	backing  string
+	data     []byte
+}
+
+type fakeDomain struct {
+	name     string
+	xml      string
+	metadata string
+	running  bool
+	// addrs is what DomainInterfaceAddresses returns per source.
+	addrs map[uint32][]libvirt.DomainInterface
+}
+
+func newFakeAPI(pools ...string) *fakeAPI {
+	f := &fakeAPI{
+		poolPath: "/var/lib/libvirt/images",
+		pools:    map[string]map[string]*fakeVol{},
+		domains:  map[string]*fakeDomain{},
+		fail:     map[string]error{},
+		failOnce: map[string]error{},
+	}
+	for _, p := range pools {
+		f.pools[p] = map[string]*fakeVol{}
+	}
+	return f
+}
+
+// enter records the call and returns any injected error for method.
+func (f *fakeAPI) enter(method string) error {
+	f.calls = append(f.calls, method)
+	if err, ok := f.failOnce[method]; ok {
+		delete(f.failOnce, method)
+		return err
+	}
+	return f.fail[method]
+}
+
+func (f *fakeAPI) callCount(method string) int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	n := 0
+	for _, c := range f.calls {
+		if c == method {
+			n++
+		}
+	}
+	return n
+}
+
+func (f *fakeAPI) hasCall(method string) bool { return f.callCount(method) > 0 }
+
+func (f *fakeAPI) vol(pool, name string) *fakeVol {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.pools[pool][name]
+}
+
+func (f *fakeAPI) putVol(pool, name, format string, data []byte) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.pools[pool][name] = &fakeVol{name: name, format: format, capacity: int64(len(data)), data: data}
+}
+
+func (f *fakeAPI) volNames(pool string) []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []string
+	for n := range f.pools[pool] {
+		out = append(out, n)
+	}
+	slices.Sort(out)
+	return out
+}
+
+func (f *fakeAPI) domain(name string) *fakeDomain {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.domains[name]
+}
+
+func (f *fakeAPI) putDomain(d *fakeDomain) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.domains[d.name] = d
+}
+
+func lvError(code libvirt.ErrorNumber, format string, args ...any) error {
+	return libvirt.Error{Code: uint32(code), Message: fmt.Sprintf(format, args...)}
+}
+
+// --- domains ---
+
+func (f *fakeAPI) DomainLookupByName(name string) (libvirt.Domain, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainLookupByName"); err != nil {
+		return libvirt.Domain{}, err
+	}
+	if _, ok := f.domains[name]; !ok {
+		return libvirt.Domain{}, lvError(libvirt.ErrNoDomain, "Domain not found: no domain with matching name '%s'", name)
+	}
+	return libvirt.Domain{Name: name}, nil
+}
+
+func (f *fakeAPI) DomainDefineXML(raw string) (libvirt.Domain, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainDefineXML"); err != nil {
+		return libvirt.Domain{}, err
+	}
+	var d domainXML
+	if err := xml.Unmarshal([]byte(raw), &d); err != nil {
+		return libvirt.Domain{}, lvError(libvirt.ErrXMLError, "%v", err)
+	}
+	meta, err := xml.Marshal(d.Metadata.Instance)
+	if err != nil {
+		return libvirt.Domain{}, lvError(libvirt.ErrXMLError, "%v", err)
+	}
+	existing := f.domains[d.Name]
+	nd := &fakeDomain{name: d.Name, xml: raw, metadata: string(meta)}
+	if existing != nil {
+		nd.running = existing.running
+		nd.addrs = existing.addrs
+	}
+	f.domains[d.Name] = nd
+	return libvirt.Domain{Name: d.Name}, nil
+}
+
+func (f *fakeAPI) DomainCreate(d libvirt.Domain) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainCreate"); err != nil {
+		return err
+	}
+	dom, ok := f.domains[d.Name]
+	if !ok {
+		return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	if dom.running {
+		return lvError(libvirt.ErrOperationInvalid, "domain is already running")
+	}
+	dom.running = true
+	return nil
+}
+
+func (f *fakeAPI) DomainDestroy(d libvirt.Domain) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainDestroy"); err != nil {
+		return err
+	}
+	dom, ok := f.domains[d.Name]
+	if !ok {
+		return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	if !dom.running {
+		return lvError(libvirt.ErrOperationInvalid, "domain is not running")
+	}
+	dom.running = false
+	return nil
+}
+
+func (f *fakeAPI) DomainUndefineFlags(d libvirt.Domain, _ libvirt.DomainUndefineFlagsValues) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainUndefineFlags"); err != nil {
+		return err
+	}
+	if _, ok := f.domains[d.Name]; !ok {
+		return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	delete(f.domains, d.Name)
+	return nil
+}
+
+func (f *fakeAPI) DomainGetState(d libvirt.Domain, _ uint32) (int32, int32, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainGetState"); err != nil {
+		return 0, 0, err
+	}
+	dom, ok := f.domains[d.Name]
+	if !ok {
+		return 0, 0, lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	if dom.running {
+		return int32(libvirt.DomainRunning), 0, nil
+	}
+	return int32(libvirt.DomainShutoff), 0, nil
+}
+
+func (f *fakeAPI) DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, _ libvirt.DomainModificationImpact) (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainGetMetadata"); err != nil {
+		return "", err
+	}
+	dom, ok := f.domains[d.Name]
+	if !ok {
+		return "", lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	if typ != int32(libvirt.DomainMetadataElement) || len(uri) != 1 || uri[0] != metadataURI || dom.metadata == "" {
+		return "", lvError(libvirt.ErrNoDomainMetadata, "Requested metadata element is not present")
+	}
+	return dom.metadata, nil
+}
+
+func (f *fakeAPI) DomainInterfaceAddresses(d libvirt.Domain, source uint32, _ uint32) ([]libvirt.DomainInterface, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainInterfaceAddresses"); err != nil {
+		return nil, err
+	}
+	dom, ok := f.domains[d.Name]
+	if !ok {
+		return nil, lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	if !dom.running {
+		return nil, lvError(libvirt.ErrOperationInvalid, "domain is not running")
+	}
+	if source == uint32(libvirt.DomainInterfaceAddressesSrcAgent) && dom.addrs[source] == nil {
+		return nil, lvError(libvirt.ErrAgentUnresponsive, "Guest agent is not responding")
+	}
+	return dom.addrs[source], nil
+}
+
+func (f *fakeAPI) ConnectListAllDomains(_ int32, _ libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("ConnectListAllDomains"); err != nil {
+		return nil, 0, err
+	}
+	names := make([]string, 0, len(f.domains))
+	for n := range f.domains {
+		names = append(names, n)
+	}
+	slices.Sort(names)
+	out := make([]libvirt.Domain, 0, len(names))
+	for _, n := range names {
+		out = append(out, libvirt.Domain{Name: n})
+	}
+	return out, uint32(len(out)), nil
+}
+
+// --- storage ---
+
+func (f *fakeAPI) StoragePoolLookupByName(name string) (libvirt.StoragePool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StoragePoolLookupByName"); err != nil {
+		return libvirt.StoragePool{}, err
+	}
+	if _, ok := f.pools[name]; !ok {
+		return libvirt.StoragePool{}, lvError(libvirt.ErrNoStoragePool, "Storage pool not found: no storage pool with matching name '%s'", name)
+	}
+	return libvirt.StoragePool{Name: name}, nil
+}
+
+func (f *fakeAPI) StoragePoolRefresh(pool libvirt.StoragePool, _ uint32) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StoragePoolRefresh"); err != nil {
+		return err
+	}
+	if _, ok := f.pools[pool.Name]; !ok {
+		return lvError(libvirt.ErrNoStoragePool, "no pool %s", pool.Name)
+	}
+	return nil
+}
+
+func (f *fakeAPI) StorageVolLookupByName(pool libvirt.StoragePool, name string) (libvirt.StorageVol, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StorageVolLookupByName"); err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	if _, ok := f.pools[pool.Name][name]; !ok {
+		return libvirt.StorageVol{}, lvError(libvirt.ErrNoStorageVol, "Storage volume not found: no storage vol with matching name '%s'", name)
+	}
+	return libvirt.StorageVol{Pool: pool.Name, Name: name, Key: f.poolPath + "/" + name}, nil
+}
+
+func (f *fakeAPI) StorageVolGetPath(vol libvirt.StorageVol) (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StorageVolGetPath"); err != nil {
+		return "", err
+	}
+	if _, ok := f.pools[vol.Pool][vol.Name]; !ok {
+		return "", lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
+	}
+	return f.poolPath + "/" + vol.Name, nil
+}
+
+func (f *fakeAPI) StorageVolCreateXML(pool libvirt.StoragePool, raw string, _ libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StorageVolCreateXML"); err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	vols, ok := f.pools[pool.Name]
+	if !ok {
+		return libvirt.StorageVol{}, lvError(libvirt.ErrNoStoragePool, "no pool %s", pool.Name)
+	}
+	var v volumeXML
+	if err := xml.Unmarshal([]byte(raw), &v); err != nil {
+		return libvirt.StorageVol{}, lvError(libvirt.ErrXMLError, "%v", err)
+	}
+	if _, exists := vols[v.Name]; exists {
+		return libvirt.StorageVol{}, lvError(libvirt.ErrStorageVolExist, "storage volume '%s' exists already", v.Name)
+	}
+	fv := &fakeVol{name: v.Name, format: v.Target.Format.Type, capacity: v.Capacity.Value}
+	if v.BackingStore != nil {
+		fv.backing = v.BackingStore.Path
+	}
+	vols[v.Name] = fv
+	return libvirt.StorageVol{Pool: pool.Name, Name: v.Name, Key: f.poolPath + "/" + v.Name}, nil
+}
+
+func (f *fakeAPI) StorageVolUpload(vol libvirt.StorageVol, r io.Reader, _, _ uint64, _ libvirt.StorageVolUploadFlags) error {
+	f.mu.Lock()
+	if err := f.enter("StorageVolUpload"); err != nil {
+		f.mu.Unlock()
+		return err
+	}
+	fv, ok := f.pools[vol.Pool][vol.Name]
+	hook := f.uploadHook
+	f.mu.Unlock()
+	if !ok {
+		return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
+	}
+	if hook != nil {
+		if err := hook(vol.Name, r); err != nil {
+			return err
+		}
+	}
+	// Read outside the lock: the stream may be a slow HTTP body.
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return fmt.Errorf("reading upload stream: %w", err)
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	fv.data = data
+	return nil
+}
+
+func (f *fakeAPI) StorageVolDownload(vol libvirt.StorageVol, w io.Writer, _, _ uint64, _ libvirt.StorageVolDownloadFlags) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StorageVolDownload"); err != nil {
+		return err
+	}
+	fv, ok := f.pools[vol.Pool][vol.Name]
+	if !ok {
+		return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
+	}
+	_, err := io.Copy(w, bytes.NewReader(fv.data))
+	return err
+}
+
+func (f *fakeAPI) StorageVolDelete(vol libvirt.StorageVol, _ libvirt.StorageVolDeleteFlags) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StorageVolDelete"); err != nil {
+		return err
+	}
+	if _, ok := f.pools[vol.Pool][vol.Name]; !ok {
+		return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
+	}
+	delete(f.pools[vol.Pool], vol.Name)
+	return nil
+}

internal/provider/kvm/id.go

@@ -0,0 +1,21 @@
+package kvm
+
+import (
+	"fmt"
+	"strings"
+)
+
+// The providerID carries the host, so Get/Delete stay correct after a
+// placement.zone edit mid-replacement — re-reading the spec then would look
+// at the wrong hypervisor.
+func formatProviderID(host, name string) string {
+	return fmt.Sprintf("hosts/%s/domains/%s", host, name)
+}
+
+func parseProviderID(id string) (host, name string, err error) {
+	parts := strings.Split(id, "/")
+	if len(parts) != 4 || parts[0] != "hosts" || parts[2] != "domains" || parts[1] == "" || parts[3] == "" {
+		return "", "", fmt.Errorf("malformed kvm providerID %q, want hosts/<host>/domains/<name>", id)
+	}
+	return parts[1], parts[3], nil
+}

internal/provider/kvm/id_test.go

@@ -0,0 +1,41 @@
+package kvm
+
+import "testing"
+
+func TestParseProviderID_roundTrip(t *testing.T) {
+	t.Parallel()
+	id := formatProviderID("nas1", "proxy-abc123def456ghij")
+	if id != "hosts/nas1/domains/proxy-abc123def456ghij" {
+		t.Fatalf("formatProviderID() = %q", id)
+	}
+	host, name, err := parseProviderID(id)
+	if err != nil {
+		t.Fatalf("parseProviderID(%q) error = %v", id, err)
+	}
+	if host != "nas1" || name != "proxy-abc123def456ghij" {
+		t.Errorf("parseProviderID(%q) = %q, %q", id, host, name)
+	}
+}
+
+func TestParseProviderID_malformed(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		id   string
+	}{
+		{"empty", ""},
+		{"gcp style", "zones/europe-west1-b/instances/proxy-x"},
+		{"missing name", "hosts/nas1/domains/"},
+		{"missing host", "hosts//domains/proxy-x"},
+		{"too many segments", "hosts/nas1/domains/proxy-x/extra"},
+		{"bare name", "proxy-x"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if _, _, err := parseProviderID(tc.id); err == nil {
+				t.Errorf("parseProviderID(%q) error = nil, want error", tc.id)
+			}
+		})
+	}
+}

internal/provider/kvm/image.go

@@ -0,0 +1,232 @@
+package kvm
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// imageSource is a resolved placement.image: either an entry from the
+// provider's images map (Key set) or a URL given directly.
+type imageSource struct {
+	Key    string
+	URL    string
+	SHA256 string
+	Format string
+}
+
+// resolveImage maps placement.image to a source. Anything that is neither a
+// configured key nor an http(s) URL is a spec error the reconciler should
+// surface, not retry.
+func resolveImage(cfg provider.KVMConfig, image string) (imageSource, error) {
+	if img, ok := cfg.Images[image]; ok {
+		return imageSource{Key: image, URL: img.URL, SHA256: img.SHA256, Format: img.Format}, nil
+	}
+	u, err := url.Parse(image)
+	if err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != "" {
+		return imageSource{URL: image, Format: formatQcow2}, nil
+	}
+	return imageSource{}, fmt.Errorf("%w: placement.image %q is neither a key in kvm.images nor an http(s) URL",
+		provider.ErrPermanent, image)
+}
+
+// volumeName is the cached base image's name in the pool: readable for
+// configured keys, a URL hash for ad-hoc URLs.
+func (s imageSource) volumeName() string {
+	if s.Key != "" {
+		return "base-" + s.Key + "." + s.Format
+	}
+	sum := sha256.Sum256([]byte(s.URL))
+	return "base-" + hex.EncodeToString(sum[:8]) + "." + s.Format
+}
+
+// markerName is the sidecar volume holding the sha256 of the verified
+// upload. Its presence is what says "the base volume is complete".
+func (s imageSource) markerName() string {
+	return s.volumeName() + ".sha256"
+}
+
+// imageCache fetches base images into a host's pool on first use. It keeps
+// no state about what is cached — the pool is the source of truth, checked
+// on every call — only a per-(host, image) lock so concurrent Creates for
+// the same image on the same host result in one download.
+type imageCache struct {
+	http *http.Client
+
+	mu    sync.Mutex
+	locks map[string]*sync.Mutex
+}
+
+func newImageCache(client *http.Client) *imageCache {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	return &imageCache{http: client, locks: make(map[string]*sync.Mutex)}
+}
+
+func (c *imageCache) lockFor(key string) *sync.Mutex {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	m, ok := c.locks[key]
+	if !ok {
+		m = &sync.Mutex{}
+		c.locks[key] = m
+	}
+	return m
+}
+
+// ensure makes src present in pool and returns the base volume's path on
+// the host. The image is downloaded synchronously through the libvirt
+// stream when the pool does not already hold a verified copy.
+func (c *imageCache) ensure(ctx context.Context, log logr.Logger, api libvirtAPI, hostName string, pool libvirt.StoragePool, src imageSource) (string, error) {
+	m := c.lockFor(hostName + "/" + src.volumeName())
+	m.Lock()
+	defer m.Unlock()
+
+	base, ready, err := c.lookup(api, pool, src)
+	if err != nil {
+		return "", err
+	}
+	if ready {
+		log.V(1).Info("base image already cached", "host", hostName, "volume", src.volumeName())
+		return api.StorageVolGetPath(base)
+	}
+	if err := deleteIfExists(api, pool, src.markerName()); err != nil {
+		return "", err
+	}
+	if err := deleteIfExists(api, pool, src.volumeName()); err != nil {
+		return "", err
+	}
+	base, err = c.fetch(ctx, log, api, hostName, pool, src)
+	if err != nil {
+		return "", err
+	}
+	return api.StorageVolGetPath(base)
+}
+
+// lookup reports whether the pool holds a complete, verified base volume:
+// the volume and its marker both exist and, when a checksum is configured,
+// the marker records that checksum.
+func (c *imageCache) lookup(api libvirtAPI, pool libvirt.StoragePool, src imageSource) (libvirt.StorageVol, bool, error) {
+	base, err := api.StorageVolLookupByName(pool, src.volumeName())
+	if hasCode(err, libvirt.ErrNoStorageVol) {
+		return libvirt.StorageVol{}, false, nil
+	}
+	if err != nil {
+		return libvirt.StorageVol{}, false, fmt.Errorf("looking up base volume: %w", err)
+	}
+	marker, err := api.StorageVolLookupByName(pool, src.markerName())
+	if hasCode(err, libvirt.ErrNoStorageVol) {
+		return base, false, nil
+	}
+	if err != nil {
+		return libvirt.StorageVol{}, false, fmt.Errorf("looking up image marker: %w", err)
+	}
+	if src.SHA256 == "" {
+		return base, true, nil
+	}
+	var buf bytes.Buffer
+	if err := api.StorageVolDownload(marker, &buf, 0, 0, 0); err != nil {
+		return libvirt.StorageVol{}, false, fmt.Errorf("reading image marker: %w", err)
+	}
+	return base, strings.EqualFold(strings.TrimSpace(buf.String()), src.SHA256), nil
+}
+
+// fetch streams the image from its URL straight into a new base volume,
+// hashing on the way, then writes the marker. Nothing is buffered in the
+// operator beyond the HTTP body's read buffer.
+func (c *imageCache) fetch(ctx context.Context, log logr.Logger, api libvirtAPI, hostName string, pool libvirt.StoragePool, src imageSource) (libvirt.StorageVol, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, src.URL, nil)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("%w: building image request: %v", provider.ErrPermanent, err)
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("%w: fetching image %s: %v", provider.ErrTransient, src.URL, err)
+	}
+	defer resp.Body.Close()
+	switch {
+	case resp.StatusCode >= 200 && resp.StatusCode < 300:
+	case resp.StatusCode >= 400 && resp.StatusCode < 500:
+		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s returned HTTP %d", provider.ErrPermanent, src.URL, resp.StatusCode)
+	default:
+		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s returned HTTP %d", provider.ErrTransient, src.URL, resp.StatusCode)
+	}
+	if resp.ContentLength < 0 {
+		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s has no Content-Length; the base volume needs a known size", provider.ErrPermanent, src.URL)
+	}
+
+	log.Info("fetching base image into host pool",
+		"host", hostName, "pool", pool.Name, "volume", src.volumeName(),
+		"url", src.URL, "bytes", resp.ContentLength)
+	start := time.Now()
+
+	baseXML, err := buildVolumeXML(volumeSpec{Name: src.volumeName(), SizeBytes: resp.ContentLength, Format: src.Format})
+	if err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	base, err := api.StorageVolCreateXML(pool, baseXML, 0)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("creating base volume: %w", err)
+	}
+	hasher := sha256.New()
+	if err := api.StorageVolUpload(base, io.TeeReader(resp.Body, hasher), 0, 0, 0); err != nil {
+		_ = api.StorageVolDelete(base, 0)
+		return libvirt.StorageVol{}, fmt.Errorf("uploading image to base volume: %w", err)
+	}
+	digest := hex.EncodeToString(hasher.Sum(nil))
+	if src.SHA256 != "" && !strings.EqualFold(digest, src.SHA256) {
+		_ = api.StorageVolDelete(base, 0)
+		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s sha256 is %s, config expects %s",
+			provider.ErrPermanent, src.URL, digest, src.SHA256)
+	}
+	// libvirt sized the volume from Content-Length; a refresh makes it
+	// re-read the real virtual size from the uploaded qcow2 header.
+	if err := api.StoragePoolRefresh(pool, 0); err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("refreshing pool after upload: %w", err)
+	}
+
+	markerXML, err := buildVolumeXML(rawVolume(src.markerName(), int64(len(digest))))
+	if err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	marker, err := api.StorageVolCreateXML(pool, markerXML, 0)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("creating image marker: %w", err)
+	}
+	if err := api.StorageVolUpload(marker, strings.NewReader(digest), 0, 0, 0); err != nil {
+		_ = api.StorageVolDelete(marker, 0)
+		return libvirt.StorageVol{}, fmt.Errorf("writing image marker: %w", err)
+	}
+	log.Info("base image cached",
+		"host", hostName, "volume", src.volumeName(), "sha256", digest,
+		"bytes", resp.ContentLength, "duration", time.Since(start).Round(time.Second))
+	return base, nil
+}
+
+func deleteIfExists(api libvirtAPI, pool libvirt.StoragePool, name string) error {
+	vol, err := api.StorageVolLookupByName(pool, name)
+	if hasCode(err, libvirt.ErrNoStorageVol) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("looking up volume %s: %w", name, err)
+	}
+	if err := api.StorageVolDelete(vol, 0); err != nil && !hasCode(err, libvirt.ErrNoStorageVol) {
+		return fmt.Errorf("deleting volume %s: %w", name, err)
+	}
+	return nil
+}

internal/provider/kvm/image_test.go

@@ -0,0 +1,385 @@
+package kvm
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/go-logr/logr"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+var testImage = bytes.Repeat([]byte("QFI\xfb fake qcow2 payload "), 1024)
+
+func testImageSHA() string {
+	sum := sha256.Sum256(testImage)
+	return hex.EncodeToString(sum[:])
+}
+
+// imageServer serves testImage and counts hits. Status overrides the
+// response, noLength forces chunked encoding (no Content-Length), delay
+// slows each response so concurrent callers overlap.
+type imageServer struct {
+	*httptest.Server
+	hits     atomic.Int32
+	status   int
+	noLength bool
+	delay    time.Duration
+	body     []byte
+}
+
+func newImageServer(t *testing.T) *imageServer {
+	t.Helper()
+	s := &imageServer{status: http.StatusOK, body: testImage}
+	s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		s.hits.Add(1)
+		time.Sleep(s.delay)
+		if s.status != http.StatusOK {
+			w.WriteHeader(s.status)
+			return
+		}
+		// Bodies larger than the server's write buffer would otherwise go
+		// out chunked without a Content-Length, unlike real image mirrors.
+		if s.noLength {
+			w.(http.Flusher).Flush()
+		} else {
+			w.Header().Set("Content-Length", strconv.Itoa(len(s.body)))
+		}
+		_, _ = w.Write(s.body)
+	}))
+	t.Cleanup(s.Close)
+	return s
+}
+
+func testSource(srv *imageServer, sha string) imageSource {
+	return imageSource{Key: "debian-13", URL: srv.URL + "/debian.qcow2", SHA256: sha, Format: formatQcow2}
+}
+
+func ensureTest(t *testing.T, api *fakeAPI, src imageSource) (string, error) {
+	t.Helper()
+	c := newImageCache(nil)
+	pool, err := api.StoragePoolLookupByName("default")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, src)
+}
+
+func TestResolveImage(t *testing.T) {
+	t.Parallel()
+	cfg := withDefaults(provider.KVMConfig{Images: map[string]provider.KVMImage{
+		"debian-13": {URL: "https://cloud.debian.org/d.qcow2", SHA256: "abc"},
+	}})
+	tests := []struct {
+		name    string
+		image   string
+		want    imageSource
+		wantErr bool
+	}{
+		{"configured key", "debian-13", imageSource{Key: "debian-13", URL: "https://cloud.debian.org/d.qcow2", SHA256: "abc", Format: "qcow2"}, false},
+		{"direct https url", "https://x.example/img.qcow2", imageSource{URL: "https://x.example/img.qcow2", Format: "qcow2"}, false},
+		{"direct http url", "http://mirror/img.qcow2", imageSource{URL: "http://mirror/img.qcow2", Format: "qcow2"}, false},
+		{"unknown key", "ubuntu", imageSource{}, true},
+		{"file url", "file:///tmp/x.qcow2", imageSource{}, true},
+		{"empty", "", imageSource{}, true},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := resolveImage(cfg, tc.image)
+			if tc.wantErr {
+				if !errors.Is(err, provider.ErrPermanent) {
+					t.Fatalf("resolveImage(%q) error = %v, want ErrPermanent", tc.image, err)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveImage(%q) error = %v", tc.image, err)
+			}
+			if got != tc.want {
+				t.Errorf("resolveImage(%q) = %+v, want %+v", tc.image, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestImageSource_volumeNames(t *testing.T) {
+	t.Parallel()
+	keyed := imageSource{Key: "debian-13", URL: "https://a/b.qcow2", Format: "qcow2"}
+	if keyed.volumeName() != "base-debian-13.qcow2" || keyed.markerName() != "base-debian-13.qcow2.sha256" {
+		t.Errorf("keyed names = %s / %s", keyed.volumeName(), keyed.markerName())
+	}
+	direct := imageSource{URL: "https://a/b.qcow2", Format: "qcow2"}
+	name := direct.volumeName()
+	if !strings.HasPrefix(name, "base-") || !strings.HasSuffix(name, ".qcow2") || len(name) != len("base-")+16+len(".qcow2") {
+		t.Errorf("direct-url name = %q, want base-<16 hex>.qcow2", name)
+	}
+	other := imageSource{URL: "https://a/c.qcow2", Format: "qcow2"}
+	if other.volumeName() == name {
+		t.Error("different URLs produced the same volume name")
+	}
+}
+
+func TestImageCache_ensure_freshFetch(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	path, err := ensureTest(t, api, testSource(srv, testImageSHA()))
+	if err != nil {
+		t.Fatalf("ensure() error = %v", err)
+	}
+	if path != "/var/lib/libvirt/images/base-debian-13.qcow2" {
+		t.Errorf("path = %q", path)
+	}
+	base := api.vol("default", "base-debian-13.qcow2")
+	if base == nil || !bytes.Equal(base.data, testImage) {
+		t.Fatalf("base volume missing or content differs")
+	}
+	if base.format != "qcow2" || base.capacity != int64(len(testImage)) {
+		t.Errorf("base volume format/capacity = %s/%d, want qcow2/%d", base.format, base.capacity, len(testImage))
+	}
+	marker := api.vol("default", "base-debian-13.qcow2.sha256")
+	if marker == nil || string(marker.data) != testImageSHA() {
+		t.Fatalf("marker missing or wrong: %+v", marker)
+	}
+	if !api.hasCall("StoragePoolRefresh") {
+		t.Error("pool was not refreshed after the upload")
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("http hits = %d, want 1", srv.hits.Load())
+	}
+}
+
+func TestImageCache_ensure_noChecksumConfigured(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	if _, err := ensureTest(t, api, testSource(srv, "")); err != nil {
+		t.Fatalf("ensure() error = %v", err)
+	}
+	if marker := api.vol("default", "base-debian-13.qcow2.sha256"); marker == nil || string(marker.data) != testImageSHA() {
+		t.Errorf("marker should still record the computed digest, got %+v", marker)
+	}
+	// Second call: ready without a checksum to compare, so no marker read.
+	if _, err := ensureTest(t, api, testSource(srv, "")); err != nil {
+		t.Fatalf("second ensure() error = %v", err)
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("http hits = %d, want 1", srv.hits.Load())
+	}
+}
+
+func TestImageCache_ensure_alreadyCached(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	api.putVol("default", "base-debian-13.qcow2", "qcow2", testImage)
+	api.putVol("default", "base-debian-13.qcow2.sha256", "raw", []byte(testImageSHA()))
+	path, err := ensureTest(t, api, testSource(srv, testImageSHA()))
+	if err != nil {
+		t.Fatalf("ensure() error = %v", err)
+	}
+	if path != "/var/lib/libvirt/images/base-debian-13.qcow2" {
+		t.Errorf("path = %q", path)
+	}
+	if srv.hits.Load() != 0 {
+		t.Errorf("http hits = %d, want 0 for a cached image", srv.hits.Load())
+	}
+	if api.hasCall("StorageVolCreateXML") || api.hasCall("StorageVolDelete") {
+		t.Errorf("cached image was touched: calls = %v", api.calls)
+	}
+}
+
+func TestImageCache_ensure_markerMissingRedownloads(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	api.putVol("default", "base-debian-13.qcow2", "qcow2", []byte("half an image"))
+	if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
+		t.Fatalf("ensure() error = %v", err)
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("http hits = %d, want 1: a base without its marker is a partial upload", srv.hits.Load())
+	}
+	if base := api.vol("default", "base-debian-13.qcow2"); base == nil || !bytes.Equal(base.data, testImage) {
+		t.Error("partial base volume was not replaced")
+	}
+}
+
+func TestImageCache_ensure_checksumChangedRedownloads(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	api.putVol("default", "base-debian-13.qcow2", "qcow2", []byte("an older release"))
+	api.putVol("default", "base-debian-13.qcow2.sha256", "raw", []byte(strings.Repeat("0", 64)))
+	if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
+		t.Fatalf("ensure() error = %v", err)
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("http hits = %d, want 1 after the configured checksum changed", srv.hits.Load())
+	}
+	if marker := api.vol("default", "base-debian-13.qcow2.sha256"); marker == nil || string(marker.data) != testImageSHA() {
+		t.Errorf("marker not updated: %+v", marker)
+	}
+}
+
+func TestImageCache_ensure_checksumMismatchIsPermanent(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	_, err := ensureTest(t, api, testSource(srv, strings.Repeat("a", 64)))
+	if !errors.Is(err, provider.ErrPermanent) {
+		t.Fatalf("ensure() error = %v, want ErrPermanent", err)
+	}
+	if names := api.volNames("default"); len(names) != 0 {
+		t.Errorf("volumes left behind after a checksum mismatch: %v", names)
+	}
+}
+
+func TestImageCache_ensure_httpErrors(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name   string
+		status int
+		want   error
+	}{
+		{"404 is permanent", http.StatusNotFound, provider.ErrPermanent},
+		{"403 is permanent", http.StatusForbidden, provider.ErrPermanent},
+		{"503 is transient", http.StatusServiceUnavailable, provider.ErrTransient},
+		{"500 is transient", http.StatusInternalServerError, provider.ErrTransient},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			srv := newImageServer(t)
+			srv.status = tc.status
+			api := newFakeAPI("default")
+			_, err := ensureTest(t, api, testSource(srv, ""))
+			if !errors.Is(err, tc.want) {
+				t.Fatalf("ensure() error = %v, want %v", err, tc.want)
+			}
+			if names := api.volNames("default"); len(names) != 0 {
+				t.Errorf("volumes created despite HTTP %d: %v", tc.status, names)
+			}
+		})
+	}
+}
+
+func TestImageCache_ensure_noContentLengthIsPermanent(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	srv.noLength = true
+	api := newFakeAPI("default")
+	_, err := ensureTest(t, api, testSource(srv, ""))
+	if !errors.Is(err, provider.ErrPermanent) || !strings.Contains(err.Error(), "Content-Length") {
+		t.Fatalf("ensure() error = %v, want ErrPermanent about Content-Length", err)
+	}
+}
+
+func TestImageCache_ensure_uploadFailureCleansUp(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	api.uploadHook = func(_ string, r io.Reader) error {
+		_, _ = io.CopyN(io.Discard, r, 100)
+		return io.ErrUnexpectedEOF
+	}
+	_, err := ensureTest(t, api, testSource(srv, testImageSHA()))
+	if !errors.Is(err, provider.ErrTransient) && classify(err) != provider.ErrTransient {
+		t.Fatalf("ensure() error = %v, want a transient failure", err)
+	}
+	if names := api.volNames("default"); len(names) != 0 {
+		t.Errorf("volumes left behind after a failed upload: %v", names)
+	}
+	// Next attempt starts clean and succeeds.
+	api.uploadHook = nil
+	if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
+		t.Fatalf("retry ensure() error = %v", err)
+	}
+}
+
+func TestImageCache_ensure_missingPoolIsPermanent(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	c := newImageCache(nil)
+	pool, _ := api.StoragePoolLookupByName("default")
+	delete(api.pools, "default")
+	_, err := c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, testSource(srv, ""))
+	if classify(err) != provider.ErrPermanent {
+		t.Fatalf("ensure() error = %v, want one classifying as Permanent", err)
+	}
+}
+
+func TestImageCache_ensure_concurrentCallersDownloadOnce(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	srv.delay = 50 * time.Millisecond
+	api := newFakeAPI("default")
+	c := newImageCache(nil)
+	pool, _ := api.StoragePoolLookupByName("default")
+	src := testSource(srv, testImageSHA())
+	var wg sync.WaitGroup
+	paths := make([]string, 4)
+	errs := make([]error, 4)
+	for i := range 4 {
+		wg.Go(func() {
+			paths[i], errs[i] = c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, src)
+		})
+	}
+	wg.Wait()
+	for i := range 4 {
+		if errs[i] != nil || paths[i] != "/var/lib/libvirt/images/base-debian-13.qcow2" {
+			t.Errorf("caller %d: path=%q err=%v", i, paths[i], errs[i])
+		}
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("http hits = %d, want 1 for four concurrent callers", srv.hits.Load())
+	}
+}
+
+func TestImageCache_ensure_differentHostsFetchIndependently(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	c := newImageCache(nil)
+	src := testSource(srv, "")
+	for _, host := range []string{"nas1", "lab2"} {
+		api := newFakeAPI("default")
+		pool, _ := api.StoragePoolLookupByName("default")
+		if _, err := c.ensure(context.Background(), logr.Discard(), api, host, pool, src); err != nil {
+			t.Fatalf("ensure(%s) error = %v", host, err)
+		}
+	}
+	if srv.hits.Load() != 2 {
+		t.Errorf("http hits = %d, want one per host", srv.hits.Load())
+	}
+}
+
+func TestImageCache_ensure_cancelledContextIsTransient(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	c := newImageCache(nil)
+	pool, _ := api.StoragePoolLookupByName("default")
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	_, err := c.ensure(ctx, logr.Discard(), api, "nas1", pool, testSource(srv, ""))
+	if classify(err) != provider.ErrTransient {
+		t.Fatalf("ensure() error = %v, want transient", err)
+	}
+	if names := api.volNames("default"); len(names) != 0 {
+		t.Errorf("volumes created despite cancelled context: %v", names)
+	}
+}

internal/provider/kvm/kvm.go

@@ -0,0 +1,560 @@
+// Package kvm implements the provider contract on libvirt/KVM hypervisors.
+// Each configured host is reached over SSH with the libvirt unix socket
+// forwarded through the connection, so the hypervisors need no libvirtd TCP
+// or TLS listener and the operator image needs no libvirt libraries. VMs
+// boot a stock cloud image (fetched into the host's storage pool on first
+// use) through a qcow2 overlay, with the Proxy's cloud-init delivered on a
+// NoCloud seed ISO built in-process.
+package kvm
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+const (
+	defaultDiskSizeGB = 10
+	defaultSocket     = "/var/run/libvirt/libvirt-sock"
+	defaultPool       = "default"
+	formatQcow2       = "qcow2"
+	formatRaw         = "raw"
+
+	// undefineFlags removes everything libvirt may keep alongside a domain
+	// definition, so an undefine never fails on leftover state.
+	undefineFlags = libvirt.DomainUndefineManagedSave |
+		libvirt.DomainUndefineSnapshotsMetadata |
+		libvirt.DomainUndefineNvram
+)
+
+func withDefaults(cfg provider.KVMConfig) provider.KVMConfig {
+	if cfg.DiskSizeGB == 0 {
+		cfg.DiskSizeGB = defaultDiskSizeGB
+	}
+	if cfg.VendorData == "" {
+		cfg.VendorData = defaultVendorData
+	}
+	images := make(map[string]provider.KVMImage, len(cfg.Images))
+	for k, img := range cfg.Images {
+		if img.Format == "" {
+			img.Format = formatQcow2
+		}
+		images[k] = img
+	}
+	cfg.Images = images
+	hosts := make([]provider.KVMHost, len(cfg.Hosts))
+	for i, h := range cfg.Hosts {
+		if h.Socket == "" {
+			h.Socket = defaultSocket
+		}
+		if h.Pool == "" {
+			h.Pool = defaultPool
+		}
+		hosts[i] = h
+	}
+	cfg.Hosts = hosts
+	return cfg
+}
+
+// Provider implements provider.Provider on one or more libvirt hosts.
+type Provider struct {
+	name   string
+	cfg    provider.KVMConfig
+	hosts  *hostPool
+	images *imageCache
+	now    func() time.Time
+}
+
+// New builds a Provider that reaches its hosts over SSH. The key and
+// known_hosts files are read here so a bad path fails at startup rather
+// than on the first Create. Satisfies registry.Constructor.
+func New(_ context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+	cfg := provider.KVMConfig{}
+	if pc.KVM != nil {
+		cfg = *pc.KVM
+	}
+	auth, err := newSSHAuth(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("kvm provider %q: %w", pc.Name, err)
+	}
+	return newWithDialer(pc, auth.dialer(), nil), nil
+}
+
+// newWithDialer is the test seam: the dialer decides what libvirtAPI each
+// host resolves to, and the HTTP client is what image downloads go through.
+func newWithDialer(pc provider.ProviderConfig, dial hostDialer, httpClient *http.Client) *Provider {
+	cfg := provider.KVMConfig{}
+	if pc.KVM != nil {
+		cfg = *pc.KVM
+	}
+	return &Provider{
+		name:   pc.Name,
+		cfg:    withDefaults(cfg),
+		hosts:  newHostPool(dial),
+		images: newImageCache(httpClient),
+		now:    time.Now,
+	}
+}
+
+func (p *Provider) logger(ctx context.Context) logr.Logger {
+	return logf.FromContext(ctx).WithName("kvm").WithValues("provider", p.name)
+}
+
+func (p *Provider) wrapErr(op, id string, err error) error {
+	return provider.Wrap(classify(err), op, p.name, id, err)
+}
+
+// host resolves placement.zone (or a providerID's host segment) to a
+// configured host. An empty zone is allowed only when exactly one host is
+// configured; anything else is a spec error.
+func (p *Provider) host(zone string) (provider.KVMHost, error) {
+	if zone == "" {
+		switch len(p.cfg.Hosts) {
+		case 0:
+			return provider.KVMHost{}, fmt.Errorf("%w: kvm.hosts is empty", provider.ErrPermanent)
+		case 1:
+			return p.cfg.Hosts[0], nil
+		default:
+			return provider.KVMHost{}, fmt.Errorf("%w: placement.zone is required when kvm.hosts lists more than one host",
+				provider.ErrPermanent)
+		}
+	}
+	for _, h := range p.cfg.Hosts {
+		if h.Name == zone {
+			return h, nil
+		}
+	}
+	return provider.KVMHost{}, fmt.Errorf("%w: host %q is not in kvm.hosts", provider.ErrPermanent, zone)
+}
+
+// Create makes the VM exist and running on the placed host, and is
+// idempotent: a defined domain is only started if needed, its volumes
+// untouched. Before the domain exists nothing can be using the volumes,
+// so an existing overlay is reused (its creation is atomic) but the seed
+// is rebuilt from scratch — a previous attempt may have been cut off mid
+// upload, and a truncated seed would boot a VM without cloud-init. The
+// first Create for an image on a host also downloads that image into the
+// host's pool, synchronously.
+func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
+	pl := req.Placement
+	host, err := p.host(pl.Zone)
+	if err != nil {
+		return "", p.wrapErr("create", "", err)
+	}
+	mt, ok := p.cfg.MachineTypes[pl.MachineType]
+	if !ok {
+		return "", p.wrapErr("create", "", fmt.Errorf("%w: placement.machineType %q is not in kvm.machineTypes",
+			provider.ErrPermanent, pl.MachineType))
+	}
+	src, err := resolveImage(p.cfg, pl.Image)
+	if err != nil {
+		return "", p.wrapErr("create", "", err)
+	}
+	id := formatProviderID(host.Name, req.Name)
+	log := p.logger(ctx).WithValues("host", host.Name, "name", req.Name)
+	log.V(2).Info("KVM create resolved",
+		"pool", host.Pool, "bridge", host.Bridge, "machineType", pl.MachineType,
+		"vcpus", mt.VCPUs, "memoryMiB", mt.MemoryMiB, "diskSizeGb", p.cfg.DiskSizeGB,
+		"image", src.volumeName(), "cloudInitBytes", len(req.CloudInit))
+
+	err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
+		dom, err := api.DomainLookupByName(req.Name)
+		if err == nil {
+			log.V(1).Info("domain already defined")
+			return p.ensureRunning(log, api, dom)
+		}
+		if !hasCode(err, libvirt.ErrNoDomain) {
+			return fmt.Errorf("looking up domain: %w", err)
+		}
+		pool, err := api.StoragePoolLookupByName(host.Pool)
+		if err != nil {
+			return fmt.Errorf("looking up pool %s: %w", host.Pool, err)
+		}
+		basePath, err := p.images.ensure(ctx, log, api, host.Name, pool, src)
+		if err != nil {
+			return err
+		}
+		if err := p.ensureOverlay(log, api, pool, req.Name, basePath, src.Format); err != nil {
+			return err
+		}
+		if err := p.writeSeed(log, api, pool, req.Name, req.CloudInit); err != nil {
+			return err
+		}
+		dom, err = p.defineDomain(log, api, host, req, mt)
+		if err != nil {
+			return err
+		}
+		return p.ensureRunning(log, api, dom)
+	})
+	if err != nil {
+		logAPIError(log, "create", err)
+		return "", p.wrapErr("create", id, err)
+	}
+	log.V(1).Info("KVM domain ensured running", "id", id)
+	return id, nil
+}
+
+func (p *Provider) ensureOverlay(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, basePath, baseFormat string) error {
+	xml, err := buildVolumeXML(overlayVolume(name, p.cfg.DiskSizeGB, basePath, baseFormat))
+	if err != nil {
+		return err
+	}
+	_, err = api.StorageVolCreateXML(pool, xml, 0)
+	switch {
+	case err == nil:
+		log.V(1).Info("overlay volume created", "volume", overlayVolumeName(name), "base", basePath)
+	case hasCode(err, libvirt.ErrStorageVolExist):
+		log.V(1).Info("overlay volume already exists", "volume", overlayVolumeName(name))
+	default:
+		return fmt.Errorf("creating overlay volume: %w", err)
+	}
+	return nil
+}
+
+// writeSeed replaces the seed volume with a freshly built NoCloud ISO.
+// Only called while no domain references it.
+func (p *Provider) writeSeed(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, userData string) error {
+	seedName := seedVolumeName(name)
+	if err := deleteIfExists(api, pool, seedName); err != nil {
+		return err
+	}
+	iso, err := buildSeedISO(name, userData, p.cfg.VendorData)
+	if err != nil {
+		return fmt.Errorf("%w: %v", provider.ErrPermanent, err)
+	}
+	xml, err := buildVolumeXML(rawVolume(seedName, int64(len(iso))))
+	if err != nil {
+		return err
+	}
+	vol, err := api.StorageVolCreateXML(pool, xml, 0)
+	if err != nil {
+		return fmt.Errorf("creating seed volume: %w", err)
+	}
+	if err := api.StorageVolUpload(vol, bytes.NewReader(iso), 0, 0, 0); err != nil {
+		return fmt.Errorf("uploading seed volume: %w", err)
+	}
+	log.V(1).Info("seed volume uploaded", "volume", seedName, "bytes", len(iso))
+	return nil
+}
+
+func (p *Provider) defineDomain(log logr.Logger, api libvirtAPI, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType) (libvirt.Domain, error) {
+	xml, err := buildDomainXML(domainParams{
+		Name:      req.Name,
+		UID:       req.UID,
+		CreatedAt: p.now(),
+		VCPUs:     mt.VCPUs,
+		MemoryMiB: mt.MemoryMiB,
+		Pool:      host.Pool,
+		Bridge:    host.Bridge,
+		DiskVol:   overlayVolumeName(req.Name),
+		SeedVol:   seedVolumeName(req.Name),
+	})
+	if err != nil {
+		return libvirt.Domain{}, err
+	}
+	dom, err := api.DomainDefineXML(xml)
+	if err != nil {
+		return libvirt.Domain{}, fmt.Errorf("defining domain: %w", err)
+	}
+	log.V(1).Info("domain defined", "uid", req.UID)
+	return dom, nil
+}
+
+func (p *Provider) ensureRunning(log logr.Logger, api libvirtAPI, dom libvirt.Domain) error {
+	state, _, err := api.DomainGetState(dom, 0)
+	if err != nil {
+		return fmt.Errorf("reading domain state: %w", err)
+	}
+	if isActive(libvirt.DomainState(state)) {
+		return nil
+	}
+	err = api.DomainCreate(dom)
+	switch {
+	case err == nil:
+		log.V(1).Info("domain started")
+	case hasCode(err, libvirt.ErrOperationInvalid):
+		log.V(1).Info("domain already running, start treated as success")
+	default:
+		return fmt.Errorf("starting domain: %w", err)
+	}
+	return nil
+}
+
+// Get reads the domain's state, metadata and addresses. The providerID
+// carries the host, so this looks at the right hypervisor even after a
+// placement.zone edit.
+func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instance, error) {
+	hostName, name, err := parseProviderID(providerID)
+	if err != nil {
+		return nil, provider.Wrap(provider.ErrPermanent, "get", p.name, providerID, err)
+	}
+	host, err := p.host(hostName)
+	if err != nil {
+		return nil, p.wrapErr("get", providerID, err)
+	}
+	log := p.logger(ctx).WithValues("host", host.Name, "name", name)
+	var inst *provider.Instance
+	err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
+		dom, err := api.DomainLookupByName(name)
+		if err != nil {
+			return fmt.Errorf("looking up domain: %w", err)
+		}
+		meta, err := p.metadata(api, dom)
+		if err != nil {
+			return err
+		}
+		inst, err = p.inspect(log, api, host.Name, dom, meta)
+		return err
+	})
+	if err != nil {
+		logAPIError(log, "get", err)
+		return nil, p.wrapErr("get", providerID, err)
+	}
+	log.V(1).Info("KVM domain fetched", "state", inst.State, "ip", inst.IP, "uid", inst.UID)
+	return inst, nil
+}
+
+// inspect assembles an Instance from a domain that is known to exist and
+// whose metadata has already been read (the zero value for a domain
+// without the operator's element, which reports an empty UID as the
+// contract requires).
+func (p *Provider) inspect(log logr.Logger, api libvirtAPI, hostName string, dom libvirt.Domain, meta instanceMetadata) (*provider.Instance, error) {
+	state, _, err := api.DomainGetState(dom, 0)
+	if err != nil {
+		return nil, fmt.Errorf("reading domain state: %w", err)
+	}
+	var ip string
+	if isActive(libvirt.DomainState(state)) {
+		ip = p.address(log, api, dom)
+	}
+	return &provider.Instance{
+		ID:        formatProviderID(hostName, dom.Name),
+		IP:        ip,
+		State:     mapState(libvirt.DomainState(state), ip),
+		UID:       meta.UID,
+		CreatedAt: meta.createdAt(),
+	}, nil
+}
+
+// metadata returns the operator's element, or the zero value when the
+// domain has none (it was not created by this operator).
+func (p *Provider) metadata(api libvirtAPI, dom libvirt.Domain) (instanceMetadata, error) {
+	raw, err := api.DomainGetMetadata(dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, libvirt.DomainAffectCurrent)
+	if hasCode(err, libvirt.ErrNoDomainMetadata) {
+		return instanceMetadata{}, nil
+	}
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("reading domain metadata: %w", err)
+	}
+	meta, err := parseInstanceMetadata(raw)
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
+	}
+	return meta, nil
+}
+
+// address returns the domain's first routable IPv4, asking the host's ARP
+// table first and the guest agent second. Both sources fail routinely while
+// the guest is still booting (no traffic seen yet, agent not installed
+// yet), so their errors mean "no address yet", not "Get failed".
+func (p *Provider) address(log logr.Logger, api libvirtAPI, dom libvirt.Domain) string {
+	for _, source := range []libvirt.DomainInterfaceAddressesSource{
+		libvirt.DomainInterfaceAddressesSrcArp,
+		libvirt.DomainInterfaceAddressesSrcAgent,
+	} {
+		ifaces, err := api.DomainInterfaceAddresses(dom, uint32(source), 0)
+		if err != nil {
+			log.V(2).Info("no addresses from source", "source", source.String(), "error", err.Error())
+			continue
+		}
+		if ip := firstIPv4(ifaces); ip != "" {
+			return ip
+		}
+	}
+	return ""
+}
+
+func firstIPv4(ifaces []libvirt.DomainInterface) string {
+	for _, iface := range ifaces {
+		for _, a := range iface.Addrs {
+			if a.Type != int32(libvirt.IPAddrTypeIpv4) {
+				continue
+			}
+			ip := net.ParseIP(a.Addr)
+			if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsUnspecified() {
+				continue
+			}
+			return a.Addr
+		}
+	}
+	return ""
+}
+
+// isActive reports whether the guest is executing (or blocked on the
+// hypervisor, which is still "up" from the outside).
+func isActive(s libvirt.DomainState) bool {
+	return s == libvirt.DomainRunning || s == libvirt.DomainBlocked
+}
+
+// mapState collapses libvirt's states onto the provider's. An active
+// domain without an address is still Provisioning: an empty IP must never
+// be published as Running. "No state" is also Provisioning — the
+// reconciler answers Stopped with delete-and-recreate, too drastic for a
+// state libvirt could not determine; the next poll decides. Everything
+// else — paused, suspended, shut off, crashed, shutting down — is Stopped.
+func mapState(s libvirt.DomainState, ip string) provider.InstanceState {
+	if s == libvirt.DomainNostate {
+		return provider.StateProvisioning
+	}
+	if !isActive(s) {
+		return provider.StateStopped
+	}
+	if ip == "" {
+		return provider.StateProvisioning
+	}
+	return provider.StateRunning
+}
+
+// Delete tears the VM down and is idempotent at every step: a stopped
+// domain is not destroyed again, a missing domain is not undefined, and
+// the per-VM volumes are removed by name even when the domain is already
+// gone, so a crash between volume creation and define is cleaned up on the
+// next reconcile.
+func (p *Provider) Delete(ctx context.Context, providerID string) error {
+	hostName, name, err := parseProviderID(providerID)
+	if err != nil {
+		return provider.Wrap(provider.ErrPermanent, "delete", p.name, providerID, err)
+	}
+	host, err := p.host(hostName)
+	if err != nil {
+		return p.wrapErr("delete", providerID, err)
+	}
+	log := p.logger(ctx).WithValues("host", host.Name, "name", name)
+	err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
+		if err := p.removeDomain(log, api, name); err != nil {
+			return err
+		}
+		return p.removeVolumes(log, api, host.Pool, name)
+	})
+	if err != nil {
+		logAPIError(log, "delete", err)
+		return p.wrapErr("delete", providerID, err)
+	}
+	log.V(1).Info("KVM domain deleted")
+	return nil
+}
+
+func (p *Provider) removeDomain(log logr.Logger, api libvirtAPI, name string) error {
+	dom, err := api.DomainLookupByName(name)
+	if hasCode(err, libvirt.ErrNoDomain) {
+		log.V(1).Info("domain already gone")
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("looking up domain: %w", err)
+	}
+	err = api.DomainDestroy(dom)
+	switch {
+	case err == nil:
+		log.V(1).Info("domain destroyed")
+	case hasCode(err, libvirt.ErrOperationInvalid):
+		log.V(1).Info("domain not running, destroy skipped")
+	case hasCode(err, libvirt.ErrNoDomain):
+		return nil
+	default:
+		return fmt.Errorf("destroying domain: %w", err)
+	}
+	if err := api.DomainUndefineFlags(dom, undefineFlags); err != nil && !hasCode(err, libvirt.ErrNoDomain) {
+		return fmt.Errorf("undefining domain: %w", err)
+	}
+	log.V(1).Info("domain undefined")
+	return nil
+}
+
+// removeVolumes deletes the overlay and seed. A missing pool means there
+// is nothing left to clean, not a failure: refusing to finish a Delete
+// over a pool that no longer exists would pin the Proxy's finalizer.
+func (p *Provider) removeVolumes(log logr.Logger, api libvirtAPI, poolName, name string) error {
+	pool, err := api.StoragePoolLookupByName(poolName)
+	if hasCode(err, libvirt.ErrNoStoragePool) {
+		log.Info("pool not found, no volumes to delete", "pool", poolName)
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("looking up pool %s: %w", poolName, err)
+	}
+	for _, vol := range []string{overlayVolumeName(name), seedVolumeName(name)} {
+		if err := deleteIfExists(api, pool, vol); err != nil {
+			return err
+		}
+	}
+	log.V(1).Info("per-VM volumes deleted")
+	return nil
+}
+
+// ListByTag inspects every domain on every host and keeps those carrying
+// the operator's managed metadata. One unreachable host fails the whole
+// sweep, so orphan GC never acts on a partial view of the fleet; a single
+// domain that cannot be read is skipped instead, which is safe in the
+// same direction — a domain missing from the list is never collected.
+func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
+	log := p.logger(ctx)
+	var out []provider.Instance
+	for _, host := range p.cfg.Hosts {
+		hlog := log.WithValues("host", host.Name)
+		err := p.hosts.call(ctx, host, func(api libvirtAPI) error {
+			doms, _, err := api.ConnectListAllDomains(1, 0)
+			if err != nil {
+				return fmt.Errorf("listing domains: %w", err)
+			}
+			for _, dom := range doms {
+				if inst := p.managedInstance(hlog, api, host.Name, dom); inst != nil {
+					out = append(out, *inst)
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			logAPIError(hlog, "list", err)
+			return nil, p.wrapErr("list", "", err)
+		}
+	}
+	log.V(1).Info("KVM domains listed", "hosts", len(p.cfg.Hosts), "managed", len(out))
+	for _, inst := range out {
+		log.V(2).Info("KVM listed instance", "id", inst.ID, "state", inst.State, "uid", inst.UID, "createdAt", inst.CreatedAt)
+	}
+	return out, nil
+}
+
+// managedInstance returns nil for domains the operator does not own, for
+// domains that vanished between the listing and the inspection, and for
+// domains whose metadata or state cannot be read (corrupt element, mid
+// teardown): one bad domain must not block GC for every other one. A
+// transport failure would also land here, but it fails the next RPC in
+// the same sweep too, so the host still counts as unreachable.
+func (p *Provider) managedInstance(log logr.Logger, api libvirtAPI, hostName string, dom libvirt.Domain) *provider.Instance {
+	log = log.WithValues("name", dom.Name)
+	meta, err := p.metadata(api, dom)
+	if err != nil {
+		log.V(1).Info("skipping domain with unreadable metadata", "error", err.Error())
+		return nil
+	}
+	if !meta.managed() {
+		return nil
+	}
+	inst, err := p.inspect(log, api, hostName, dom, meta)
+	if err != nil {
+		log.V(1).Info("skipping domain that could not be inspected", "error", err.Error())
+		return nil
+	}
+	return inst
+}

internal/provider/kvm/kvm_test.go

@@ -0,0 +1,880 @@
+package kvm
+
+import (
+	"context"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func TestWithDefaults(t *testing.T) {
+	t.Parallel()
+	cfg := withDefaults(provider.KVMConfig{
+		Images: map[string]provider.KVMImage{
+			"debian": {URL: "https://x/d.qcow2"},
+			"raw":    {URL: "https://x/r.img", Format: "raw"},
+		},
+		Hosts: []provider.KVMHost{
+			{Name: "a", Address: "a:22", User: "virt", Bridge: "br0"},
+			{Name: "b", Address: "b:22", User: "virt", Bridge: "br1", Socket: "/run/s", Pool: "fast"},
+		},
+	})
+	if cfg.DiskSizeGB != defaultDiskSizeGB {
+		t.Errorf("diskSizeGb = %d, want %d", cfg.DiskSizeGB, defaultDiskSizeGB)
+	}
+	if cfg.VendorData != defaultVendorData {
+		t.Errorf("vendorData = %q, want the default", cfg.VendorData)
+	}
+	if cfg.Images["debian"].Format != "qcow2" || cfg.Images["raw"].Format != "raw" {
+		t.Errorf("image formats = %q/%q, want qcow2 default and raw kept", cfg.Images["debian"].Format, cfg.Images["raw"].Format)
+	}
+	if a := cfg.Hosts[0]; a.Socket != defaultSocket || a.Pool != defaultPool {
+		t.Errorf("host a = %+v, want default socket and pool", a)
+	}
+	if b := cfg.Hosts[1]; b.Socket != "/run/s" || b.Pool != "fast" {
+		t.Errorf("host b = %+v, want explicit socket and pool kept", b)
+	}
+}
+
+func TestWithDefaults_keepsExplicitValues(t *testing.T) {
+	t.Parallel()
+	cfg := withDefaults(provider.KVMConfig{DiskSizeGB: 42, VendorData: "#cloud-config\n"})
+	if cfg.DiskSizeGB != 42 || cfg.VendorData != "#cloud-config\n" {
+		t.Errorf("cfg = %+v, want explicit values kept", cfg)
+	}
+}
+
+// --- provider over fake hosts ---
+
+const (
+	testName      = "proxy-abc123def456ghij"
+	testUID       = "11111111-2222-3333-4444-555555555555"
+	testImageURL  = "https://images.example/debian-13.qcow2"
+	testCloudInit = "#cloud-config\npackages: [squid]\n"
+)
+
+var (
+	testNow    = time.Date(2026, 9, 8, 10, 0, 0, 0, time.UTC)
+	errHostOff = errors.New("ssh dial: connection refused")
+)
+
+// hostFakes routes each configured host name to its own fakeAPI; a host
+// with no entry is unreachable.
+type hostFakes map[string]*fakeAPI
+
+func (h hostFakes) dial(_ context.Context, host provider.KVMHost) (*hostClient, error) {
+	api, ok := h[host.Name]
+	if !ok {
+		return nil, errHostOff
+	}
+	return &hostClient{api: api, close: func() {}}, nil
+}
+
+func testConfig(hosts ...string) provider.ProviderConfig {
+	cfg := &provider.KVMConfig{
+		SSHKeyFile:     "/k",
+		KnownHostsFile: "/kh",
+		Images:         map[string]provider.KVMImage{"debian-13": {URL: testImageURL}},
+		MachineTypes:   map[string]provider.KVMMachineType{"small": {VCPUs: 2, MemoryMiB: 2048}},
+	}
+	for _, h := range hosts {
+		cfg.Hosts = append(cfg.Hosts, provider.KVMHost{Name: h, Address: h + ":22", User: "virt", Bridge: "br0"})
+	}
+	return provider.ProviderConfig{Name: "kvm-home", Type: "kvm", KVM: cfg}
+}
+
+func newTestProvider(fakes hostFakes, hosts ...string) *Provider {
+	p := newWithDialer(testConfig(hosts...), fakes.dial, nil)
+	p.now = func() time.Time { return testNow }
+	return p
+}
+
+// cacheImage pre-populates the pool with the configured image and its
+// marker so Create never touches the network.
+func cacheImage(api *fakeAPI) {
+	src := imageSource{Key: "debian-13", URL: testImageURL, Format: formatQcow2}
+	api.putVol("default", src.volumeName(), formatQcow2, []byte("qcow2"))
+	api.putVol("default", src.markerName(), formatRaw, []byte(testImageSHA()))
+}
+
+func readyHost() *fakeAPI {
+	api := newFakeAPI("default")
+	cacheImage(api)
+	return api
+}
+
+func testRequest() provider.CreateRequest {
+	return provider.CreateRequest{
+		Name:      testName,
+		UID:       testUID,
+		Namespace: "default",
+		ProxyName: "home-proxy-1",
+		Placement: provider.Placement{Zone: "nas1", MachineType: "small", Image: "debian-13"},
+		CloudInit: testCloudInit,
+		Port:      3128,
+	}
+}
+
+func metadataXML(t *testing.T, uid string, created time.Time) string {
+	t.Helper()
+	out, err := xml.Marshal(newInstanceMetadata(uid, created))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(out)
+}
+
+func ipv4(addr string) libvirt.DomainIPAddr {
+	return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv4), Addr: addr, Prefix: 24}
+}
+
+func ipv6(addr string) libvirt.DomainIPAddr {
+	return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv6), Addr: addr, Prefix: 64}
+}
+
+// managedDomain seeds a domain as Create would have left it.
+func managedDomain(t *testing.T, api *fakeAPI, name, uid string, running bool, addrs map[uint32][]libvirt.DomainInterface) {
+	t.Helper()
+	api.putDomain(&fakeDomain{name: name, metadata: metadataXML(t, uid, testNow), running: running, addrs: addrs})
+	api.putVol("default", overlayVolumeName(name), formatQcow2, nil)
+	api.putVol("default", seedVolumeName(name), formatRaw, []byte("iso"))
+}
+
+func wantClass(t *testing.T, err error, class error) {
+	t.Helper()
+	if err == nil {
+		t.Fatalf("got nil error, want %v", class)
+	}
+	if !errors.Is(err, class) {
+		t.Fatalf("error class = %v, want %v (err: %v)", provider.Class(err), class, err)
+	}
+}
+
+func assertCreated(t *testing.T, api *fakeAPI) {
+	t.Helper()
+	dom := api.domain(testName)
+	if dom == nil {
+		t.Fatal("domain not defined")
+	}
+	if !dom.running {
+		t.Error("domain not running")
+	}
+	overlay := api.vol("default", overlayVolumeName(testName))
+	if overlay == nil {
+		t.Fatal("overlay volume missing")
+	}
+	if overlay.backing != "/var/lib/libvirt/images/base-debian-13.qcow2" || overlay.format != formatQcow2 {
+		t.Errorf("overlay = %+v, want qcow2 backed by the cached base", overlay)
+	}
+	if overlay.capacity != defaultDiskSizeGB*bytesPerGiB {
+		t.Errorf("overlay capacity = %d, want %d", overlay.capacity, defaultDiskSizeGB*bytesPerGiB)
+	}
+	seed := api.vol("default", seedVolumeName(testName))
+	if seed == nil {
+		t.Fatal("seed volume missing")
+	}
+	if seed.format != formatRaw || len(seed.data) == 0 || int64(len(seed.data)) != seed.capacity {
+		t.Errorf("seed = format %s, %d bytes, capacity %d; want raw with capacity == uploaded size", seed.format, len(seed.data), seed.capacity)
+	}
+}
+
+func TestCreate_freshHost(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	id, err := p.Create(context.Background(), testRequest())
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if want := "hosts/nas1/domains/" + testName; id != want {
+		t.Errorf("providerID = %s, want %s", id, want)
+	}
+	assertCreated(t, api)
+
+	var d domainXML
+	if err := xml.Unmarshal([]byte(api.domain(testName).xml), &d); err != nil {
+		t.Fatal(err)
+	}
+	if d.VCPU != 2 || d.Memory.Value != 2048 || d.Memory.Unit != "MiB" {
+		t.Errorf("domain size = %d vcpu / %d %s, want 2 / 2048 MiB", d.VCPU, d.Memory.Value, d.Memory.Unit)
+	}
+	if d.Devices.Interfaces[0].Source.Bridge != "br0" {
+		t.Errorf("bridge = %q, want br0", d.Devices.Interfaces[0].Source.Bridge)
+	}
+	if m := d.Metadata.Instance; m.UID != testUID || !m.managed() || !m.createdAt().Equal(testNow) {
+		t.Errorf("metadata = %+v, want managed, uid %s, createdAt %s", m, testUID, testNow)
+	}
+	if api.hasCall("StorageVolDownload") || api.callCount("StorageVolUpload") != 1 {
+		t.Errorf("calls = %v, want no image download and exactly one upload (the seed)", api.calls)
+	}
+}
+
+func TestCreate_fetchesImageOnFirstUse(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	cfg := testConfig("nas1")
+	cfg.KVM.Images["debian-13"] = provider.KVMImage{URL: srv.URL + "/debian.qcow2", SHA256: testImageSHA()}
+	p := newWithDialer(cfg, hostFakes{"nas1": api}.dial, nil)
+
+	if _, err := p.Create(context.Background(), testRequest()); err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("image server hits = %d, want 1", srv.hits.Load())
+	}
+	base := api.vol("default", "base-debian-13.qcow2")
+	if base == nil || string(base.data) != string(testImage) {
+		t.Fatal("base image not cached in the pool")
+	}
+	if api.domain(testName) == nil || !api.domain(testName).running {
+		t.Error("domain not running after image fetch")
+	}
+
+	// A second Proxy with the same image on the same host reuses the cache.
+	req := testRequest()
+	req.Name, req.UID = "proxy-second", "uid-2"
+	if _, err := p.Create(context.Background(), req); err != nil {
+		t.Fatalf("second Create: %v", err)
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("image server hits after second Create = %d, want still 1", srv.hits.Load())
+	}
+}
+
+func TestCreate_isIdempotentWhenComplete(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+	if _, err := p.Create(context.Background(), testRequest()); err != nil {
+		t.Fatal(err)
+	}
+	before := len(api.calls)
+
+	id, err := p.Create(context.Background(), testRequest())
+	if err != nil {
+		t.Fatalf("second Create: %v", err)
+	}
+	if id != "hosts/nas1/domains/"+testName {
+		t.Errorf("providerID = %s", id)
+	}
+	for _, m := range []string{"DomainDefineXML", "DomainCreate", "StorageVolUpload"} {
+		if n := api.callCount(m); n != 1 {
+			t.Errorf("%s called %d times across two Creates, want 1", m, n)
+		}
+	}
+	if len(api.calls) == before {
+		t.Error("second Create made no calls at all; expected lookups")
+	}
+}
+
+// Each case breaks Create after a different step, then re-runs it and
+// expects the second attempt to pick up where the first stopped without
+// duplicating what already exists.
+func TestCreate_resumesAfterPartialFailure(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name     string
+		sabotage func(api *fakeAPI)
+		class    error
+		check    func(t *testing.T, api *fakeAPI)
+	}{
+		{
+			name: "seed upload fails after overlay exists",
+			sabotage: func(api *fakeAPI) {
+				api.uploadHook = func(vol string, _ io.Reader) error {
+					api.uploadHook = nil
+					return io.ErrUnexpectedEOF
+				}
+			},
+			class: provider.ErrTransient,
+			check: func(t *testing.T, api *fakeAPI) {
+				if n := api.callCount("StorageVolUpload"); n != 2 {
+					t.Errorf("seed uploads = %d, want 2 (failed, then redone)", n)
+				}
+				if n := api.callCount("StorageVolCreateXML"); n != 4 {
+					t.Errorf("volume creates = %d, want 4 (overlay, seed; overlay rejected as existing, seed again)", n)
+				}
+				if names := api.volNames("default"); len(names) != 4 {
+					t.Errorf("pool holds %v, want base, marker, one overlay, one seed", names)
+				}
+			},
+		},
+		{
+			name:     "define fails after volumes exist",
+			sabotage: func(api *fakeAPI) { api.failOnce["DomainDefineXML"] = lvError(libvirt.ErrInternalError, "boom") },
+			class:    provider.ErrTransient,
+			check: func(t *testing.T, api *fakeAPI) {
+				if n := api.callCount("StorageVolUpload"); n != 2 {
+					t.Errorf("seed uploads = %d, want 2 (seed rebuilt while no domain exists)", n)
+				}
+				if n := api.callCount("DomainDefineXML"); n != 2 {
+					t.Errorf("defines = %d, want 2", n)
+				}
+			},
+		},
+		{
+			name:     "start fails after define",
+			sabotage: func(api *fakeAPI) { api.failOnce["DomainCreate"] = lvError(libvirt.ErrInternalError, "boom") },
+			class:    provider.ErrTransient,
+			check: func(t *testing.T, api *fakeAPI) {
+				if n := api.callCount("DomainDefineXML"); n != 1 {
+					t.Errorf("defines = %d, want 1 (existing domain reused)", n)
+				}
+				if n := api.callCount("DomainCreate"); n != 2 {
+					t.Errorf("starts = %d, want 2", n)
+				}
+			},
+		},
+		{
+			name:     "transport drops mid-create",
+			sabotage: func(api *fakeAPI) { api.failOnce["DomainLookupByName"] = io.EOF },
+			class:    provider.ErrTransient,
+			check: func(t *testing.T, api *fakeAPI) {
+				if n := api.callCount("DomainDefineXML"); n != 1 {
+					t.Errorf("defines = %d, want 1", n)
+				}
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			api := readyHost()
+			tc.sabotage(api)
+			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+			_, err := p.Create(context.Background(), testRequest())
+			wantClass(t, err, tc.class)
+			if _, err := p.Create(context.Background(), testRequest()); err != nil {
+				t.Fatalf("retry Create: %v", err)
+			}
+			assertCreated(t, api)
+			tc.check(t, api)
+		})
+	}
+}
+
+// A reconcile cancelled mid-upload leaves a seed volume whose content is
+// short; the compensating delete cannot run because the pool has already
+// closed the connection. The next Create must not trust that volume.
+func TestCreate_replacesLeftoverSeed(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	api.putVol("default", seedVolumeName(testName), formatRaw, []byte("trunc"))
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	if _, err := p.Create(context.Background(), testRequest()); err != nil {
+		t.Fatal(err)
+	}
+	assertCreated(t, api)
+	if seed := api.vol("default", seedVolumeName(testName)); string(seed.data) == "trunc" {
+		t.Error("leftover seed was kept")
+	}
+	if n := api.callCount("StorageVolUpload"); n != 1 {
+		t.Errorf("uploads = %d, want 1 (the rebuilt seed)", n)
+	}
+}
+
+func TestCreate_startsStoppedDomain(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	managedDomain(t, api, testName, testUID, false, nil)
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	if _, err := p.Create(context.Background(), testRequest()); err != nil {
+		t.Fatal(err)
+	}
+	if !api.domain(testName).running {
+		t.Error("domain still stopped")
+	}
+	if api.hasCall("DomainDefineXML") || api.hasCall("StorageVolUpload") || api.hasCall("StorageVolDelete") {
+		t.Errorf("calls = %v, want no define, upload or volume delete for an existing domain", api.calls)
+	}
+}
+
+func TestCreate_singleHostNeedsNoZone(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+	req := testRequest()
+	req.Placement.Zone = ""
+
+	id, err := p.Create(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if id != "hosts/nas1/domains/"+testName {
+		t.Errorf("providerID = %s, want the single host filled in", id)
+	}
+}
+
+func TestCreate_placementErrors(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		hosts []string
+		edit  func(*provider.CreateRequest)
+		want  string
+	}{
+		{"unknown host", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Zone = "nas9" }, "not in kvm.hosts"},
+		{"empty zone with two hosts", []string{"nas1", "nas2"}, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "placement.zone is required"},
+		{"empty zone with no hosts", nil, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "kvm.hosts is empty"},
+		{"unknown machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "huge" }, "not in kvm.machineTypes"},
+		{"empty machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "" }, "not in kvm.machineTypes"},
+		{"unknown image", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Image = "windows" }, "neither a key"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fakes := hostFakes{}
+			for _, h := range tc.hosts {
+				fakes[h] = readyHost()
+			}
+			p := newTestProvider(fakes, tc.hosts...)
+			req := testRequest()
+			tc.edit(&req)
+
+			_, err := p.Create(context.Background(), req)
+			wantClass(t, err, provider.ErrPermanent)
+			if !strings.Contains(err.Error(), tc.want) {
+				t.Errorf("error = %q, want it to mention %q", err, tc.want)
+			}
+			for _, api := range fakes {
+				if len(api.calls) != 0 {
+					t.Errorf("host was called: %v", api.calls)
+				}
+			}
+		})
+	}
+}
+
+func TestCreate_hostFailures(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		setup func() (hostFakes, string)
+		class error
+	}{
+		{
+			name:  "host unreachable",
+			setup: func() (hostFakes, string) { return hostFakes{}, "nas1" },
+			class: provider.ErrTransient,
+		},
+		{
+			name: "pool missing",
+			setup: func() (hostFakes, string) {
+				api := newFakeAPI("other")
+				return hostFakes{"nas1": api}, "nas1"
+			},
+			class: provider.ErrPermanent,
+		},
+		{
+			name: "domain xml rejected",
+			setup: func() (hostFakes, string) {
+				api := readyHost()
+				api.fail["DomainDefineXML"] = lvError(libvirt.ErrConfigUnsupported, "no q35 here")
+				return hostFakes{"nas1": api}, "nas1"
+			},
+			class: provider.ErrPermanent,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fakes, host := tc.setup()
+			p := newTestProvider(fakes, host)
+			_, err := p.Create(context.Background(), testRequest())
+			wantClass(t, err, tc.class)
+		})
+	}
+}
+
+func TestCreate_cancelledContext(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	_, err := p.Create(ctx, testRequest())
+	wantClass(t, err, provider.ErrTransient)
+	if !errors.Is(err, context.Canceled) {
+		t.Errorf("error = %v, want it to unwrap to context.Canceled", err)
+	}
+}
+
+func TestGet(t *testing.T) {
+	t.Parallel()
+	arp := map[uint32][]libvirt.DomainInterface{
+		uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Name: "vnet0", Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.50")}}},
+	}
+	agentOnly := map[uint32][]libvirt.DomainInterface{
+		uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
+			{Name: "lo", Addrs: []libvirt.DomainIPAddr{ipv4("127.0.0.1"), ipv6("::1")}},
+			{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv6("fe80::1"), ipv4("192.168.1.51")}},
+		},
+	}
+	linkLocalOnly := map[uint32][]libvirt.DomainInterface{
+		uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
+			{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv4("169.254.10.10")}},
+		},
+	}
+	cases := []struct {
+		name    string
+		running bool
+		addrs   map[uint32][]libvirt.DomainInterface
+		state   provider.InstanceState
+		ip      string
+	}{
+		{"running with arp address", true, arp, provider.StateRunning, "192.168.1.50"},
+		{"running, agent only, skips loopback and v6", true, agentOnly, provider.StateRunning, "192.168.1.51"},
+		{"running, no address yet", true, nil, provider.StateProvisioning, ""},
+		{"running, only link-local", true, linkLocalOnly, provider.StateProvisioning, ""},
+		{"shut off", false, arp, provider.StateStopped, ""},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			api := readyHost()
+			managedDomain(t, api, testName, testUID, tc.running, tc.addrs)
+			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+			inst, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
+			if err != nil {
+				t.Fatalf("Get: %v", err)
+			}
+			want := provider.Instance{ID: "hosts/nas1/domains/" + testName, IP: tc.ip, State: tc.state, UID: testUID, CreatedAt: testNow}
+			if *inst != want {
+				t.Errorf("instance = %+v, want %+v", *inst, want)
+			}
+		})
+	}
+}
+
+func TestGet_unmanagedDomainHasNoUID(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	api.putDomain(&fakeDomain{name: "hand-made", running: true})
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	inst, err := p.Get(context.Background(), "hosts/nas1/domains/hand-made")
+	if err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	if inst.UID != "" || !inst.CreatedAt.IsZero() {
+		t.Errorf("instance = %+v, want empty UID and zero CreatedAt", inst)
+	}
+}
+
+func TestGet_errors(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		id    string
+		setup func(t *testing.T, api *fakeAPI)
+		class error
+	}{
+		{"missing domain", "hosts/nas1/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrNotFound},
+		{"malformed id", "zones/x/instances/y", func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
+		{"host not configured", "hosts/nas9/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
+		{
+			"state read fails", "hosts/nas1/domains/" + testName,
+			func(t *testing.T, api *fakeAPI) {
+				managedDomain(t, api, testName, testUID, true, nil)
+				api.fail["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
+			},
+			provider.ErrTransient,
+		},
+		{
+			"corrupt metadata", "hosts/nas1/domains/" + testName,
+			func(_ *testing.T, api *fakeAPI) {
+				api.putDomain(&fakeDomain{name: testName, metadata: "<instance", running: true})
+			},
+			provider.ErrPermanent,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			api := readyHost()
+			tc.setup(t, api)
+			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+			_, err := p.Get(context.Background(), tc.id)
+			wantClass(t, err, tc.class)
+		})
+	}
+}
+
+func TestGet_hostUnreachable(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(hostFakes{}, "nas1")
+	_, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
+	wantClass(t, err, provider.ErrTransient)
+}
+
+func assertGone(t *testing.T, api *fakeAPI) {
+	t.Helper()
+	if api.domain(testName) != nil {
+		t.Error("domain still defined")
+	}
+	if names := api.volNames("default"); len(names) != 2 {
+		t.Errorf("pool holds %v, want only the cached base image and its marker", names)
+	}
+}
+
+func TestDelete(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		setup func(t *testing.T, api *fakeAPI)
+		check func(t *testing.T, api *fakeAPI)
+	}{
+		{
+			name:  "running domain",
+			setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, true, nil) },
+			check: func(t *testing.T, api *fakeAPI) {
+				if !api.hasCall("DomainDestroy") || !api.hasCall("DomainUndefineFlags") {
+					t.Errorf("calls = %v, want destroy and undefine", api.calls)
+				}
+			},
+		},
+		{
+			name:  "stopped domain",
+			setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, false, nil) },
+			check: func(t *testing.T, api *fakeAPI) {
+				if !api.hasCall("DomainUndefineFlags") {
+					t.Errorf("calls = %v, want undefine despite destroy being invalid", api.calls)
+				}
+			},
+		},
+		{
+			name: "domain gone, volumes left",
+			setup: func(_ *testing.T, api *fakeAPI) {
+				api.putVol("default", overlayVolumeName(testName), formatQcow2, nil)
+				api.putVol("default", seedVolumeName(testName), formatRaw, nil)
+			},
+			check: func(t *testing.T, api *fakeAPI) {
+				if api.hasCall("DomainDestroy") || api.hasCall("DomainUndefineFlags") {
+					t.Errorf("calls = %v, want no domain operations", api.calls)
+				}
+			},
+		},
+		{
+			name:  "nothing exists",
+			setup: func(*testing.T, *fakeAPI) {},
+			check: func(t *testing.T, api *fakeAPI) {
+				if api.hasCall("StorageVolDelete") {
+					t.Errorf("calls = %v, want no deletes", api.calls)
+				}
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			api := readyHost()
+			tc.setup(t, api)
+			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+			if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
+				t.Fatalf("Delete: %v", err)
+			}
+			assertGone(t, api)
+			tc.check(t, api)
+			if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
+				t.Fatalf("second Delete: %v", err)
+			}
+		})
+	}
+}
+
+func TestDelete_missingPoolIsNotAnError(t *testing.T) {
+	t.Parallel()
+	api := newFakeAPI("other")
+	api.putDomain(&fakeDomain{name: testName, metadata: metadataXML(t, testUID, testNow), running: true})
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
+		t.Fatalf("Delete: %v", err)
+	}
+	if api.domain(testName) != nil {
+		t.Error("domain still defined")
+	}
+}
+
+func TestDelete_errors(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		id    string
+		fakes func(t *testing.T) hostFakes
+		class error
+	}{
+		{"malformed id", "nope", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
+		{"host not configured", "hosts/nas9/domains/x", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
+		{"host unreachable", "hosts/nas1/domains/x", func(*testing.T) hostFakes { return hostFakes{} }, provider.ErrTransient},
+		{
+			"undefine fails", "hosts/nas1/domains/" + testName,
+			func(t *testing.T) hostFakes {
+				api := readyHost()
+				managedDomain(t, api, testName, testUID, true, nil)
+				api.fail["DomainUndefineFlags"] = lvError(libvirt.ErrInternalError, "boom")
+				return hostFakes{"nas1": api}
+			},
+			provider.ErrTransient,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			p := newTestProvider(tc.fakes(t), "nas1")
+			wantClass(t, p.Delete(context.Background(), tc.id), tc.class)
+		})
+	}
+}
+
+func TestListByTag_acrossHosts(t *testing.T) {
+	t.Parallel()
+	nas1, nas2 := readyHost(), readyHost()
+	managedDomain(t, nas1, "proxy-one", "uid-1", true, map[uint32][]libvirt.DomainInterface{
+		uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.10")}}},
+	})
+	nas1.putDomain(&fakeDomain{name: "hand-made", running: true})
+	managedDomain(t, nas2, "proxy-two", "uid-2", false, nil)
+	nas2.putDomain(&fakeDomain{name: "foreign", metadata: `<instance xmlns="` + metadataURI + `"><managed>false</managed></instance>`})
+	p := newTestProvider(hostFakes{"nas1": nas1, "nas2": nas2}, "nas1", "nas2")
+
+	got, err := p.ListByTag(context.Background())
+	if err != nil {
+		t.Fatalf("ListByTag: %v", err)
+	}
+	want := []provider.Instance{
+		{ID: "hosts/nas1/domains/proxy-one", IP: "192.168.1.10", State: provider.StateRunning, UID: "uid-1", CreatedAt: testNow},
+		{ID: "hosts/nas2/domains/proxy-two", State: provider.StateStopped, UID: "uid-2", CreatedAt: testNow},
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d instances %+v, want %d", len(got), got, len(want))
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("instance[%d] = %+v, want %+v", i, got[i], want[i])
+		}
+	}
+}
+
+func TestListByTag_oneHostDownFailsSweep(t *testing.T) {
+	t.Parallel()
+	nas1 := readyHost()
+	managedDomain(t, nas1, "proxy-one", "uid-1", true, nil)
+	p := newTestProvider(hostFakes{"nas1": nas1}, "nas1", "nas2")
+
+	got, err := p.ListByTag(context.Background())
+	wantClass(t, err, provider.ErrTransient)
+	if got != nil {
+		t.Errorf("got %+v alongside the error, want nil (no partial view)", got)
+	}
+}
+
+// A domain that cannot be read is left out of the sweep rather than
+// failing it: GC never deletes what it does not see, so skipping is the
+// safe direction, while one bad domain blocking GC for all others is not.
+func TestListByTag_skipsUnreadableDomain(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		setup func(t *testing.T, api *fakeAPI)
+	}{
+		{
+			"deleted mid-sweep",
+			func(t *testing.T, api *fakeAPI) {
+				managedDomain(t, api, "proxy-one", "uid-1", true, nil)
+				api.failOnce["DomainGetMetadata"] = lvError(libvirt.ErrNoDomain, "gone")
+			},
+		},
+		{
+			"corrupt metadata",
+			func(_ *testing.T, api *fakeAPI) {
+				api.putDomain(&fakeDomain{name: "proxy-one", metadata: "<instance", running: true})
+			},
+		},
+		{
+			"state unreadable",
+			func(t *testing.T, api *fakeAPI) {
+				managedDomain(t, api, "proxy-one", "uid-1", true, nil)
+				api.failOnce["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			nas1 := readyHost()
+			tc.setup(t, nas1)
+			managedDomain(t, nas1, "proxy-two", "uid-2", false, nil)
+			p := newTestProvider(hostFakes{"nas1": nas1}, "nas1")
+
+			got, err := p.ListByTag(context.Background())
+			if err != nil {
+				t.Fatalf("ListByTag: %v", err)
+			}
+			if len(got) != 1 || got[0].UID != "uid-2" {
+				t.Errorf("got %+v, want only proxy-two", got)
+			}
+		})
+	}
+}
+
+func TestListByTag_noHosts(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(hostFakes{})
+	got, err := p.ListByTag(context.Background())
+	if err != nil || len(got) != 0 {
+		t.Errorf("got %v, %v; want empty, nil", got, err)
+	}
+}
+
+func TestMapState(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		state libvirt.DomainState
+		ip    string
+		want  provider.InstanceState
+	}{
+		{libvirt.DomainRunning, "10.0.0.1", provider.StateRunning},
+		{libvirt.DomainRunning, "", provider.StateProvisioning},
+		{libvirt.DomainBlocked, "10.0.0.1", provider.StateRunning},
+		{libvirt.DomainPaused, "10.0.0.1", provider.StateStopped},
+		{libvirt.DomainShutdown, "", provider.StateStopped},
+		{libvirt.DomainShutoff, "", provider.StateStopped},
+		{libvirt.DomainCrashed, "", provider.StateStopped},
+		{libvirt.DomainPmsuspended, "", provider.StateStopped},
+		{libvirt.DomainNostate, "", provider.StateProvisioning},
+	}
+	for _, tc := range cases {
+		t.Run(fmt.Sprintf("%s/%q", tc.state, tc.ip), func(t *testing.T) {
+			t.Parallel()
+			if got := mapState(tc.state, tc.ip); got != tc.want {
+				t.Errorf("mapState(%s, %q) = %s, want %s", tc.state, tc.ip, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestNew_badKeyPathFailsFast(t *testing.T) {
+	t.Parallel()
+	cfg := testConfig("nas1")
+	cfg.KVM.SSHKeyFile = t.TempDir() + "/missing"
+	if _, err := New(context.Background(), cfg); err == nil {
+		t.Fatal("New succeeded with a missing key file")
+	}
+}

internal/provider/kvm/metadata.go

@@ -0,0 +1,57 @@
+package kvm
+
+import (
+	"encoding/xml"
+	"fmt"
+	"time"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// metadataURI namespaces the operator's element inside the domain's
+// <metadata>. libvirt has no labels or tags, but it stores any namespaced XML
+// there and hands it back through DomainGetMetadata, which is how the GC
+// contract (managed flag + owning UID) and the creation time survive across
+// operator restarts.
+const metadataURI = "https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm"
+
+type instanceMetadata struct {
+	XMLName   xml.Name `xml:"https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm instance"`
+	Managed   string   `xml:"managed"`
+	UID       string   `xml:"uid"`
+	CreatedAt string   `xml:"createdAt"`
+}
+
+func newInstanceMetadata(uid string, createdAt time.Time) instanceMetadata {
+	return instanceMetadata{
+		Managed:   provider.LabelManagedYes,
+		UID:       uid,
+		CreatedAt: createdAt.UTC().Format(time.RFC3339),
+	}
+}
+
+// parseInstanceMetadata decodes the element as DomainGetMetadata returns it.
+// libvirt may re-serialise it with a namespace prefix rather than the default
+// namespace it was defined with; encoding/xml matches on namespace URI, so
+// both spellings decode.
+func parseInstanceMetadata(s string) (instanceMetadata, error) {
+	var m instanceMetadata
+	if err := xml.Unmarshal([]byte(s), &m); err != nil {
+		return instanceMetadata{}, fmt.Errorf("parsing domain metadata: %w", err)
+	}
+	return m, nil
+}
+
+func (m instanceMetadata) managed() bool {
+	return m.Managed == provider.LabelManagedYes
+}
+
+// createdAt returns the zero time on a malformed timestamp: orphan GC treats
+// zero as "old", so a corrupt value can never protect a domain forever.
+func (m instanceMetadata) createdAt() time.Time {
+	t, err := time.Parse(time.RFC3339, m.CreatedAt)
+	if err != nil {
+		return time.Time{}
+	}
+	return t
+}

internal/provider/kvm/metadata_test.go

@@ -0,0 +1,79 @@
+package kvm
+
+import (
+	"encoding/xml"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestInstanceMetadata_roundTrip(t *testing.T) {
+	t.Parallel()
+	created := time.Date(2026, 9, 4, 12, 30, 0, 0, time.FixedZone("CEST", 2*3600))
+	raw, err := xml.Marshal(newInstanceMetadata("uid-1", created))
+	if err != nil {
+		t.Fatalf("Marshal() error = %v", err)
+	}
+	if !strings.Contains(string(raw), `xmlns="`+metadataURI+`"`) {
+		t.Errorf("marshalled metadata lacks the namespace declaration: %s", raw)
+	}
+	m, err := parseInstanceMetadata(string(raw))
+	if err != nil {
+		t.Fatalf("parseInstanceMetadata() error = %v", err)
+	}
+	if !m.managed() || m.UID != "uid-1" {
+		t.Errorf("parsed = %+v, want managed with uid-1", m)
+	}
+	if got := m.createdAt(); !got.Equal(created) || got.Location() != time.UTC {
+		t.Errorf("createdAt() = %v, want %v in UTC", got, created)
+	}
+}
+
+func TestParseInstanceMetadata_libvirtPrefixedForm(t *testing.T) {
+	t.Parallel()
+	// libvirt re-serialises the stored element with whatever prefix it
+	// chooses; only the namespace URI is contractual.
+	in := `<op:instance xmlns:op="` + metadataURI + `">` +
+		`<op:managed>true</op:managed><op:uid>uid-2</op:uid>` +
+		`<op:createdAt>2026-09-04T10:00:00Z</op:createdAt></op:instance>`
+	m, err := parseInstanceMetadata(in)
+	if err != nil {
+		t.Fatalf("parseInstanceMetadata() error = %v", err)
+	}
+	if !m.managed() || m.UID != "uid-2" || m.createdAt().Hour() != 10 {
+		t.Errorf("parsed = %+v", m)
+	}
+}
+
+func TestParseInstanceMetadata_notManaged(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name    string
+		in      string
+		wantErr bool
+	}{
+		{"managed false", `<instance xmlns="` + metadataURI + `"><managed>false</managed><uid>u</uid></instance>`, false},
+		{"foreign element", `<foo xmlns="urn:other"><managed>true</managed></foo>`, true},
+		{"garbage", `not xml`, true},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			m, err := parseInstanceMetadata(tc.in)
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("error = %v, wantErr %v", err, tc.wantErr)
+			}
+			if m.managed() {
+				t.Errorf("managed() = true for %q", tc.in)
+			}
+		})
+	}
+}
+
+func TestInstanceMetadata_createdAtMalformedIsZero(t *testing.T) {
+	t.Parallel()
+	m := instanceMetadata{CreatedAt: "yesterday"}
+	if !m.createdAt().IsZero() {
+		t.Errorf("createdAt() = %v, want zero time", m.createdAt())
+	}
+}

internal/provider/kvm/seed.go

@@ -0,0 +1,53 @@
+package kvm
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+
+	"github.com/kdomanski/iso9660"
+)
+
+// seedLabel is the volume label cloud-init's NoCloud datasource looks for.
+const seedLabel = "cidata"
+
+// defaultVendorData is the cloud-init vendor-data shipped on every seed
+// unless the config overrides it. cloud-init merges it under the Proxy's own
+// user-data, so stock images grow a guest agent (which lets the provider
+// read the VM's IP before ARP sees traffic) without the user's cloud-init
+// having to know about it.
+const defaultVendorData = `#cloud-config
+packages:
+  - qemu-guest-agent
+runcmd:
+  - [systemctl, enable, --now, qemu-guest-agent]
+`
+
+// buildSeedISO renders a NoCloud seed: user-data verbatim, vendor-data as
+// given, and a meta-data whose instance-id is the deterministic VM name, so
+// cloud-init never mistakes a reboot for a new instance. The ISO's volume
+// descriptor carries the build time, so two builds differ in a few
+// timestamp bytes; the files and label are identical.
+func buildSeedISO(name, userData, vendorData string) ([]byte, error) {
+	w, err := iso9660.NewWriter()
+	if err != nil {
+		return nil, fmt.Errorf("creating iso writer: %w", err)
+	}
+	defer w.Cleanup() //nolint:errcheck // temp-dir cleanup; nothing to do on failure
+
+	files := []struct{ name, body string }{
+		{"user-data", userData},
+		{"meta-data", fmt.Sprintf("instance-id: %s\nlocal-hostname: %s\n", name, name)},
+		{"vendor-data", vendorData},
+	}
+	for _, f := range files {
+		if err := w.AddFile(strings.NewReader(f.body), f.name); err != nil {
+			return nil, fmt.Errorf("adding %s to seed iso: %w", f.name, err)
+		}
+	}
+	var buf bytes.Buffer
+	if err := w.WriteTo(&buf, seedLabel); err != nil {
+		return nil, fmt.Errorf("writing seed iso: %w", err)
+	}
+	return buf.Bytes(), nil
+}

internal/provider/kvm/seed_test.go

@@ -0,0 +1,100 @@
+package kvm
+
+import (
+	"bytes"
+	"io"
+	"maps"
+	"strings"
+	"testing"
+
+	"github.com/kdomanski/iso9660"
+)
+
+func readSeed(t *testing.T, iso []byte) (label string, files map[string]string) {
+	t.Helper()
+	img, err := iso9660.OpenImage(bytes.NewReader(iso))
+	if err != nil {
+		t.Fatalf("OpenImage() error = %v", err)
+	}
+	label, err = img.Label()
+	if err != nil {
+		t.Fatalf("Label() error = %v", err)
+	}
+	root, err := img.RootDir()
+	if err != nil {
+		t.Fatalf("RootDir() error = %v", err)
+	}
+	children, err := root.GetChildren()
+	if err != nil {
+		t.Fatalf("GetChildren() error = %v", err)
+	}
+	files = make(map[string]string, len(children))
+	for _, c := range children {
+		body, err := io.ReadAll(c.Reader())
+		if err != nil {
+			t.Fatalf("reading %s: %v", c.Name(), err)
+		}
+		files[c.Name()] = string(body)
+	}
+	return label, files
+}
+
+func TestBuildSeedISO_contents(t *testing.T) {
+	t.Parallel()
+	userData := "#cloud-config\npackages: [squid]\n"
+	iso, err := buildSeedISO("proxy-abc123def456ghij", userData, defaultVendorData)
+	if err != nil {
+		t.Fatalf("buildSeedISO() error = %v", err)
+	}
+	label, files := readSeed(t, iso)
+	if !strings.EqualFold(label, seedLabel) {
+		t.Errorf("label = %q, want %q", label, seedLabel)
+	}
+	if len(files) != 3 {
+		t.Errorf("files = %v, want exactly user-data, meta-data, vendor-data", files)
+	}
+	if files["user-data"] != userData {
+		t.Errorf("user-data = %q, want verbatim cloud-init", files["user-data"])
+	}
+	if files["vendor-data"] != defaultVendorData {
+		t.Errorf("vendor-data = %q", files["vendor-data"])
+	}
+	if got, want := files["meta-data"], "instance-id: proxy-abc123def456ghij\nlocal-hostname: proxy-abc123def456ghij\n"; got != want {
+		t.Errorf("meta-data = %q, want %q", got, want)
+	}
+}
+
+// The writer stamps the build time into the volume descriptor, so two
+// builds are not byte-identical; what must not change between them is
+// what cloud-init reads: the label and the three files.
+func TestBuildSeedISO_deterministicContent(t *testing.T) {
+	t.Parallel()
+	a, err := buildSeedISO("proxy-x", "#cloud-config\n", "")
+	if err != nil {
+		t.Fatalf("first buildSeedISO() error = %v", err)
+	}
+	b, err := buildSeedISO("proxy-x", "#cloud-config\n", "")
+	if err != nil {
+		t.Fatalf("second buildSeedISO() error = %v", err)
+	}
+	labelA, filesA := readSeed(t, a)
+	labelB, filesB := readSeed(t, b)
+	if labelA != labelB || !maps.Equal(filesA, filesB) {
+		t.Errorf("two builds for the same inputs differ: %q %v vs %q %v", labelA, filesA, labelB, filesB)
+	}
+	if len(a) != len(b) {
+		t.Errorf("sizes differ: %d vs %d; the seed volume is created with the ISO's exact size", len(a), len(b))
+	}
+}
+
+func TestBuildSeedISO_emptyUserData(t *testing.T) {
+	t.Parallel()
+	iso, err := buildSeedISO("proxy-x", "", defaultVendorData)
+	if err != nil {
+		t.Fatalf("buildSeedISO() error = %v", err)
+	}
+	_, files := readSeed(t, iso)
+	if body, ok := files["user-data"]; !ok || body != "" {
+		t.Errorf("user-data = %q (present=%v), want an empty file to still exist", body, ok)
+	}
+}

internal/provider/kvm/volume.go

@@ -0,0 +1,78 @@
+package kvm
+
+import (
+	"encoding/xml"
+	"fmt"
+)
+
+const bytesPerGiB = 1 << 30
+
+func overlayVolumeName(name string) string { return name + ".qcow2" }
+
+func seedVolumeName(name string) string { return name + "-seed.iso" }
+
+// volumeSpec describes a storage volume to create. A non-empty BackingPath
+// makes it a copy-on-write overlay of that file.
+type volumeSpec struct {
+	Name          string
+	SizeBytes     int64
+	Format        string
+	BackingPath   string
+	BackingFormat string
+}
+
+func overlayVolume(name string, sizeGB int64, basePath, baseFormat string) volumeSpec {
+	return volumeSpec{
+		Name:          overlayVolumeName(name),
+		SizeBytes:     sizeGB * bytesPerGiB,
+		Format:        formatQcow2,
+		BackingPath:   basePath,
+		BackingFormat: baseFormat,
+	}
+}
+
+func rawVolume(name string, sizeBytes int64) volumeSpec {
+	return volumeSpec{Name: name, SizeBytes: sizeBytes, Format: formatRaw}
+}
+
+// buildVolumeXML renders the volume for StorageVolCreateXML. Capacity is
+// always given in bytes so no libvirt unit-suffix ambiguity (G vs GiB) can
+// creep in.
+func buildVolumeXML(v volumeSpec) (string, error) {
+	x := volumeXML{
+		Type:     "file",
+		Name:     v.Name,
+		Capacity: sizeXML{Unit: "bytes", Value: v.SizeBytes},
+		Target:   volumeTargetXML{Format: formatXML{Type: v.Format}},
+	}
+	if v.BackingPath != "" {
+		x.BackingStore = &backingStoreXML{Path: v.BackingPath, Format: formatXML{Type: v.BackingFormat}}
+	}
+	out, err := xml.Marshal(x)
+	if err != nil {
+		return "", fmt.Errorf("marshalling volume xml: %w", err)
+	}
+	return string(out), nil
+}
+
+type volumeXML struct {
+	XMLName      xml.Name         `xml:"volume"`
+	Type         string           `xml:"type,attr"`
+	Name         string           `xml:"name"`
+	Capacity     sizeXML          `xml:"capacity"`
+	Target       volumeTargetXML  `xml:"target"`
+	BackingStore *backingStoreXML `xml:"backingStore"`
+}
+
+type volumeTargetXML struct {
+	Format formatXML `xml:"format"`
+}
+
+type formatXML struct {
+	Type string `xml:"type,attr"`
+}
+
+type backingStoreXML struct {
+	Path   string    `xml:"path"`
+	Format formatXML `xml:"format"`
+}

internal/provider/kvm/volume_test.go

@@ -0,0 +1,62 @@
+package kvm
+
+import (
+	"encoding/xml"
+	"strings"
+	"testing"
+)
+
+func TestVolumeNames(t *testing.T) {
+	t.Parallel()
+	if got := overlayVolumeName("proxy-x"); got != "proxy-x.qcow2" {
+		t.Errorf("overlayVolumeName = %q", got)
+	}
+	if got := seedVolumeName("proxy-x"); got != "proxy-x-seed.iso" {
+		t.Errorf("seedVolumeName = %q", got)
+	}
+}
+
+func TestBuildVolumeXML_overlay(t *testing.T) {
+	t.Parallel()
+	raw, err := buildVolumeXML(overlayVolume("proxy-x", 10, "/var/lib/libvirt/images/base-debian-13.qcow2", "qcow2"))
+	if err != nil {
+		t.Fatalf("buildVolumeXML() error = %v", err)
+	}
+	var v volumeXML
+	if err := xml.Unmarshal([]byte(raw), &v); err != nil {
+		t.Fatalf("output is not well-formed XML: %v", err)
+	}
+	if v.Type != "file" || v.Name != "proxy-x.qcow2" {
+		t.Errorf("type/name = %s/%s", v.Type, v.Name)
+	}
+	if v.Capacity.Unit != "bytes" || v.Capacity.Value != 10*bytesPerGiB {
+		t.Errorf("capacity = %d %s, want %d bytes", v.Capacity.Value, v.Capacity.Unit, 10*bytesPerGiB)
+	}
+	if v.Target.Format.Type != "qcow2" {
+		t.Errorf("format = %q", v.Target.Format.Type)
+	}
+	if v.BackingStore == nil {
+		t.Fatalf("backingStore missing:\n%s", raw)
+	}
+	if v.BackingStore.Path != "/var/lib/libvirt/images/base-debian-13.qcow2" || v.BackingStore.Format.Type != "qcow2" {
+		t.Errorf("backingStore = %+v", v.BackingStore)
+	}
+}
+
+func TestBuildVolumeXML_rawHasNoBackingStore(t *testing.T) {
+	t.Parallel()
+	raw, err := buildVolumeXML(rawVolume("proxy-x-seed.iso", 372736))
+	if err != nil {
+		t.Fatalf("buildVolumeXML() error = %v", err)
+	}
+	if strings.Contains(raw, "backingStore") {
+		t.Errorf("raw volume must not declare a backing store:\n%s", raw)
+	}
+	var v volumeXML
+	if err := xml.Unmarshal([]byte(raw), &v); err != nil {
+		t.Fatalf("output is not well-formed XML: %v", err)
+	}
+	if v.Name != "proxy-x-seed.iso" || v.Capacity.Value != 372736 || v.Target.Format.Type != "raw" {
+		t.Errorf("volume = %+v", v)
+	}
+}

internal/provider/metrics.go

@@ -0,0 +1,66 @@
+package provider
+
+import "context"
+
+// RequestRecorder receives one record per provider API call. Implemented
+// by internal/metrics; defined here so this package needs no metrics
+// dependency.
+type RequestRecorder interface {
+	ProviderRequest(provider, op, result string)
+}
+
+// WithMetrics wraps a Provider so every call is recorded with its
+// classified result — zero-cost instrumentation for the next five
+// providers, and the one place Class is called purely for observability.
+func WithMetrics(name string, p Provider, rec RequestRecorder) Provider {
+	return &instrumented{name: name, inner: p, rec: rec}
+}
+
+type instrumented struct {
+	name  string
+	inner Provider
+	rec   RequestRecorder
+}
+
+func (i *instrumented) Create(ctx context.Context, req CreateRequest) (string, error) {
+	id, err := i.inner.Create(ctx, req)
+	i.record("create", err)
+	return id, err
+}
+
+func (i *instrumented) Get(ctx context.Context, providerID string) (*Instance, error) {
+	inst, err := i.inner.Get(ctx, providerID)
+	i.record("get", err)
+	return inst, err
+}
+
+func (i *instrumented) Delete(ctx context.Context, providerID string) error {
+	err := i.inner.Delete(ctx, providerID)
+	i.record("delete", err)
+	return err
+}
+
+func (i *instrumented) ListByTag(ctx context.Context) ([]Instance, error) {
+	instances, err := i.inner.ListByTag(ctx)
+	i.record("list", err)
+	return instances, err
+}
+
+func (i *instrumented) record(op string, err error) {
+	i.rec.ProviderRequest(i.name, op, resultLabel(err))
+}
+
+func resultLabel(err error) string {
+	switch Class(err) {
+	case nil:
+		return "ok"
+	case ErrNotFound:
+		return "not_found"
+	case ErrQuotaExceeded:
+		return "quota_exceeded"
+	case ErrPermanent:
+		return "permanent"
+	default:
+		return "transient"
+	}
+}

internal/provider/metrics_test.go

@@ -0,0 +1,100 @@
+package provider
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+type recordedCall struct{ provider, op, result string }
+
+type fakeRecorder struct{ calls []recordedCall }
+
+func (f *fakeRecorder) ProviderRequest(provider, op, result string) {
+	f.calls = append(f.calls, recordedCall{provider, op, result})
+}
+
+// staticProvider returns canned values; only the classification of its
+// errors matters here.
+type staticProvider struct {
+	createErr, deleteErr, getErr, listErr error
+}
+
+func (s *staticProvider) Create(context.Context, CreateRequest) (string, error) {
+	return "id-1", s.createErr
+}
+func (s *staticProvider) Get(context.Context, string) (*Instance, error) {
+	return &Instance{ID: "id-1"}, s.getErr
+}
+func (s *staticProvider) Delete(context.Context, string) error          { return s.deleteErr }
+func (s *staticProvider) ListByTag(context.Context) ([]Instance, error) { return nil, s.listErr }
+
+func TestWithMetrics_recordsClassifiedResults(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name       string
+		inner      *staticProvider
+		call       func(p Provider) error
+		wantOp     string
+		wantResult string
+	}{
+		{
+			name:       "successful create is ok",
+			inner:      &staticProvider{},
+			call:       func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
+			wantOp:     "create",
+			wantResult: "ok",
+		},
+		{
+			name:       "get NotFound",
+			inner:      &staticProvider{getErr: Wrap(ErrNotFound, "get", "x", "id-1", nil)},
+			call:       func(p Provider) error { _, err := p.Get(context.Background(), "id-1"); return err },
+			wantOp:     "get",
+			wantResult: "not_found",
+		},
+		{
+			name:       "create quota",
+			inner:      &staticProvider{createErr: Wrap(ErrQuotaExceeded, "create", "x", "", nil)},
+			call:       func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
+			wantOp:     "create",
+			wantResult: "quota_exceeded",
+		},
+		{
+			name:       "delete permanent",
+			inner:      &staticProvider{deleteErr: Wrap(ErrPermanent, "delete", "x", "id-1", nil)},
+			call:       func(p Provider) error { return p.Delete(context.Background(), "id-1") },
+			wantOp:     "delete",
+			wantResult: "permanent",
+		},
+		{
+			name:       "unclassified list error is transient",
+			inner:      &staticProvider{listErr: errors.New("connection reset")},
+			call:       func(p Provider) error { _, err := p.ListByTag(context.Background()); return err },
+			wantOp:     "list",
+			wantResult: "transient",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			rec := &fakeRecorder{}
+			p := WithMetrics("gcp-eu", tc.inner, rec)
+
+			callErr := tc.call(p)
+
+			if len(rec.calls) != 1 {
+				t.Fatalf("recorded %d calls, want 1", len(rec.calls))
+			}
+			want := recordedCall{provider: "gcp-eu", op: tc.wantOp, result: tc.wantResult}
+			if rec.calls[0] != want {
+				t.Errorf("recorded %+v, want %+v", rec.calls[0], want)
+			}
+			// The decorator must be transparent: errors pass through.
+			if (tc.wantResult == "ok") != (callErr == nil) {
+				t.Errorf("error passthrough broken: result %s but err %v", tc.wantResult, callErr)
+			}
+		})
+	}
+}

internal/version/version.go

@@ -0,0 +1,47 @@
+// Package version reports which commit the binary was built from. Docker
+// builds stamp it via -ldflags (the build context has no .git, so Go's
+// automatic VCS stamp is absent there); host builds fall back to that
+// automatic stamp.
+package version
+
+import "runtime/debug"
+
+// Commit is set at link time via
+// -ldflags "-X <module>/internal/version.Commit=<hash>".
+var Commit string
+
+// Resolve returns the commit the binary was built from, or "unknown" when
+// neither the ldflags stamp nor build info is available (e.g. go test).
+func Resolve() string {
+	return resolve(Commit, debug.ReadBuildInfo)
+}
+
+func resolve(ldflagsCommit string, readBuildInfo func() (*debug.BuildInfo, bool)) string {
+	if ldflagsCommit != "" {
+		return ldflagsCommit
+	}
+	bi, ok := readBuildInfo()
+	if !ok {
+		return "unknown"
+	}
+	var revision string
+	var modified bool
+	for _, s := range bi.Settings {
+		switch s.Key {
+		case "vcs.revision":
+			revision = s.Value
+		case "vcs.modified":
+			modified = s.Value == "true"
+		}
+	}
+	if revision == "" {
+		return "unknown"
+	}
+	if len(revision) > 12 {
+		revision = revision[:12]
+	}
+	if modified {
+		revision += "-dirty"
+	}
+	return revision
+}

internal/version/version_test.go

@@ -0,0 +1,74 @@
+package version
+
+import (
+	"runtime/debug"
+	"testing"
+)
+
+func buildInfoWith(settings ...debug.BuildSetting) func() (*debug.BuildInfo, bool) {
+	return func() (*debug.BuildInfo, bool) {
+		return &debug.BuildInfo{Settings: settings}, true
+	}
+}
+
+func TestResolve_precedenceAndFallback(t *testing.T) {
+	t.Parallel()
+
+	noBuildInfo := func() (*debug.BuildInfo, bool) { return nil, false }
+
+	tests := []struct {
+		name          string
+		ldflagsCommit string
+		readBuildInfo func() (*debug.BuildInfo, bool)
+		want          string
+	}{
+		{
+			name:          "ldflags stamp wins over build info",
+			ldflagsCommit: "abc123def456-dirty",
+			readBuildInfo: buildInfoWith(debug.BuildSetting{Key: "vcs.revision", Value: "ffffffffffffffffffffffffffffffffffffffff"}),
+			want:          "abc123def456-dirty",
+		},
+		{
+			name:          "no stamp, no build info",
+			readBuildInfo: noBuildInfo,
+			want:          "unknown",
+		},
+		{
+			name:          "build info without vcs settings",
+			readBuildInfo: buildInfoWith(),
+			want:          "unknown",
+		},
+		{
+			name: "full revision truncated to 12 chars",
+			readBuildInfo: buildInfoWith(
+				debug.BuildSetting{Key: "vcs.revision", Value: "0123456789abcdef0123456789abcdef01234567"},
+				debug.BuildSetting{Key: "vcs.modified", Value: "false"},
+			),
+			want: "0123456789ab",
+		},
+		{
+			name: "modified tree gets dirty suffix",
+			readBuildInfo: buildInfoWith(
+				debug.BuildSetting{Key: "vcs.revision", Value: "0123456789abcdef0123456789abcdef01234567"},
+				debug.BuildSetting{Key: "vcs.modified", Value: "true"},
+			),
+			want: "0123456789ab-dirty",
+		},
+		{
+			name: "short revision kept as-is",
+			readBuildInfo: buildInfoWith(
+				debug.BuildSetting{Key: "vcs.revision", Value: "abc123"},
+			),
+			want: "abc123",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := resolve(tc.ldflagsCommit, tc.readBuildInfo); got != tc.want {
+				t.Errorf("resolve() = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}