Log the provider docs split in the changelog

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

.claude/settings.json

@@ -83,7 +83,23 @@
       "Bash(kubectl -n egress-proxies-operator-system get pods -o wide)",
       "Bash(kubectl -n egress-proxies-operator-system get deploy egress-proxies-operator-controller-manager -o jsonpath='{.spec.template.spec.containers[0].args}')",
       "Bash(kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager)",
-      "Bash(python3 -c \"import json; d=json.load\\(open\\('docs/deploy/sa_key.json'\\)\\); print\\(d.get\\('type'\\), d.get\\('client_email'\\)\\)\")"
+      "Bash(python3 -c \"import json; d=json.load\\(open\\('docs/deploy/sa_key.json'\\)\\); print\\(d.get\\('type'\\), d.get\\('client_email'\\)\\)\")",
+      "Bash(tea pr *)",
+      "Bash(git worktree *)",
+      "Bash(python3 -c \"import yaml; yaml.safe_load\\(open\\('.gitea/workflows/build.yaml'\\)\\); print\\('YAML OK'\\)\")",
+      "Bash(ruby -ryaml -e \"YAML.load_file\\('.gitea/workflows/build.yaml'\\); puts 'YAML OK'\")",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator status --short --branch)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator log --oneline -1)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator tag 0.01)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator push origin 0.01)",
+      "Bash(chmod +x docs/demo/run-demo.sh)",
+      "Bash(bash -n docs/demo/run-demo.sh)",
+      "Bash(git add *)",
+      "Bash(git commit -q -m 'Add plan: KVM/libvirt provider *)",
+      "Bash(go get *)",
+      "Bash(go build *)",
+      "Bash(go test *)",
+      "Bash(git commit *)"
     ],
     "additionalDirectories": [
       "/Users/jan.novak/srv/go/egress-proxies-operator/.claude",

CHANGELOG.md

@@ -1,5 +1,132 @@
 # Changelog
 
+## 2026-09-16 11:46 UTC — Provider blueprint split into contract, spec template and appendix
+
+- **`docs/provider-contract.md`** is the generic provider specification: MUST/SHOULD/MAY
+  rules with the caller behind each one, 19 behaviour-profile axes with stable codes
+  (where providers may differ, such as blocking vs submit-and-return Create, or a
+  backend with no provisioning state), error taxonomy, credential hygiene, testing
+  obligations, the adding-a-provider checklist and the contract's known limits. It
+  names no existing provider.
+- **`docs/provider-spec-template.md`** is the fill-in questionnaire a new provider's
+  design answers before implementation; entry numbers match the contract's profile
+  table.
+- **`docs/providers-existing.md`** maps kubernetes, gcp and kvm to every profile and
+  keeps their per-provider reference notes, shared gaps and proposed features.
+- **`docs/providers.md`** is now an index; README, CLAUDE.md and architecture.md point
+  at the new files. This supersedes the single-file blueprint written earlier on the
+  same branch.
+- Two rules the callers enforce that were not written down before: orphan GC sweeps
+  providers sequentially with no deadline, so a hung `ListByTag` stalls all collection;
+  and the reconciler deletes an instance a backend reports as Stopped before start.
+
+## 2026-09-16 08:45 UTC — Provider health checks: ProviderHealth CRD, per-provider checks, gauges
+
+- **Each configured provider is now checked on a schedule** and reported in a
+  cluster-scoped `ProviderHealth` object (`kubectl get ph`), one per provider, with a
+  `Ready` condition, a reason, and per-target verdicts.
+- **Checks:** kubernetes lists one object from the API server; GCP fetches a single
+  instance page; KVM checks SSH/libvirt reachability and the storage pool on every
+  host, in parallel. Failures map to fixed reasons (`Unreachable`,
+  `SSHHandshakeFailed`, `PoolNotFound`, `Timeout`, …).
+- **The KVM check can't hurt reconciles:** it uses `hostPool.probe`, which gives up on
+  a slow RPC without dropping the shared connection, and waiting for a host slot can
+  be interrupted.
+- **Metrics:** `proxy_operator_provider_target_up`, `_check_duration_seconds` and
+  `_check_timestamp_seconds`. The flags are `--provider-check-interval` and
+  `--provider-check-timeout` (defaults to `min(15s, interval)`).
+- Validated with `make test`, `make test-e2e` and the real-hypervisor KVM run against
+  beelink. `make lint` is blocked by a Go 1.27 toolchain/staticcheck panic that has
+  nothing to do with this change.
+- Key files: `api/v1alpha1/providerhealth_types.go`, `internal/provider/check.go`,
+  `internal/provider/kvm/`, `internal/providerhealth/`, `docs/architecture.md`.
+
+## 2026-09-10 22:58 UTC — Code-review fixes on the KVM branch: deletion, validation, lazy SSH, guest agent
+
+Five findings from a review of `feat/kvm-provider`, all fixed on the branch.
+
+- **A Proxy can always finish deleting.** The delete path used to drop the finalizer
+  only when `Get` answered not-found, so a permanent error there retried forever —
+  a KVM host removed from the config was enough. It now calls `Delete` anyway, since
+  that is idempotent and can finish where `Get` cannot, and
+  `crawl.example.com/skip-cleanup: "true"` abandons the instance for whatever stays
+  stuck. In the KVM provider, deleting over an unconfigured host is a no-op success;
+  `Get` still fails permanently there so a typo cannot look like a missing VM.
+- **The guest agent ships as a vendor-data script**, superseding the cloud-config
+  merge in the entry below. Merging could never be made safe: cloud-init drops
+  vendor-data keys the user-data defines, and the strategy cannot be overridden from
+  either document. The Proxy's cloud-init now reaches the seed byte for byte in any
+  format — Jinja, MIME, gzip, scripts — where the round-trip previously re-typed
+  unquoted scalars (`version: 1.10` arrived as `1.1`), dropped comments and reordered
+  keys. New `installGuestAgent` setting, default on.
+- **A missing `kvm-ssh` Secret no longer crash-loops the manager.** The SSH material
+  is read on first use and cached, so only KVM Proxies fail and they recover when the
+  Secret appears. A missing file is transient, an unparseable key permanent, and
+  startup still logs a warning.
+- **Host names and image keys are validated.** Either can end up in a providerID or a
+  volume file name, where a slash made the VM unaddressable for good.
+- Verified end to end: unit suite, the integration test against a real hypervisor
+  (22 s, agent reporting the IP), both in-cluster Proxies Ready in 2 min with 204
+  through squid, the guest showing `qemu-guest-agent` active with its cloud-config
+  comments intact, and the manager staying up with the Secret pointed away.
+- Key files: `internal/provider/kvm/{seed.go,conn.go,kvm.go}`,
+  `internal/controller/proxy_controller.go`, `internal/provider/config_kvm.go`,
+  `api/v1alpha1/proxy_types.go`.
+
+## 2026-09-10 21:04 UTC — KVM provider passes in-cluster; guest agent merged into cloud-config
+
+- Two KVM Proxies with real cloud-configs reach `Ready` from inside the kind cluster
+  (2 min from apply, IP via the guest agent, 204 through squid from the LAN).
+- The `qemu-guest-agent` bootstrap is now merged into each Proxy's cloud-config
+  (`packages` + `runcmd`) instead of shipped as vendor-data: cloud-init's default
+  merger drops vendor-data keys the user-data also defines, so any Proxy with its own
+  `packages:` never got the agent and stayed `Provisioning` forever. `vendorData` is
+  now plain extra vendor-data with no default; non-cloud-config user-data (scripts,
+  `#include`) still gets the bootstrap as vendor-data. A cloud-config that is not a
+  YAML mapping fails the Proxy permanently instead of booting without its config.
+- The manager pod mounts a 64 MiB `emptyDir` on `/tmp`: the seed ISO writer stages
+  files on disk and the root filesystem is read-only.
+- Errors already classified with a provider sentinel no longer drop the cached
+  libvirt SSH connection; only libvirt and transport errors do.
+- Key files: `internal/provider/kvm/{seed.go,conn.go,errors.go}`,
+  `config/manager/manager.yaml`, `docs/kvm-provider.md`.
+
+## 2026-09-10 10:07 UTC — KVM provider validated on a real hypervisor; UEFI default; libvirt call tracing
+
+- `make test-integration-kvm` passes end to end against a libvirt 11.6 / QEMU 10.1
+  host (Debian 13 image, UEFI, guest agent reporting the IP after ~20 s). Four
+  defects found only on the real host, all fixed on the branch:
+  - domain disks are now `type="file"` with resolved paths: pool-volume references
+    get no AppArmor rule from `virt-aa-helper`, so QEMU was denied the disks;
+  - domain metadata is parsed by local name: libvirt returns the element without
+    its namespace declaration;
+  - guests boot **UEFI by default** (`images.<key>.firmware: uefi|bios`, OVMF now a
+    host prerequisite): Debian 13's image hangs in GRUB under SeaBIOS on a headless
+    VM with a serial console (Debian bug #1111240, open); bookworm is unaffected;
+  - the image cache's raw-upload path is confirmed to produce a byte-identical file.
+- Every libvirt RPC is logged at `--zap-log-level=debug` as one `libvirt call` line
+  (`rpc=virDomainCreate`, identifiers, duration, `libvirtError` on failure); level 2
+  adds XML, metadata and addresses; cloud-init is never logged. The integration test
+  shows the trace in `-v` output (`KVM_TEST_LOG_LEVEL`, default 1).
+- Key files: `internal/provider/kvm/{domainxml.go,metadata.go,trace.go,conn.go}`,
+  `internal/provider/config_kvm.go`, `docs/kvm-provider.md`.
+
+## 2026-09-08 11:00 UTC — KVM/libvirt provider
+
+- New `kvm` provider type: Managed proxies as VMs on your own libvirt/KVM hosts. Each
+  host is reached over SSH with the libvirt unix socket forwarded through the
+  connection (pure Go, nothing to expose on the hypervisors beyond sshd); one provider
+  instance can span several hosts, selected by `placement.zone`.
+- Base images (`images:` map or a direct URL in `placement.image`) are downloaded by the
+  operator into each host's storage pool on first use, verified by sha256 when configured,
+  and cached there. Per-VM disks are qcow2 overlays; cloud-init arrives on a NoCloud seed
+  ISO built in-process. GC identity lives in the domain's `<metadata>` element.
+- Wiring: `"kvm": kvm.New` in `cmd/main.go`; optional `kvm-ssh` Secret mounted at
+  `/etc/proxy-operator/kvm`; samples `config/samples/proxy_kvm.yaml` and a `kvm-home`
+  block in `providers-config.yaml`; `docs/kvm-provider.md`; opt-in integration test
+  (`make test-integration-kvm`) against a real hypervisor.
+- Not validated on a real host at the time of this entry; see 2026-09-10 above.
+
 ## 2026-08-10 09:34 CEST — kind e2e verified; fix Squid OOM in containers; quickstart goes in-cluster
 
 - Full end-to-end pass on a throwaway kind cluster: Squid pod Ready with a real CONNECT

CLAUDE.md

@@ -25,10 +25,16 @@ entrypoint yet, so there's nothing to run locally.
 
 ## Architecture
 
-Not yet built. TODO: once the controller-runtime manager and reconcile loop exist,
-document the data-flow diagram and package/role table here, following the standard
-reconcile-loop shape: fetch → resolve → list related → compute desired state → merge
-→ patch only on diff.
+Documented in [docs/architecture.md](docs/architecture.md). It holds the component
+table, the reconcile flow, and one section per runnable: health engine, discovery and
+lease API, orphan GC, metrics, tracing, and provider health checks. The generic
+provider contract (MUST/SHOULD rules, behaviour profiles, checklist for adding a
+provider, known limits) is in [docs/provider-contract.md](docs/provider-contract.md);
+a new provider starts from [docs/provider-spec-template.md](docs/provider-spec-template.md);
+how the existing providers meet the contract, with their gaps, is in
+[docs/providers-existing.md](docs/providers-existing.md); [docs/providers.md](docs/providers.md)
+indexes them. Provider-specific detail is in `docs/kvm-provider.md` and the GCP
+docs; the test layout is in `docs/testing.md`.
 
 ## Conventions
 
@@ -214,7 +220,9 @@ A lightweight `check` job (`go vet` / `go build` / `go test -short`) gates the b
 
 ## Gotchas
 
-- This repo currently contains only `go.mod` (module
-  `gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator`) — no controller code,
-  `main.go`, Dockerfile, k8s manifests, Makefile, or CI config exist yet. Nothing in
-  this file describes working software; it's the convention scaffold to build against.
+- The "What this is" and "Commands" sections above predate the code and are stale.
+  The Makefile targets (`make test`, `make manifests generate`, `make test-e2e`,
+  `make test-integration-kvm`) and `docs/testing.md` are authoritative.
+- `+kubebuilder:rbac` markers are package-level. Keep them in their own comment block,
+  never inside a type's or function's doc comment: controller-gen silently ignores
+  them there. After `make manifests`, check `git diff config/rbac/role.yaml`.

Makefile

@@ -66,6 +66,10 @@ vet: ## Run go vet against code.
 test: manifests generate fmt vet setup-envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test -race $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
+.PHONY: test-integration-kvm
+test-integration-kvm: ## Run the KVM provider test against a real hypervisor. Needs KVM_TEST_HOST/USER/SSH_KEY/BRIDGE/IMAGE_URL; see docs/testing.md.
+	go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/
+
 # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
 # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
 # kubectl kuberc is disabled by default for test isolation; enable with:
@@ -90,7 +94,8 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist
 
 .PHONY: test-e2e
 test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
-	KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v
+	KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) TEMPO_URL=$(TEMPO_URL) OTLP_ENDPOINT=$(OTLP_ENDPOINT) \
+		go test -tags=e2e ./test/e2e/ -v -ginkgo.v -timeout 30m
 	$(MAKE) cleanup-test-e2e
 
 .PHONY: cleanup-test-e2e
@@ -173,6 +178,20 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 	@out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \
 	if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi
 
+KVM_SSH_KEY ?= $(HOME)/.ssh/proxy-operator_ed25519
+KVM_KNOWN_HOSTS ?= $(HOME)/.ssh/kvm_known_hosts
+KVM_SSH_NAMESPACE ?= egress-proxies-operator-system
+
+.PHONY: kvm-ssh-secret
+kvm-ssh-secret: ## Create or update the kvm-ssh Secret from KVM_SSH_KEY (private key) and KVM_KNOWN_HOSTS.
+	@test -f "$(KVM_SSH_KEY)" || { echo "KVM_SSH_KEY=$(KVM_SSH_KEY) not found"; exit 1; }
+	@test -f "$(KVM_KNOWN_HOSTS)" || { echo "KVM_KNOWN_HOSTS=$(KVM_KNOWN_HOSTS) not found"; exit 1; }
+	@grep -q 'PRIVATE KEY' "$(KVM_SSH_KEY)" || { echo "KVM_SSH_KEY=$(KVM_SSH_KEY) is not a private key (did you pass the .pub?)"; exit 1; }
+	"$(KUBECTL)" -n $(KVM_SSH_NAMESPACE) create secret generic kvm-ssh \
+		--from-file=id_ed25519="$(KVM_SSH_KEY)" \
+		--from-file=known_hosts="$(KVM_KNOWN_HOSTS)" \
+		--dry-run=client -o yaml | "$(KUBECTL)" apply -f -
+
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG}

PROJECT

@@ -18,4 +18,11 @@ resources:
   kind: Proxy
   path: gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+  domain: example.com
+  group: crawl
+  kind: ProviderHealth
+  path: gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1
+  version: v1alpha1
 version: "3"

README.md

@@ -15,9 +15,10 @@ itself**, and hands out to crawler clients via an HTTP list/lease API.
   cattle**: any meaningful spec change (placement, cloud-init, port)
   deletes and recreates the VM — never in-place mutation.
 - **Providers** behind one minimal interface: `kubernetes` (a real Squid
-  pod in this cluster — local dev/CI) and `gcp` (Compute Engine VMs with
-  ephemeral external IPs — the real egress fleet). Config is a YAML file
-  (`--providers-config`) with named instances (`gcp-eu`, `gcp-us`, ...).
+  pod in this cluster — local dev/CI), `gcp` (Compute Engine VMs with
+  ephemeral external IPs — the real egress fleet) and `kvm` (VMs on your
+  own libvirt hypervisors, reached over SSH). Config is a YAML file
+  (`--providers-config`) with named instances (`gcp-eu`, `kvm-home`, ...).
 - **Health engine** probes every proxy by fetching a URL *through* it (a
   real CONNECT tunnel — a proxy that accepts TCP but can't egress goes
   Unhealthy), with threshold logic and transition-only status writes.
@@ -59,7 +60,8 @@ kubectl get px -w
 # proxy-kubernetes-sample   Managed   kubernetes   Ready   10.244.x.x   True
 ```
 
-Once it's `Ready`, port-forward the discovery API and use it:
+Once it's `Ready`, port-forward the discovery API and use it (full
+reference with schemas and error codes: [docs/api.md](docs/api.md)):
 
 ```sh
 kubectl -n egress-proxies-operator-system port-forward \
@@ -102,7 +104,8 @@ make deploy IMG=<registry>/egress-proxies-operator:dev
 
 - Provider config comes from the `providers-config` ConfigMap
   ([config/manager/providers_config.yaml](config/manager/providers_config.yaml));
-  the default ships only the kubernetes provider.
+  the default ships the kubernetes provider plus a `kvm-home` entry that needs
+  the `kvm-ssh` Secret (without it the manager starts and warns).
 - The discovery API is exposed by the
   `controller-manager-discovery-service` Service on port 8090.
 - Auth: create the token Secret, or the API serves **unauthenticated**
@@ -130,11 +133,92 @@ make deploy IMG=<registry>/egress-proxies-operator:dev
    which also installs Squid via cloud-init). A missing field fails the
    Proxy with a message naming it.
 
+## KVM setup
+
+Managed proxies as VMs on libvirt/KVM hosts you own. Full guide:
+[docs/kvm-provider.md](docs/kvm-provider.md).
+
+1. Each host needs `libvirtd`, an SSH user in the `libvirt` group, a
+   bridge to a network with DHCP, and a storage pool.
+2. Create the `kvm-ssh` Secret from the operator's **private** key and a
+   `known_hosts` with each host's key; the manager mounts it at
+   `/etc/proxy-operator/kvm`:
+
+   ```sh
+   ssh-keyscan -H <host> >> ~/.ssh/kvm_known_hosts
+   kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
+     --from-file=id_ed25519=$HOME/.ssh/proxy-operator_ed25519 \
+     --from-file=known_hosts=$HOME/.ssh/kvm_known_hosts
+   # or: make kvm-ssh-secret   (same paths by default; KVM_SSH_KEY / KVM_KNOWN_HOSTS to override)
+   ```
+
+3. Add a `kvm` entry to the providers config with `images`,
+   `machineTypes` and `hosts` (see
+   [config/samples/providers-config.yaml](config/samples/providers-config.yaml)).
+   Base images are downloaded into each host's pool on first use.
+4. A Proxy names a host (`placement.zone`, optional with one host), a
+   size (`placement.machineType`) and an image (`placement.image`) — see
+   [config/samples/proxy_kvm.yaml](config/samples/proxy_kvm.yaml).
+
 Cloud-init from a Secret: the Secret **must** carry the label
 `crawl.example.com/cloud-init: "true"` — the operator's cache only holds
 labelled Secrets, so an unlabelled one is invisible (the Proxy reports
 `CloudInitError`). Rotating the Secret's content triggers VM replacement.
 
+Stuck deletion: a Proxy stays `Terminating` while the operator keeps failing
+to clean its instance up — credentials revoked, a provider or a KVM host
+dropped from the config. Annotate it with
+`crawl.example.com/skip-cleanup: "true"` to abandon the instance and release
+the object. Whatever is still running then has to be removed by hand.
+
+## Providers
+
+`Managed` proxies are provisioned by a provider — a small compute backend
+behind one minimal interface. Providers are configured in the
+`--providers-config` YAML file as **named instances**: `spec.provider` on a
+Proxy refers to an entry's `name`, not its `type`, so `gcp-eu` and `gcp-us`
+can be two differently-configured instances of the same `gcp` type (see
+[config/samples/providers-config.yaml](config/samples/providers-config.yaml)).
+
+### Implemented providers
+
+| Type | Creates | Per-instance config | Notes |
+| --- | --- | --- | --- |
+| `kubernetes` | A real Squid pod (`ubuntu/squid:6.6-24.04_edge` by default) in the same cluster the operator runs in | `image` (optional) | Needs no cloud account — local dev, CI, and the kind quickstart. Pods share the cluster's egress IP, so it exercises the full lifecycle but not distinct egress paths. |
+| `gcp` | A Compute Engine VM with an ephemeral external IP | `project` (required), `network`, `networkTag`, `diskSizeGb` | The real egress fleet. Deliberately uses only four API calls (Insert / Get / Delete / AggregatedList), all fire-and-forget: `Create` returns as soon as the operation is submitted and the reconciler discovers progress by polling `Get`. Auth is Application Default Credentials — workload identity in-cluster, `gcloud` ADC locally; no key-file plumbing. |
+| `kvm` | A libvirt VM on one of your own hypervisors, reached over SSH | `sshKeyFile`, `knownHostsFile` (unless `insecureIgnoreHostKey`), `machineTypes`, `hosts` (required); `images`, `diskSizeGb`, `vendorData`, `installGuestAgent` | Home-lab or on-prem egress. The base image is downloaded into each host's pool on first use; `Create` waits for the VM to start. See [docs/kvm-provider.md](docs/kvm-provider.md). |
+
+### Adding a provider
+
+The generic contract is [docs/provider-contract.md](docs/provider-contract.md),
+the questionnaire a new provider's design fills in is
+[docs/provider-spec-template.md](docs/provider-spec-template.md), and how the
+existing providers meet the contract is
+[docs/providers-existing.md](docs/providers-existing.md);
+[docs/providers.md](docs/providers.md) indexes them. Start there: several rules (a
+self-contained `providerID`, `Get` returning `ErrNotFound` only when the
+instance is really gone, `CreatedAt` read back for orphan GC) are enforced by
+callers, not by the interface.
+
+## Tracing
+
+OpenTelemetry tracing is built in but **off by default** — the operator
+only exports spans when pointed at an OTLP receiver:
+
+```bash
+# In config/manager/manager.yaml (commented-out block is already there):
+#   OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger-collector.observability:4318
+# Local dev: print spans to stdout instead
+OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml
+```
+
+Every reconcile, discovery API request, and GC sweep becomes a trace, with
+child spans for provider and Kubernetes API calls; log lines inside a traced
+operation carry matching `traceID`/`spanID` fields (point Grafana/Loki
+derived fields at `traceID`). Sampling and endpoints follow the standard
+`OTEL_*` env vars; `--trace-health-probes` additionally traces each health
+probe (high volume). Details in `docs/architecture.md` §10.
+
 ## Caveats — read these two
 
 **Changing a proxy changes its IP.** Proxies are immutable cattle: editing

api/v1alpha1/providerhealth_types.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ConditionReady summarizes a ProviderHealth's targets: True when every
+// target passed its last check.
+const ConditionReady = "Ready"
+
+// ProviderTarget is the latest check verdict for one thing a provider
+// depends on: a KVM hypervisor, the Kubernetes API server, a GCP project.
+type ProviderTarget struct {
+	// name identifies the target within its provider: the KVM host name,
+	// "apiserver", or the GCP project.
+	// +kubebuilder:validation:MinLength=1
+	// +required
+	Name string `json:"name"`
+
+	// address is where the check connected to.
+	// +optional
+	Address string `json:"address,omitempty"`
+
+	// ready is true when the last check succeeded.
+	// +required
+	Ready bool `json:"ready"`
+
+	// reason is a CamelCase verdict such as Reachable, Unreachable,
+	// Forbidden, SSHHandshakeFailed or PoolNotActive.
+	// +optional
+	Reason string `json:"reason,omitempty"`
+
+	// message is human-readable detail, usually the error of a failed check.
+	// +kubebuilder:validation:MaxLength=1024
+	// +optional
+	Message string `json:"message,omitempty"`
+
+	// latencyMillis is how long the last check took.
+	// +optional
+	LatencyMillis int64 `json:"latencyMillis,omitempty"`
+
+	// lastTransitionTime is when ready last changed.
+	// +required
+	LastTransitionTime metav1.Time `json:"lastTransitionTime"`
+
+	// details are provider-specific facts from the last check, e.g. the
+	// libvirt version and free storage-pool space of a KVM host.
+	// +optional
+	Details map[string]string `json:"details,omitempty"`
+}
+
+// ProviderHealthStatus is the result of the latest check of one provider.
+type ProviderHealthStatus struct {
+	// type is the provider's type from the providers config: kubernetes,
+	// gcp or kvm.
+	// +optional
+	Type string `json:"type,omitempty"`
+
+	// conditions hold Ready, which summarizes the targets.
+	// +listType=map
+	// +listMapKey=type
+	// +optional
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+	// targets hold one verdict per checked target.
+	// +listType=map
+	// +listMapKey=name
+	// +optional
+	Targets []ProviderTarget `json:"targets,omitempty"`
+
+	// targetsReady counts the ready targets as "ready/total".
+	// +optional
+	TargetsReady string `json:"targetsReady,omitempty"`
+
+	// lastCheckTime is when the latest check finished.
+	// +optional
+	LastCheckTime *metav1.Time `json:"lastCheckTime,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster,shortName=ph
+// +kubebuilder:printcolumn:name="Type",type=string,JSONPath=`.status.type`
+// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`
+// +kubebuilder:printcolumn:name="Targets",type=string,JSONPath=`.status.targetsReady`
+// +kubebuilder:printcolumn:name="Last Check",type=date,JSONPath=`.status.lastCheckTime`
+// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
+
+// ProviderHealth reports the periodic health check of one configured
+// provider. It is named after the provider, and the operator creates,
+// updates and deletes it itself; there is no spec to edit.
+type ProviderHealth struct {
+	metav1.TypeMeta `json:",inline"`
+
+	// metadata is a standard object metadata
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitzero"`
+
+	// status is the result of the latest check
+	// +optional
+	Status ProviderHealthStatus `json:"status,omitzero"`
+}
+
+// +kubebuilder:object:root=true
+
+// ProviderHealthList contains a list of ProviderHealth
+type ProviderHealthList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitzero"`
+	Items           []ProviderHealth `json:"items"`
+}

api/v1alpha1/proxy_types.go

@@ -62,6 +62,15 @@ const (
 	// must be replaced.
 	AnnotationSpecHash = "crawl.example.com/spec-hash"
 
+	// AnnotationSkipCleanup abandons the provider-side instance: set it to
+	// "true" and deletion removes the finalizer without asking the provider
+	// to clean anything up. It is the escape hatch for a cleanup that cannot
+	// succeed — a provider dropped from the config, credentials revoked, a
+	// backend that answers permanently — where the object would otherwise
+	// stay in Terminating forever. The instance is left running and becomes
+	// the administrator's to remove.
+	AnnotationSkipCleanup = "crawl.example.com/skip-cleanup"
+
 	// LabelCloudInit must be set (to "true") on every Secret referenced by
 	// spec.cloudInit.secretRef: the manager's cache only holds Secrets
 	// carrying this label, so an unlabelled Secret is invisible to the
@@ -352,7 +361,7 @@ type ProxyList struct {
 
 func init() {
 	SchemeBuilder.Register(func(s *runtime.Scheme) error {
-		s.AddKnownTypes(SchemeGroupVersion, &Proxy{}, &ProxyList{})
+		s.AddKnownTypes(SchemeGroupVersion, &Proxy{}, &ProxyList{}, &ProviderHealth{}, &ProviderHealthList{})
 		return nil
 	})
 }

api/v1alpha1/zz_generated.deepcopy.go

@@ -95,6 +95,120 @@ func (in *PlacementSpec) DeepCopy() *PlacementSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderHealth) DeepCopyInto(out *ProviderHealth) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderHealth.
+func (in *ProviderHealth) DeepCopy() *ProviderHealth {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderHealth)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProviderHealth) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderHealthList) DeepCopyInto(out *ProviderHealthList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]ProviderHealth, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderHealthList.
+func (in *ProviderHealthList) DeepCopy() *ProviderHealthList {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderHealthList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProviderHealthList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderHealthStatus) DeepCopyInto(out *ProviderHealthStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]v1.Condition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Targets != nil {
+		in, out := &in.Targets, &out.Targets
+		*out = make([]ProviderTarget, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.LastCheckTime != nil {
+		in, out := &in.LastCheckTime, &out.LastCheckTime
+		*out = (*in).DeepCopy()
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderHealthStatus.
+func (in *ProviderHealthStatus) DeepCopy() *ProviderHealthStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderHealthStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProviderTarget) DeepCopyInto(out *ProviderTarget) {
+	*out = *in
+	in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
+	if in.Details != nil {
+		in, out := &in.Details, &out.Details
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderTarget.
+func (in *ProviderTarget) DeepCopy() *ProviderTarget {
+	if in == nil {
+		return nil
+	}
+	out := new(ProviderTarget)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Proxy) DeepCopyInto(out *Proxy) {
 	*out = *in

cmd/main.go

@@ -16,8 +16,8 @@ limitations under the License.
 
 // Package main is the composition root: it loads the provider config (fail
 // fast), assembles the provider registry, and wires the reconciler, health
-// engine, lease store, discovery API, orphan GC, and metrics onto one
-// controller-runtime manager.
+// engine, lease store, discovery API, orphan GC, provider health checks and
+// metrics onto one controller-runtime manager.
 package main
 
 import (
@@ -42,6 +42,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -57,7 +58,10 @@ import (
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kvm"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/providerhealth"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version"
 	// +kubebuilder:scaffold:imports
 )
@@ -93,6 +97,8 @@ func main() {
 	var leaseCooldown, maxLeaseTTL time.Duration
 	var showVersion bool
 	var gcpWireFullPayloads bool
+	var traceHealthProbes bool
+	var providerCheckInterval, providerCheckTimeout time.Duration
 
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -133,6 +139,16 @@ func main() {
 		"Print the commit the binary was built from and exit.")
 	flag.BoolVar(&gcpWireFullPayloads, "gcp-wire-log-full-payloads", false,
 		"Log GCP V(5) wire payloads verbatim instead of eliding fields larger than 1KiB.")
+	flag.BoolVar(&traceHealthProbes, "trace-health-probes", false,
+		"Emit one trace span per health probe. Off by default: probes run about "+
+			"once per second per proxy and would dominate trace volume.")
+	flag.DurationVar(&providerCheckInterval, "provider-check-interval", time.Minute,
+		"Interval between provider health checks (can the cluster reach each hypervisor, API "+
+			"and account), published as ProviderHealth objects and metrics. 0 turns them off; "+
+			"otherwise at least 10s.")
+	flag.DurationVar(&providerCheckTimeout, "provider-check-timeout", 0,
+		"Time limit for one round of provider health checks: 15s, or the interval when that is "+
+			"shorter. If set, at most the interval.")
 
 	opts := zap.Options{
 		Development: true,
@@ -148,6 +164,17 @@ func main() {
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 	setupLog.Info("Starting egress-proxies-operator",
 		"commit", version.Resolve(), "goVersion", goruntime.Version())
+
+	// Off (no-op spans, unchanged logs) unless OTEL_* env opts in; see
+	// internal/tracing. Shutdown is called explicitly after mgr.Start
+	// returns — the os.Exit paths below skip defers.
+	tracingShutdown, err := tracing.Setup(logf.IntoContext(context.Background(), setupLog),
+		"egress-proxies-operator", version.Resolve())
+	if err != nil {
+		setupLog.Error(err, "Failed to set up tracing")
+		os.Exit(1)
+	}
+
 	ctx := ctrl.SetupSignalHandler()
 
 	// Providers load first and fail fast: a manager that comes up without
@@ -156,24 +183,42 @@ func main() {
 		setupLog.Error(nil, "--providers-config is required")
 		os.Exit(1)
 	}
+	if err := providerhealth.ValidateSchedule(providerCheckInterval, providerCheckTimeout); err != nil {
+		setupLog.Error(err, "Invalid provider check flags")
+		os.Exit(1)
+	}
 	cfg, err := provider.LoadConfigFile(providersConfig)
 	if err != nil {
 		setupLog.Error(err, "Failed to load providers config", "path", providersConfig)
 		os.Exit(1)
 	}
 	providers, err := registry.Build(ctx, cfg, map[string]registry.Constructor{
-		"kubernetes": kubernetes.New,
+		"kubernetes": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+			// The kubernetes provider builds its own uncached client;
+			// wrap its transport so its API calls join the caller's trace.
+			return kubernetes.NewWithTransportWrapper(ctx, pc, tracing.RestConfigWrapper())
+		},
 		"gcp": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
 			return gcp.NewWithWireOptions(ctx, pc, gcp.WireLogOptions{FullPayloads: gcpWireFullPayloads})
 		},
+		"kvm": kvm.New,
 	})
 	if err != nil {
 		setupLog.Error(err, "Failed to build providers")
 		os.Exit(1)
 	}
 	m := metrics.New()
+	// Checkers come from the unwrapped providers, in config order: the
+	// metrics and tracing wrappers implement only Provider, and a check is
+	// not a provider request to be counted anyway.
+	checkTargets := make([]providerhealth.Target, 0, len(cfg.Providers))
+	for _, pc := range cfg.Providers {
+		checker, _ := providers[pc.Name].(provider.Checker)
+		checkTargets = append(checkTargets, providerhealth.Target{Name: pc.Name, Type: pc.Type, Checker: checker})
+	}
 	for name, p := range providers {
-		providers[name] = provider.WithMetrics(name, p, m)
+		// Tracing outermost: the span covers the metrics recording too.
+		providers[name] = provider.WithTracing(name, provider.WithMetrics(name, p, m))
 	}
 
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
@@ -236,7 +281,9 @@ func main() {
 		cacheOpts.DefaultNamespaces = map[string]cache.Config{proxyNamespace: {}}
 	}
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	restCfg := ctrl.GetConfigOrDie()
+	restCfg.Wrap(tracing.RestConfigWrapper())
+	mgr, err := ctrl.NewManager(restCfg, ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
@@ -258,6 +305,7 @@ func main() {
 	engine := health.NewEngine(mgr.GetClient())
 	engine.Workers = healthWorkers
 	engine.Metrics = m
+	engine.TraceProbes = traceHealthProbes
 	if err := mgr.Add(engine); err != nil {
 		setupLog.Error(err, "Failed to add health engine")
 		os.Exit(1)
@@ -287,6 +335,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	monitor := &providerhealth.Monitor{
+		Client:   mgr.GetClient(),
+		Reader:   mgr.GetAPIReader(),
+		Targets:  checkTargets,
+		Interval: providerCheckInterval,
+		Timeout:  providerCheckTimeout,
+	}
+	if err := mgr.Add(monitor); err != nil {
+		setupLog.Error(err, "Failed to add provider health monitor")
+		os.Exit(1)
+	}
+
 	if err := m.Register(ctrlmetrics.Registry,
 		proxyPhaseCounts(mgr.GetClient()),
 		func() int {
@@ -300,6 +360,10 @@ func main() {
 		setupLog.Error(err, "Failed to register metrics")
 		os.Exit(1)
 	}
+	if err := metrics.RegisterProviderTargets(ctrlmetrics.Registry, providerTargetSamples(monitor)); err != nil {
+		setupLog.Error(err, "Failed to register provider check metrics")
+		os.Exit(1)
+	}
 
 	if err := (&controller.ProxyReconciler{
 		Client:       mgr.GetClient(),
@@ -323,8 +387,19 @@ func main() {
 	}
 
 	setupLog.Info("Starting manager")
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "Failed to run manager")
+	startErr := mgr.Start(ctx)
+
+	// Flush pending spans on the way out, error path included. Fresh
+	// context: the signal ctx is already cancelled by the time Start
+	// returns.
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	if err := tracingShutdown(shutdownCtx); err != nil {
+		setupLog.Error(err, "Failed to flush traces on shutdown")
+	}
+	cancel()
+
+	if startErr != nil {
+		setupLog.Error(startErr, "Failed to run manager")
 		os.Exit(1)
 	}
 }
@@ -351,3 +426,23 @@ func proxyPhaseCounts(c client.Reader) func() map[string]int {
 		return counts
 	}
 }
+
+// providerTargetSamples flattens the monitor's latest results for the
+// scrape-time provider-check gauges.
+func providerTargetSamples(mon *providerhealth.Monitor) func() []metrics.ProviderTarget {
+	return func() []metrics.ProviderTarget {
+		var out []metrics.ProviderTarget
+		for _, r := range mon.Snapshot() {
+			for _, ts := range r.Targets {
+				out = append(out, metrics.ProviderTarget{
+					Provider:  r.Provider,
+					Target:    ts.Name,
+					Up:        ts.Ready,
+					Duration:  ts.Latency,
+					CheckedAt: r.CheckedAt,
+				})
+			}
+		}
+		return out
+	}
+}

config/crd/bases/crawl.example.com_providerhealths.yaml

@@ -0,0 +1,193 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.21.0
+  name: providerhealths.crawl.example.com
+spec:
+  group: crawl.example.com
+  names:
+    kind: ProviderHealth
+    listKind: ProviderHealthList
+    plural: providerhealths
+    shortNames:
+    - ph
+    singular: providerhealth
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .status.type
+      name: Type
+      type: string
+    - jsonPath: .status.conditions[?(@.type=="Ready")].status
+      name: Ready
+      type: string
+    - jsonPath: .status.targetsReady
+      name: Targets
+      type: string
+    - jsonPath: .status.lastCheckTime
+      name: Last Check
+      type: date
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: |-
+          ProviderHealth reports the periodic health check of one configured
+          provider. It is named after the provider, and the operator creates,
+          updates and deletes it itself; there is no spec to edit.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          status:
+            description: status is the result of the latest check
+            properties:
+              conditions:
+                description: conditions hold Ready, which summarizes the targets.
+                items:
+                  description: Condition contains details for one aspect of the current
+                    state of this API Resource.
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - type
+                x-kubernetes-list-type: map
+              lastCheckTime:
+                description: lastCheckTime is when the latest check finished.
+                format: date-time
+                type: string
+              targets:
+                description: targets hold one verdict per checked target.
+                items:
+                  description: |-
+                    ProviderTarget is the latest check verdict for one thing a provider
+                    depends on: a KVM hypervisor, the Kubernetes API server, a GCP project.
+                  properties:
+                    address:
+                      description: address is where the check connected to.
+                      type: string
+                    details:
+                      additionalProperties:
+                        type: string
+                      description: |-
+                        details are provider-specific facts from the last check, e.g. the
+                        libvirt version and free storage-pool space of a KVM host.
+                      type: object
+                    lastTransitionTime:
+                      description: lastTransitionTime is when ready last changed.
+                      format: date-time
+                      type: string
+                    latencyMillis:
+                      description: latencyMillis is how long the last check took.
+                      format: int64
+                      type: integer
+                    message:
+                      description: message is human-readable detail, usually the error
+                        of a failed check.
+                      maxLength: 1024
+                      type: string
+                    name:
+                      description: |-
+                        name identifies the target within its provider: the KVM host name,
+                        "apiserver", or the GCP project.
+                      minLength: 1
+                      type: string
+                    ready:
+                      description: ready is true when the last check succeeded.
+                      type: boolean
+                    reason:
+                      description: |-
+                        reason is a CamelCase verdict such as Reachable, Unreachable,
+                        Forbidden, SSHHandshakeFailed or PoolNotActive.
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - name
+                  - ready
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
+              targetsReady:
+                description: targetsReady counts the ready targets as "ready/total".
+                type: string
+              type:
+                description: |-
+                  type is the provider's type from the providers config: kubernetes,
+                  gcp or kvm.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

config/crd/kustomization.yaml

@@ -3,4 +3,5 @@
 # It should be run by config/default
 resources:
 - bases/crawl.example.com_proxies.yaml
+- bases/crawl.example.com_providerhealths.yaml
 # +kubebuilder:scaffold:crdkustomizeresource

config/manager/kustomization.yaml

@@ -5,5 +5,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: example.com/egress-proxies-operator
-  newTag: v0.0.1
+  newName: egress-proxies-operator
+  newTag: kvm-dev

config/manager/manager.yaml

@@ -64,6 +64,12 @@ spec:
           - --leader-elect
           - --health-probe-bind-address=:8081
           - --providers-config=/etc/proxy-operator/providers.yaml
+          # Add --trace-health-probes to emit one span per health probe
+          # (high volume: ~1/s per proxy).
+          # Provider health checks (kubectl get providerhealth) run every
+          # minute; tune with --provider-check-interval (0 turns them off,
+          # otherwise at least 10s) and --provider-check-timeout (15s, or
+          # the interval if that is shorter).
         env:
           # Bearer token for the discovery API. Optional: without the
           # Secret the API serves unauthenticated (with a loud warning).
@@ -76,6 +82,33 @@ spec:
                 name: discovery-token
                 key: token
                 optional: true
+          # --- OpenTelemetry tracing ---
+          # Tracing is OFF until OTEL_EXPORTER_OTLP_ENDPOINT (or
+          # OTEL_TRACES_EXPORTER) is set; without it the operator runs
+          # exactly as before. The downward-API vars must stay listed
+          # before OTEL_RESOURCE_ATTRIBUTES: $(VAR) expansion only sees
+          # earlier-listed vars.
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: OTEL_SERVICE_NAME
+            value: egress-proxies-operator
+          - name: OTEL_RESOURCE_ATTRIBUTES
+            value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)
+          # Point at an OTLP receiver (Tempo, Jaeger, otel-collector) to
+          # enable tracing; http/protobuf (4318) is the default protocol,
+          # set OTEL_EXPORTER_OTLP_PROTOCOL=grpc for 4317.
+          # - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          #   value: http://jaeger-collector.observability:4318
+          # - name: OTEL_TRACES_SAMPLER
+          #   value: parentbased_traceidratio
+          # - name: OTEL_TRACES_SAMPLER_ARG
+          #   value: "0.1"
         image: controller:latest
         name: manager
         ports:
@@ -116,9 +149,30 @@ spec:
         - name: providers-config
           mountPath: /etc/proxy-operator
           readOnly: true
+        # SSH material for kvm providers (sshKeyFile / knownHostsFile in the
+        # providers config point here). Optional: the mount is empty when the
+        # Secret does not exist, and only kvm entries need it. Create it with:
+        #   kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
+        #     --from-file=id_ed25519=<private key> \
+        #     --from-file=known_hosts=<known_hosts with every host's key>
+        - name: kvm-ssh
+          mountPath: /etc/proxy-operator/kvm
+          readOnly: true
+        # Scratch space: the root filesystem is read-only, and the kvm
+        # provider builds each VM's cloud-init seed ISO through a temp
+        # directory (a few hundred KB, deleted right after upload).
+        - name: tmp
+          mountPath: /tmp
       volumes:
       - name: providers-config
         configMap:
           name: providers-config
+      - name: kvm-ssh
+        secret:
+          secretName: kvm-ssh
+          optional: true
+      - name: tmp
+        emptyDir:
+          sizeLimit: 64Mi
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10

config/manager/providers_config.yaml

@@ -9,9 +9,46 @@ metadata:
 data:
   # Mounted at /etc/proxy-operator/providers.yaml (--providers-config).
   # The default ships only the kubernetes-pod provider so the operator runs
-  # out of the box; add gcp entries (type: gcp, gcp.project: ...) for real
-  # egress fleets — see config/samples/providers-config.yaml.
+  # out of the box; add gcp entries (type: gcp, gcp.project: ...) or kvm
+  # entries (type: kvm, hosts reached over SSH) for real egress fleets —
+  # see config/samples/providers-config.yaml.
   providers.yaml: |
     providers:
       - name: kubernetes
         type: kubernetes
+      - name: kvm-home
+        type: kvm
+        kvm:
+          # SSH to each host with the libvirt unix socket forwarded through the
+          # connection; nothing to open on the hypervisors beyond sshd. Both
+          # files come from the optional kvm-ssh Secret mounted by
+          # config/manager/manager.yaml — see docs/kvm-provider.md.
+          sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+          knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+          # insecureIgnoreHostKey: true   # skip host-key checks (not recommended)
+          # diskSizeGb: 10                # per-VM overlay; cloud-init grows the root fs into it
+          # vendorData: |                 # extra vendor-data; the qemu-guest-agent bootstrap is merged
+          #   #cloud-config               # into each Proxy's cloud-config, not configured here
+          #   ...
+          images:
+            # placement.image names a key here (or is an https:// URL directly).
+            # Fetched into each host's pool on first use and cached there as
+            # base-<key>.qcow2; do not change an entry while Proxies use it.
+            debian-13:
+              url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+              # sha256: "<64 hex>"        # optional; verified after download
+              # firmware: uefi            # default; "bios" only for images with no EFI partition
+              #                           # (Debian 13 hangs in GRUB under BIOS headless: Debian bug #1111240)
+            ubuntu-24.04:
+              url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+          machineTypes:                   # placement.machineType names a key here
+            small:  {vcpus: 1, memoryMiB: 1024}
+            medium: {vcpus: 2, memoryMiB: 2048}
+          hosts:                          # placement.zone names a host; optional with one host
+            - name: beelink
+              address: beelink.tail8c69c.ts.net:22
+              user: novakj                  # must be in the libvirt group
+              bridge: br0                 # to a LAN with DHCP; the VM's IP comes from there
+              # socket: /var/run/libvirt/libvirt-sock
+              # pool: default
+

config/rbac/kustomization.yaml

@@ -25,4 +25,7 @@ resources:
 - proxy_admin_role.yaml
 - proxy_editor_role.yaml
 - proxy_viewer_role.yaml
+# ProviderHealth objects are written only by the operator, so they get a
+# viewer role and nothing else.
+- providerhealth_viewer_role.yaml
 

config/rbac/providerhealth_viewer_role.yaml

@@ -0,0 +1,29 @@
+# This rule is not used by the project egress-proxies-operator itself.
+# It is provided to allow the cluster admin to help manage permissions for users.
+#
+# Grants read-only access to ProviderHealth objects, the results of the
+# operator's periodic provider checks. The operator creates, updates and
+# deletes them itself, so no admin or editor role is provided.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: egress-proxies-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: providerhealth-viewer-role
+rules:
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths/status
+  verbs:
+  - get

config/rbac/role.yaml

@@ -22,6 +22,22 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+- apiGroups:
+  - crawl.example.com
+  resources:
+  - providerhealths/status
+  verbs:
+  - get
+  - patch
 - apiGroups:
   - crawl.example.com
   resources:

config/samples/kustomization.yaml

@@ -4,5 +4,6 @@
 resources:
 - proxy_kubernetes.yaml
 - proxy_gcp.yaml
+- proxy_kvm.yaml
 - proxy_external.yaml
 # +kubebuilder:scaffold:manifestskustomizesamples

config/samples/providers-config.yaml

@@ -19,3 +19,39 @@ providers:
       # diskSizeGb: 10
       # auth: Application Default Credentials (workload identity
       # in-cluster, gcloud ADC locally). No key-file plumbing.
+  - name: kvm-home
+    type: kvm
+    kvm:
+      # SSH to each host with the libvirt unix socket forwarded through the
+      # connection; nothing to open on the hypervisors beyond sshd. Both
+      # files come from the optional kvm-ssh Secret mounted by
+      # config/manager/manager.yaml — see docs/kvm-provider.md.
+      sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+      knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+      # insecureIgnoreHostKey: true   # skip host-key checks (not recommended)
+      # diskSizeGb: 10                # per-VM overlay; cloud-init grows the root fs into it
+      # installGuestAgent: false      # skip the vendor-data script that installs qemu-guest-agent
+      # vendorData: |                 # extra vendor-data, carried beside the agent script as a
+      #   #cloud-config               # MIME multipart. The Proxy's own cloud-init is never rewritten.
+      #   ...
+      images:
+        # placement.image names a key here (or is an https:// URL directly).
+        # Fetched into each host's pool on first use and cached there as
+        # base-<key>.qcow2; do not change an entry while Proxies use it.
+        debian-13:
+          url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+          # sha256: "<64 hex>"        # optional; verified after download
+          # firmware: uefi            # default; "bios" only for images with no EFI partition
+          #                           # (Debian 13 hangs in GRUB under BIOS headless: Debian bug #1111240)
+        ubuntu-24.04:
+          url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+      machineTypes:                   # placement.machineType names a key here
+        small:  {vcpus: 1, memoryMiB: 1024}
+        medium: {vcpus: 2, memoryMiB: 2048}
+      hosts:                          # placement.zone names a host; optional with one host
+        - name: beelink
+          address: beelink:22
+          user: novakj                  # must be in the libvirt group
+          bridge: br0                 # to a LAN with DHCP; the VM's IP comes from there
+          # socket: /var/run/libvirt/libvirt-sock
+          # pool: default

config/samples/proxy_kvm.yaml

@@ -0,0 +1,43 @@
+# A Managed proxy backed by a libvirt/KVM hypervisor on the home network:
+# the operator boots a stock Debian cloud image as a VM on the named host,
+# with a NIC on the host's bridge (IP from the LAN's DHCP), and installs
+# Squid via cloud-init. Requires a providers-config entry named "kvm-home"
+# (see providers-config.yaml) and the kvm-ssh Secret (docs/kvm-provider.md).
+#
+# placement.zone is the host name from kvm.hosts (optional when only one
+# host is configured); machineType and image are keys in the provider's
+# machineTypes and images maps. The first Proxy for an image on a host
+# waits for the image download; later ones reuse the cached copy.
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-kvm-sample
+spec:
+  mode: Managed
+  provider: kvm-home
+  placement:
+    zone: nas1
+    machineType: small
+    image: debian-13
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_port 3128
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+      # Optional: log in with `ssh debian@<ip>` (the Debian image's default
+      # user) to look around; `virsh console <name>` on the host works too.
+      # ssh_authorized_keys:
+      #   - ssh-ed25519 AAAA... you@laptop
+  attributes:
+    geo: home
+    purpose: crawl

docs/api.md

@@ -0,0 +1,323 @@
+# Discovery API reference
+
+The operator serves an HTTP API (the *discovery API*) that crawler clients
+use to find and lease egress proxies: list healthy proxies filtered by
+attributes, acquire a TTL-based lease on one, release it early, and report
+how a target site treated the proxy. It is implemented in
+[`internal/discovery`](../internal/discovery/) with lease state in
+[`internal/lease`](../internal/lease/); the only Kubernetes interaction is
+reading `Proxy` resources from the manager's cache.
+
+## Base URL
+
+The API listens on `:8090` (`--discovery-addr`) inside the manager pod and
+is exposed by a Service
+([config/default/discovery_service.yaml](../config/default/discovery_service.yaml)).
+
+In-cluster:
+
+```text
+http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090
+```
+
+From a workstation, port-forward:
+
+```sh
+kubectl -n egress-proxies-operator-system port-forward \
+  svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090 &
+```
+
+Set `BASE_URL` to wherever you reach the API; all examples below use it:
+
+```sh
+export BASE_URL=localhost:8090   # via the port-forward above
+# or, from inside the cluster:
+# export BASE_URL=http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090
+```
+
+## Authentication
+
+A single static bearer token, read from the `DISCOVERY_TOKEN` environment
+variable at startup. The shipped Deployment populates it from the
+`discovery-token` Secret (key `token`), which is **optional** — if the
+Secret is absent or the token is empty, the API serves **unauthenticated**
+(the manager logs a loud warning at startup). Create the Secret:
+
+```sh
+kubectl -n egress-proxies-operator-system create secret generic discovery-token \
+  --from-literal=token="$(openssl rand -hex 24)"
+```
+
+Send the token on every request:
+
+```sh
+export TOKEN=<the token>
+curl -s -H "Authorization: Bearer $TOKEN" "$BASE_URL/v1/proxies" | jq
+```
+
+A missing or wrong token gets `401 {"error":"unauthorized",...}`.
+`GET /healthz` is always exempt.
+
+The curl examples below omit the `-H "Authorization: Bearer $TOKEN"` flag
+for brevity — add it to every call when auth is enabled.
+
+## Conventions
+
+- Requests and responses are JSON. Errors share one envelope:
+
+  ```json
+  {"error": "<machine_code>", "message": "<human-readable text>"}
+  ```
+
+- Request bodies are capped at **64 KiB** (larger bodies fail the JSON
+  decode with `400 invalid_body`).
+- Proxies with a deletion timestamp (being finalized) are excluded from
+  every response and never offered for lease.
+
+## Configuration
+
+| Setting | Default | Meaning |
+|---|---|---|
+| `--discovery-addr` | `:8090` | Listen address of the API |
+| `--max-lease-ttl` | `1h` | Maximum `ttlSeconds` a client may request |
+| `--lease-cooldown` | `15m` | Cooldown window applied on `rate_limited`/`banned` reports |
+| `DISCOVERY_TOKEN` (env) | empty | Bearer token; empty disables auth |
+
+The shipped Deployment passes none of these flags, so the defaults apply.
+
+## Endpoints
+
+### `GET /healthz`
+
+Liveness check. Unauthenticated, always `200` with body `ok`.
+
+```sh
+curl -s "$BASE_URL/healthz"
+```
+
+### `GET /v1/proxies` — list proxies
+
+Query parameters (all optional):
+
+| Parameter | Values | Effect |
+|---|---|---|
+| `healthy` | `true` \| `false` | Keep only proxies whose `Healthy` condition matches. Any other value → `400 invalid_query`. |
+| `attr.<key>` | any string | Exact match on `spec.attributes[<key>]`. Repeatable; **all** given pairs must match. |
+
+List everything:
+
+```sh
+curl -s "$BASE_URL/v1/proxies" | jq
+```
+
+List healthy proxies in a given geo:
+
+```sh
+curl -s "$BASE_URL/v1/proxies?healthy=true&attr.geo=eu" | jq
+```
+
+Response — `200`, proxies sorted by `id`, an empty match is `200` with
+`"count": 0` (never `404`):
+
+```json
+{
+  "proxies": [
+    {
+      "id": "default/proxy-kubernetes-sample",
+      "ip": "10.244.1.7",
+      "port": 3128,
+      "attributes": {"geo": "local"},
+      "phase": "Ready",
+      "healthy": true,
+      "latencyMillis": 42,
+      "activeLeases": 1,
+      "maxLeases": 5
+    }
+  ],
+  "count": 1
+}
+```
+
+Proxy object fields (the same shape appears inside lease responses):
+
+| Field | Meaning |
+|---|---|
+| `id` | `namespace/name` of the `Proxy` resource; used as the stable key everywhere |
+| `ip` | Effective host — `spec.endpoint.host` for `External` proxies, `status.ip` for `Managed` (empty until the backing VM/pod is up) |
+| `port` | Effective port (default `3128`) |
+| `attributes` | `spec.attributes` — free-form selection labels (`geo`, `asn`, `purpose`, …); omitted when empty |
+| `phase` | `Pending` \| `Provisioning` \| `Ready` \| `Unhealthy` \| `Deleting` \| `Failed` |
+| `healthy` | `true` iff the `Healthy` condition is `True` (the through-the-proxy health probe passes) |
+| `latencyMillis` | Latency of the last status-affecting health probe |
+| `activeLeases` | Currently active leases on this proxy |
+| `maxLeases` | Lease capacity (default `5`; an explicit `0` means unleasable) |
+
+### `POST /v1/leases` — acquire a lease
+
+Picks a healthy proxy with free capacity matching the selector and grants
+an exclusive-slot, TTL-based lease on it.
+
+Request body (every field optional; `{}` is valid):
+
+```json
+{
+  "selector": {"geo": "eu"},
+  "ttlSeconds": 300,
+  "target": "example.com"
+}
+```
+
+| Field | Default | Meaning |
+|---|---|---|
+| `selector` | none | Attribute equality filter, same semantics as `attr.<key>` above |
+| `ttlSeconds` | `300` (5 min) | Lease lifetime; must be ≤ `--max-lease-ttl` (default 1 h), else `400 invalid_ttl` |
+| `target` | none | The site you intend to crawl; enables per-target cooldowns (see below) |
+
+```sh
+curl -s -XPOST "$BASE_URL/v1/leases" \
+  -d '{"selector":{"geo":"eu"},"ttlSeconds":300,"target":"example.com"}' | jq
+```
+
+Success — `201`:
+
+```json
+{
+  "leaseID": "P3X6HHQTPCM5UTGVGE3B5UPS3A",
+  "proxy": {
+    "id": "default/proxy-eu-1",
+    "ip": "34.88.10.20",
+    "port": 3128,
+    "attributes": {"geo": "eu"},
+    "phase": "Ready",
+    "healthy": true,
+    "latencyMillis": 42,
+    "activeLeases": 1,
+    "maxLeases": 5
+  },
+  "expiresAt": "2026-08-11T22:05:00Z",
+  "ttlSeconds": 300
+}
+```
+
+Use `proxy.ip` and `proxy.port` as an HTTP proxy for the lease's lifetime:
+
+```sh
+curl -x http://34.88.10.20:3128 https://example.com
+```
+
+No match — `409` with diagnostic counts explaining why nothing qualified:
+
+```json
+{
+  "error": "no_match",
+  "message": "no healthy proxy with free capacity matched the selector",
+  "considered": 3,
+  "atCapacity": 1,
+  "inCooldown": 1,
+  "unhealthy": 1
+}
+```
+
+| Count | Meaning |
+|---|---|
+| `considered` | Proxies that matched the selector (before health/capacity checks) |
+| `atCapacity` | Skipped because `activeLeases >= maxLeases` |
+| `inCooldown` | Skipped because of an active cooldown for this target (or a global one) |
+| `unhealthy` | Skipped because the `Healthy` condition is not `True` |
+
+Leases expire on their own — releasing is only needed to free the slot
+early. There is no renew/extend endpoint; acquire a new lease instead.
+
+### `DELETE /v1/leases/{id}` — release early
+
+Frees the lease's capacity slot immediately. Idempotent: always `204`,
+including for unknown or already-expired lease IDs.
+
+```sh
+curl -si -XDELETE "$BASE_URL/v1/leases/P3X6HHQTPCM5UTGVGE3B5UPS3A"
+```
+
+### `POST /v1/leases/{id}/report` — report an outcome
+
+Tell the operator how the target site treated the proxy. This is the
+feedback signal that drives cooldowns.
+
+Request body:
+
+```json
+{"result": "rate_limited", "target": "example.com"}
+```
+
+| Field | Values | Meaning |
+|---|---|---|
+| `result` | `ok` \| `rate_limited` \| `banned` | Anything else → `400 invalid_result` |
+| `target` | optional | Which site produced the result; falls back to the lease's `target`, then to global |
+
+```sh
+curl -si -XPOST "$BASE_URL/v1/leases/P3X6HHQTPCM5UTGVGE3B5UPS3A/report" \
+  -d '{"result":"rate_limited","target":"example.com"}'
+```
+
+Responses: `204` on success, `404 unknown_lease` if the lease ID was never
+issued or has aged out.
+
+Semantics:
+
+- `ok` is a pure acknowledgement — nothing is recorded.
+- `rate_limited` and `banned` currently behave **identically**: both put
+  the proxy in one cooldown window (default 15 min, `--lease-cooldown`)
+  for the resolved target.
+- An expired lease remains reportable for one cooldown window past its
+  TTL, so a late "we got rate-limited" still lands.
+
+## Proxy selection and cooldowns
+
+How `POST /v1/leases` picks among eligible proxies (healthy, matching the
+selector, not being deleted, not at capacity, not in cooldown), in order:
+
+1. fewest `activeLeases` (least-loaded),
+2. lowest `latencyMillis`,
+3. lexicographic `id` (deterministic tie-break).
+
+Cooldowns are keyed by **(proxy, target)**:
+
+- A report **with a target** blocks that proxy only for lease requests
+  naming the **same target**. Other targets — and requests with no
+  target — still get the proxy.
+- A report **without a target**, on a lease that also had no target,
+  creates a **global** cooldown: the proxy is blocked for *all* lease
+  requests until the window passes. Always pass `target` on leases and
+  reports unless you really mean "this proxy is bad for everyone".
+
+## End-to-end example
+
+```sh
+# 1. Acquire a lease for crawling example.com through an EU proxy
+LEASE=$(curl -s -XPOST "$BASE_URL/v1/leases" \
+  -H "Authorization: Bearer $TOKEN" \
+  -d '{"selector":{"geo":"eu"},"ttlSeconds":600,"target":"example.com"}')
+LEASE_ID=$(echo "$LEASE" | jq -r .leaseID)
+PROXY=$(echo "$LEASE" | jq -r '"\(.proxy.ip):\(.proxy.port)"')
+
+# 2. Crawl through the leased proxy
+curl -x "http://$PROXY" https://example.com/some/page
+
+# 3. Got a 429? Report it — example.com-bound leases will avoid this
+#    proxy for the next 15 minutes
+curl -s -XPOST "$BASE_URL/v1/leases/$LEASE_ID/report" \
+  -H "Authorization: Bearer $TOKEN" \
+  -d '{"result":"rate_limited","target":"example.com"}'
+
+# 4. Done early? Release the slot (otherwise the TTL frees it)
+curl -s -XDELETE "$BASE_URL/v1/leases/$LEASE_ID" \
+  -H "Authorization: Bearer $TOKEN"
+```
+
+## Caveats
+
+- **Lease and cooldown state is in-memory and per-process.** An operator
+  restart drops all active leases and cooldowns. Clients must tolerate a
+  granted lease disappearing (a subsequent report returns `404`).
+- **Run a single replica.** The API is served by every manager replica but
+  is not leader-elected, and lease state is not shared between replicas;
+  the shipped Deployment pins `replicas: 1`.

docs/architecture.md

@@ -4,15 +4,17 @@
 
 | Component | Package | Runs as | Leader-elected | Role |
 |---|---|---|---|---|
-| Proxy CRD + helpers | `api/v1alpha1` | types | — | `Proxy` spec/status, CEL validation, defaulting, pure helpers |
+| Proxy + ProviderHealth CRDs | `api/v1alpha1` | types | — | `Proxy` spec/status, CEL validation, defaulting, pure helpers; the status-only, cluster-scoped `ProviderHealth` |
 | Reconciler | `internal/controller` | controller | yes (with the manager) | the state machine: provision, replace, delete, represent health |
-| Provider contract | `internal/provider` | library | — | `Provider` interface, error taxonomy, deterministic naming, config, metrics decorator |
+| Provider contract | `internal/provider` | library | — | `Provider` interface, optional `Checker`, error taxonomy, deterministic naming, config, metrics decorator; rules in [provider-contract.md](provider-contract.md), per-provider profile table in [providers-existing.md](providers-existing.md) |
 | kubernetes provider | `internal/provider/kubernetes` | library | — | real Squid pods in this cluster (local dev/CI) |
 | gcp provider | `internal/provider/gcp` | library | — | Compute Engine VMs, four API calls, fire-and-forget ops |
+| kvm provider | `internal/provider/kvm` | library | — | VMs on libvirt hosts over SSH-forwarded sockets; image cache, overlay disks, NoCloud seeds |
 | Health engine | `internal/health` | Runnable | yes | through-the-proxy probes, thresholds, transition events |
 | Lease store | `internal/lease` | Runnable (expiry sweep) | no | in-memory leases + cooldowns, single mutex |
 | Discovery API | `internal/discovery` | Runnable | no | HTTP list/lease/release/report on `:8090` |
 | Orphan GC | `internal/gc` | Runnable | yes | deletes tagged instances whose CR is gone |
+| Provider health monitor | `internal/providerhealth` | Runnable | yes | periodic provider checks → one `ProviderHealth` per provider, target gauges, transition logs |
 | Metrics | `internal/metrics` | library | — | explicit registration, scrape-time collectors |
 | Composition root | `cmd/main.go` | binary | — | flags, provider registry, wires everything onto one manager |
 
@@ -147,6 +149,25 @@ prov.ListByTag ─► AggregatedList(label filter,                   ─┘ prov
                     ReturnPartialSuccess: true)
 providerID = zones/<zone>/instances/<name> — zone-qualified, so Get/Delete
 stay correct even mid-replacement after a zone edit
+
+kvm provider (internal/provider/kvm/) — libvirt RPC over an SSH-forwarded unix socket, per host
+prov.Create ──► DomainLookupByName: exists → start if needed, done      ─┐ every step tolerates
+                else: ensure base image in pool (download on first use) │ the previous attempt
+                      overlay volume (exists = reuse)                    │ having got that far;
+                      seed ISO (always rebuilt — a cut-off upload must   │ the seed is the one
+                        never be trusted)                                │ thing redone
+                      DomainDefineXML + DomainCreate                    ─┘
+prov.Get    ──► DomainGetState + DomainGetMetadata + DomainInterfaceAddresses
+                  (ARP, then guest agent; first routable IPv4)
+                  running without IP = Provisioning; nostate = Provisioning
+prov.Delete ──► destroy (not running = fine) → undefine → delete overlay + seed by name
+prov.ListByTag ─► ConnectListAllDomains on every host, keep domains whose
+                    <metadata> carries managed=true; one host down fails the sweep,
+                    one unreadable domain is skipped
+providerID = hosts/<host>/domains/<name>; GC identity lives in the domain's
+<metadata> element (libvirt has no labels). All RPCs run under a context-
+bounded call: go-libvirt never times out on its own, so a cancelled reconcile
+closes the SSH connection to unblock it.
 ```
 
 The reconciler never watches provider-side resources (Pods or GCP VMs).
@@ -213,6 +234,8 @@ Kubernetes interaction is reading Proxies from the manager's cache. The
 server is a non-leader-elected Runnable (all replicas would serve, but the
 deployment ships `replicas: 1` because lease state is per-process — an
 operator restart drops all leases and cooldowns, a documented caveat).
+Client-facing reference with request/response schemas and curl examples:
+[api.md](api.md).
 
 ```text
 crawler client
@@ -284,6 +307,8 @@ registries). Two kinds:
 - **Scrape-time collectors** — `proxy_operator_proxies{phase}` and
   `proxy_operator_leases_active` read the cache / lease store at every
   scrape; reconcile-incremented gauges inevitably drift and leak series.
+  The provider-check gauges (§11) read the health monitor's snapshot the
+  same way.
 - **Fed vectors** — `healthcheck_duration_seconds{proxy}` and
   `healthcheck_failures_total{proxy}` observe EVERY probe (status writes
   are transition-only; metrics carry the high-frequency signal), and the
@@ -298,6 +323,103 @@ Each consuming package defines its own small recorder interface
 `metrics.Metrics` satisfies all of them structurally, so no package other
 than `cmd/main.go` imports the metrics package.
 
+### 10. Tracing (`internal/tracing/`)
+
+OpenTelemetry tracing, integrated with — not replacing — the logr/zap
+logging. Everything hangs off standard `OTEL_*` env vars: with no
+`OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` set, no SDK is
+installed, spans are no-ops, and logs are byte-for-byte what they were.
+`OTEL_SDK_DISABLED=true` and `OTEL_TRACES_EXPORTER=none` force it off;
+`console` prints spans to stdout for local dev; sampling follows
+`OTEL_TRACES_SAMPLER(_ARG)`.
+
+Span topology — each unit of work is a new trace (watch events carry no
+incoming trace context):
+
+- `Reconcile Proxy` (root, via a `reconcile.Reconciler` decorator) →
+  `reconcile.managed` / `reconcile.replaceInstance` / `reconcile.delete`,
+  plus `status.patch` from the deferred flush. `reconcileExternal` does no
+  I/O and is unspanned.
+- `provider.create|get|delete|list` (client spans, `provider.WithTracing`
+  decorator wired outermost around `WithMetrics`); the GCP SDK's own
+  otelhttp transport contributes HTTP child spans automatically.
+- Kubernetes API calls become client child spans via a wrapped
+  `rest.Config` transport — gated on an existing parent span, so informer
+  list/watch long-polls and leader-election renewals never create root
+  spans. The kube-apiserver ignores incoming `traceparent` by design;
+  these spans are leaves.
+- Discovery API: one server span per request (named from the route
+  pattern), incoming W3C `traceparent` honored so clients' traces continue
+  into the operator; `/healthz` excluded.
+- `gc.sweep` per GC pass; `providerhealth.check` per round of provider
+  checks; `health.probe` per probe **only** with
+  `--trace-health-probes` (default off — probes run ~1/s per proxy).
+
+Log correlation: logr sinks never see a context, so trace IDs ride on the
+logger — `tracing.Start` re-derives the ctx logger from a captured base
+with `traceID`/`spanID` values (lowerCamel, matching `reconcileID`;
+Grafana/Loki derived-field regexes must match `traceID`, not `trace_id`).
+Re-deriving from the base rather than layering keeps zap from emitting
+duplicate keys on nested spans. GCP V(5) wire logs get the same keys from
+the slog handler's ctx.
+
+### 11. Provider health checks (`internal/providerhealth/`)
+
+Answers "can this cluster reach each provider's backend right now?"
+without waiting for a Proxy to fail. A provider opts in by implementing
+`provider.Checker`: `Check(ctx) []TargetStatus` returns one verdict per
+target and never fails as a whole. All three providers do:
+
+| Provider | Target | What one check does |
+|---|---|---|
+| kvm | each `kvm.hosts` entry | over the pooled SSH+libvirt connection: libvirt version and hostname, then the configured pool must exist and be running; reports pool capacity and free space ([details](kvm-provider.md#health-check)) |
+| kubernetes | `apiserver` | lists at most one managed Pod cluster-wide: the verb and scope `ListByTag` and GC rely on |
+| gcp | the project | one page of an aggregated instance list with `maxResults=1`: credentials, Compute API enabled, `compute.instances.list` |
+
+```text
+Monitor.Start(ctx)   ── leader-only; --provider-check-interval (1m; 0 = off,
+  │                     otherwise ≥ 10s), --provider-check-timeout (15s, or
+  │                     the interval if that is shorter)
+  ├─ once: delete managed-by-labelled ProviderHealth objects whose provider
+  │   is not configured (all of them when checks are off) — the providers
+  │   config is read once per process
+  └─ at start, then every Interval: checkOnce
+       │ every provider's Check concurrently, under one context.WithTimeout
+       │ snapshot → the scrape-time gauges
+       │ log transitions: down or reason changed → "WARNING: provider target
+       │   not ready"; back up → "Provider target ready"; steady → silent
+       └ per provider: APIReader.Get → Create if missing → status merge patch
+```
+
+- **One status write per provider per round**, since `lastCheckTime`
+  moves every round. At the 10s minimum interval that is still far below
+  leader-election renewals. Proxy status avoids writes like this only
+  because its probes run once a second per proxy.
+- **Reads go through the API reader**, not the cache. The objects are
+  cluster-scoped; a cached Get would start a cluster-wide informer (and
+  need `watch`) even with `--proxy-namespace`.
+- **Checks bypass the provider decorators.** `cmd/main.go` collects the
+  Checkers before wrapping the providers in `WithTracing(WithMetrics(...))`,
+  so a check never counts as a provider request. KVM RPCs still appear in
+  the V(1) `libvirt call` trace, and each round is one
+  `providerhealth.check` span.
+- **Not wired to `/readyz`.** A dead hypervisor must not make the manager
+  unready while the other providers keep working.
+- **Gauges** `proxy_operator_provider_target_up`,
+  `proxy_operator_provider_check_duration_seconds` and
+  `proxy_operator_provider_check_timestamp_seconds`, each labelled
+  `{provider,target}`. Only the leader exports them.
+
+Illustrative output:
+
+```sh
+$ kubectl get providerhealth          # short name: ph
+NAME         TYPE         READY   TARGETS   LAST CHECK   AGE
+kubernetes   kubernetes   True    1/1       12s          3d
+kvm-home     kvm          False   0/1       12s          3d
+$ kubectl get ph kvm-home -o yaml     # per target: reason, message, details
+```
+
 ## Decisions
 
 Judgment calls the spec left open, and deliberate deviations — recorded so
@@ -374,6 +496,41 @@ they read as choices, not accidents. Chronological by build step.
 - **Unknown GCP instance statuses map to `Stopped`:** the reconciler's
   answer to Stopped is delete-and-recreate, the always-safe move for
   cattle when the API grows a new state.
+- **KVM hosts are reached over SSH with the libvirt socket forwarded,**
+  not libvirtd's TCP/TLS listener: nothing to configure or expose on the
+  hypervisors beyond sshd, pure Go on the operator side (no cgo, no
+  libvirt libraries in the image), and the hosts list can grow by a
+  config edit because the providerID carries the host name.
+- **KVM base images are fetched by the operator into the host's pool**
+  through libvirt's upload stream, rather than expected to be pre-placed:
+  the operator then needs no filesystem access on the host and a new
+  host is usable with zero manual staging. The cost is a synchronous
+  first-time download inside Create.
+- **The KVM seed ISO is rebuilt on every Create attempt** before the
+  domain exists, instead of "exists → skip": a reconcile cancelled mid
+  upload leaves a truncated seed, and the compensating delete cannot run
+  because the connection was already closed to unblock the RPC. Skipping
+  it would boot a VM with no cloud-init and no retry would ever repair it.
+- **KVM `nostate` maps to `Provisioning`, not `Stopped`:** unlike GCP's
+  unknown statuses, `nostate` means libvirt could not determine the
+  state at all; answering that with delete-and-recreate is too drastic,
+  so the next poll decides.
+- **KVM guests boot UEFI by default, secure boot off.** Found on the
+  first real host: Debian 13's generic image hangs in GRUB under SeaBIOS
+  on a headless VM (Debian bug #1111240: `terminal_output gfxterm
+  serial` with a serial console and no video device) regardless of CPU
+  model or seed, and boots under OVMF with the serial console working.
+  The alternatives were a virtio console (loses the kernel's serial
+  output) or a video device the VMs have no use for. Every
+  current distro cloud image supports UEFI, so it is the default and
+  `firmware: bios` is the per-image escape; libvirt's firmware
+  autoselection (`<os firmware="efi">`) keeps the OVMF path out of the
+  operator's config. Secure boot is off so unsigned images boot too.
+- **A KVM domain GC cannot read is skipped, a host it cannot reach fails
+  the sweep.** GC never deletes what it does not see, so leaving one
+  domain out is safe; one corrupt domain blocking GC for every other one
+  is not. A whole host missing from the view would hide many, so that
+  still aborts.
 - **Kubernetes 403s classify as `ErrPermanent`** even though quota
   exhaustion also surfaces as 403 (indistinguishable from RBAC denial in
   `apierrors`): not hammering an API server that may never allow the
@@ -393,6 +550,15 @@ they read as choices, not accidents. Chronological by build step.
   cluster-wide Secret read RBAC — an unrestricted cache would hold every
   Secret in scope). Unlabelled referenced Secrets are invisible by
   construction, surfacing as `CloudInitError`.
+- **Deletion never gives up on its own, but can be released by hand:** a
+  permanent cleanup failure keeps retrying with the finalizer on, because
+  dropping it automatically would leak a running instance on any provider
+  that answers "permission denied". Where `Get` fails permanently the
+  reconciler still calls `Delete`, since it is idempotent and may finish
+  where `Get` cannot converge. `crawl.example.com/skip-cleanup: "true"`
+  abandons the instance and releases the object; it is checked before the
+  provider lookup, so it also frees a Proxy whose provider is gone from the
+  config.
 - **Events RBAC from the plan is omitted:** nothing wires an
   EventRecorder in the prototype, and granting verbs nothing uses would
   be RBAC lint noise. Add the marker together with the recorder if events
@@ -402,3 +568,48 @@ they read as choices, not accidents. Chronological by build step.
   logger to everything running under the manager — fighting that would
   mean two logging systems in one process. Noted as a deviation rather
   than silently ignored.
+- **Tracing is env-gated, not flag-gated:** it activates only when
+  `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` is set (user
+  decision). No collector configured means no SDK installed, no-op spans,
+  unchanged logs — the safe default for every existing deployment. The
+  one flag is `--trace-health-probes`, off by default, because probes at
+  ~1/s per proxy would dominate trace volume.
+- **Trace keys are `traceID`/`spanID`**, lowerCamel like `reconcileID` and
+  `providerID`, deliberately not the `trace_id` many Grafana derived-field
+  examples assume — configure the derived-field regex accordingly. logr
+  sinks can't read ctx, so the IDs ride on the ctx logger, re-derived from
+  a captured base per span so zap never emits duplicate keys.
+- **Probe→reconcile trace links are not attempted:** the health engine's
+  `GenericEvent` carries only namespace/name (no ctx), and the workqueue
+  coalesces events, so any link would be a guess. A health-triggered
+  reconcile starts a fresh trace; the probe that caused it is findable via
+  its own (opt-in) span and shared proxy attributes.
+- **No `traceparent` toward probe targets:** probe transports stay
+  uninstrumented so trace headers can never leak through a proxy to
+  external sites. The kube-apiserver ignores incoming `traceparent` by
+  design (public endpoint), so k8s client spans are leaves — in-process
+  traces, not cross-process ones.
+- **Provider health is a status-only, cluster-scoped CRD** (user
+  decision). `kubectl get providerhealth` works without Prometheus, which
+  the deployment does not ship (its ServiceMonitor is commented out). There
+  is no spec: the providers config file stays the one place providers are
+  configured, and the object only reports on it. The gauges carry the same
+  verdicts for alerting.
+- **KVM checks reuse the pooled connection instead of dialing fresh.** A
+  fresh dial would not pick up a rotated Secret anyway, since the SSH
+  material is cached. It would also open an sshd session per host per
+  minute. And a working cached connection is exactly what reconciles use.
+  Sharing must not cost the reconciles anything, so the check calls
+  through `hostPool.probe`. A check that runs out of time abandons its
+  RPCs instead of closing the connection, which would kill an upload or a
+  Create running on it. At most one abandoned check per host is left
+  waiting, and until it returns, the next check reports `Timeout` at once.
+  Waiting for a host's connection slot also ends with the caller's
+  deadline, so a reconcile redialing a dead host cannot stall the round.
+- **Provider transitions are logs, not Events**, in line with the
+  EventRecorder decision above. They log at Info with a `WARNING:` prefix,
+  and each target's `lastTransitionTime` in the object records when it
+  flipped.
+- **A provider name that is not a valid object name** gets metrics but no
+  `ProviderHealth` object, with a warning at start. Tightening config
+  validation instead would crash-loop existing deployments on upgrade.

docs/demo/create-gcp-proxies.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+# Demo: create N Managed proxies backed by the gcp provider, named
+# proxy-gcp-demo-1 .. proxy-gcp-demo-N, each in a randomly picked EU zone
+# so the fleet gets egress IPs from different locations.
+#
+# Usage:
+#   ./create-gcp-proxies.sh <count>
+#
+# Optional environment:
+#   NAMESPACE     namespace to create the proxies in (default: current context)
+#   GCP_PROVIDER  provider NAME from providers.yaml (default: gcp-eu)
+#   ZONES         space-separated zone list to pick from (default: EU zones below)
+set -euo pipefail
+
+if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
+    echo "usage: $(basename "$0") <count>  (positive integer)" >&2
+    exit 1
+fi
+count="$1"
+provider="${GCP_PROVIDER:-gcp-eu}"
+
+# GCP zones in the EU where e2-micro is generally available. Override with
+# ZONES="zone1 zone2 ..." if your project has quota only in some of them.
+default_zones=(
+    europe-west1-b europe-west1-c europe-west1-d       # Belgium
+    europe-west2-a europe-west2-b europe-west2-c       # London
+    europe-west3-a europe-west3-b europe-west3-c       # Frankfurt
+    europe-west4-a europe-west4-b europe-west4-c       # Netherlands
+    europe-west6-a europe-west6-b europe-west6-c       # Zurich
+    europe-west8-a europe-west8-b europe-west8-c       # Milan
+    europe-west9-a europe-west9-b europe-west9-c       # Paris
+    europe-central2-a europe-central2-b europe-central2-c   # Warsaw
+    europe-north1-a europe-north1-b europe-north1-c    # Finland
+    europe-southwest1-a europe-southwest1-b europe-southwest1-c   # Madrid
+)
+if [ -n "${ZONES:-}" ]; then
+    read -r -a zones <<< "${ZONES}"
+else
+    zones=("${default_zones[@]}")
+fi
+
+if ! command -v kubectl &> /dev/null; then
+    echo "ERROR: kubectl is required but not installed" >&2
+    exit 1
+fi
+
+ns_args=()
+if [ -n "${NAMESPACE:-}" ]; then
+    ns_args=(-n "${NAMESPACE}")
+fi
+
+for i in $(seq 1 "${count}"); do
+    zone="${zones[RANDOM % ${#zones[@]}]}"
+    echo "Creating proxy-gcp-demo-${i} in ${zone} ..."
+    kubectl apply "${ns_args[@]}" -f - << EOF
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-gcp-demo-${i}
+spec:
+  mode: Managed
+  provider: ${provider}              # must match a provider NAME in providers.yaml
+  placement:
+    zone: ${zone}
+    machineType: e2-micro
+    # debian-cloud images have no cloud-init, so spec.cloudInit (passed as
+    # user-data metadata) would be silently ignored there. Ubuntu images do.
+    image: projects/ubuntu-os-cloud/global/images/family/ubuntu-2404-lts-amd64
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      package_update: true
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+  attributes:
+    geo: eu
+    zone: ${zone}
+    purpose: crawl
+EOF
+done
+
+echo ""
+echo "Created ${count} proxies. VMs take a few minutes to provision and pass"
+echo "the health check. Watch them come up with:"
+echo "  kubectl get px ${ns_args[*]:-} -w"

docs/demo/create-kubernetes-proxies.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Demo: create N Managed proxies backed by the kubernetes-pod provider,
+# named proxy-kubernetes-demo-1 .. proxy-kubernetes-demo-N. Pods share the
+# cluster's egress IP — this exercises the full lifecycle, not distinct
+# egress paths (use create-gcp-proxies.sh for that).
+#
+# Usage:
+#   ./create-kubernetes-proxies.sh <count>
+#
+# Optional environment:
+#   NAMESPACE  namespace to create the proxies in (default: current context)
+set -euo pipefail
+
+if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
+    echo "usage: $(basename "$0") <count>  (positive integer)" >&2
+    exit 1
+fi
+count="$1"
+
+if ! command -v kubectl &> /dev/null; then
+    echo "ERROR: kubectl is required but not installed" >&2
+    exit 1
+fi
+
+ns_args=()
+if [ -n "${NAMESPACE:-}" ]; then
+    ns_args=(-n "${NAMESPACE}")
+fi
+
+for i in $(seq 1 "${count}"); do
+    echo "Creating proxy-kubernetes-demo-${i} ..."
+    kubectl apply "${ns_args[@]}" -f - << EOF
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-kubernetes-demo-${i}
+spec:
+  mode: Managed
+  provider: kubernetes
+  attributes:
+    geo: local
+    purpose: crawl
+EOF
+done
+
+echo ""
+echo "Created ${count} proxies. Watch them come up with:"
+echo "  kubectl get px ${ns_args[*]:-} -w"

docs/demo/run-demo.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Demo driver: opens a tmux session with a 2x2 pane grid:
+#
+#   top-left:     show-egress-ips-table.sh in a 10s loop, run inside the
+#                 netshoot pod against the in-cluster discovery Service
+#   top-right:    watch -n3 kubectl get px
+#   bottom-left:  create-kubernetes-proxies.sh <count>
+#   bottom-right: create-gcp-proxies.sh <count>
+#
+# Usage:
+#   ./run-demo.sh
+#
+# Optional environment:
+#   COUNT         proxies each create script makes (default: 4)
+#   SESSION       tmux session name (default: proxy-demo; an existing
+#                 session with this name is killed and recreated)
+#   NETSHOOT_POD  pod to exec into for the egress-IP loop (default: netshoot)
+#   DEMO_DIR      where the demo scripts live (default: this script's dir)
+set -euo pipefail
+
+COUNT="${COUNT:-4}"
+SESSION="${SESSION:-proxy-demo}"
+NETSHOOT_POD="${NETSHOOT_POD:-netshoot}"
+DEMO_DIR="${DEMO_DIR:-$(cd "$(dirname "$0")" && pwd)}"
+BASE_URL="http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090"
+
+for tool in tmux kubectl; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+if ! kubectl get pod "${NETSHOOT_POD}" &> /dev/null; then
+    echo "ERROR: pod ${NETSHOOT_POD} not found — start one with:" >&2
+    echo "  kubectl run netshoot --image=nicolaka/netshoot -- sleep infinity" >&2
+    exit 1
+fi
+
+echo "Copying show-egress-ips-table.sh into pod ${NETSHOOT_POD} ..."
+kubectl cp "${DEMO_DIR}/show-egress-ips-table.sh" "${NETSHOOT_POD}:/tmp/show-egress-ips-table.sh"
+
+if tmux has-session -t "${SESSION}" 2> /dev/null; then
+    echo "Killing existing tmux session ${SESSION}"
+    tmux kill-session -t "${SESSION}"
+fi
+
+# 2x2 grid: after these splits pane indexes are 0 top-left, 1 top-right,
+# 2 bottom-left, 3 bottom-right; tiled layout evens them into quarters.
+tmux new-session -d -s "${SESSION}"
+tmux split-window -h -t "${SESSION}:0"
+tmux split-window -v -t "${SESSION}:0.0"
+tmux split-window -v -t "${SESSION}:0.1"
+tmux select-layout -t "${SESSION}:0" tiled
+
+loop_cmd="BASE_URL=${BASE_URL}; while true; do bash /tmp/show-egress-ips-table.sh \"\$BASE_URL\"; echo; sleep 10; done"
+tmux send-keys -t "${SESSION}:0.0" "kubectl exec -it ${NETSHOOT_POD} -- bash -c '${loop_cmd}'" C-m
+tmux send-keys -t "${SESSION}:0.1" "watch -n3 kubectl get px" C-m
+tmux send-keys -t "${SESSION}:0.2" "bash ${DEMO_DIR}/create-kubernetes-proxies.sh ${COUNT}" C-m
+tmux send-keys -t "${SESSION}:0.3" "bash ${DEMO_DIR}/create-gcp-proxies.sh ${COUNT}" C-m
+
+tmux select-pane -t "${SESSION}:0.2"
+if [ -n "${TMUX:-}" ]; then
+    tmux switch-client -t "${SESSION}"
+else
+    tmux attach-session -t "${SESSION}"
+fi

docs/demo/show-egress-ips-table.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+# Demo: condensed-table variant of show-egress-ips.sh, made to fit a small
+# tmux pane. One line per proxy: which proxy the request goes through, its
+# endpoint, location (zone/geo attribute), and the egress IP the IP-echo
+# site saw — or unhealthy/FAILED.
+#
+# Usage:
+#   ./show-egress-ips-table.sh <BASE_URL>    e.g. ./show-egress-ips-table.sh localhost:8090
+#
+# Optional environment:
+#   TOKEN        bearer token for the discovery API (see docs/api.md)
+#   IP_ECHO_URL  site that returns the caller's IP as JSON with an "ip" field
+#                (default: https://api.ipify.org?format=json)
+set -euo pipefail
+
+if [ $# -ne 1 ]; then
+    echo "usage: $(basename "$0") <BASE_URL>  (e.g. localhost:8090)" >&2
+    exit 1
+fi
+BASE_URL="$1"
+IP_ECHO_URL="${IP_ECHO_URL:-https://api.ipify.org?format=json}"
+
+for tool in curl jq; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+auth_args=()
+if [ -n "${TOKEN:-}" ]; then
+    auth_args=(-H "Authorization: Bearer ${TOKEN}")
+fi
+
+if ! proxies_json=$(curl -sS --fail "${auth_args[@]}" "${BASE_URL}/v1/proxies") \
+    || ! echo "${proxies_json}" | jq -e . > /dev/null 2>&1; then
+    echo "ERROR: could not fetch proxy list from ${BASE_URL}" >&2
+    exit 1
+fi
+
+total=$(echo "${proxies_json}" | jq -r '.count')
+fmt="%-31s %-21s %-21s %s\n"
+
+echo "${total} proxies @ $(date +%H:%M:%S)"
+# shellcheck disable=SC2059
+printf "${fmt}" "PROXY" "ENDPOINT" "LOCATION" "EGRESS-IP"
+
+probed=0
+skipped=0
+failed=0
+while IFS= read -r proxy; do
+    id=$(echo "${proxy}" | jq -r '.id')
+    endpoint=$(echo "${proxy}" | jq -r '"\(.ip):\(.port)"')
+    location=$(echo "${proxy}" | jq -r '.attributes.zone // .attributes.geo // "-"')
+    healthy=$(echo "${proxy}" | jq -r '.healthy')
+
+    if [ "${healthy}" != "true" ]; then
+        # shellcheck disable=SC2059
+        printf "${fmt}" "${id}" "${endpoint}" "${location}" "(unhealthy)"
+        skipped=$((skipped + 1))
+        continue
+    fi
+
+    if response=$(curl -sS --max-time 10 -x "http://${endpoint}" "${IP_ECHO_URL}" 2> /dev/null); then
+        egress=$(echo "${response}" | jq -r '.ip // "?"' 2> /dev/null || echo "?")
+        probed=$((probed + 1))
+    else
+        egress="FAILED"
+        failed=$((failed + 1))
+    fi
+    # shellcheck disable=SC2059
+    printf "${fmt}" "${id}" "${endpoint}" "${location}" "${egress}"
+done < <(echo "${proxies_json}" | jq -c '.proxies[]')
+
+echo "-- ${probed} probed, ${skipped} unhealthy, ${failed} failed --"

docs/demo/show-egress-ips.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Demo: list proxies from the discovery API and show the egress IP each
+# healthy one provides, by calling an IP-echo site through it.
+#
+# Usage:
+#   ./show-egress-ips.sh <BASE_URL>          e.g. ./show-egress-ips.sh localhost:8090
+#
+# Optional environment:
+#   TOKEN        bearer token for the discovery API (see docs/api.md)
+#   IP_ECHO_URL  site that returns the caller's IP as JSON
+#                (default: https://api.ipify.org?format=json)
+set -euo pipefail
+
+if [ $# -ne 1 ]; then
+    echo "usage: $(basename "$0") <BASE_URL>  (e.g. localhost:8090)" >&2
+    exit 1
+fi
+BASE_URL="$1"
+IP_ECHO_URL="${IP_ECHO_URL:-https://api.ipify.org?format=json}"
+
+for tool in curl jq; do
+    if ! command -v "${tool}" &> /dev/null; then
+        echo "ERROR: ${tool} is required but not installed" >&2
+        exit 1
+    fi
+done
+
+auth_args=()
+if [ -n "${TOKEN:-}" ]; then
+    auth_args=(-H "Authorization: Bearer ${TOKEN}")
+fi
+
+echo "Fetching proxies from ${BASE_URL}/v1/proxies ..."
+if ! proxies_json=$(curl -sS --fail "${auth_args[@]}" "${BASE_URL}/v1/proxies"); then
+    echo "ERROR: could not fetch proxy list from ${BASE_URL}" >&2
+    exit 1
+fi
+if ! echo "${proxies_json}" | jq -e . > /dev/null; then
+    echo "ERROR: response from ${BASE_URL}/v1/proxies is not valid JSON" >&2
+    exit 1
+fi
+
+total=$(echo "${proxies_json}" | jq -r '.count')
+echo "Found ${total} proxies"
+echo ""
+
+probed=0
+skipped=0
+failed=0
+while IFS= read -r proxy; do
+    id=$(echo "${proxy}" | jq -r '.id')
+    ip=$(echo "${proxy}" | jq -r '.ip')
+    port=$(echo "${proxy}" | jq -r '.port')
+    healthy=$(echo "${proxy}" | jq -r '.healthy')
+
+    if [ "${healthy}" != "true" ]; then
+        echo "--- skipping ${id} (unhealthy) ---"
+        echo ""
+        skipped=$((skipped + 1))
+        continue
+    fi
+
+    echo "=== via ${id} — http://${ip}:${port} ==="
+    if response=$(curl -sS --max-time 10 -x "http://${ip}:${port}" "${IP_ECHO_URL}"); then
+        echo "${response}" | jq . 2> /dev/null || echo "${response}"
+        probed=$((probed + 1))
+    else
+        echo "WARNING: request through ${id} failed" >&2
+        failed=$((failed + 1))
+    fi
+    echo ""
+done < <(echo "${proxies_json}" | jq -c '.proxies[]')
+
+echo "Done: ${total} proxies — ${probed} probed, ${skipped} skipped (unhealthy), ${failed} failed"

docs/kvm-provider.md

@@ -0,0 +1,373 @@
+# The KVM provider
+
+Runs Managed proxies as VMs on your own libvirt/KVM hypervisors. The
+operator reaches each host over SSH, forwards the libvirt unix socket
+through that connection, and talks libvirt's own RPC over it — so the
+hosts need nothing beyond `sshd` and `libvirtd`, and the operator image
+carries no libvirt libraries. VMs boot a stock distro cloud image that the
+operator downloads into the host's storage pool on first use, with the
+Proxy's cloud-init delivered on a NoCloud seed ISO built in-process.
+
+## Hypervisor prerequisites
+
+Per host, once:
+
+1. **libvirt + KVM** (`libvirtd` running, `/dev/kvm` present).
+2. **An SSH user in the `libvirt` group**, so it may open
+   `/var/run/libvirt/libvirt-sock`. The operator authenticates with a
+   private key; nothing else on the host is touched.
+3. **A bridge to a network with DHCP** (e.g. `br0` bridged onto the LAN).
+   Each VM gets one virtio NIC on it and takes its IP from that network's
+   DHCP; the cluster must be able to reach those IPs for health probes
+   and for crawlers to use the proxy.
+4. **A storage pool** (default `default`, usually `/var/lib/libvirt/images`)
+   with room for one base image per distro (a few hundred MB) plus the
+   per-VM overlays (`diskSizeGb` each, thin-provisioned).
+5. **UEFI firmware for guests**: the `ovmf` package on Debian/Ubuntu,
+   `edk2-ovmf` on Fedora/RHEL. VMs boot UEFI by default (libvirt picks
+   the OVMF binary itself); a domain start fails with "no firmware
+   found" without it. Only `firmware: bios` images skip this.
+6. Optionally, **outbound internet from the VMs**: a vendor-data script
+   installs `qemu-guest-agent` from the distro mirrors on first boot.
+   Without it the VM still works — the operator falls back to the host's
+   ARP table to find the IP, which fills in once the guest sends traffic —
+   but the ARP table only sees traffic to the host itself, so on a plain
+   bridge the agent is in practice required.
+
+Check from the operator's point of view before wiring it in:
+
+```sh
+ssh virt@nas1.home.lan virsh -c qemu:///system list --all   # user can drive libvirt
+ssh virt@nas1.home.lan virsh -c qemu:///system pool-info default
+ssh virt@nas1.home.lan ip link show br0
+```
+
+## Cluster-side setup
+
+1. Create the SSH Secret from a key pair dedicated to the operator and a
+   `known_hosts` file holding every host's key. The worked example below
+   uses `~/.ssh/proxy-operator_ed25519` and `~/.ssh/kvm_known_hosts`,
+   the same paths the integration test and `make kvm-ssh-secret` default
+   to.
+
+   ```sh
+   # once: a key pair for the operator; the .pub goes on every host
+   ssh-keygen -t ed25519 -f ~/.ssh/proxy-operator_ed25519 -N '' -C proxy-operator
+   ssh-copy-id -i ~/.ssh/proxy-operator_ed25519.pub virt@nas1.home.lan
+
+   # once per host: its host key, under the address the *operator* will
+   # dial (hosts[].address), which may differ from what your laptop uses
+   ssh-keyscan -H nas1.home.lan >> ~/.ssh/kvm_known_hosts
+
+   # the Secret: the PRIVATE key (not the .pub) plus known_hosts
+   kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
+     --from-file=id_ed25519=$HOME/.ssh/proxy-operator_ed25519 \
+     --from-file=known_hosts=$HOME/.ssh/kvm_known_hosts
+   ```
+
+   `make kvm-ssh-secret` runs that last command as an apply, so it also
+   updates an existing Secret; override the paths with
+   `KVM_SSH_KEY=... KVM_KNOWN_HOSTS=...`. Putting the `.pub` in the
+   Secret is the classic mistake: the operator then fails at startup
+   with a "not a private key" error. Without `known_hosts`, set
+   `insecureIgnoreHostKey: true` and accept that a MITM could hand you a
+   fake hypervisor.
+
+   `config/manager/manager.yaml` mounts the Secret (optional) at
+   `/etc/proxy-operator/kvm` as two files, `id_ed25519` and
+   `known_hosts`, which is what `sshKeyFile` and `knownHostsFile` in the
+   provider config point at. The deployment starts fine without the
+   Secret, even with a `kvm` provider configured: only that provider's
+   proxies fail, and they recover once the Secret exists. Rotating the
+   key: re-run the command (or the make target) and restart the manager;
+   both files are read once, on the first connection, and cached.
+
+2. Add a provider entry to the providers config
+   (`config/manager/providers_config.yaml` in-cluster; full sample in
+   `config/samples/providers-config.yaml`):
+
+   ```yaml
+   - name: kvm-home
+     type: kvm
+     kvm:
+       sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+       knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+       images:
+         debian-13:
+           url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+       machineTypes:
+         small: {vcpus: 1, memoryMiB: 1024}
+       hosts:
+         - name: nas1
+           address: nas1.home.lan:22
+           user: virt
+           bridge: br0
+   ```
+
+   The operator checks both files at startup and logs a `WARNING:` line
+   when either is unreadable, but it does not refuse to start: the
+   provider set is built all-or-nothing, so exiting here would crash-loop
+   the manager and take the `kubernetes` and `gcp` providers with it.
+   The files are read for real on the first connection. A missing file is
+   a transient failure, so proxies heal by themselves once the Secret
+   lands; a key that does not parse is permanent and needs a fix.
+
+3. Apply a Proxy (`config/samples/proxy_kvm.yaml`) and watch it:
+
+   ```sh
+   kubectl get px -w
+   # NAME               MODE      PROVIDER   PHASE          IP             HEALTHY
+   # proxy-kvm-sample   Managed   kvm-home   Provisioning
+   # proxy-kvm-sample   Managed   kvm-home   Running        192.168.1.73
+   # proxy-kvm-sample   Managed   kvm-home   Ready          192.168.1.73   True
+   ```
+
+   The first Proxy for a given image on a given host sits in
+   `Provisioning` for the length of the image download; the operator log
+   shows `fetching base image into host pool` and `base image cached`
+   with the size and duration.
+
+## Config reference
+
+| Field | Required | Default | Meaning |
+|---|---|---|---|
+| `sshKeyFile` | yes | | private key used for every host |
+| `knownHostsFile` | unless `insecureIgnoreHostKey` | | OpenSSH `known_hosts` to verify host keys against |
+| `insecureIgnoreHostKey` | | `false` | skip host-key verification entirely |
+| `diskSizeGb` | | `10` | capacity of each VM's overlay disk; cloud-init's growpart expands the root fs into it |
+| `vendorData` | | | extra cloud-init vendor-data placed on every seed next to the Proxy's user-data. Beware that cloud-init drops any top-level key of a vendor-data **cloud-config** that the user-data also defines. Shipped as one part of a MIME multipart alongside the agent script when `installGuestAgent` is on |
+| `installGuestAgent` | | `true` | add the vendor-data script that installs and starts `qemu-guest-agent`. Turn off for images that ship it already or cannot reach a package mirror; IP discovery then depends on the host's ARP table |
+| `images.<key>` | | | the key names the image in `placement.image`. Letters, digits, dots, dashes and underscores only — it becomes the cached volume name `base-<key>.<format>` |
+| `images.<key>.url` | yes per entry | | http(s) URL of a cloud image (qcow2 or raw) |
+| `images.<key>.sha256` | | | expected digest; verified after download, a change triggers a re-download |
+| `images.<key>.format` | | `qcow2` | `qcow2` or `raw` |
+| `images.<key>.firmware` | | `uefi` | `uefi` (OVMF, secure boot off) or `bios` (SeaBIOS) for images without an EFI system partition; Debian 13 hangs in GRUB under BIOS on a headless VM, see *Known gaps* |
+| `machineTypes.<key>` | ≥ 1 | | `{vcpus, memoryMiB}` size profile |
+| `hosts[].name` | yes | | the `placement.zone` value that selects this host; unique. Letters, digits, dots, dashes and underscores only — it becomes part of the providerID `hosts/<name>/domains/<vm>` |
+| `hosts[].address` | yes | | SSH endpoint, `host:port` |
+| `hosts[].user` | yes | | SSH user, member of the `libvirt` group |
+| `hosts[].socket` | | `/var/run/libvirt/libvirt-sock` | libvirt socket path on the host |
+| `hosts[].pool` | | `default` | storage pool for base images, overlays and seeds |
+| `hosts[].bridge` | yes | | host bridge for the VM's NIC |
+
+## Placement
+
+| Proxy field | Meaning for kvm |
+|---|---|
+| `placement.zone` | host name from `hosts`. May be omitted when exactly one host is configured; otherwise required. Unknown → `Failed` |
+| `placement.machineType` | key in `machineTypes`. Unknown → `Failed` |
+| `placement.image` | key in `images`, or an `http(s)://` URL used directly. Anything else → `Failed` |
+| `placement.region` | ignored |
+
+`Failed` here means the reconciler stops retrying and the message names
+the offending value; fix the spec (which replaces the VM) or the config.
+
+## What lands on the host
+
+For a Proxy whose deterministic name is `proxy-abc…`, on the chosen
+host's pool:
+
+| Object | Name | Lifetime |
+|---|---|---|
+| base image | `base-<key>.qcow2` (or `base-<url-hash>.qcow2` for a URL) | cached; survives Proxy deletion |
+| image marker | `base-<key>.qcow2.sha256` | with the base image |
+| overlay disk | `proxy-abc….qcow2`, qcow2 backed by the base | with the Proxy |
+| seed ISO | `proxy-abc…-seed.iso` | with the Proxy |
+| domain | `proxy-abc…` | with the Proxy |
+
+The domain is a q35 machine with host-passthrough CPU, the overlay on
+virtio, the seed on a SATA cdrom, one virtio NIC on the bridge, a serial
+console, the guest-agent channel, a virtio RNG and no graphics. Its
+`<metadata>` carries the operator's element (managed flag, owning UID,
+creation time) — that is how orphan GC recognises its own VMs, since
+libvirt has no labels.
+
+The seed's `user-data` is the Proxy's `cloudInit`, **byte for byte**. Any
+format cloud-init accepts works, including a Jinja template, a MIME
+archive, a shell script or a gzipped payload; the operator never parses or
+re-renders it.
+
+The guest agent rides along in `vendor-data`, as a **shell script** rather
+than a cloud-config. That is the only shape cloud-init cannot drop:
+vendor-data cloud-config is merged under the user's own document with
+mergers that keep the user-data's value for every key both define and never
+append lists, so a Proxy carrying its own `packages:` silently lost the
+agent and sat in `Provisioning` forever. The strategy cannot be overridden
+from either document — `merge_how` in user-data governs only the first
+source, and `merge_how` in vendor-data is consumed and discarded before the
+document is stored. A vendor-data script skips all of it: cloud-init gives
+vendor-data its own handler chain, writes scripts to a separate directory
+and runs them from `cc_scripts_vendor`, after the user's packages are
+installed and before the user's `runcmd`. The script is idempotent, picks
+whichever of `apt-get`, `dnf`, `yum`, `zypper` or `apk` the image has, and
+logs into `/var/log/cloud-init-output.log`. Set `installGuestAgent: false`
+to leave it out. When `vendorData` is also configured, the two travel as
+separate parts of a MIME multipart vendor-data.
+
+## The image cache
+
+The base image is downloaded **by the operator, through libvirt**, into
+the host's pool the first time a Proxy asks for it on that host — nothing
+is pre-placed on the hypervisor, and the operator needs no filesystem
+access there. The bytes stream from the URL through the operator into
+the pool with a running sha256; nothing is buffered. The marker volume is
+written last and holds the digest, so a crash mid-download leaves a base
+volume without a marker, which the next attempt deletes and redoes.
+
+A re-download happens when: the marker is missing, or `sha256` is
+configured and differs from the marker's content. Otherwise the cached
+copy is used for as long as it exists, even if the URL's content changed
+upstream (a `latest` URL is pinned at the moment of first fetch, per
+host).
+
+To evict manually:
+
+```sh
+ssh virt@nas1 virsh -c qemu:///system vol-delete --pool default base-debian-13.qcow2.sha256
+ssh virt@nas1 virsh -c qemu:///system vol-delete --pool default base-debian-13.qcow2
+```
+
+⚠️ **Do not change an image entry's `url` or `sha256`, and do not evict
+its base volume, while Proxies using that image exist on the host.** The
+overlays back onto the base file; deleting or replacing it corrupts every
+VM that uses it. Delete those Proxies first, or add the new image under a
+new key and move Proxies over by changing their `placement.image` (which
+replaces the VM).
+
+## Logging into a VM
+
+- Put your key in the Proxy's cloud-init (`ssh_authorized_keys`) and
+  `ssh debian@<ip>` (Debian) / `ssh ubuntu@<ip>` (Ubuntu).
+- Or on the host: `virsh console proxy-abc…` attaches to the serial
+  console; cloud-init's output scrolls by there on first boot.
+- `virsh domifaddr proxy-abc… --source arp` / `--source agent` shows what
+  the operator sees when it looks for the IP.
+
+## Logging
+
+The provider logs through controller-runtime's logger, named `kvm`, with
+`provider`, `host` and `name` (the domain) on every line.
+
+- `--zap-log-level=debug` (logr V(1)): one line per provider operation
+  (`domain defined`, `KVM domain ensured running`, ...) **and one
+  `libvirt call` line per libvirt RPC**, named `kvm.libvirt`, with
+  `rpc` (the libvirt API function, e.g. `virDomainLookupByName`), the
+  identifiers it was given (`domain`, `pool`, `volume`), curated results
+  (`state`, `path`, `interfaces`), `duration`, and on failure `error`
+  plus `libvirtError` (the code name, e.g. `ErrNoDomain`) and
+  `libvirtCode`. Connection events (`libvirt host connected`, dial
+  failures, `libvirt connection dropped` with a reason) log at the same
+  level. A Create is about ten RPCs; a Get is four.
+- `--zap-log-level=2`: the same lines also carry the domain and volume
+  `xml`, the `metadata` element, the interface `addresses` and the
+  domain `names` from a listing.
+- Cloud-init is never logged at any level: the seed ISO upload reports
+  only `length` and `duration`. There is no raw wire tier like the GCP
+  provider's V(5); go-libvirt exposes no hook for it.
+
+## Health check
+
+The operator checks every configured host every `--provider-check-interval`
+(default 1m). It publishes the result in two places: the `ProviderHealth`
+object named after the provider (`kubectl get ph kvm-home -o yaml`), and the
+`proxy_operator_provider_target_up{provider,target}` gauge.
+
+All hosts are checked concurrently, each over the same pooled connection
+that Create and Get use:
+
+1. The SSH and libvirt handshakes. These only happen on the first check, or
+   after the link dropped.
+2. `virConnectGetLibVersion` and `virConnectGetHostname`.
+3. `virStoragePoolLookupByName` and `virStoragePoolGetInfo` on the host's
+   `pool`.
+
+A passing target's `details` carry `hostname`, `libvirtVersion`, `pool`,
+`poolState`, `poolCapacity` and `poolAvailable` (in GiB). The free space is
+the closest thing this provider has to a capacity signal. Every target also
+shows its SSH `user`. A failing target says where the check broke:
+
+| Reason | Meaning | Look at |
+|---|---|---|
+| `SSHMaterialUnavailable` | the key or known_hosts file is missing or does not parse | the `kvm-ssh` Secret (see *Cluster-side setup*) |
+| `Unreachable` | the TCP connect failed, or the link died mid-check | DNS, Tailscale or a firewall between the cluster and `address` |
+| `SSHHandshakeFailed` | the host key is missing from known_hosts or has changed, or `user` rejected the key | the known_hosts entry; `user`'s `authorized_keys` |
+| `LibvirtConnectFailed` | SSH worked, the libvirt socket did not | `user` in the `libvirt` group, `libvirtd` running, the `socket` path |
+| `Timeout` | no answer within `--provider-check-timeout` (default 15s), or the previous check of the host is still waiting | a hung `libvirtd`, a saturated link, or a reconcile stuck redialing the host |
+| `PoolNotFound` | there is no pool named `pool` | `virsh pool-list --all` |
+| `PoolNotActive` | the pool exists but is not running | `virsh pool-start <pool>` and `virsh pool-autostart <pool>` |
+| `LibvirtError` | any other libvirt error | the message carries libvirt's own text |
+
+Sharing the connection has two consequences:
+
+- The leader now keeps one SSH connection per host open from startup, with a
+  keepalive every 15s. Before, the first connection opened with the first KVM
+  reconcile.
+- A check that runs out of time gives up on its RPCs but leaves the
+  connection alone, so an image upload or a Create running on it is not cut
+  off. At most one abandoned check per host keeps waiting; until it
+  returns, the next check reports `Timeout` straight away. A link that is
+  really dead is closed by the keepalive within about two keepalive
+  intervals.
+- A check never waits longer than its timeout for the host's connection,
+  even while a reconcile holds it to redial a host that is down.
+
+## Known gaps
+
+- **No quota or capacity detection.** libvirt reports an out-of-space
+  pool or an over-committed host as an ordinary error; the reconciler
+  treats it as transient and retries with backoff rather than the slower
+  quota backoff. The health check does report each pool's free space
+  (`poolAvailable`), so a pool that is filling up shows before Creates
+  start failing.
+- **A rotated `kvm-ssh` Secret needs a manager restart.** The key and
+  known_hosts files are read on first use and cached for the life of the
+  process. Replacing them in the Secret therefore changes nothing for
+  later connections, health checks included. After a host key change, the
+  check keeps reporting `SSHHandshakeFailed` until the manager restarts.
+- **Orphan volumes without a domain are invisible to GC.** A crash
+  between creating the volumes and defining the domain leaves an overlay
+  and seed that `ListByTag` cannot see (it lists domains). They are
+  cleaned when the same Proxy reconciles again (Delete removes volumes by
+  name even with no domain), but a Proxy CR that is gone for good leaves
+  them for `virsh vol-list` to find.
+- **A host removed from the config takes its VMs with it.** `Delete` for a
+  providerID naming an unconfigured host is a no-op success, so the Proxy
+  finishes deleting instead of pinning its finalizer, but the domain and its
+  volumes stay on the hypervisor and orphan GC cannot see them either (it
+  only sweeps configured hosts). Delete the Proxies before retiring a host,
+  or clean up afterwards with `virsh undefine --nvram` plus `virsh vol-delete`.
+  `Get` still fails permanently for such a providerID on purpose: a typo in
+  `hosts` must not look like a missing VM and trigger a rebuild elsewhere.
+- **The first image fetch is synchronous.** It holds a reconcile worker
+  for the download (LAN speed, a few hundred MB, once per image per
+  host). Other Proxies keep reconciling on the remaining workers.
+- **Replacing an image in use breaks running VMs** — see the warning
+  under *The image cache*.
+- **IP discovery needs either guest traffic or the agent.** A VM on a
+  bridge with no DHCP never gets an address and stays `Provisioning`
+  forever; check `virsh console`. The same symptom with a booted, addressed
+  VM means the guest agent never came up: check `systemctl status
+  qemu-guest-agent`, `cloud-init status --long` and the vendor script's
+  output in `/var/log/cloud-init-output.log` (a VM without outbound
+  internet cannot install it). A user-data that disables vendor-data
+  (`vendor_data: {enabled: false}`) also removes the agent, by design.
+- **Debian 13 cloud images hang in GRUB under BIOS on a headless VM**
+  ([Debian bug #1111240](https://bugs.debian.org/1111240), open, in
+  `grub2`): trixie's image ships `terminal_output gfxterm serial` in its
+  GRUB config, and with a serial console but no video device (exactly
+  the domain this provider defines) GRUB prints "Booting" and never
+  hands over to the kernel. Not host-specific; reproduced by the Debian
+  cloud team. The same image boots under UEFI with the serial console
+  intact, which is why `firmware` defaults to `uefi`. Debian 12
+  (bookworm) images are unaffected and boot either way. Images that
+  need `firmware: bios` are the exception, not the rule.
+
+## Testing
+
+Unit tests run over an in-memory fake of the libvirt API. The optional
+integration test against a real host is described in
+[testing.md](testing.md#the-kvm-integration-test-real-hypervisor); its
+`-v` output includes the `libvirt call` trace above, at the verbosity
+`KVM_TEST_LOG_LEVEL` selects (default 1). `TestKVMCheck_realHost` runs the
+health check against the same host and creates nothing:
+`go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/`.

docs/plans-executions/2026-08-11-2152-discovery-api-docs.md

@@ -0,0 +1,38 @@
+# Execution: Discovery API documentation
+
+Plan: [2026-08-11-2152-discovery-api-docs.md](../plans/2026-08-11-2152-discovery-api-docs.md)
+
+- [x] Step 1 — Write `docs/api.md` full API reference
+- [x] Step 2 — Add pointers in README and architecture.md
+
+## Step 1 — docs/api.md
+
+Wrote the full reference: base URL (in-cluster FQDN + port-forward), bearer
+auth, error envelope, configuration table, all five routes with schemas,
+status codes and curl examples, the selection/cooldown semantics section,
+an end-to-end curl walkthrough, and the in-memory/single-replica caveats.
+All facts were taken from the code, not from memory of prior docs.
+
+Worth noting: the doc explicitly calls out two things no earlier doc
+stated for clients — that a report **without** a target on a targetless
+lease creates a *global* cooldown (blocking the proxy for everyone), and
+that `rate_limited` and `banned` currently behave identically. Both came
+straight from `internal/lease/store.go` and are easy to trip over.
+
+## Step 2 — Pointers + verification
+
+Added one-line links to the new doc in README's quickstart (above the curl
+block) and in `docs/architecture.md` §7. Verified the documented behavior
+against the tree rather than trusting the write-up:
+
+```sh
+go build ./... && go test -short ./internal/discovery/ ./internal/lease/
+```
+
+Both pass; a grep of `server_test.go` confirmed every documented status
+code and error code (`invalid_ttl`, `invalid_query`, `invalid_result`,
+`no_match`, `unknown_lease`, 201/204/401/404/409) is asserted by tests.
+
+Worth noting: CHANGELOG entry deliberately deferred until the user
+confirms the docs read well, per the CHANGELOG convention's
+"once the user confirms it works" clause.

docs/plans-executions/2026-08-11-2220-demo-scripts.md

@@ -0,0 +1,55 @@
+# Execution: Demo scripts
+
+Plan: [2026-08-11-2220-demo-scripts.md](../plans/2026-08-11-2220-demo-scripts.md)
+
+- [x] Step 0 — Branch `feat/demo-scripts` + plan commit
+- [x] Step 1 — `docs/demo/show-egress-ips.sh`
+- [x] Extra (added iteratively, not in the original plan) — proxy-creation scripts
+
+## Step 0 + Step 1
+
+Branched off `main`, committed the plan alone, then wrote
+`docs/demo/show-egress-ips.sh`: takes `BASE_URL` as its argument, lists
+`/v1/proxies` (bearer auth via optional `TOKEN` env), probes each healthy
+proxy with `curl -x http://ip:port` against an IP-echo site
+(`IP_ECHO_URL`, default ipify JSON), banners which proxy each request goes
+through, skips unhealthy ones, and ends with a probed/skipped/failed
+summary. Per user request the script was left uncommitted for iteration
+and no verification beyond `bash -n` was run.
+
+## Extra — create-kubernetes-proxies.sh, create-gcp-proxies.sh
+
+Added on the same branch before the first commit:
+
+- `create-kubernetes-proxies.sh <count>` — creates
+  `proxy-kubernetes-demo-1..N` with the kubernetes provider, spec taken
+  from `config/samples/proxy_kubernetes.yaml`, applied via
+  `kubectl apply -f -` heredocs. Optional `NAMESPACE` env.
+- `create-gcp-proxies.sh <count>` — creates `proxy-gcp-demo-1..N` from the
+  user-supplied gcp-eu manifest (e2-micro, Ubuntu 24.04, Squid
+  cloud-init), each with a zone picked randomly from a hardcoded list of
+  27 EU zones so the fleet gets egress IPs from different locations.
+  Env overrides: `ZONES`, `GCP_PROVIDER` (default `gcp-eu`), `NAMESPACE`.
+
+## Extra — run-demo.sh, show-egress-ips-table.sh
+
+Second round of iterative additions:
+
+- `run-demo.sh` — tmux demo driver: 2x2 tiled grid with the egress-IP
+  table looping every 10s inside a netshoot pod (script `kubectl cp`'d
+  into the pod, `BASE_URL` set to the in-cluster Service FQDN),
+  `watch -n3 kubectl get px`, and both create scripts auto-running with
+  `COUNT` proxies each (default 4). Env knobs: `COUNT`, `SESSION`,
+  `NETSHOOT_POD`, `DEMO_DIR` (defaults to the script's own dir).
+- `show-egress-ips-table.sh` — condensed one-line-per-proxy variant of
+  `show-egress-ips.sh` sized for a tmux pane: PROXY / ENDPOINT /
+  LOCATION (zone→geo attribute fallback) / EGRESS-IP columns, with
+  `(unhealthy)` and `FAILED` inline instead of verbose output. The
+  verbose script stays for standalone use; the demo driver uses the
+  table variant.
+
+Worth noting: beyond the user's sample manifest, the gcp script also
+writes the picked zone into `attributes.zone`, so the discovery API
+exposes each proxy's location and leases can select on it. The zone list
+is static — if a project lacks quota in some region, `ZONES` narrows the
+pool; nothing validates zones against the live project.

docs/plans-executions/2026-08-24-1025-otel-tracing.md

@@ -0,0 +1,183 @@
+# Execution log: OpenTelemetry tracing integrated with logr/zap logging
+
+Plan: `docs/plans/2026-08-24-1025-otel-tracing.md`
+
+- [x] Step 1 — Dependencies
+- [x] Step 2 — New package `internal/tracing`
+- [x] Step 3 — `provider.WithTracing` decorator
+- [x] Step 4 — `cmd/main.go` wiring
+- [x] Step 5 — Reconciler spans
+- [x] Step 6 — Discovery server
+- [x] Step 7 — GC + health
+- [x] Step 8 — GCP wire-log enrichment
+- [x] Step 9 — Manifests + docs
+
+## Step 1 — Dependencies
+
+Aligned the pre-existing indirect skew (otel core v1.44.0 vs otlptrace exporters
+v1.40.0) and added the new direct deps in one shot:
+
+```bash
+go get go.opentelemetry.io/otel@v1.45.0 \
+  go.opentelemetry.io/otel/sdk@v1.45.0 \
+  go.opentelemetry.io/otel/trace@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/stdout/stdouttrace@v1.45.0 \
+  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0
+go mod tidy
+```
+
+MVS side-effects worth recording: `logr v1.4.3→v1.4.4`, `httpsnoop
+v1.0.4→v1.1.0`, `grpc-gateway/v2 v2.27.7→v2.29.0`, `proto/otlp v1.9.0→v1.11.0`,
+plus a `genproto/googleapis/api` pseudo-version bump. Full `make test` passed
+against the bumped graph (`google.golang.org/api v0.292.0` and k8s v0.36
+tolerate otelhttp v0.70.0).
+
+Worth noting: the first `go mod tidy` ran *before* any first-party code
+imported `otlptracehttp`/`stdouttrace`, so it silently dropped those two
+modules again; the Step 2 tidy re-added them. The plan's semconv question
+resolved to `semconv/v1.43.0` — that's what `sdk@v1.45.0/resource/builtin.go`
+imports, so first-party code uses the same version to avoid
+`ErrSchemaURLConflict` in the common path.
+
+## Step 2 — `internal/tracing` package
+
+Landed as planned: `tracing.go` (env-gated `Setup`, hand-rolled exporter
+selection), `logger.go` (`Start`/`StartSpan`/`ContextWithLogger` with the
+base-logger ctx key that prevents duplicate `traceID` zap fields on nested
+spans), `reconciler.go`, `transport.go`, `http.go`, `options.go`; tests for
+all of it (first use of `sdk/trace/tracetest` in the repo).
+
+Two deviations from the plan's letter:
+
+- **k8s transport gating is a custom RoundTripper, not `otelhttp.WithFilter`**
+  (`parentGatedTransport`): requests without a parent span bypass the otel
+  transport entirely, so the no-root-spans-from-informers guarantee doesn't
+  depend on otelhttp filter semantics for transports.
+- **`HTTPMiddleware` must copy the route pattern back.** The logger-injecting
+  inner handler wraps the request via `WithContext` (a shallow copy), so the
+  mux records the matched pattern on the copy while otelhttp's post-routing
+  span rename reads the original. Found by the middleware test (span named
+  `"GET"` instead of `"GET /v1/things/{id}"`); fixed with `r.Pattern =
+  r2.Pattern` after `next.ServeHTTP`.
+
+Also: both the transport and the middleware pass explicit W3C propagators
+instead of relying on the global, so behavior is deterministic under tests
+and when tracing is disabled. `Setup` tests reset the global provider to a
+fresh noop per case — restoring otel's own default delegate triggers a
+"Setting tracer provider to its current value" warning from the SDK.
+
+## Step 3 — `provider.WithTracing`
+
+`internal/provider/tracing.go` mirrors `metrics.go` exactly (same wrap shape,
+same `resultLabel` classification reused as the `provider.result` span
+attribute); tests reuse `staticProvider` from `metrics_test.go`. `ErrNotFound`
+maps to span status Ok as planned — the reconciler polls `Get` to NotFound
+during replacement/deletion, so it's an answer, not a failure.
+
+One small API addition to `internal/tracing` for this: exported
+`tracing.Tracer(opts...)`, since the decorator lives in package `provider`
+and couldn't reach the unexported option resolver. The `cmd/main.go` wiring
+(`WithTracing` outermost around `WithMetrics`) lands with Step 4's commit —
+same file, one commit.
+
+## Step 4 — `cmd/main.go` wiring
+
+As planned: `tracing.Setup` right after `ctrl.SetLogger`; manager rest.Config
+wrapped via `restCfg := ctrl.GetConfigOrDie(); restCfg.Wrap(...)` (the
+previous inline call left nowhere to wrap); providers decorated tracing-
+outermost; explicit trace flush after `mgr.Start` returns on both the error
+and clean paths, with a fresh 10s context since the signal ctx is already
+cancelled by then.
+
+The kubernetes provider grew `NewWithTransportWrapper(ctx, cfg, wrap)` (its
+client is built from its own `ctrl.GetConfig()`, invisible to the manager's
+wrapped config); `New` now delegates with a nil wrapper, so `newWithClient`
+tests stayed untouched, and main registers a closure — the same pattern the
+gcp constructor already used for wire-log options.
+
+Deviation: the plan put `--trace-health-probes` here, but the flag needs the
+`health.Engine.TraceProbes` field that Step 7 introduces — moved there to
+keep every commit compiling.
+
+## Step 5 — Reconciler spans
+
+`SetupWithManager` completes with `tracing.NewReconciler("Proxy", r)`;
+sub-spans `reconcile.managed` / `reconcile.replaceInstance` /
+`reconcile.delete` open at the top of each state machine, and `status.patch`
+opens inside the deferred flush closure so it stays within the root span
+while its error still folds into the recorded result. `reconcileExternal`
+left unspanned as planned (no I/O).
+
+Two small judgment calls: the sub-spans carry no extra attributes — the
+provider decorator already records `provider.id`, and the root span carries
+the object identity, so duplicating them was noise; and `status.patch` is
+emitted every reconcile even when nothing changed (the no-op compare is the
+span's content — a real PATCH shows up as its k8s HTTP child). Tests that
+call `r.Reconcile` directly bypass the wrapper; with no global tracer set
+they see no-op spans, so the existing fake-client and envtest suites run
+unchanged.
+
+## Step 6 — Discovery server
+
+Middleware chain is now recover → tracing (server span + request logger) →
+request-log → body cap → auth; the request-log middleware and the three
+handler error sites log via `logf.FromContext(r.Context())` — which is
+`s.log` enriched with the request's traceID/spanID by the tracing
+middleware, not a different logger (logr sinks can't read ctx at log time,
+so per-request values must ride on the logger instance in the ctx; user
+asked, answered in-session).
+
+Deviation from the plan's letter: `recoverMiddleware` keeps `s.log`. It sits
+*outside* the tracing layer, so its request ctx never has the enriched
+logger — switching it to `FromContext` would silently drop the `"discovery"`
+name and gain nothing.
+
+## Step 7 — GC + health
+
+`gc.sweep` runs in a root span with `gc.providers` / `gc.deleted` counters
+(a failed orphan delete no longer counts as deleted — the loop grew an
+explicit `continue`). The health engine got the deferred `--trace-health-probes`
+flag and `TraceProbes` field; the probe call moved into `runProbe`, which
+wraps it in a `health.probe` client span only when enabled, ending before
+`record` (which stays ctx-free by design). Probe transports remain
+uninstrumented so no traceparent can leak through a proxy to external
+targets. Probe→reconcile span links stay out of scope (GenericEvent carries
+no ctx; the workqueue coalesces events) — recorded in the architecture
+Decisions in Step 9.
+
+## Step 8 — GCP wire-log enrichment
+
+`wireFilterHandler.Handle` clones the record and appends `traceID`/`spanID`
+when the ctx carries a valid span — same keys as the logr enrichment, added
+before the elision pass (both values are well under the 1KiB threshold).
+The new `wirelog_test.go` covers the enrichment, that the JWT/token security
+filter still drops non-wire Debug records even with a span present, and that
+spanless records stay unchanged.
+
+## Step 9 — Manifests, docs, lint
+
+`config/manager/manager.yaml` gained the OTel env block (downward-API
+`POD_NAME`/`POD_NAMESPACE` deliberately listed *before*
+`OTEL_RESOURCE_ATTRIBUTES` — `$(VAR)` expansion only sees earlier vars),
+with the OTLP endpoint and sampler left as commented examples so deployments
+stay tracing-off by default. `docs/architecture.md` got "### 10. Tracing"
+plus five Decisions entries; README got a Tracing section.
+
+`make lint` surfaced 16 new-on-branch issues (checked with
+`golangci-lint run --new-from-rev main`; the other ~43 pre-date this work):
+
+- one gofmt slip, 13 goconst repeats — fixed with `env*`/`exporter*`
+  constants in `internal/tracing/tracing.go` shared by the tests;
+- two logcheck hits ("function takes both a context and a logger"):
+  `Setup` now reads its logger from ctx (`logf.FromContext`, caller seeds
+  via `logf.IntoContext` — the linter's sanctioned pattern), and
+  `ContextWithLogger` carries a justified `//nolint:logcheck`, since being
+  the IntoContext-analogue is its purpose.
+
+Worth noting: pre-existing lint debt (50 goconst etc. on `main`) was left
+untouched — `make lint` still fails overall; only branch-introduced issues
+were cleaned. The lint fixes changed `Setup`'s signature after Step 2/4 had
+landed, so those commits show the two-arg version; this commit is where it
+settles.

docs/plans-executions/2026-08-24-1224-tracing-e2e.md

@@ -0,0 +1,68 @@
+# Execution log: e2e test — OTel tracing against real Tempo
+
+Plan: `docs/plans/2026-08-24-1224-tracing-e2e.md`
+
+- [x] Step 1 — `test/e2e/tracing_test.go`
+- [x] Step 2 — Makefile
+- [x] Step 3 — Docs
+
+## Steps 1–3 — spec, Makefile, docs (one commit)
+
+The three steps landed together — the spec is one new file and the other
+two are its wiring. `Describe("OTel tracing", Ordered)` is fully
+self-contained (own ns create → `make install`/`make deploy` → teardown)
+because Ginkgo randomizes top-level container order, so it cannot share the
+Manager Describe's deployment. It reuses the package-level `namespace` /
+`managerImage` and the suite's idioms (`utils.Run`, curl-pod with the
+restricted-PSS overrides JSON, log-substring `Eventually`s).
+
+Judgment calls beyond the plan's letter:
+
+- The squid pre-pull (`docker pull` + `kind load`) is **best effort** — a
+  missing docker binary logs a note and continues rather than failing the
+  spec; the 5m Ready timeout still covers an in-cluster pull.
+- The OTLP preflight pod prints per-attempt HTTP codes and a final
+  `OTLP_OK`/`OTLP_UNREACHABLE` marker; the assertion quotes the pod's
+  output, so an unreachable endpoint names itself in the failure.
+- `kubectl set env` is passed the literal
+  `OTEL_RESOURCE_ATTRIBUTES=...$(POD_NAME)...` string via `exec.Command` —
+  no shell involved, kubectl stores `$()` verbatim, and the in-place update
+  keeps the var after the downward-API vars it references.
+- Tempo helpers are stdlib-only; `/api/traces/<id>` is decoded as
+  OTLP-JSON (`batches[].scopeSpans[].spans[].name`), which is Tempo's
+  actual shape (not Jaeger's).
+
+Verified so far: `go vet -tags=e2e ./...` clean.
+
+## Live run against homelab Tempo
+
+First attempt failed before the suite started — Docker Desktop wasn't
+running (`kind` could not create the cluster), and the failure was masked
+to exit 0 by a `| tail` pipe on the make invocation (no pipefail in that
+shell). Rerun with Docker started first and no pipe:
+
+```bash
+TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e
+```
+
+Result: **SUCCESS — 4/4 specs (Manager smoke + both tracing Its), 219 s**,
+kind cluster auto-deleted. Run id `e2e-1787567624494810000`; the traces are
+findable in Grafana with TraceQL
+`{resource.test.run.id="e2e-1787567624494810000"}`. Both proxies reached
+`Ready` on real squid pods; the create trace carried
+`Reconcile Proxy` / `reconcile.managed` / `provider.create` /
+`status.patch` and the expected resource attrs; the deletion trace carried
+`reconcile.delete` / `provider.delete`.
+
+Worth noting: the OTLP preflight and both Tempo polls succeeded on the
+in-cluster → LAN path (kind on macOS reaches 192.168.0.30 through Docker
+Desktop's NAT), so no extra networking setup is needed on this machine.
+
+The plan's negative verification also ran: with a deliberately wrong
+endpoint (`OTLP_ENDPOINT=http://192.168.0.30:9999`) the tracing spec
+failed in the BeforeAll preflight after ~52 s with
+`OTLP endpoint http://192.168.0.30:9999 is not reachable from inside the
+kind cluster; curl output: OTLP_UNREACHABLE` — a named, fast failure
+instead of a 2-minute opaque search timeout — and teardown still deleted
+the kind cluster (`make cleanup-test-e2e` run explicitly, since a failing
+`go test` skips the Makefile's cleanup step).

docs/plans-executions/2026-09-04-1404-kvm-provider.md

@@ -0,0 +1,673 @@
+# Execution: KVM/libvirt provider
+
+Plan: [2026-09-04-1404-kvm-provider.md](../plans/2026-09-04-1404-kvm-provider.md)
+
+- [x] Step 0 — Save plan + branch `feat/kvm-provider`
+- [x] Step 1 — Config (`KVMConfig` & friends, `validate()` case, tests)
+- [x] Step 2 — Pure builders (`domainxml.go`, `volume.go`, `seed.go`, `id.go`, `metadata.go`)
+- [x] Step 3 — Transport + errors (`conn.go`, `errors.go`)
+- [x] Step 3b — Image cache (`image.go`)
+- [x] Step 4 — Provider methods (`kvm.go`) over a fake API
+- [x] Step 5 — Wiring, manifests, samples, docs, CHANGELOG
+- [x] Step 6 — Integration test against a real hypervisor (written and compiled; first real run is part of Step 7)
+- [x] Step 7 — Manual validation + MR
+
+## Step 0 — Save plan + branch
+
+Plan committed alone on `feat/proxy-operator`, then branched:
+
+```bash
+git checkout -b feat/kvm-provider
+```
+
+## Step 1 — Config
+
+The new structs (`KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage`) and
+`validateKVM` went into their own file, `internal/provider/config_kvm.go`,
+rather than being appended to `config.go` as the plan said: the KVM block has
+far more fields and rules than the other two providers, and keeping it
+separate leaves `config.go` readable. Tests likewise live in
+`config_kvm_test.go` instead of extending the table in `config_test.go`; the
+invalid-config table covers every required field, duplicate host names, image
+URL scheme, sha256 shape, and the disk format enum.
+
+While adding the `kvm` case to `validate()`, the pairwise "type is X but a Y
+block is set" checks (which would have needed six branches with three types)
+were replaced by a `setBlocks()` helper that lists the non-nil blocks and
+rejects any that don't match `type`. The error strings are unchanged, so the
+existing GCP/kubernetes tests still pass.
+
+Dependencies were fetched up front so the API signatures in the plan could be
+checked with `go doc`; they stay `// indirect` in `go.mod` until Step 2 imports
+them:
+
+```bash
+go get github.com/digitalocean/go-libvirt@latest github.com/kdomanski/iso9660@v0.4.0
+# go-libvirt resolved to v0.0.0-20260814190004-1a83157e1858 (no tagged release)
+# side effect: golang.org/x/crypto 0.54.0 → 0.56.0, x/mod, x/text, x/tools bumped
+```
+
+Worth noting: `go doc` showed that go-libvirt's `New(net.Conn)` is deprecated
+in favour of `NewWithDialer(socket.Dialer)`, where `Dialer` is a one-method
+`Dial() (net.Conn, error)` interface. Step 3 should wrap the SSH client's unix
+forward in that interface instead of using `New`; the plan's transport design
+is otherwise unaffected.
+
+## Step 2 — Pure builders
+
+`internal/provider/kvm/` now has the five builders from the plan plus
+`kvm.go` holding the package doc and `withDefaults` (disk size 10 GiB, the
+default vendor-data, `/var/run/libvirt/libvirt-sock`, pool `default`, image
+format `qcow2`). Everything is a pure function over `encoding/xml` structs or
+strings, unit-tested by unmarshalling the output back and checking fields.
+
+Two design choices differ from the plan text:
+
+- **Domain disks reference pool volumes, not host paths.** The plan had
+  `buildDomainXML` take the overlay and seed paths (needing two
+  `StorageVolGetPath` round-trips per Create). The domain XML now uses
+  `<disk type="volume"><source pool=... volume=.../>`, so only the *base*
+  image path is ever looked up (the overlay's `<backingStore>` needs a real
+  path). Fewer RPCs and the builder's inputs are all names.
+  **Reverted in Step 7**: volume-referenced disks cannot start on an
+  AppArmor host, see there. The domain XML is back to `type="file"` with
+  the two `StorageVolGetPath` calls the plan had.
+- **Capacities are always `unit="bytes"`.** libvirt's `G` means GiB but
+  `GB` means 10^9; spelling the byte count out removes the ambiguity from
+  both the overlay and the seed/base volumes.
+
+The metadata element is written with a default namespace
+(`<instance xmlns="…/kvm">`) because `encoding/xml` cannot emit prefixes on
+marshal. libvirt may hand it back prefixed; `parseInstanceMetadata` is
+tested against both spellings since `encoding/xml` matches on the namespace
+URI. Whether libvirt accepts the default-namespace form on `DomainDefineXML`
+is confirmed in Step 6 against a real host, not here. (Step 7: it does,
+but `DomainGetMetadata` returns the element with no namespace at all; the
+parser now matches on local name only.)
+
+Checked before relying on `github.com/kdomanski/iso9660` for the seed: its
+filename mangling lowercases and keeps `-`, so `user-data`/`meta-data`/
+`vendor-data` come out as `user-data;1` etc. in the directory record, which
+the Linux isofs driver (and therefore cloud-init's NoCloud mount) presents as
+`user-data`. The seed test opens the ISO with the same library's reader and
+asserts label `cidata`, all three files, and byte-identical output for
+identical inputs. `NewWriter()` stages files in a temp directory, so
+`Cleanup()` is deferred.
+
+`go mod tidy` ran as part of this step; it promoted `kdomanski/iso9660` to a
+direct dependency and dropped `go-libvirt` again because nothing imports it
+yet — Step 3 brings it back.
+
+Worth noting: `withDefaults` copies the `Images` map and `Hosts` slice
+instead of mutating the config in place, so the registry's shared
+`ProviderConfig` stays untouched. The default vendor-data installs
+`qemu-guest-agent` via `packages:` + `runcmd`, which means the first boot
+needs network access to the distro mirrors; a Proxy on an isolated bridge
+would still work (ARP discovery), just without the agent.
+
+## Step 3 — Transport + errors
+
+`conn.go` holds the `libvirtAPI` seam (17 methods, signatures as in the
+plan), a per-host connection pool, and the SSH dialer; `errors.go` holds
+`classify`, `isLibvirtError`, `hasCode` and `logAPIError`. Both are
+unit-tested; the whole package still runs in about a second under `-race`.
+
+The one thing the plan did not anticipate: **go-libvirt RPCs take no
+context and never time out**. A hung hypervisor would pin a reconcile worker
+forever. So the pool exposes `call(ctx, host, fn)` rather than handing out
+raw clients: `fn` runs in a goroutine, and if `ctx` ends first the pool
+closes the SSH connection out from under it (which makes the blocked RPC
+return) and reports `ctx.Err()`. The same helper (`awaitOrCancel`) bounds
+`lv.Connect()` during dial. Two more liveness measures for the same reason:
+the SSH handshake gets a deadline (ctx's, or 30 s), and each connection runs
+an OpenSSH `keepalive@openssh.com` loop every 15 s that closes the client
+when a request errors or goes unanswered for one interval.
+
+Connection-drop policy in `call`: any error that is *not* a `libvirt.Error`
+(so io.EOF, "connection closed", SSH failures) drops the cached connection
+and the next call redials; a daemon-reported `libvirt.Error` keeps it. Dials
+are serialised per host (concurrent Creates on one host share a dial) but
+hosts don't block each other.
+
+As Step 1 noted, the dialer uses `libvirt.NewWithDialer` with a
+`socket.Dialer` that wraps `ssh.Client.Dial("unix", socket)` rather than the
+deprecated `New(net.Conn)`.
+
+Tests: the pool is exercised through a fake dialer (reuse, redial after
+`Disconnected`, drop on transport error, keep on libvirt error, cancel
+releases a blocked call, concurrent callers share one dial, hosts
+independent). The SSH layer is tested against an in-process
+`x/crypto/ssh` server that serves `direct-streamlocal@openssh.com` channels
+by dialing the requested unix path (an echo socket standing in for
+`libvirt-sock`), with keys generated per test and a real `known_hosts` file
+written via `knownhosts.Line`. Covered: forward round-trip and the socket
+path the server saw, unknown host key rejected with `knownhosts.KeyError`,
+`insecureIgnoreHostKey`, wrong client key, cancelled ctx, missing key or
+known_hosts file, and keepalive both closing a stalled client and leaving a
+healthy one alone. The libvirt handshake (`Connect()`) itself is *not*
+covered here — faking libvirt's XDR RPC just for that is not worth it; the
+Step 6 integration test covers it against a real host.
+
+`go mod tidy` promoted `go-libvirt` and `golang.org/x/crypto` to direct
+dependencies.
+
+Worth noting: `classify` maps `ErrOperationInvalid` (e.g. destroying a
+domain that is already off) to Transient; Delete in Step 4 must check that
+code itself via `hasCode` and treat it as success rather than relying on
+the classifier. Nothing maps to `ErrQuotaExceeded` — libvirt has no quota
+concept; an out-of-space pool surfaces as a Transient `ErrInternalError`.
+
+## Step 3b — Image cache
+
+`image.go` follows the plan closely: `resolveImage` maps `placement.image`
+to a configured key or a direct http(s) URL (else Permanent);
+`imageCache.ensure` checks base volume + marker (+ marker content when a
+checksum is configured), tears down and re-fetches when anything is off,
+streams the HTTP body through `StorageVolUpload` with a `sha256` tee,
+refreshes the pool, then writes the marker last. A per-`(host, volume)`
+mutex serialises concurrent callers; the pool itself is the only cache
+state.
+
+Errors from this file are pre-tagged with the taxonomy sentinel via
+`fmt.Errorf("%w: …", provider.ErrPermanent, …)` and `classify` now honours
+such tags before looking at libvirt codes. That keeps one `wrapErr` path in
+the provider instead of special-casing image errors.
+
+`fake_test.go` landed with this step rather than Step 4: an in-memory
+`libvirtAPI` with pools, volumes (content, format, capacity, backing path),
+domains (state, metadata, per-source interface addresses), per-method error
+injection (`fail` always / `failOnce`) and an `uploadHook` to break a stream
+mid-way. It models the libvirt error codes the provider branches on
+(`ErrStorageVolExist`, `ErrNoStorageVol`, `ErrNoDomain`,
+`ErrOperationInvalid`, `ErrNoDomainMetadata`, `ErrAgentUnresponsive`).
+
+Tests cover every case in the plan plus: no checksum configured (marker is
+still written with the computed digest; second call needs no marker read),
+403/500 alongside 404/503, missing pool → Permanent, cancelled ctx →
+Transient with nothing created, and different hosts fetching independently.
+
+One test-harness lesson worth keeping: `httptest`'s server drops
+`Content-Length` and switches to chunked encoding for bodies larger than
+its 2 KiB write buffer, so the fake mirror sets the header explicitly.
+Real distro mirrors send it; a server that doesn't is reported as
+Permanent with a message naming the header.
+
+Worth noting: the base volume is created with `capacity = Content-Length`
+and the *image's* format (`qcow2`), and the raw bytes are uploaded over it.
+That relies on libvirt's upload writing the file verbatim (which is how
+`virsh vol-upload` populates volumes); the following `StoragePoolRefresh`
+makes libvirt re-read the real virtual size from the qcow2 header. Step 6
+confirms this on a real host. Also, the whole download runs inside one
+`pool.call`, so cancellation aborts the HTTP body (ctx-bound) and the
+upload stream fails; the half-written base volume is deleted on the spot
+and, failing that, on the next attempt via the missing marker.
+
+## Step 4 — Provider methods
+
+`kvm.go` now holds `Provider`, `New` (loads the SSH key and known_hosts
+up front via `newSSHAuth`, so a bad path fails at startup), the
+`newWithDialer` seam, and `Create`/`Get`/`Delete`/`ListByTag`. Each
+method resolves its host from config, then runs the whole libvirt
+conversation inside one `hostPool.call`, so a cancelled reconcile drops
+the connection and the blocked RPC returns. Errors pass through `classify`
+in one `wrapErr`, as the Step 3b note intended.
+
+Behaviour that goes beyond the plan text, or that a future reader should
+know before touching this file:
+
+- **Create looks up the domain first.** If it exists, only the running
+  check runs and the volumes are never touched. If not, nothing can be
+  using the volumes yet: the overlay is reused when it exists (its
+  creation is atomic; `ErrStorageVolExist` → reuse) but the **seed is
+  always deleted and rebuilt**. The plan said "exists → skip upload"; the
+  reviewer pointed out that a reconcile cancelled mid-upload leaves a
+  truncated seed that the compensating delete cannot remove, because
+  `hostPool.call` has already closed the connection the delete would go
+  over. Skipping such a seed would boot the VM without cloud-init and no
+  retry would ever repair it. Rebuilding costs one ~40 KiB upload per
+  attempt before the domain exists, nothing afterwards. The table test
+  breaks Create after each step and re-runs it, and a separate test
+  starts from a leftover truncated seed.
+- **Get swallows address-source errors.** `DomainInterfaceAddresses` from
+  ARP returns nothing until the guest has sent traffic and the agent
+  source errors with `ErrAgentUnresponsive` until `qemu-guest-agent` is
+  up, so both are logged at V(2) and mean "no IP yet" → `Provisioning`.
+  Loopback, link-local and unspecified addresses are skipped when
+  choosing the first IPv4 (the agent reports `lo` too).
+- **State mapping:** `running`/`blocked` are active; `nostate` →
+  Provisioning (the reconciler answers Stopped with delete-and-recreate,
+  too drastic for a state libvirt could not determine); everything else
+  (paused, pmsuspended, shutdown, shutoff, crashed) → Stopped.
+- **Delete treats a missing pool as "nothing to clean"** rather than
+  Permanent: failing there would pin the Proxy's finalizer on a config
+  edit that renamed the pool. Domain removal is destroy (`ErrOperationInvalid`
+  on a stopped domain is fine) → undefine with managed-save, snapshot
+  metadata and nvram flags → overlay and seed deleted by name.
+- **ListByTag** reads each domain's metadata once (passed into the same
+  `inspect` that Get uses, so no second `DomainGetMetadata` round-trip)
+  and skips domains without the operator's element. A domain that cannot
+  be read at all — vanished mid-sweep, corrupt metadata, state call
+  failing — is *skipped*, not fatal: GC never deletes what it does not
+  see, so leaving one out is safe, whereas one bad domain blocking GC for
+  every other one (the first version's behaviour, caught in review) is
+  not. An unreachable host still fails the whole sweep.
+  `ConnectListAllDomains` is called with flags 0 (active and inactive).
+- A host named in a providerID but no longer in `kvm.hosts` is Permanent
+  for Get and Delete: the operator cannot reach it, and retrying forever
+  would hide the config problem.
+
+Tests run the provider over a `hostDialer` that maps host names to
+`fakeAPI` instances (an unmapped host is "unreachable"). The image is
+pre-seeded in the pool for most tests; one test runs the full first-use
+download through `httptest` and confirms a second Proxy on the same host
+does not download again.
+
+Found while running the package: `TestBuildSeedISO_deterministic` from
+Step 2 was wrong. `iso9660.ImageWriter` stamps `time.Now()` into the
+volume descriptor, so two builds in different seconds differ by six
+timestamp bytes; the test had only passed because both builds landed in
+the same second. Byte identity is nothing the provider depends on (an
+existing seed is never re-uploaded), so the test now compares label,
+files and size, and the comment on `buildSeedISO` no longer claims
+identical output.
+
+The `go-operator-reviewer` agent ran on the first version of this step
+and found the two bugs above (truncated seed, wedged GC sweep) plus the
+double metadata read and the `nostate` mapping; all were fixed before
+committing. One finding it raised is deliberately **not** fixed here
+because it lives in Step 3b's `image.go` and changes the volume naming
+the plan chose: re-fetching a base image (checksum edited in config, or
+upstream content changed) deletes `base-<key>.<fmt>` while existing
+overlays still back onto it, breaking those VMs' disks. Content-addressed
+base names would avoid it. Flagged for the user; until decided, Step 5's
+docs must warn that an image's `sha256`/`url` must not change while
+Proxies using it exist.
+
+Worth noting: `go test -cover` reports 88.2 % for the package; the
+uncovered lines are the real SSH/libvirt dial path and log-only
+branches. `go vet` and `-race` (three runs) are clean. Wiring into
+`cmd/main.go` is Step 5, so `New` is not yet reachable from a running
+operator.
+
+## Step 6 — Integration test
+
+Done before Step 5, at the user's request, after a discussion of what the
+unit tests can and cannot prove: they run over an in-memory fake of the
+libvirt calls, so every assumption about libvirt's real behaviour (domain
+XML accepted, raw qcow2 upload + pool refresh usable as a backing image,
+guest boots and reports an address) needed a test against a real host.
+The wiring in Step 5 is not a prerequisite — the test drives the provider
+directly.
+
+`internal/provider/kvm/kvm_integration_test.go` is behind
+`//go:build integration` and skips itself under `-short` or when any of
+`KVM_TEST_HOST`, `KVM_TEST_USER`, `KVM_TEST_SSH_KEY`, `KVM_TEST_BRIDGE`,
+`KVM_TEST_IMAGE_URL` is unset. Optional knobs beyond the plan's list:
+`KVM_TEST_KNOWN_HOSTS` (without it the run disables host-key checking
+and logs that), `KVM_TEST_IMAGE_SHA256`, `KVM_TEST_POOL`,
+`KVM_TEST_SOCKET`, `KVM_TEST_TIMEOUT` (default 20 m, must cover a cold
+image download) and `KVM_TEST_KEEP` to leave a failed run's VM behind for
+`virsh console`.
+
+One sequential test, `TestKVM_lifecycle`, not subtests: each stage
+depends on hypervisor state the previous one left. Create (timed, so the
+log shows the image fetch), re-Create returns the same ID, poll Get to
+Running with an IPv4 (Stopped fails fast with a hint to use the console),
+UID and CreatedAt survive the metadata round-trip, ListByTag includes it,
+Delete, Get → NotFound, second Delete is a no-op, ListByTag excludes it,
+and the pool is inspected through the provider's own connection pool
+(`p.hosts.call`, reachable because the test is in package `kvm`): overlay
+and seed gone, base image and marker kept. A TCP connect to the guest's
+port 22 is attempted but only logged, since the test machine may not
+route to the bridge network. Cleanup deletes the VM on failure with its
+own two-minute budget, because the test context may already be spent.
+
+Makefile target, so the invocation is not something to remember:
+
+```bash
+make test-integration-kvm
+# = go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/
+```
+
+`docs/testing.md` gained the kvm package rows in the suite table, a
+"live libvirt" bullet under the deliberately-not-covered list, and a
+section on running this test, with the deviation from the testcontainers
+convention called out as the plan asked.
+
+Worth noting: **the test has not been executed yet.** This session had
+no hypervisor credentials; it was verified with
+`go vet -tags=integration ./...` and by running it without the
+environment (self-skip). Its first real run is the opening act of
+Step 7's manual validation, and it is the point at which the Step 2 and
+Step 3b "confirm on a real host" notes get answered. The test's config
+is built directly rather than through the provider package's validation
+(unexported), so a typo in an env var surfaces from `New` or from
+libvirt, not from a validation message.
+
+## Step 5 — Wiring, manifests, samples, docs, CHANGELOG
+
+Done after Step 6 (see that entry for why the order flipped). Everything
+the plan listed, plus a few things it did not:
+
+- `cmd/main.go`: `"kvm": kvm.New` in the builtins map. No RBAC change —
+  the provider talks to hypervisors, not the API server.
+- `config/manager/manager.yaml`: `kvm-ssh` Secret mounted read-only at
+  `/etc/proxy-operator/kvm` with `optional: true`, so deployments without
+  a kvm provider are unaffected. It nests under the providers-config
+  mount at `/etc/proxy-operator`; kustomize renders it (`kubectl
+  kustomize config/default` checked). The container runs as uid 65532
+  and Secret files default to 0644, so the key is readable without an
+  `fsGroup`; the `kubectl create secret` recipe is in a comment next to
+  the mount, in `docs/kvm-provider.md` and in the README.
+- `config/samples/providers-config.yaml`: `kvm-home` block with Debian 13
+  and Ubuntu 24.04 official image URLs, two machine types, one host, and
+  every optional field shown commented out. Parsed through
+  `provider.LoadConfigFile` in a throwaway test to make sure the sample
+  validates. `config/samples/proxy_kvm.yaml` uses the GCP sample's squid
+  cloud-init and shows `ssh_authorized_keys` commented out rather than
+  set — a sample with a placeholder key would either fail cloud-init or
+  invite pasting a real key into a committed file. Added to the samples
+  kustomization; `config/manager/providers_config.yaml`'s comment now
+  mentions kvm.
+- `docs/kvm-provider.md`: prerequisites with a three-line preflight
+  (`virsh list`, `pool-info`, `ip link` over ssh), cluster setup, the full
+  config reference table, placement table, a table of what lands on the
+  host and its lifetime, how the image cache works and how to evict, the
+  **warning not to change or evict an image while Proxies use it** (the
+  reviewer's finding from Step 4, carried as a documented rule since the
+  content-addressed-naming alternative was not chosen), how to log in,
+  and the known gaps.
+- README: providers bullet and a "KVM setup" section mirroring the GCP
+  one. `docs/architecture.md`: components row, a §5 block for the kvm
+  call flow in the same style as the other two, and five decision-log
+  entries (SSH transport, operator-fetched images, seed always rebuilt,
+  `nostate` → Provisioning, skip-vs-fail in the GC sweep).
+- `CHANGELOG.md` entry at the top, stating explicitly that the provider
+  is not yet validated on a real host.
+
+Verification: `go build ./...`, `go vet ./...`, `go test -short` on
+`cmd` and `internal/provider/...`, and both kustomizations rendered.
+`hack/providers-dev.yaml` untouched, as planned.
+
+Worth noting: the CRD has no rule requiring all three placement fields
+(that is GCP's own check), so the single-host "zone may be omitted"
+behaviour works without an API change. And with this step the branch is
+feature-complete but **unproven**: Step 7 runs the integration test and
+the end-to-end checklist against a real hypervisor before the MR.
+
+## Step 7 — Manual validation + MR (in progress)
+
+Host: `beelink` (Ubuntu, libvirt 11.6.0, QEMU 10.1.0, pool `default` at
+`/srv/vms/images` on local ext4, bridge `br0`), reached over Tailscale as
+`beelink.tail8c69c.ts.net`, SSH user `novakj`. Not the `nas1.home.lan`
+the plan and samples name; that host does not resolve from the dev box.
+
+```bash
+export KVM_TEST_HOST=beelink.tail8c69c.ts.net:22
+export KVM_TEST_USER=novakj
+export KVM_TEST_SSH_KEY=$HOME/.ssh/proxy-operator_ed25519
+export KVM_TEST_KNOWN_HOSTS=$HOME/.ssh/kvm_known_hosts
+export KVM_TEST_BRIDGE=br0
+export KVM_TEST_IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+make test-integration-kvm
+```
+
+### Run 1 and 2: `Could not open '/srv/vms/images/base-test-image.qcow2': Permission denied`
+
+Everything up to `DomainCreate` worked on the first try against a real
+host, which answers the open notes from Steps 2 and 3b: the SSH-forwarded
+libvirt handshake, the raw qcow2 upload plus pool refresh as a backing
+image, the default-namespace metadata element, and `DomainDefineXML` for
+the generated XML were all accepted by libvirt 11.6. QEMU then exited at
+start with `Permission denied` on the base image.
+
+Ownership was ruled out the slow way: the file was `root:root 0600` (the
+dir-pool default; libvirt's DAC driver chowns the chain at start and
+restores it after a failed start, so that observation meant nothing),
+`qemu.conf` was all defaults (`libvirt-qemu`, `dynamic_ownership=1`), and
+after a manual `chown libvirt-qemu:kvm` the second run failed identically.
+
+The actual cause is the Step 2 choice of `<disk type="volume">`.
+libvirt's AppArmor helper builds each domain's profile from the domain XML
+without consulting the storage driver, and its `add_file_path` only emits a
+rule when the source has a `path`; a volume-referenced disk has pool and
+volume names instead, so neither the overlay nor its backing image gets a
+rule and QEMU is denied on the first one it opens. Every stock Debian and
+Ubuntu hypervisor confines QEMU this way. The unit tests could not have
+caught it: the fake accepts either form.
+
+Fix, in this step: `buildDomainXML` takes `DiskPath`/`SeedPath` and emits
+`type="file"` disks; `ensureOverlay` and `writeSeed` return the volume
+handles (the exists-already branch now does a `StorageVolLookupByName`),
+and Create resolves both paths with `StorageVolGetPath` before defining
+the domain. Two extra cheap RPCs per Create. `TestBuildDomainXML_fieldByField`
+now fails on any `type="volume"` or `pool=` attribute so the design cannot
+drift back, and `TestCreate_freshHost` asserts the disk sources equal the
+paths the fake reports.
+
+Worth noting: the first two runs left the base image and its marker in
+the pool as designed, so later runs skip the 435 MB download.
+
+### Run 3: `parsing domain metadata: expected element <instance> in name space … but have no name space`
+
+With file-backed disks, Create returned in 1 s (image cached) and the
+domain started. Get then failed to parse the metadata: libvirt 11.6 finds
+the element by our URI but returns it as a bare `<instance>` with no
+namespace declaration at all. Step 2 had tested the default-namespace and
+prefixed spellings and left "does libvirt accept the default-namespace
+form" for the real host; it does, it just does not give it back that way.
+
+Fix: `parseInstanceMetadata` matches on the local name only, since the
+lookup already selected the element by URI. The fake now strips the
+declaration when storing metadata, so the provider's Get and ListByTag
+tests parse the shape libvirt really returns, and a new
+`TestParseInstanceMetadata_libvirtBareForm` pins it.
+
+### Run 4: guest hangs at GRUB's "Booting" line
+
+Create and start passed; Get polled `Provisioning` with no IP while the
+VM sat at 100 % CPU after GRUB printed "Booting `Debian GNU/Linux`" and
+before the kernel wrote anything to the serial console. The domain XML
+matches a working VM on the same host (q35, BIOS, host-passthrough,
+virtio disk, SATA cdrom, br0), so the difference is the disk content:
+the cached image was downloaded without a checksum and never verified.
+Diagnosis was handed to the user (sha512 against Debian's published
+sums, `qemu-img check`, and a `virt-install --import` of the same image
+outside the operator). Because the provider's own logs were invisible in
+the test output, this run also produced a side plan:
+[2026-09-10-0805-kvm-libvirt-tracing](2026-09-10-0805-kvm-libvirt-tracing.md),
+which traces every libvirt RPC at V(1) and routes it into `-v` output.
+
+### Run 5: the guest hang is Debian 13's BIOS boot path; UEFI fixes it
+
+The cached image's sha512 matched Debian's published sum, and
+`qemu-img info` showed a sound 3 GiB qcow2, which also confirms that
+the raw upload over a libvirt-created volume plus pool refresh yields a
+byte-identical file (the Step 3b note is answered). Bisected by hand on
+beelink with `virt-install --import` on a fresh overlay of the same
+image:
+
+| variant | seed | CPU | firmware | result |
+|---|---|---|---|---|
+| operator's domain | ours | host-passthrough | SeaBIOS | hangs after GRUB "Booting", 100 % CPU |
+| bare | none | host-passthrough | SeaBIOS | same hang |
+| bare | none | host-model | SeaBIOS | same hang |
+| bare | none | host-passthrough | **OVMF (`--boot uefi`)** | **boots to login** |
+
+So neither the seed ISO, the domain XML nor the CPU model was the
+problem: the image's legacy GRUB → kernel handoff does not work on this
+host, while its EFI stub path does. Not diagnosed further; it is
+Debian's chain, not ours, and every current cloud image boots UEFI.
+
+Change: `images.<key>.firmware` (`uefi` default, `bios` escape),
+threaded through `withDefaults` → `imageSource` → `domainParams`, and
+`buildDomainXML` emits `<os firmware="efi"><firmware><feature
+enabled="no" name="secure-boot"/></firmware>` so libvirt selects the
+OVMF binary and creates the NVRAM file; the `nvram` undefine flag Delete
+already passes removes it (seen on the host: `virsh undefine` without
+`--nvram` refuses such a domain). Direct-URL images default to UEFI
+too. Docs: OVMF is a new hypervisor prerequisite, config table row,
+sample comment, known-gaps entry, decision-log entry.
+
+Worth noting: the first image download went through with no checksum
+configured, and the cache reuses a base image by key, so switching
+`KVM_TEST_IMAGE_URL` requires deleting `base-test-image.qcow2` and its
+marker by hand first. That is the reviewer's Step 4 finding showing up
+in practice; still documented rather than fixed.
+
+Follow-up, same day: the cause is known upstream. [Debian bug
+#1111240](https://bugs.debian.org/1111240) (filed 2025-08-16 against
+the cloud image, reassigned to `grub2`, still open as of 2026-09-10;
+trixie ships grub2 2.12-9+deb13u2) reports exactly this with a domain
+XML almost identical to ours: BIOS, virtio disk, SATA cdrom seed, bridge
+NIC, `<console><target type='serial'/>`, no `<graphics>`/`<video>`.
+The cloud team reproduced it: trixie's grub.cfg has `terminal_output
+gfxterm serial`, and with a serial port present but no video adapter
+GRUB never hands over to the kernel. Their fix in the image
+(`terminal_output serial`) is not something an operator can apply;
+the reporter's workaround (console `target type='virtio'`) boots but
+takes the kernel's `console=ttyS0` output with it. UEFI sidesteps the
+gfxterm path and keeps the serial console, so the decision stands, but
+the docs now name the bug rather than blaming the host, and note that
+bookworm images are unaffected. `latest` on cloud.debian.org was the
+2026-08-31 build when this was hit, so every trixie point release to
+date carries it.
+
+### Run 6: pass
+
+With UEFI the full lifecycle passed in 23 s on a warm cache: Create 2 s
+(domain start 0.9 s), guest agent answering with `192.168.0.89` after
+20 s (ARP never did; the host had not talked to the guest), TCP 22
+reachable from the dev machine, ListByTag seeing it, Delete leaving only
+the base image and marker in the pool, second Delete a no-op. Trace
+excerpt worth keeping for the record:
+
+```
+09:44:28.603 libvirt virDomainDefineXML xmlBytes=1509 domain=proxy-hk5oavoqk45n3twq duration=13.51ms
+09:44:28.603 kvm domain defined uid=9bbceb41-… firmware=uefi
+09:44:29.542 libvirt virDomainCreate domain=proxy-hk5oavoqk45n3twq duration=938.705ms
+09:44:49.573 libvirt virDomainInterfaceAddresses domain=proxy-hk5oavoqk45n3twq source=DomainInterfaceAddressesSrcAgent interfaces=2 duration=1.742ms
+09:44:49.573 kvm KVM domain fetched state=Running ip=192.168.0.89 uid=9bbceb41-…
+```
+
+Worth noting: the sweep listed a second managed domain,
+`proxy-qxge3dbm2mnpc3ih`, shut off, left behind by an earlier failed
+run whose cleanup never ran. That is exactly the orphan GC exists for;
+on the host it needs `virsh undefine --nvram` plus its two volumes by
+hand. The `ListByTag` sweep costs one `virDomainGetMetadata` per
+domain on the host (21 here, ~0.4 ms each over the tunnel), which is
+fine at this scale but is the call to watch on a busy hypervisor.
+
+### Run 7: in-cluster, two defects the integration test could not see
+
+The e2e pass ran the operator inside the kind cluster
+(`egress-proxies-operator:kvm-dev`, built with `make docker-build`,
+loaded with `kind load docker-image`, deployed via the kustomize
+`config/manager` with the `kvm-home` provider pointing at beelink) and
+two Proxies with real cloud-configs: one plain squid, one with a debug
+login user. Both defects come from the difference between the test
+harness and the real thing, not from libvirt.
+
+**Read-only root filesystem.** The manager runs with
+`readOnlyRootFilesystem: true`, and `iso9660.NewWriter` stages the seed
+in a temp directory, so the first Create failed with `creating iso
+writer: read-only file system`. Fix: a 64 MiB `emptyDir` on `/tmp` in
+`config/manager/manager.yaml`. Side finding: the error is classified
+`ErrPermanent` by `writeSeed`, yet `hostPool.call` treated any
+non-libvirt error as a transport failure and dropped the SSH connection,
+so every retry redialled. `call` now leaves the connection alone for
+errors already carrying a provider sentinel (`isClassified`), with a
+test.
+
+**The guest agent never got installed.** Both VMs booted (confirmed on
+the console by hand) but sat in `Provisioning` for 25 minutes: the
+trace showed `virDomainInterfaceAddresses` from the agent source failing
+with `ErrAgentUnresponsive` on every poll and the ARP source empty. The
+integration test had passed with `CloudInit: "#cloud-config\n"`; the
+real Proxies have `packages: [squid]` and `runcmd:`. cloud-init merges
+vendor-data under user-data with its default mergers, which keep the
+user-data's value for any key both define and never append lists, so the
+vendor-data's `packages`/`runcmd` were dropped wholesale. Confirmed
+locally with the dev box's cloud-init 26.1 before changing anything:
+
+```sh
+python3 - <<'EOF'
+from cloudinit import mergers
+user = {"packages": ["squid"], "runcmd": ["systemctl restart squid"]}
+vendor = {"packages": ["qemu-guest-agent"], "runcmd": [["systemctl", "enable", "--now", "qemu-guest-agent"]]}
+print(mergers.construct(mergers.default_mergers()).merge(dict(user), dict(vendor)))
+EOF
+# {'packages': ['squid'], 'runcmd': ['systemctl restart squid']}
+```
+
+Fix: `withGuestAgent` in `seed.go` merges the bootstrap into the Proxy's
+cloud-config itself (append to `packages` and `runcmd`, skip if present,
+re-render with `sigs.k8s.io/yaml`), so vendor-data is no longer defaulted
+and `vendorData` is plain extra data. User-data that is not a cloud-config
+document (script, `#include`, MIME) still gets the bootstrap as
+vendor-data since there is nothing to collide with; a cloud-config that
+does not parse as a mapping fails the Proxy as `ErrPermanent`. The
+rendered document for the squid Proxy passed `cloud-init schema
+--config-file`. The logging test's fixture (`#cloud-config` + a bare
+sentinel line) is not a mapping and had to become `hostname: <sentinel>`.
+
+Rebuilt, reloaded, restarted, deleted and recreated both Proxies: both
+`Ready`/`HEALTHY True` in 2 min 1 s from `kubectl apply`, IP via the
+agent 110 s after `virDomainCreate` (the guest installs squid and the
+agent from the mirrors first), and `curl -x http://<ip>:3128
+https://www.gstatic.com/generate_204` from the dev box returns 204
+through both.
+
+Worth noting: the `#cloud-config` round-trip drops comments and reorders
+keys alphabetically; semantics are preserved (numbers, booleans and
+block scalars survive, checked in tests) but a user diffing the seed
+against their CR will see a different document. Vendor-data supplied via
+`vendorData` is subject to the same cloud-init merge rule and is now
+documented as such. The kind pod also restarted twice during the session
+from "Pod sandbox changed" (the node briefly lost the API server), which
+is unrelated to the operator. Still uncommitted on purpose: the
+`kustomization.yaml` image override (`egress-proxies-operator:kvm-dev`)
+and the `beelink` host entries in `config/manager/providers_config.yaml`
+and `config/samples/providers-config.yaml`, which are this machine's dev
+setup rather than the repo's defaults.
+
+### Run 8: review fixes
+
+A `/code-review high` pass over the branch found five defects, fixed in
+[plan 2026-09-10-2222](../plans/2026-09-10-2222-kvm-review-fixes.md) with
+its own execution log. Two of them land on Run 7's conclusions above and
+supersede them:
+
+- The guest-agent **merge described in Run 7 is gone**. It could not be made
+  safe for user-data that is not a plain cloud-config, and its YAML
+  round-trip re-typed unquoted scalars. The bootstrap now ships as a
+  vendor-data shell script, which cloud-init never merges, and the Proxy's
+  cloud-init reaches the seed byte for byte.
+- The read-only-filesystem fix from Run 7 still stands, but the eager SSH
+  key load it sat next to does not: a missing `kvm-ssh` Secret aborted the
+  whole provider set and crash-looped the manager. The material is read on
+  first use now.
+
+Re-validated the same way: integration test against beelink, both Proxies
+recreated in kind, 204 through squid.
+
+### MR opened
+
+[PR #5](https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/pulls/5),
+`feat/kvm-provider` → `main`, 19 commits. `tea` is not installed on the
+dev box and the HTTPS remote has no credential helper, so the push used a
+one-off helper reading a Gitea access token from `~/.config/gitea-token`,
+and the MR was created with the API instead of `tea pr create`:
+
+```sh
+git -c credential.helper='!f() { echo username=kacerr; echo "password=$(tr -d "[:space:]" < ~/.config/gitea-token)"; }; f' \
+  push -u origin feat/kvm-provider
+curl -s -H "Authorization: token $(tr -d '[:space:]' < ~/.config/gitea-token)" \
+  -H 'Content-Type: application/json' -d @pr.json \
+  https://gitea.home.hrajfrisbee.cz/api/v1/repos/kacerr/egress-proxies-operator/pulls
+# pr.json: {"title": ..., "head": "feat/kvm-provider", "base": "main", "body": ...}
+```
+
+The push printed a libsecret "store failed" warning from the system
+credential store; harmless, the token was not persisted anywhere but the
+file.

docs/plans-executions/2026-09-10-0805-kvm-libvirt-tracing.md

@@ -0,0 +1,109 @@
+# Execution: Trace every libvirt call in the KVM provider
+
+Plan: [2026-09-10-0805-kvm-libvirt-tracing.md](../plans/2026-09-10-0805-kvm-libvirt-tracing.md)
+
+- [x] Step 0 — Save plan
+- [x] Step 1 — `errors.go`: shared error fields
+- [x] Step 2 — `trace.go`: the `tracingAPI` decorator
+- [x] Step 3 — `conn.go`: pool logging and wrapping
+- [x] Step 4 — Call sites
+- [x] Step 5 — Integration test output
+- [x] Step 6 — Unit tests
+- [x] Step 7 — Docs, execution log, CHANGELOG
+
+## Steps 1–6 — Tracer, pool logging, tests, integration wiring
+
+Done as one commit on `feat/kvm-provider`; the plan's assumption that
+this belongs on the KVM branch rather than its own feature branch was
+accepted at plan approval.
+
+The code follows the plan closely. Things that came out differently
+or are worth knowing:
+
+- **Logger names join with `/` in funcr, `.` in zap.** The trace lines
+  are `kvm/libvirt` in the unit-test capture and in `testr` output, and
+  `kvm.libvirt` under the operator's zap logger. The docs use the zap
+  spelling; the unit test asserts the funcr one. First run of the tier
+  test failed on exactly this.
+- **Cancel is not reported as a transport error.** The original `call`
+  dropped the connection twice on cancellation (once from the abort
+  closure, once from the "not a libvirt error" branch, harmlessly, since
+  drop is idempotent). With a log line on each drop that would have
+  printed both `context cancelled` and `transport error` for one event,
+  so the transport branch now skips `ctx.Err()`. Covered by
+  `TestHostPool_call_logsDropReason`.
+- **`ConnectListAllDomains` in the fake includes every defined domain**,
+  so the trace test's fixture (one running managed domain plus one
+  stopped domain for `DomainCreate` to start) lists two; the V(2)
+  assertion checks for the `names` key rather than an exact list.
+- **`libvirtErrorFields` also feeds `logAPIError`**, which therefore
+  gains `libvirtCodeName` (e.g. `ErrNoDomain`) next to the numeric code.
+- The seam-coverage guard (`reflect` count of `libvirtAPI` methods vs
+  the test table) is the only thing standing between a future 18th RPC
+  and a silently untraced call; keep it.
+
+Verification run before committing:
+
+```bash
+go build ./... && go vet ./... && go vet -tags=integration ./internal/provider/kvm/
+go test -short ./...
+go test -race -count=1 -cover ./internal/provider/kvm/   # 90.2 %, up from 88.2
+go test -tags=integration -count=1 -run TestKVM ./internal/provider/kvm/   # self-skips without env
+```
+
+Worth noting: the live run (`make test-integration-kvm` against beelink)
+had not happened when this was written; it is what Step 7 of the KVM
+provider plan is waiting on, and its output is the acceptance test for
+this feature. Expected shape is in the plan's Verification section.
+
+## Revision after the first live run — less noise, libvirt names
+
+The first live run on beelink showed the trace working but hard to read:
+`testr` printed `"ts"`, `"level"`, `"msg"` and the three per-run
+constants on every line, the error text appeared twice (`error` and
+`libvirtMessage`), `host` appeared twice on image-cache lines (the
+logger already carried it), and `method=DomainLookupByName` was
+go-libvirt's name rather than something to look up in libvirt's docs.
+The user asked for "less verbose but more informative". Changes:
+
+- The tracer's headline field is now `rpc=virDomainLookupByName`: the
+  public libvirt API function, which go-libvirt's method names are
+  exactly minus the `vir` prefix (they come from the same
+  `remote_protocol.x`). Not gRPC, for the record: libvirt's own
+  XDR-based RPC over the forwarded unix socket.
+- Errors are one `error` field plus `libvirtError=ErrNoDomain` and
+  `libvirtCode=42`; `libvirtMessage` is gone from both the tracer and
+  `logAPIError`, since it is the error text.
+- `image.go` no longer adds `host` to lines whose logger has it.
+- The integration test replaces `testr` with a small `compactSink`
+  (`logr.LogSink` plus `GetCallStackHelper`, so `t.Log` still
+  attributes lines to `trace.go`): `HH:MM:SS.mmm libvirt
+  virDomainCreate domain=… duration=… error=… libvirtError=…`. The
+  `rpc` value becomes the headline, `provider`/`host`/`name` are
+  dropped (the run's first `t.Log` line states them), and values with
+  spaces are quoted. Production zap output is unchanged apart from the
+  field renames; structured logs are meant to repeat context.
+
+Previewed by driving the sink through the fake provider with an
+injected `DomainCreate` failure (a throwaway integration-tagged test,
+not committed):
+
+```
+trace.go:55: 09:04:33.165 libvirt virDomainLookupByName domain=proxy-abc123def456ghij duration=4µs error="Domain not found: …" libvirtError=ErrNoDomain libvirtCode=42
+trace.go:55: 09:04:33.165 libvirt virStorageVolUpload pool=default volume=proxy-abc123def456ghij-seed.iso offset=0 length=0 duration=71µs
+kvm.go:262:  09:04:33.165 kvm seed volume uploaded volume=proxy-abc123def456ghij-seed.iso bytes=45056
+trace.go:55: 09:04:33.165 libvirt virDomainCreate domain=proxy-abc123def456ghij duration=0s error="internal error: process exited…" libvirtError=ErrResourceBusy libvirtCode=87
+```
+
+Worth noting: `length=0` on `virStorageVolUpload` is what the provider
+passes (libvirt reads to EOF); the following `seed volume uploaded`
+line has the byte count.
+
+## Step 7 — Live run confirmed; CHANGELOG
+
+The compact trace was accepted on the first passing live run
+(2026-09-10, beelink): one line per RPC with the libvirt function name,
+the provider's own lines between them, `Provisioning` polls readable at
+a glance (`SrcArp interfaces=0`, `SrcAgent … ErrAgentUnresponsive`
+until the agent came up at +20 s), and a full `ListByTag` sweep over 21
+domains in 15 ms. CHANGELOG entry added under the KVM validation entry.

docs/plans-executions/2026-09-10-2222-kvm-review-fixes.md

@@ -0,0 +1,229 @@
+# Execution: fix the five code-review findings on `feat/kvm-provider`
+Plan: [docs/plans/2026-09-10-2222-kvm-review-fixes.md](../plans/2026-09-10-2222-kvm-review-fixes.md)
+
+- [x] Step 0 — Save plan into the repo
+- [x] Step 1 — Unwedge deletion (finding 1)
+- [x] Step 2 — Validate names that become identifiers (finding 2)
+- [x] Step 3 — Read the SSH material lazily (finding 3)
+- [x] Step 4 — Guest agent via vendor-data script (findings 4 and 5)
+- [x] Step 5 — Validate, record, push
+
+## Step 0 — plan committed
+
+Copied out of `~/.claude/plans/` and committed alone before touching code, as
+the repo's plan rules require.
+
+## Step 1 — Unwedge deletion
+
+Three changes, in the order the bug unfolds.
+
+`reconcileDelete` calls `Get` first and only drops the finalizer when it
+answers not-found, so any permanent error there retried forever. It now
+switches on the class: a permanent `Get` still calls `Delete`, which is
+idempotent, and releases the object when that succeeds. Polling a `Get`
+that cannot converge was the wedge; `Delete` is the only call that can
+still make progress.
+
+`crawl.example.com/skip-cleanup: "true"` abandons the instance and removes
+the finalizer. It is checked **before** the provider lookup, which also
+frees a Proxy whose provider was deleted from the config — a second wedge
+sitting three lines above the reported one, with the same shape. No
+`EventRecorder` exists on the reconciler and none was added, so the
+abandonment is an Info log; `docs/architecture.md` already records that
+decision for the whole controller.
+
+In the KVM provider, `Delete` for a providerID naming an unconfigured host
+now logs and returns success. The precedent is two functions further down:
+`removeVolumes` treats a missing pool the same way, with a comment saying
+outright that refusing "would pin the Proxy's finalizer".
+
+**`Get` deliberately still fails permanently for that case.** Making it
+answer not-found would have been a one-line fix for the wedge, but the live
+reconcile path reads not-found as "the VM is gone" and rebuilds it
+elsewhere. A typo in `hosts` would then quietly duplicate every Proxy on
+that host, with the originals still running where GC cannot see them, since
+`ListByTag` only sweeps configured hosts. The asymmetry is now pinned by a
+test that asserts both halves at once.
+
+## Step 2 — Validate names that become identifiers
+
+One regexp, `^[A-Za-z0-9][A-Za-z0-9_.-]*$`, applied to `hosts[].name` and to
+the `images` map keys. Dots stay legal because a host name like
+`nas1.home.lan` is ordinary and harmless; the character that actually
+breaks things is `/`.
+
+The review only reported the host name. The image key has the same defect
+and is arguably worse: it is not merely embedded in an ID but pasted into a
+file name, `base-<key>.<format>`, so `../etc/x` would be a path traversal
+into the storage pool rather than a parse failure. Both are validated in the
+same pass. Machine-type keys are left alone deliberately — they are only
+ever compared against `placement.machineType`, never embedded anywhere.
+
+Worth noting: this is validation of the operator's own config file, which
+only a cluster admin can write, so the traversal was never a privilege
+boundary. It is still cheaper to reject at load than to explain a stuck
+Proxy later. Both new table cases assert the message text, so the wording
+that names the consequence is now pinned by tests.
+
+## Step 3 — Read the SSH material lazily
+
+`kvm.New` no longer loads the key; `lazyAuth` does, on the first connection,
+caching the result once it works. `registry.Build` is all-or-nothing by
+design, so the old constructor error did not merely fail the KVM provider,
+it exited the manager before any controller started.
+
+The plan said "lazy dialer", and the first attempt was exactly that — one
+closure wrapping both the loading and the dial. It could not be tested: the
+fixture's SSH server is real, but the socket behind it is an echo server, so
+every successful dial dies in the libvirt handshake and the assertion could
+not tell "the key worked" from "the key did not". Splitting the seam in two,
+`lazyAuth` for the material and `lazyDialer` for the wiring, made both
+testable: the healing path against `lazyAuth`, and a separate test that the
+dialer really authenticates, asserted by the error being past SSH and by the
+server having seen the forward request.
+
+Error classes carry the intent. A missing file stays unclassified, so
+`provider.Class` defaults it to transient and a Proxy heals on its own when
+the Secret appears — the common case, since the Secret is usually created
+after the manager. A key that does not parse wraps `ErrPermanent`, because
+retrying cannot fix it. `newSSHAuth` distinguishes the two with
+`errors.Is(err, fs.ErrNotExist)`.
+
+Worth noting: the startup check is kept, just non-fatally, logged with the
+`WARNING:` prefix the repo already uses where logr has no warn level. That
+preserves the diagnostic the eager version was written for. The caching test
+deliberately deletes the key file after a successful load and asserts the
+cached value is reused, which pins the rotation behaviour the docs promise:
+a rotated Secret needs a manager restart. `TestNew_badKeyPathFailsFast` was
+inverted rather than deleted, so the old contract stays visible in the diff
+next to the new one.
+
+## Step 4 — Guest agent via vendor-data script
+
+The review filed two findings against `withGuestAgent` — detection too
+narrow, and YAML re-typing on the round-trip. Reading cloud-init 26.1's
+sources while planning turned both into one finding: the operator should
+not be editing the user's document at all.
+
+Vendor-data gets its own handler chain (`stages.py:_default_vendordata_handlers`),
+with separate script and cloud-config paths. A vendor-data **script** is
+therefore written to the vendor script directory and run by
+`cc_scripts_vendor`, never touching the config merger that drops colliding
+keys. On the final-stage module list it lands after
+`package_update_upgrade_install` and before the user's `runcmd`, so the
+agent now comes up *earlier* than the old `runcmd` append did.
+
+That also closed the door the previous approach had left ajar. The merge
+cannot be rescued from inside either document: `merge_how` in user-data
+governs only the first source, and `merge_how` in vendor-data is popped by
+`dict_extract_mergers` and discarded before the document is written. There
+was no fix that kept the merge.
+
+`withGuestAgent`, `isCloudConfig` and `appendUnique` are gone, along with
+seed.go's `sigs.k8s.io/yaml` dependency. `installGuestAgent` (default true)
+turns the script off. When `vendorData` is also configured the two ship as
+a MIME multipart, with the user's document declared `text/plain` — a type
+cloud-init re-sniffs from the content's own header line, so this code never
+classifies what the user wrote.
+
+Before deleting the old path, its damage was captured rather than taken on
+trust. Round-tripping a plausible cloud-config through it:
+
+```sh
+# temporary test in the package, deleted afterwards
+go test -run TestZZProbeRetyping -v ./internal/provider/kvm/
+# in:  version: 1.10
+# out: version: 1.1
+```
+
+`1.10` arriving as `1.1` is worse than the reviewer's `passwd` example,
+since nothing about it looks wrong in the seed. Comments were dropped and
+keys reordered on the same pass.
+
+The generated vendor-data was then checked against the cloud-init actually
+installed on this machine, rather than against the docs:
+
+```sh
+python3 -c 'from cloudinit import user_data, handlers; ...'
+# vendor-script:    multipart=False  sniffed=text/x-shellscript
+# vendor-multipart: multipart=True   part 1 text/plain -> text/cloud-config
+#                                    part 2 text/x-shellscript
+```
+
+and the script itself through `sh -n`.
+
+Worth noting: the script has to choose a package manager where cloud-init's
+`packages:` key was distro-agnostic. It covers apt, dnf, yum, zypper and
+apk, and both systemd and OpenRC for the enable step, but an image outside
+that set now needs `installGuestAgent: false` and the agent baked in. The
+seed's user-data is byte-identical to the Proxy's `cloudInit` from here on,
+which is what the new `TestBuildSeedISO_userDataIsNeverRewritten` pins
+across seven formats. One consequence worth remembering: a user-data that
+sets `vendor_data: {enabled: false}` disables the agent, and cloud-init
+documents that as intended behaviour.
+
+## Step 5 — Validation
+
+Unit suite, `go vet` and `gofmt` clean across the repo. Then the two real
+runs.
+
+**Integration against beelink**, unchanged environment from the KVM plan's
+log, passed in 22 s on a warm cache, with
+`virDomainInterfaceAddresses source=DomainInterfaceAddressesSrcAgent
+interfaces=2` — the agent installed by the new vendor script answering, on
+a stock Debian 13 image.
+
+**In-cluster**, both Proxies deleted and recreated against the rebuilt
+image: Ready with an address 2 min 3 s after `kubectl apply`, 204 through
+squid from the dev box for both. Inside the debug VM:
+
+```sh
+ssh novakj@192.168.0.94 'systemctl is-active qemu-guest-agent; cloud-init status; \
+  sudo head -4 /var/lib/cloud/instance/user-data.txt; sudo ls /var/lib/cloud/instance/scripts/vendor/'
+# active
+# status: done
+# #cloud-config
+# # Login for debugging: user "novakj", password "kvm-dev", passwordless sudo.
+# ...
+# part-001
+```
+
+Two things worth having seen with your own eyes there: the user's **comment
+lines survived** into the guest, which the old merge stripped, and the
+script really did land in the vendor directory rather than being merged
+into the user's config.
+
+**Negative check for the Secret**, done by pointing the deployment's volume
+at a name that does not exist rather than deleting the real Secret:
+
+```sh
+kubectl -n egress-proxies-operator-system patch deploy egress-proxies-operator-controller-manager \
+  --type=json -p '[{"op":"replace","path":"/spec/template/spec/volumes/1/secret/secretName","value":"kvm-ssh-absent"}]'
+```
+
+The manager came up and stayed up, 0 restarts, logging
+`WARNING: kvm ssh material is not usable ... open /etc/proxy-operator/kvm/id_ed25519:
+no such file or directory`. The Kubernetes Proxies kept reconciling; the
+KVM ones failed per reconcile and retried. Before this branch the same
+config exited the process before any controller started. Patched back
+afterwards, everything Ready.
+
+Worth noting: the KVM Proxies kept reporting `Ready` during that window
+rather than degrading visibly. That is existing behaviour, not a
+regression — a transient provider error rides the workqueue and does not
+clear a healthy status — but it does mean the only signal of a missing
+Secret is the log and the reconcile errors. Also, the ListByTag sweep on
+beelink still reports `managed=2` where one domain is an orphan from an
+older failed run; unrelated to this work, and the same leftover noted in
+the KVM plan's Run 6.
+
+Worth noting: the review filed this as a KVM finding, but the fix is mostly
+in the controller, because KVM is the only provider that resolves a
+providerID segment against operator config at all. GCP and Kubernetes hand
+the lookup to the backend, which answers 404 for a retired zone or a deleted
+namespace, so they reach not-found and release on their own. Anything else
+that classifies permanently on the delete path — revoked RBAC on the
+Kubernetes provider is the realistic one — was equally stuck before this
+step and is what the annotation is really for. The KVM table case
+`"host not configured"` moved out of `TestDelete_errors` into its own named
+test, since it is no longer an error.

docs/plans-executions/2026-09-11-1526-provider-health-checks.md

@@ -0,0 +1,378 @@
+# Execution: provider health checks
+Plan: [docs/plans/2026-09-11-1526-provider-health-checks.md](../plans/2026-09-11-1526-provider-health-checks.md)
+
+- [x] Step 0 — Save the plan
+- [x] Step 1 — ProviderHealth API and CRD
+- [x] Step 2 — Contract plus the kubernetes and GCP checks
+- [x] Step 3 — KVM check
+- [x] Step 4 — Monitor
+- [x] Step 5 — Metrics and wiring
+- [x] Step 6 — Docs
+- [x] Step 7 — Validate, record, push, open the PR
+
+## Step 0 — Save the plan
+
+Branched off `main` (at the PR #5 merge, `eb8f075`). The plan was copied out of
+`~/.claude/plans/` and committed on its own, before any code, as the repo's plan rules
+require:
+
+```bash
+git checkout -b feat/provider-health main
+cp ~/.claude/plans/now-i-would-like-flickering-wall.md docs/plans/2026-09-11-1526-provider-health-checks.md
+```
+
+The uncommitted `.claude/settings.json` change was left unstaged and stays out of this
+branch.
+
+## Step 1 — ProviderHealth API and CRD
+
+`kubebuilder` isn't installed here (only `bin/controller-gen`, `bin/kustomize` and
+`bin/setup-envtest` are), so `api/v1alpha1/providerhealth_types.go` and the `PROJECT`
+entry were written by hand instead of with `kubebuilder create api`. For a cluster-scoped
+resource `PROJECT` simply omits `namespaced`, which is what kubebuilder itself writes.
+
+The new types are registered in the existing `AddKnownTypes` call in `proxy_types.go`,
+not in a second `init()`.
+
+The deepcopy code, the CRD and the RBAC come from the usual generators:
+
+```bash
+make generate manifests
+go build ./... && go vet ./api/... && go test -short ./api/...
+```
+
+To confirm the new CRD and the viewer role actually reach the rendered install, I built
+the kustomization and grepped for them:
+
+```bash
+bin/kustomize build config/default | grep -n providerhealth
+# providerhealths.crawl.example.com (scope: Cluster, shortName ph)
+# egress-proxies-operator-providerhealth-viewer-role
+```
+
+Worth noting:
+- The only role added is the viewer role. The operator is the object's only writer, so
+  the admin and editor roles kubebuilder scaffolds for Proxy were not added.
+- The type file has no unit test, since it has no logic. The schema, including the
+  required `ready`/`lastTransitionTime` and the list-map keys, is proven by the envtest
+  round-trip in Step 4.
+- `config/rbac/role.yaml` is unchanged so far. The manager's own permissions arrive in
+  Step 4 with the Monitor's RBAC markers.
+
+## Step 2 — Contract plus the kubernetes and GCP checks
+
+`provider.Checker` is an optional interface in `internal/provider/check.go`, kept out of
+`Provider`. Each implementation has a compile-time assertion, because the monitor's
+type assertion would otherwise fail silently.
+
+- **kubernetes:** lists at most one managed Pod cluster-wide, using the same client, verb
+  and scope as `ListByTag`. The target is `apiserver`, at the `restCfg.Host` now stored
+  in the new `apiHost` field.
+- **GCP:** calls a new seam method, `Probe`. `realInstances.Probe` lives in `check.go`,
+  next to its only caller. `ListByTag`'s `Sprintf` filter became the shared
+  `managedFilter` constant, so the check and the GC sweep can't drift apart.
+
+Before relying on "MaxResults is only a page size", I checked the SDK's REST
+`AggregatedList` in the module cache:
+
+```bash
+grep -n "func (c \*instancesRESTClient) AggregatedList" -A80 \
+  "$(go list -m -f '{{.Dir}}' cloud.google.com/go/compute)"/apiv1/instances_client.go
+# InternalFetch: req.MaxResults = pageSize ... return elems, resp.GetNextPageToken(), nil
+```
+
+```bash
+go build ./... && go vet ./internal/provider/... && go test -race -count=1 ./internal/provider/...
+```
+
+Worth noting:
+- The planned `TestRealInstances_Probe_fetchesOnePage` became a three-case
+  `TestRealInstances_Probe` table.
+  - It keeps the first page's `nextPageToken`, and asserts exactly one request with
+    `maxResults=1` for each case.
+  - "empty project" covers `iterator.Done`.
+  - "api error keeps its status" proves the real client returns a `*googleapi.Error`, so
+    a disabled Compute API is reported as `Forbidden` rather than `Unreachable`.
+- Once a context has ended, both checks report `Timeout` whatever the error says. The
+  monitor's timeout is the usual cause, and the underlying error only repeats it.
+
+## Step 3 — KVM check
+
+`conn.go` now wraps each of the four dial stages in a `dialError`: ssh-material, tcp,
+ssh-handshake and libvirt-connect. `Error()` returns the inner message unchanged and
+`Unwrap()` returns the cause, so `classify`/`isClassified` and every existing
+message-matching test behave as before. `TestDialError_keepsClassification` pins this
+down.
+
+The three new RPCs went into `libvirtAPI`, `tracingAPI` and `fakeAPI`. The method-count
+guard in `TestTracingAPI_logsEveryMethod` fails until a traced row exists for each new
+method, which is what forced the three new rows.
+
+`Check` runs one goroutine per host, over the shared connection, through
+`hostPool.call`. The pool verdict is decided only after `call` returns, because
+returning it as an error from inside the callback would make `call` drop a perfectly
+good connection.
+
+Build, vet (also with the integration tag, since that file changed) and race tests:
+
+```bash
+go build ./internal/provider/... && go vet ./internal/provider/kvm/ \
+  && go vet -tags=integration ./internal/provider/kvm/ \
+  && go test -race -count=1 ./internal/provider/kvm/
+```
+
+Worth noting:
+- **Pool state wording:** `poolState` uses virsh's words (`running`, `inactive`, …)
+  rather than go-libvirt's constant names, so it reads like `virsh pool-list` output.
+- **Details:** each host also reports its SSH `user`. That is the first thing to check
+  when the reason is `SSHHandshakeFailed`.
+- **Tests renamed or merged:**
+  - The planned `TestSSHAuth_dial_stages` is `TestLazySSH_dial_tagsTheFailingStage`. It
+    dials through `lazySSH`, so the ssh-material stage is covered as well as the three
+    stages the in-process SSH server can trigger.
+  - The pool-inactive and pool-missing cases are merged into `TestCheck_libvirtVerdicts`,
+    which also covers a daemon error. Each case runs two checks and asserts a single
+    dial.
+- **Tests added beyond the plan:**
+  - `TestCheck_timeout`: a daemon that stops answering gives `Timeout`.
+  - `TestDialError_keepsClassification`.
+- **Not run yet:** `TestKVMCheck_realHost` in the integration file only compiles here.
+  It needs beelink, so it goes on the Step 7 checklist for the user to run.
+
+## Step 4 — Monitor
+
+`internal/providerhealth` has the `Monitor` runnable plus the pure `buildStatus`.
+
+- **Scheduling:** the Monitor is leader-only. It cleans up once at start, checks
+  immediately, then checks on a ticker.
+- **Each round:** providers are checked concurrently under one timeout, then logged and
+  published one after another. A failed write is logged, and the next round writes
+  again.
+- **Reads:** objects are read through the reader it is given. `main.go` passes the
+  manager's API reader (Step 5).
+
+The first `make manifests` left `config/rbac/role.yaml` unchanged, with no error.
+I had put the `+kubebuilder:rbac` markers inside `Monitor`'s doc comment, and
+controller-gen only reads RBAC markers from package-level comment groups. So they were
+silently attached to the type and ignored. An empty `git diff config/rbac/role.yaml` was
+the only sign. Moving them into their own comment block fixed it:
+
+```bash
+make manifests && git diff config/rbac/role.yaml
+# + providerhealths: create, delete, get, list
+# + providerhealths/status: get, patch
+```
+
+Vet, race tests and the envtest round-trip (using the `bin/k8s/1.36.2-linux-amd64`
+binaries `make setup-envtest` had already fetched):
+
+```bash
+go vet ./internal/providerhealth/ && go test -race -count=1 -v ./internal/providerhealth/
+# --- PASS: TestMonitor_envtestRoundTrip (6.45s) ... ok
+```
+
+Worth noting:
+- **Fake client scope:** the fake client needs a RESTMapper that registers
+  ProviderHealth as `RESTScopeRoot`. Its default mapper treats every kind as namespaced.
+- **Why the envtest calls `publish` directly:** `checkOnce` only logs write errors, so a
+  schema rejection would look like a missing object. Calling `publish` makes the API
+  server's answer the test failure.
+  - Its first round writes a 2000-character message; the stored message is truncated to
+    1024.
+  - Its second round proves that only the target that flipped gets a new
+    `lastTransitionTime`.
+- **No guard against a hung checker:** the Monitor trusts `Checker`'s contract to return
+  once the context ends, and doesn't abandon a checker that ignores it. All three
+  checkers stop when the context ends: KVM through `awaitOrCancel`, and kubernetes and
+  GCP through their clients.
+- **Test renamed:** the planned `TestMonitor_timeoutBoundsBlockingChecker` is
+  `TestMonitor_checkOnce_timeoutBoundsEachRound`. It also asserts that a fast provider
+  still publishes next to a hung one.
+
+## Step 5 — Metrics and wiring
+
+`metrics.RegisterProviderTargets` registers a scrape-time collector for three gauges,
+each labelled `{provider,target}`: `up`, check duration and check timestamp.
+`Register`'s signature is unchanged.
+
+`cmd/main.go`:
+- gains `--provider-check-interval` (1m) and `--provider-check-timeout` (15s);
+- validates them before even loading the providers config;
+- collects the `Checker`s from the unwrapped providers in config order, just before the
+  metrics and tracing wrap loop;
+- adds the Monitor with the manager's API reader;
+- registers the collector through a small `providerTargetSamples` closure next to
+  `proxyPhaseCounts`.
+
+Build, vet and metrics tests:
+
+```bash
+go build ./... && go vet ./cmd/... ./internal/metrics/ && go test -race -count=1 ./internal/metrics/
+```
+
+To confirm bad flags fail fast, before anything touches the cluster, I ran the binary
+with a 5s interval. zap's development mode prints a stack trace after the error, hence
+the grep:
+
+```bash
+go run ./cmd --providers-config=/dev/null --provider-check-interval=5s 2>&1 | grep -m1 "Invalid provider check flags"
+# ERROR setup Invalid provider check flags {"error": "--provider-check-interval must be 0 (off) or at least 10s, got 5s"}
+go vet -tags=e2e ./test/e2e/
+```
+
+The e2e suite gets a new ordered spec, "should publish provider health". It waits for
+`providerhealth kubernetes` to report `Ready=True`. It runs before the existing metrics
+spec, which now also asserts
+`proxy_operator_provider_target_up{provider="kubernetes",target="apiserver"} 1`. By then
+the first round is known to be done, so the series is certain to exist.
+
+Worth noting:
+- **Main-only wiring:** `providerTargetSamples` is an 8-line loop in `package main`,
+  which has no test file. The e2e metrics assertion is its only coverage.
+- **The KVM entry isn't asserted in e2e:** the default providers config also ships
+  `kvm-home` (beelink), which would report `SSHMaterialUnavailable` in kind. That entry
+  is the user's own host, and asserting it would break the suite as soon as the default
+  config changes.
+- **Manifest:** `config/manager/manager.yaml` only gained a comment about the two flags.
+  The defaults need no args.
+
+## Step 6 — Docs
+
+- **`docs/architecture.md`:**
+  - component rows for the CRD, the `Checker` contract and the monitor;
+  - a new §11 covering the provider table, the loop diagram, the write, read and
+    decorator choices, the gauges, and sample `kubectl get ph` output;
+  - cross-references from the metrics (§9) and tracing (§10) sections;
+  - four Decisions entries.
+- **`docs/kvm-provider.md`:**
+  - a "Health check" section with the reason → meaning → where-to-look table and the
+    two consequences of sharing the connection;
+  - under Known gaps, the capacity bullet now points at `poolAvailable`, and a new
+    bullet says a rotated `kvm-ssh` Secret needs a restart;
+  - the Testing section names `TestKVMCheck_realHost`.
+- **`docs/testing.md`:**
+  - suite rows for the three `check_test.go` files, `internal/providerhealth` (unit and
+    envtest) and the metrics gauges;
+  - "17 libvirt calls" is now 20;
+  - the integration and e2e sections describe the new tests.
+- **`CLAUDE.md`:**
+  - the Architecture TODO now points at `docs/architecture.md`;
+  - the stale "only `go.mod` exists" gotcha is replaced by a note that the "What this
+    is" and "Commands" sections predate the code, plus Step 4's RBAC-marker gotcha.
+
+Worth noting:
+- `docs/api.md` is untouched. It documents the discovery HTTP API, which this change
+  doesn't alter.
+- `CLAUDE.md`'s "What this is" and "Commands" sections were left stale on purpose.
+  Rewriting them is outside this plan, so the new gotcha flags them instead of leaving
+  them silently wrong.
+
+## Step 7 — Validate, record, push, open the PR
+
+The full suite runs codegen, `go fmt` and `go vet`, then `go test -race` on every
+package, with both envtest suites included:
+
+```bash
+make test
+# every package ok; internal/providerhealth 92.5 %, internal/provider/kvm 92.0 % coverage
+```
+
+`go fmt` rewrote nothing, and `git status` showed only the Step 6 docs.
+
+`make lint` could not run, for a reason unrelated to this branch. The Makefile rebuilt
+the custom golangci-lint, which reports "built with go1.27.0", and its bundled
+staticcheck (honnef.co/go/tools v0.7.0) panics on the standard library before it
+reaches any repo package:
+
+```bash
+make lint
+# buildir: package "poll" (isInitialPkg: false, needAnalyzeSource: true): unexpected expr: *ast.KeyValueExpr
+# make: *** [Makefile:107: lint] Error 3
+```
+
+The local toolchain is go1.27.0 while `go.mod` says 1.26.0; the otel-tracing plan's
+log shows `make lint` running normally before the toolchain moved. Retrying with
+`GOTOOLCHAIN=go1.26.0` would have meant downloading a toolchain, and the user declined
+that. `go vet` is clean in all three configurations: plain inside `make test`, plus
+`-tags=integration` and `-tags=e2e`.
+
+The kind e2e suite creates and deletes its own `egress-proxies-operator-test-e2e`
+cluster. I checked `KIND_CLUSTER` first, because the two clusters already on the
+machine (`kind`, `proxy-operator-demo`) must not be torn down by `cleanup-test-e2e`:
+
+```bash
+make test-e2e > <scratch>/e2e.log 2>&1
+# Ran 3 of 5 Specs in 86.210 seconds
+# SUCCESS! -- 3 Passed | 0 Failed | 0 Pending | 2 Skipped
+```
+
+In the deployed manager, the `kubernetes` ProviderHealth reported `Ready=True` about a
+second after the pod was up (20:30:05.6 → 20:30:06.7). The metrics scrape that followed
+contained `proxy_operator_provider_target_up{provider="kubernetes",target="apiserver"} 1`.
+That exercises the generated RBAC, the API-reader path and the `main.go` gauge wiring
+end to end. The two skipped specs are the Tempo-gated tracing specs; `TEMPO_URL` was not
+set.
+
+### Follow-up — the check must not cost the reconciles their connection
+
+Reviewing the shared-connection decision after the automated validation turned up two
+ways the health check could hurt the work it shares the link with, both of which the
+plan's "a check against a hung host drops the shared connection" line accepted too
+easily:
+
+- A check that timed out went through `hostPool.call`, whose cancel path drops the
+  connection. A `libvirtd` slow to answer one `ConnectGetLibVersion` would therefore
+  cut off an image upload or a `Create` running on the same link.
+- `hostEntry` was guarded by a `sync.Mutex`, so waiting for a host's slot could not be
+  interrupted. A reconcile redialing a host that is down holds the slot for as long as
+  the SSH dial takes — minutes on a blackholed address — and the whole check round
+  would wait behind it, past its own timeout.
+
+So the check now calls `hostPool.probe` instead of `call`: on `ctx` ending it abandons
+the RPC and leaves the connection alone (a link that is really dead is closed by the
+keepalive anyway, which ends the abandoned `fn` too). One abandoned probe per host is
+allowed to stay waiting — `hostEntry.probing` is an `atomic.Bool`, not a counter, so a
+daemon that never answers cannot pile up goroutines; further probes get the new
+`errProbeBusy`, which `checkReason` maps to `Timeout`. `hostEntry.mu` became a one-slot
+channel with an `acquire(ctx)` that only bounds *waiting*: an uncontended caller takes a
+free slot even with an expired context, so `call`'s behaviour is unchanged where nothing
+is contended.
+
+The second change came out of the same pass: `--provider-check-timeout` defaulted to a
+flat 15s while the flag validation rejected a timeout longer than the interval, so
+`--provider-check-interval=12s` alone failed to start. The flag now defaults to 0 and
+`Monitor.setDefaults` derives `min(15s, Interval)`; `ValidateSchedule` accepts 0 and
+rejects only negatives and timeouts past the interval.
+
+Re-verified with the same commands as above:
+
+```bash
+go vet ./...
+go test -race -count=1 ./internal/provider/kvm/ ./internal/providerhealth/
+make test
+# every package ok; internal/provider/kvm 92.2 %, internal/providerhealth 93.7 % coverage
+```
+
+Worth noting:
+- `make lint` still cannot run, for the toolchain reason recorded above; unchanged by
+  this follow-up.
+- The two new `kvm` tests are timing-shaped by necessity — they assert that a caller
+  with a 50ms deadline does not wait out a dial that is blocked on a channel. The
+  assertions are one-sided (`> 1s` fails) so a slow machine does not make them flaky.
+- `docs/architecture.md`, `docs/kvm-provider.md` and `docs/testing.md` were corrected in
+  the same commit: the old text promised the drop-on-timeout behaviour that is now gone.
+- The real-hypervisor commands from the plan's Verification section are still the user's
+  to run; nothing here replaces them.
+
+### Real-hypervisor validation and PR
+
+The user ran the plan's real-hypervisor checks against beelink (the `TestKVMCheck`
+integration test plus the manual `ProviderHealth` and metrics checks) and confirmed
+they passed. After that the CHANGELOG entry went in, and the branch was pushed with
+the token-file credential helper. The PR against `main` was opened through the Gitea
+API because `tea` isn't installed.
+
+Worth noting:
+- The branch also carries `docs/plans/2026-09-16-0841-provider-blueprint.md`, on
+  purpose: that plan reaches `main` with this merge, and the blueprint work starts on a
+  new branch afterwards.

docs/plans-executions/2026-09-16-0841-provider-blueprint.md

@@ -0,0 +1,177 @@
+# Execution: provider blueprint
+Plan: [docs/plans/2026-09-16-0841-provider-blueprint.md](../plans/2026-09-16-0841-provider-blueprint.md)
+
+- [x] Step 0 — Save the plan; wait for the merge
+- [x] Step 1 — Collect evidence (read-only, in parallel)
+- [x] Step 2 — Write `docs/providers.md`
+- [x] Step 3 — Verify, then open the MR
+
+## Step 0 — Save the plan; wait for the merge
+
+The plan was committed on `feat/provider-health` (`80b2e9c`) and reached `main`
+through PR #6. After the user confirmed the merge, the branch was cut from the
+updated `main`:
+
+```bash
+git switch main && git pull --ff-only
+git switch -c feat/provider-blueprint
+```
+
+Worth noting: `.claude/settings.json` has local, uncommitted permission edits
+that carried over through the switch. They are deliberately left out of this
+branch.
+
+## Step 1 — Collect evidence (read-only, in parallel)
+
+Three Explore agents answered the plan's 12-item questionnaire, one each for
+kubernetes, gcp and kvm. Meanwhile I read the shared layer:
+`internal/provider/*.go`, `registry.go`, the provider wiring in `cmd/main.go`,
+`reconcileManaged`/`replaceInstance`/`reconcileDelete` in the controller,
+`internal/gc/gc.go`, and how the monitor uses `Checker`.
+
+Before relying on the agents' claims, I spot-checked them against the code. One
+was wrong and did not go into the doc: the kvm agent said `Delete` skips volume
+removal when `DomainDestroy` returns `ErrNoDomain`. In fact `removeDomain`
+returns nil there, and `Delete` still calls `removeVolumes` afterwards
+(`kvm.go:477-482`).
+
+Rules that exist only in the callers, and not in any comment, were found by
+reading the controller and GC. They went into the doc as MUST rules:
+- A zero `CreatedAt` makes an instance immediately collectable (`gc.go:120`).
+- `Get` returning `ErrPermanent` on the deletion path makes the reconciler call
+  `Delete` anyway.
+- Replacement waits for `Get` to return NotFound before creating an instance
+  with the same name.
+- `Stopped` and `Terminated` both lead to delete-and-recreate.
+
+Worth noting, found outside this plan's scope and not changed here:
+- `docs/gcp-vm-validation.md` filters on `labels.proxy-operator-managed=yes`,
+  but the value is `true`.
+- According to the gcp agent, `docs/gcp-in-specific-project.md` holds a real
+  project ID, IAM output and an egress IP. The user was told.
+
+## Step 2 — Write `docs/providers.md`
+
+Wrote the eight sections the plan lists. Linked the new doc from the Provider
+contract row in `docs/architecture.md` and from the CLAUDE.md "Architecture"
+section.
+
+Judgment calls:
+- The file:line evidence sits in per-provider notes under the matrix, as
+  planned. The contract rules in §2 cite the reconciler and GC lines each rule
+  protects.
+- The gap list includes documentation drift (kvm-provider.md's ARP order and
+  log keys, gcp-vm-validation.md's label value and its claim that async errors
+  reach status), as well as code gaps. Both would mislead the next
+  provider author in the same way.
+- "Proposed features" stays a flat list with no design, as the user decided.
+
+Worth noting: the line numbers in the doc are a snapshot. The doc says so and
+asks readers to look for the named function when a line has moved.
+
+## Step 3 — Verify, then open the MR
+
+An Explore agent re-checked the doc against the code: about 150 claims,
+including every matrix cell. It found four mismatches, and all four were fixed:
+- **Startup dial:** I had listed kvm-provider.md's "connection kept open from
+  startup" as doc drift. It isn't: the pool dials lazily, but the provider
+  health monitor runs a check as soon as the leader starts
+  (`monitor.go:147`, default interval 1m), and that check opens the connection.
+  The bullet was removed and the kvm notes now say this.
+- **Checker wording:** §5 said the monitor type-asserts `Checker`. It is
+  `cmd/main.go:216`; the monitor only sees nil or not.
+- **Reasons, not conditions:** §3 called `QuotaExceeded` and `PermanentError`
+  conditions. They are reasons on `Provisioned`.
+- **Latch lines:** the line range is `:133-139`, not `:130-135`.
+
+It also noted that none of the providers assert `var _ provider.Provider`, and
+that the kubernetes provider's RBAC markers live in the controller package. The
+§7 checklist now says both.
+
+Link check (every relative `](…)` target and anchor in the new and edited docs
+resolves), then the sanity build:
+
+```bash
+go build ./... && go test -short ./...
+# all packages ok
+```
+
+Worth noting: no CHANGELOG entry yet. Per CLAUDE.md it goes in after the user
+confirms the doc.
+
+### Follow-up — tracing was missing from the blueprint
+
+The user pointed out, after the PR was opened, that the blueprint said almost
+nothing about tracing. The plan's questionnaire item 10 had collected the
+evidence, but the doc reduced it to a matrix row about logging and a checklist
+line telling authors not to open spans. That line even suggests tracing needs
+no work. A provider author needs the opposite message: the operation span comes
+free, and child spans for backend calls are theirs to add.
+
+Added a new §6, "Observability: tracing, logs and metrics", and renumbered the
+matrix, checklist and gaps sections to §7–§9. It covers:
+- **Three layers:** decorator operation spans; child spans per backend call,
+  with parent-gated HTTP transports or a span decorator over the SDK seam; and
+  logs.
+- **The ctx and logger rules** that keep `traceID` on log lines.
+- **A per-provider status table:**
+  - kubernetes: parent-gated `RestConfigWrapper` spans.
+  - gcp: the Google SDK's own `otelhttp`, never disabled by the provider.
+  - kvm: no spans. `trace.go` is a log decorator.
+
+Also added two tracing rows to the §7 matrix, rewrote checklist item 8 as
+"Tracing and logging" with an in-memory span-recorder test, and added gaps (kvm
+has no child spans; only kubernetes is checked against Tempo, and only at
+operation level). A proposal notes that the kvm `libvirtAPI` seam takes no
+`ctx`, so per-RPC spans need that plumbed in.
+
+The links were re-checked, including the new `architecture.md#10-tracing-internaltracing` anchor.
+
+### Follow-up — a completeness sweep
+
+The user asked for one more pass over everything provider-related. After the
+tracing miss, this pass read beyond the provider packages:
+- the Proxy API types and the spec hash;
+- the health engine and discovery call sites that consume `Instance.IP`;
+- the manager Deployment, the providers ConfigMap and the samples;
+- README, testing.md, kvm-provider.md and the GCP docs;
+- the initial prompt, the proxy-operator and kvm plans, and the 2026-08-10 PR
+  review;
+- the CHANGELOG.
+
+What the blueprint was missing, now added:
+- **What the instance is for.** The IP and port must be reachable from the
+  operator's health probes and from discovery clients. Providers don't know
+  which proxy runs; the proxy comes from cloud-init, except for kubernetes.
+- **Concurrency.** Three reconciles, GC and checks can call a provider at once,
+  so it MUST be safe for concurrent use.
+- **The runtime envelope:** read-only root filesystem, a 64Mi `/tmp`, a 128Mi
+  memory limit, credentials as optional mounts.
+- **Create:** `placement` can't be validated by the CRD, so error messages must
+  name the field. Catalogue edits in config aren't in the spec hash.
+- **Delete** is re-called every `DeletionPoll` while a delete is in progress.
+- **Provider names** must be DNS-1123 for `ProviderHealth`, and `spec.provider`
+  is immutable.
+- **Metric names** in §6.
+- **Three matrix rows:** proxy software, IP reachability, distinct egress.
+- **A deployment checklist item:** credentials mount, config entries, sample
+  Proxy, Makefile helper.
+- **Gaps:**
+  - still open from the PR review: GC has no installation identity; an
+    unconfigured provider latches Proxies; kubernetes Pod RBAC always installed
+    and no `OwnerReference` on Pods;
+  - the duplicated type list;
+  - samples that can't work: `proxy_gcp.yaml` uses a debian-cloud image without
+    cloud-init, and `proxy_kvm.yaml` places on `nas1` where configs define
+    `beelink`;
+  - the `"mock"` example in the `spec.provider` doc.
+
+Outside the plan's "leave other docs alone" line, README.md changed:
+- Its "Adding a provider" section restated the contract, with the old "Create
+  never blocks" rule that kvm contradicts. It now points to `docs/providers.md`
+  instead of keeping a second copy that disagrees.
+- The "Implemented providers" table gained the missing kvm row.
+- The deploy note no longer claims the default ConfigMap ships only the
+  kubernetes provider.
+
+The links were re-checked for providers.md and README.md.

docs/plans-executions/2026-09-16-1108-provider-docs-split.md

@@ -0,0 +1,228 @@
+# Execution: Split the provider blueprint into contract, appendix, template and index
+
+Plan: [docs/plans/2026-09-16-1108-provider-docs-split.md](../plans/2026-09-16-1108-provider-docs-split.md)
+
+- [x] Step 0 — Save and commit the plan
+- [x] Step 1 — Write `docs/provider-contract.md`
+- [x] Step 2 — Write `docs/provider-spec-template.md`
+- [x] Step 3 — Write `docs/providers-existing.md`
+- [x] Step 4 — Rewrite `docs/providers.md` as the index; link edits
+- [x] Step 5 — Verification sweep
+- [x] Step 6 — Final summary, CHANGELOG after confirmation
+
+## Step 0 — Save and commit the plan
+
+Committed as `cf89ac4` on the existing `feat/provider-blueprint` branch, on
+its own, before any document work. The pre-existing local modification of
+`.claude/settings.json` was left out of the commit.
+
+## Step 1 — Write `docs/provider-contract.md`
+
+2026-09-16 11:17. Written from scratch rather than edited down from the old
+blueprint, so that nothing provider-specific could survive by accident. Every
+MUST names its caller by function (`reconcileManaged`, `replaceInstance`,
+`reconcileDelete`, `providerFailure`, `deletionFailure`, `Sweeper.sweep`,
+`Monitor.checkOnce`, `Monitor.run`, `registry.Build`) and states what breaks.
+
+**Axis codes frozen** (19, §4.1 to §4.19): `readiness`, `create-return`,
+`delete-return`, `ip-source`, `gc-mark`, `created-at`, `list-partial`,
+`quota`, `not-found`, `primary-key`, `provisioning-state`, `async-ops`,
+`secondary-resources`, `credentials`, `client-kind`, `timeouts`, `logging`,
+`child-spans`, `backend-test`. The appendix table and the template are keyed
+on these strings; the §4.20 summary table's last column gives the template
+entry number each axis maps to.
+
+**Two rules that were not in the old blueprint**, both found by reading the
+callers while verifying:
+
+- `Sweeper.sweep` loops providers sequentially in one goroutine, and no
+  caller puts a deadline on the context (only `Monitor.checkOnce` uses
+  `WithTimeout`). So a hung `ListByTag` stalls orphan GC for every provider.
+  Recorded in §1.5 and the `timeouts` axis.
+- The Stopped branch of `reconcileManaged` calls `Delete` and requeues
+  without clearing `ProviderID`, then deletes again on every poll until
+  NotFound. A backend that reports a freshly created, never started instance
+  as stopped would have it deleted right after creation. Recorded as a MUST
+  in §2.2 and as the `provisioning-state` axis with three allowed profiles.
+
+Checks run on the draft (all clean):
+
+```sh
+grep -niE "kubernetes|gcp|kvm|libvirt|squid|google|compute engine" docs/provider-contract.md
+grep -n "—" docs/provider-contract.md
+grep -nE '\.go:[0-9]+|:[0-9]+-[0-9]+' docs/provider-contract.md
+grep -ohE "(internal|cmd|config|docs|api|hack|test)/[A-Za-z0-9_./-]+" docs/provider-contract.md | sort -u | while read p; do test -e "$p" || echo "MISSING $p"; done
+```
+
+The path check reports three expected misses: `config/samples/proxy_<type>.yaml`
+is a pattern, `docs/provider-spec-template.md` is written in Step 2, and
+`internal/provider/providertest` is a proposal named in §13.
+
+Worth noting: the contract refers to "the operator pod" and "the cluster's
+own API" rather than naming the platform, so that the abstractness grep stays
+meaningful. The `hack/providers-dev.yaml` rule is phrased as "only if the type
+needs no credentials" because that file exists for `make run-dev` without any
+Secret.
+
+## Step 2 — Write `docs/provider-spec-template.md`
+
+2026-09-16 11:28. Twenty-four entries in a fixed shape (Question, Allowed
+answers, Obligations, Answer, Evidence). Thirteen of them carry an axis code
+and their numbers are the ones the contract's §4.20 table points at; the
+other eleven (identity, placement, image, cloud-init, machine type, config
+block, Checker, concurrency, deployment, docs, accepted limits) are decisions
+the contract leaves open without being a profile axis. Entries 9, 13 and 15
+carry an empty table to fill in (state mapping, error classification, config
+fields); entry 21 carries the contract §11 test list as checkboxes so a
+filled spec doubles as the test plan.
+
+Check that every axis code appears in the template:
+
+```sh
+for c in $(grep -oE '^### 4\.[0-9]+ `[a-z-]+`' docs/provider-contract.md | grep -oE '`[a-z-]+`' | tr -d '`'); do
+  grep -q "\`$c\`" docs/provider-spec-template.md || echo "template missing $c"
+done
+```
+
+Clean. Worth noting: the preamble allows "verify against the target version"
+only on an Evidence line, never as an Answer, because the whole point of the
+file is that the implementer does not guess.
+
+## Step 3 — Write `docs/providers-existing.md`
+
+2026-09-16 11:35. The profile table has exactly one row per contract axis
+code, checked by the same loop as Step 2 with a row count. Everything else
+comes from the old blueprint's §6, §7 and §9 plus a fresh read of the three
+packages, with every `file:line` replaced by a symbol name.
+
+Corrections to earlier notes applied while writing: kvm's Checker `Details`
+also carry `user` and `pool`; the kubernetes provider has no seam interface
+(`newWithClient` injects a `client.Client`, the fake client stands in); kvm's
+`classify` has an `isClassified` pass-through. Timeouts were checked per
+provider for the `timeouts` row: kubernetes sets no `rest.Config.Timeout`,
+gcp relies on SDK defaults, kvm bounds only the SSH handshake
+(`defaultHandshakeTimeout` in `dialSSH`), so all three are weaker than the
+contract's SHOULD and the table says so.
+
+Symbol check on the appendix (every backticked identifier grepped for in
+Go, Makefile and YAML sources):
+
+```sh
+grep -ohE '`[A-Za-z_][A-Za-z0-9_.]*`' docs/providers-existing.md | tr -d '`' | sort -u | while read s; do
+  grep -rqwF --include='*.go' --include='Makefile' --include='*.yaml' -- "$s" . || echo "$s"
+done
+```
+
+The residue is file names, dotted `Type.method` forms, the `timeouts` axis
+code, libvirt's `nostate`, and the API names `OwnerReference` and
+`rest.Config.Timeout`; nothing unexpected.
+
+Worth noting: matrix rows that the profile table now carries (Create
+non-blocking, GC marks, IP source, quota detection, Checker presence,
+credentials, logging, tracing, tests) were dropped from "Other differences"
+rather than kept twice.
+
+## Step 4 — Rewrite `docs/providers.md` as the index; link edits
+
+2026-09-16 11:40. `docs/providers.md` was rewritten in place (same path, so
+the inbound links from `README.md`, `CLAUDE.md`, `docs/architecture.md` and
+the historical plan files still resolve, and the pre-split blueprint stays
+reachable in the path's git history). The three inbound sentences were
+updated to name the new files:
+
+- `README.md` "### Adding a provider": the first sentence now names the
+  contract, the template and the appendix, with `docs/providers.md` as the
+  index; the "Start there" sentence is unchanged.
+- `CLAUDE.md` "## Architecture": the providers.md sentence now names all
+  three files and the index.
+- `docs/architecture.md` components row "Provider contract": "rules in
+  provider-contract.md, per-provider profile table in providers-existing.md".
+
+The historical files under `docs/plans/` and `docs/plans-executions/` keep
+their `providers.md` links on purpose.
+
+## Step 5 — Verification sweep
+
+2026-09-16 11:45. All ten checks from the plan, run against the working tree
+after Step 4; nothing needed fixing.
+
+Links (file targets and anchors) across the four new files plus `README.md`,
+`CLAUDE.md` and `docs/architecture.md`, with anchors compared against
+slugified headings of the target file:
+
+```sh
+python3 - <<'PY'
+import re, pathlib
+files = ["docs/provider-contract.md","docs/provider-spec-template.md","docs/providers-existing.md","docs/providers.md","README.md","CLAUDE.md","docs/architecture.md"]
+def slug(h):
+    h = re.sub(r'`','',h).strip().lower(); h = re.sub(r'[^\w\s-]','',h)
+    return re.sub(r'\s+','-',h).strip('-')
+def headings(p):
+    return {slug(m.group(2)) for m in (re.match(r'^(#{1,6})\s+(.*)$', l) for l in open(p)) if m}
+bad = 0
+for f in files:
+    for m in re.finditer(r'\]\(([^)\s]+)\)', open(f).read()):
+        t = m.group(1)
+        if t.startswith(('http://','https://','mailto:')): continue
+        path, _, anchor = t.partition('#')
+        target = pathlib.Path(f) if path == '' else pathlib.Path(f).parent/path
+        if not target.exists(): print(f, "missing", t); bad += 1; continue
+        if anchor and target.suffix == '.md' and anchor not in headings(target): print(f, "bad anchor", t); bad += 1
+print("link problems:", bad)
+PY
+```
+
+Output: `link problems: 0`.
+
+Em-dashes, line references and the contract's abstractness:
+
+```sh
+grep -n "—" docs/provider-contract.md docs/provider-spec-template.md docs/providers-existing.md docs/providers.md
+git diff 2d959af -- README.md CLAUDE.md docs/architecture.md | grep '^+' | grep "—"
+grep -nE '\.go:[0-9]+|:[0-9]+-[0-9]+' docs/provider-contract.md docs/provider-spec-template.md docs/providers-existing.md docs/providers.md
+grep -niE "kubernetes|gcp|kvm|libvirt|squid|google|compute engine" docs/provider-contract.md
+```
+
+The only em-dash hit is the pre-existing "Leader-elected" cell of the
+`architecture.md` components row, which shows as an added line because the
+row's last cell was edited; no new em-dash was written. The other three
+greps are empty.
+
+Nothing lost from the old blueprint's §9: the bold leads of every gap and
+proposal bullet in `git show 0bb699b:docs/providers.md` were looked up in
+the new files. The literal-prefix match reported 22 misses, all caused by
+rewording ("isn't" to "is not", bold removed), so each topic was re-checked
+by a distinctive phrase and all 22 are present in `providers-existing.md`
+or contract §13.
+
+Paths and symbols: every `path/...` token in the four files exists except
+the `config/samples/proxy_<type>.yaml` pattern and the
+`internal/provider/providertest` proposal; every backticked identifier in
+the contract and template resolves in Go, Makefile or YAML sources except
+file names, dotted `Type.method` forms, profile names and the `NewWith...`
+placeholder.
+
+Axis alignment: the 19 axis codes from contract §4 each appear once as a
+row in the appendix table (six cells per row) and at least once in the
+template.
+
+`go build ./... && go test -short ./...`: all packages `ok`.
+
+## Step 6 — Final summary
+
+The split is complete on `feat/provider-blueprint`: five commits after the
+plan commit, one per step. The CHANGELOG entry is deliberately not written
+yet; per `CLAUDE.md` it is added once the user confirms the documents read
+the way they want. Nothing has been pushed.
+
+Worth noting for whoever picks this up:
+
+- The contract's §4.20 table maps axis codes to template entry numbers; if
+  entries are renumbered, that column must follow.
+- The `CHANGELOG.md` heading format in `CLAUDE.md` uses an em-dash; the
+  "no em-dashes" rule applied to the four new documents only.
+- The stale `mock` comments in `internal/provider/provider.go`,
+  `internal/provider/registry/registry.go` and `api/v1alpha1/proxy_types.go`
+  are still listed as gaps, not fixed here.
+- The next planned use of these files is a Proxmox provider: copy the
+  template to `docs/proxmox-provider-spec.md` and fill it in before any code.

docs/plans/2026-08-11-2152-discovery-api-docs.md

@@ -0,0 +1,58 @@
+# Plan: Discovery API documentation (docs/api.md)
+**Created:** 2026-08-11 21:52
+
+
+## Context
+
+The operator serves an HTTP discovery/lease API on `:8090` ([internal/discovery/](internal/discovery/)) that crawler clients use to list proxies, acquire TTL leases, release them, and report rate-limiting. There is no dedicated API reference today: README has four quickstart curls (no auth header, no schemas), and `docs/architecture.md` §7 has an ASCII route map. The user wants full documentation with curl examples for every feature.
+
+**Decisions made with user:** doc lives in a new `docs/api.md`; commit straight to `main` (no MR).
+
+## Deliverable
+
+### 1. New file `docs/api.md` — full API reference
+
+Content (all facts verified against code during planning):
+
+- **Overview & base URL** — what the API is; in-cluster FQDN `http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090` (Service: `config/default/discovery_service.yaml`); local access via `kubectl port-forward svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090`.
+- **Authentication** — static bearer token from `DISCOVERY_TOKEN` env var (populated from the optional `discovery-token` Secret, key `token`; `config/manager/manager.yaml`). Empty token ⇒ auth disabled with startup warning. Curl: `-H "Authorization: Bearer $TOKEN"` on every example. `/healthz` always exempt.
+- **Conventions** — JSON everywhere; error envelope `{"error":"<code>","message":"<text>"}`; request bodies capped at 64 KiB; proxies with a deletion timestamp are excluded from all responses.
+- **Configuration table** — `--discovery-addr` (default `:8090`), `--max-lease-ttl` (default 1h), `--lease-cooldown` (default 15m), `DISCOVERY_TOKEN`. Note the shipped Deployment passes none of these flags, so defaults apply.
+- **Endpoints**, each with request/response schema, status codes, and a copy-pasteable curl example:
+  - `GET /healthz` — liveness, unauthenticated.
+  - `GET /v1/proxies` — filters `healthy=true|false` (else 400 `invalid_query`) and repeatable `attr.<key>=<value>` (verbatim equality on `spec.attributes`, all pairs must match). Response `{"proxies":[proxyView...],"count":N}` sorted by id. Full `proxyView` field table: `id` (ns/name), `ip`, `port`, `attributes`, `phase` (Pending/Provisioning/Ready/Unhealthy/Deleting/Failed), `healthy` (condition `Healthy` == True), `latencyMillis`, `activeLeases`, `maxLeases` (default 5; explicit 0 = unleasable).
+  - `POST /v1/leases` — body `{selector, ttlSeconds, target}` all optional; TTL defaults 5m, capped at max-lease-ttl (else 400 `invalid_ttl`). 201 `{leaseID, proxy, expiresAt, ttlSeconds}`; 409 `no_match` with `considered/atCapacity/inCooldown/unhealthy` counts (documented meanings).
+  - `DELETE /v1/leases/{id}` — early release; always 204, idempotent.
+  - `POST /v1/leases/{id}/report` — body `{result: ok|rate_limited|banned, target}`; 204, 404 `unknown_lease`, 400 `invalid_result`. `ok` is a pure ack; `rate_limited` and `banned` behave identically today (both start one cooldown window).
+- **Selection & cooldown semantics** (short section — this is the non-obvious part clients need):
+  - Selection order: fewest active leases → lowest latency → lexicographic id; deterministic; only healthy, non-deleting proxies with free capacity are candidates.
+  - Cooldown: 15m default (`--lease-cooldown`), keyed `{proxy, target}`. Report with a target blocks only leases requesting that target; report without a target (and lease without one) creates a **global** cooldown blocking all acquisitions of that proxy — call this footgun out explicitly.
+  - Expired leases stay reportable for one cooldown window past TTL.
+- **End-to-end workflow example** — numbered curl walkthrough: acquire → use `proxy.ip:port` as HTTP proxy (`curl -x`) → report `rate_limited` on 429 → release. Using a `jq`-extracted `leaseID`.
+- **Caveats** — lease/cooldown state is in-memory and per-process: single replica only, operator restart drops all leases and cooldowns.
+
+### 2. Pointers to the new doc (small edits)
+
+- `README.md`: one-line link near the quickstart curl section ("full reference: docs/api.md").
+- `docs/architecture.md` §7: one-line link to `docs/api.md` as the detailed reference.
+
+### 3. Housekeeping per CLAUDE.md
+
+- First action post-approval: copy this plan to `docs/plans/<timestamp>-discovery-api-docs.md` (timestamp from `date "+%Y-%m-%d-%H%M"`), commit it alone.
+- Then write the docs, commit to `main` with `Co-Authored-By: Claude <noreply@anthropic.com>` trailer, push.
+- Append execution summary + status checklist to `docs/plans-executions/<same-timestamp>-discovery-api-docs.md` in the docs commit.
+- Add `CHANGELOG.md` entry (timestamp via `date "+%Y-%m-%d %H:%M %Z"`) once the user confirms.
+
+## Key source files (facts source of truth)
+
+- [internal/discovery/handlers.go](internal/discovery/handlers.go), [internal/discovery/server.go](internal/discovery/server.go) — routes, schemas, status codes, auth, limits.
+- [internal/lease/store.go](internal/lease/store.go) — selection order, cooldown/retention, stats.
+- [api/v1alpha1/proxy_types.go](api/v1alpha1/proxy_types.go), [api/v1alpha1/helpers.go](api/v1alpha1/helpers.go) — defaults (port 3128, maxLeases 5), phases, conditions.
+- [cmd/main.go](cmd/main.go) — flags/env defaults.
+- [config/default/discovery_service.yaml](config/default/discovery_service.yaml), [config/manager/manager.yaml](config/manager/manager.yaml) — service DNS, token secret.
+
+## Verification
+
+- Cross-check every documented status code / field name against `internal/discovery/server_test.go` expectations.
+- Sanity-run `go build ./... && go test -short ./internal/discovery/ ./internal/lease/` (no code changes expected — confirms docs match current behavior, not a stale tree).
+- Optionally lint the curl JSON bodies by piping each through `jq .` locally.

docs/plans/2026-08-11-2220-demo-scripts.md

@@ -0,0 +1,33 @@
+# Plan: Demo script — egress IP check through each healthy proxy
+**Created:** 2026-08-11 22:20
+
+
+## Context
+
+First of a planned series of demo scripts under `docs/demo/`. This one showcases the discovery API end to end without leases: list proxies from `$BASE_URL/v1/proxies`, and for each **healthy** one, call an IP-echo site through it (`curl -x`) to show the egress IP that proxy provides — clearly labeling which proxy each request goes through. The user explicitly asked to start by switching to a new branch. The script will be iterated on: **write it but do not commit it** — the user wants to add things to it before anything is committed.
+
+## Steps
+
+### Step 0 — Branch
+
+Create `feat/demo-scripts` off `main`. Commit only the plan copy (`docs/plans/<timestamp>-demo-scripts.md`, timestamp via `date "+%Y-%m-%d-%H%M"`) per CLAUDE.md — nothing else gets committed this round.
+
+### Step 1 — `docs/demo/show-egress-ips.sh` (new file, executable, left uncommitted)
+
+Style: match [.devcontainer/post-install.sh](.devcontainer/post-install.sh) — `#!/bin/bash`, `set -euo pipefail`, `ERROR:`/`WARNING:` messages, `${VAR}` braces.
+
+Behavior:
+
+1. **Args/env:** `BASE_URL` is `$1` (required; missing → usage text + exit 1, e.g. `usage: show-egress-ips.sh <BASE_URL>  (e.g. localhost:8090)`). Optional env: `TOKEN` (bearer token, same name docs/api.md uses; when set, send `Authorization: Bearer $TOKEN`), `IP_ECHO_URL` (default `https://api.ipify.org?format=json` — returns `{"ip":"..."}`).
+2. **Dependency check:** `command -v curl`, `command -v jq` → `ERROR` + exit 1 if missing.
+3. **Fetch** `"$BASE_URL/v1/proxies"` once (no server-side `healthy` filter — fetch all so unhealthy ones can be shown as skipped, which makes the demo more informative). Fail with a clear error if curl or JSON parsing fails.
+4. **Iterate** proxies with `jq -c '.proxies[]'`; for each, extract `id`, `ip`, `port`, `healthy`:
+   - unhealthy → print `--- skipping <id> (unhealthy) ---`
+   - healthy → print a clear banner naming the proxy before the request, e.g. `=== via <id> — http://<ip>:<port> ===`, then `curl -sS --max-time 10 -x "http://${ip}:${port}" "$IP_ECHO_URL"`; print the JSON response. A failed probe prints `WARNING: request through <id> failed` and continues (guard so `set -e` doesn't kill the loop).
+5. Finish with a one-line summary: N proxies, M probed, K skipped/failed.
+
+Reference for API shapes: [docs/api.md](docs/api.md) (`proxies[].id/ip/port/healthy`; `curl -x http://ip:port` usage is already documented there and in README).
+
+### Deliberately deferred (user will iterate on the script first)
+
+- No commit of the script, no push beyond the plan commit, no MR, no execution summary, no CHANGELOG — all wait until the user says the script (or script set) is ready.

docs/plans/2026-08-24-1025-otel-tracing.md

@@ -0,0 +1,107 @@
+# Plan: OpenTelemetry tracing integrated with the existing logr/zap logging
+**Created:** 2026-08-24 10:25
+
+## Context
+
+The operator has structured logging (controller-runtime zap → logr, `logf.FromContext(ctx)` everywhere) and Prometheus metrics, but no tracing. A single reconcile fans out into k8s API calls, provider calls (GCP VMs / squid Pods), and status patches; the discovery API serves lease requests; GC and health engines run on tickers. Today correlating one operation across those hops means grepping for `reconcileID` or `providerID`. Goal: proper OTel traces — a new trace per unit of work (reconcile, HTTP request, GC sweep, optionally health probe), child spans for k8s/provider/HTTP calls — **and every log line inside a traced operation enriched with the trace context**, without disturbing the existing logr/zap system.
+
+User decisions (confirmed):
+- **Enablement:** auto from standard `OTEL_*` env vars — no collector configured ⇒ tracing fully off, zero new flags for the common case.
+- **Health probes:** not traced by default; new `--trace-health-probes` flag enables one root span per probe.
+- **Backend:** none exists yet — verification uses the console exporter and a throwaway Jaeger all-in-one in kind; manifests ship commented-out OTLP env examples.
+
+## Key design facts (verified, non-obvious)
+
+- **logr can't read ctx:** `logr.LogSink` never sees `context.Context`, so trace IDs cannot be added inside the zap sink. They must be attached as `WithValues` at span-creation points via `logf.IntoContext`. zap does **not** dedupe repeated keys, so enrichment must happen exactly once per ctx chain (see `tracing.Start` below).
+- Log keys: **`traceID` / `spanID`** (lowerCamel matches `reconcileID`/`providerID` house style; note in docs that Grafana/Loki derived-field regexes must match `traceID`, not the `trace_id` default).
+- **GCP calls trace themselves:** `google.golang.org/api/transport/http` already wraps its transport in `otelhttp.NewTransport` — once a global TracerProvider is set, every Compute call emits an HTTP client span. Do not add HTTP-level spans in the gcp provider; the `provider.*` decorator span becomes their parent.
+- **kube-apiserver ignores incoming `traceparent`** by design; our k8s-API client spans are leaves. Don't promise cross-process traces into the apiserver.
+- The Go SDK does **not** implement `OTEL_SDK_DISABLED` — handle it ourselves.
+- `contrib/exporters/autoexport` drags in metric/log/prometheus exporters (binary + deps on a 128Mi pod) — hand-roll trace-exporter selection (~40 lines) instead.
+- Span events are deprecated (OTel spec, Mar 2026): use `span.SetStatus` + `error.type`-style attributes, not `RecordError`/`AddEvent`.
+- `rest.Config.Wrap` puts our transport innermost (after auth), so spans see the final request — correct; but informer list/watch + leader election run outside any span ⇒ the transport must **filter out requests with no parent span** or every watch becomes a long-lived root span.
+- Manager runnables receive no logger in ctx (only HTTP servers do) — `logf.FromContext` falls back to the global logger there, which is fine; the discovery server must keep seeding from `s.log`.
+
+## Steps
+
+### Step 1 — Dependencies
+
+`go get` (aligning the existing indirect skew: otel core v1.44 / otlptrace v1.40):
+`go.opentelemetry.io/otel@v1.45.0`, `otel/sdk@v1.45.0`, `otel/trace@v1.45.0`,
+`otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0`, `.../otlptracehttp@v1.45.0`,
+`otel/exporters/stdout/stdouttrace@v1.45.0`,
+`go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0`, plus the
+`semconv` version matching the SDK's `resource.Default()` (treat `ErrSchemaURLConflict` from `resource.New` as non-fatal if versions drift).
+`go mod tidy`; full build + `make test` to confirm `google.golang.org/api v0.292.0` / k8s v0.36 tolerate the bump (MVS picks the higher otelhttp).
+
+### Step 2 — New package `internal/tracing`
+
+- **`tracing.go` — `Setup(ctx, log logr.Logger, service, version string) (shutdown func(context.Context) error, err error)`**
+  - Disabled (returns no-op shutdown, installs nothing) when `OTEL_SDK_DISABLED=true`, or when neither `OTEL_TRACES_EXPORTER` nor `OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is set, or `OTEL_TRACES_EXPORTER=none`. One startup Info line either way stating the effective state/exporter.
+  - Exporter selection (hand-rolled, no autoexport): `OTEL_TRACES_EXPORTER` ∈ `otlp` (default) | `console`; for otlp, `OTEL_EXPORTER_OTLP_(TRACES_)PROTOCOL` ∈ `http/protobuf` (default) | `grpc` → `otlptracehttp` / `otlptracegrpc` (endpoint/TLS/headers via their native env handling).
+  - `sdktrace.NewTracerProvider` with batch processor (`WithMaxQueueSize(512)` — bounded on the 128Mi pod; sampler stays the SDK default so `OTEL_TRACES_SAMPLER(_ARG)` works), resource = `resource.New(WithAttributes(service.name, service.version=commit), WithFromEnv())` with env last so it wins.
+  - Globals: `otel.SetTracerProvider`, `otel.SetTextMapPropagator(TraceContext+Baggage)`, `otel.SetLogger(log.WithName("otel"))`, `otel.SetErrorHandler` → logr at `V(1)` (an unreachable collector errors every few seconds; Error level would spam).
+- **`logger.go` — the one logger-enrichment mechanism** (used by everything; prevents duplicate `traceID` keys):
+  - private ctx key holding the *base* (pre-enrichment) logr.Logger.
+  - `Start(ctx, spanName, opts...) (context.Context, trace.Span)`: starts a span from the global tracer; if the resulting span context is valid, capture base = stored base or `logf.FromContext(ctx)` on first call, then `logf.IntoContext(ctx, base.WithValues("traceID", ..., "spanID", ...))`. Nested `Start` re-derives from base — never stacks. Documented trade-off: `WithValues` pushed via `logf.IntoContext` *between* two `Start` calls is dropped (nothing first-party does that; controller-runtime's `reconcileID` logger is the base and survives).
+  - `ContextWithLogger(ctx, base)` seeds the base explicitly (discovery middleware).
+- **`reconciler.go` — `NewReconciler(kind string, inner reconcile.Reconciler, opts...) reconcile.Reconciler`**: root span `Reconcile <kind>` (SpanKind Internal) via `Start`; attrs `k8s.namespace.name`, `k8s.object.name`, `reconcile.id` (`controller.ReconcileIDFromContext`); on return records `reconcile.requeue_after`, sets Error status from err. `WithTracerProvider` option for tests.
+- **`transport.go` — `RestConfigWrapper() transport.WrapperFunc`**: `otelhttp.NewTransport` with `WithFilter` requiring `trace.SpanContextFromContext(r.Context()).IsValid()` — k8s API spans only under an existing trace (excludes informers, leader election, metrics authn).
+- **`http.go` — `HTTPMiddleware(operation string, base logr.Logger) func(http.Handler) http.Handler`**: `otelhttp.NewHandler` (filter `/healthz`; otelhttp renames the span from `r.Pattern` after routing — no custom naming needed) + inner layer calling `ContextWithLogger`-then-enrich so handler logs carry `traceID`. Accepts incoming `traceparent` (clients continue their traces).
+
+Tests (all with `sdktrace` + `tracetest.NewSpanRecorder` via the `WithTracerProvider` options, parallel; `Setup` env tests use `t.Setenv`, not parallel):
+- `Start`: nested spans → child span recorded; **funcr-captured log line contains `traceID` exactly once** (pattern: `internal/provider/gcp/gcp_test.go:277`); disabled tracer → no `traceID` in logs.
+- `Setup`: table over env combinations (unset→disabled, `none`→disabled, endpoint set→otlp, `console`, `OTEL_SDK_DISABLED`).
+- Reconciler decorator: fake inner returns result/err → span name/attrs/status.
+- Transport filter: request without parent span produces no span.
+- Middleware: `httptest` request → server span named from route pattern; `/healthz` unspanned; handler logger carries `traceID`.
+
+### Step 3 — `provider.WithTracing` decorator
+
+`internal/provider/tracing.go`, mirroring `metrics.go`: `WithTracing(name string, p Provider, opts...) Provider` — one Client span per call (`provider.create|get|delete|list`), attrs `provider.name`, `provider.id` (where present), result class attr from `resultLabel(err)`; status Error only for quota/permanent/transient — `ErrNotFound` stays Ok (expected during deletion polling). Returns the inner error **unmodified**. Uses `tracing.Start` so provider-internal logs (gcp `p.logger(ctx)`) inherit `traceID`. Tests in `internal/provider/tracing_test.go` reusing `staticProvider` from `metrics_test.go:19`.
+
+Wiring in `cmd/main.go:174-177` — tracing outermost: `provider.WithTracing(name, provider.WithMetrics(name, p, m))`.
+
+### Step 4 — `cmd/main.go` wiring
+
+- `tracing.Setup(...)` immediately after `ctrl.SetLogger` (cmd/main.go:148); on error: log + `os.Exit(1)`.
+- Explicit `shutdown(context.WithTimeout(context.Background(), 10*time.Second))` (fresh ctx — signal ctx is already cancelled) after `mgr.Start` returns, on **both** the error path before `os.Exit(1)` and the clean path. Earlier `os.Exit` sites have nothing to flush.
+- `cfg := ctrl.GetConfigOrDie(); cfg.Wrap(tracing.RestConfigWrapper())` before `ctrl.NewManager` (cmd/main.go:239).
+- kubernetes provider: add `kubernetes.NewWithTransportWrapper(ctx, cfg, wrap transport.WrapperFunc)` beside `New` (internal/provider/kubernetes/kubernetes.go:42) and register a closure in main's constructor map (same pattern as gcp at cmd/main.go:166) passing `tracing.RestConfigWrapper()` — keeps `internal/tracing` out of provider packages and `newWithClient` tests untouched.
+- New flag `--trace-health-probes` (default false) → `engine.TraceProbes`.
+
+### Step 5 — Reconciler spans
+
+- `SetupWithManager` (proxy_controller.go:412): `b.Complete(tracing.NewReconciler("Proxy", r))`.
+- Sub-spans via `tracing.Start`: `reconcile.managed` (:116), `reconcile.replaceInstance` (:226), `reconcile.delete` (:254), and `status.patch` opened **inside the deferred flush closure** (:98-104, runs within the root span; its error already folds into `err`). `reconcileExternal` stays unspanned — no I/O, pure noise.
+- Useful attrs where cheap: phase transitions, `providerID`.
+
+### Step 6 — Discovery server
+
+- `handler()` (internal/discovery/server.go:151-156): chain becomes recover → `tracing.HTTPMiddleware("discovery", s.log)` → log → maxBytes → auth (otelhttp must stay outside `maxBytesMiddleware`).
+- Handlers and `logMiddleware`/`recoverMiddleware` switch `s.log` → `logf.FromContext(r.Context())` (handlers.go:85, :141, :234; server.go:163, :191) so request logs carry `traceID`. `s.log` remains for startup lines.
+- k8s `List` calls in handlers already use `r.Context()` → child spans appear via the wrapped rest transport.
+
+### Step 7 — GC + health
+
+- `gc.Sweeper.sweep` (internal/gc/gc.go:83): wrap in root span `gc.sweep` via `tracing.Start` (before the `.WithName("orphan-gc")` derivation so it inherits), attrs: providers swept, orphans deleted. Provider/k8s child spans come free from Steps 3/4.
+- `health.Engine`: new field `TraceProbes bool`. In the worker loop (engine.go:158) when enabled: root span `health.probe` (SpanKind Client) around `probeFn`, attrs proxy key/ok/latency/error class, ended before `record` (which stays ctx-free). No transport wrap and no propagation — never inject `traceparent` toward external targets through the proxies. Probe→reconcile span links are **out of scope**: `GenericEvent` (engine.go:330) carries no ctx and the workqueue coalesces events; documented in architecture Decisions.
+- Lease store sweep: not traced (in-memory only).
+
+### Step 8 — GCP wire-log enrichment
+
+`wireFilterHandler.Handle` (internal/provider/gcp/wirelog.go:46) receives the real request ctx from the SDK's slog calls: append `traceID`/`spanID` attrs when `trace.SpanContextFromContext(ctx)` is valid — V(5) wire logs become correlatable with the `provider.*` span. Add `wirelog_test.go` case.
+
+### Step 9 — Manifests + docs
+
+- `config/manager/manager.yaml` env (after `DISCOVERY_TOKEN`): `POD_NAME`/`POD_NAMESPACE` via downward API **first**, then `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)` ($(VAR) expansion only sees earlier vars); commented-out `OTEL_EXPORTER_OTLP_ENDPOINT` + `OTEL_TRACES_SAMPLER` examples with a "tracing is off until an endpoint is set" comment. Note `--trace-health-probes` beside the args list.
+- `docs/architecture.md`: new "### 10. Tracing" section after Metrics (:281) — span topology, enablement, `traceID` log-key choice; Decisions entries (:303): env-gated enablement, probes off by default, no probe→reconcile links (why), `traceID` naming vs Grafana defaults, no traceparent into apiserver.
+- README ops note; CHANGELOG entry (dated via `date`) once the user confirms it works.
+- Run `make lint` (CI has no lint step; `lll`/`revive` will hit new files).
+
+## Verification
+
+1. `make test` (unit, includes all new `tracetest` assertions) and `make lint`.
+2. Console smoke: `OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml ...` against kind — create a Proxy, confirm (a) `Reconcile Proxy` span JSON on stdout with nested `provider.create` + k8s PATCH spans, (b) the reconcile log lines carry the same `traceID` as the span, (c) no spans from informer watches or leader election.
+3. End-to-end in kind: deploy Jaeger all-in-one (`jaegertracing/all-in-one`, OTLP 4318), set `OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318` on the manager Deployment; exercise a reconcile + a `POST /v1/leases` (with and without a client `traceparent`) + wait one GC interval; in the Jaeger UI: reconcile trace shows provider+k8s children; lease trace shows route-named server span with k8s child; grep manager logs for a `traceID` and resolve it in Jaeger.
+4. Negative: run with no `OTEL_*` env — startup line says tracing disabled, no export-error spam, logs unchanged (no `traceID` keys).

docs/plans/2026-08-24-1224-tracing-e2e.md

@@ -0,0 +1,150 @@
+# Plan: e2e test — OTel tracing against real Tempo
+**Created:** 2026-08-24 12:24
+
+## Context
+
+The tracing feature (MR #4, branch `feat/otel-tracing`) is covered by unit
+tests with an in-memory span recorder, but nothing proves the full pipeline:
+operator in a real cluster → OTLP export → Tempo ingest → queryable traces
+with the documented span topology. The homelab Tempo lives at
+`http://192.168.0.30:3200` (query API — verified live from this host:
+`/api/echo`, TraceQL `GET /api/search?q=…`, `GET /api/traces/<id>`), with
+OTLP ingest on `:4318` (HTTP, verified 200) and `:4317` (gRPC, open). User
+decisions: **Go Ginkgo e2e test** (not a shell script), **structural
+assertions** (span tree, not just trace-exists), flow per the user's sketch:
+deploy operator in kind with tracing on → create kubernetes-provider proxies
+→ delete them → shut down — with traces tagged for easy discovery per test
+run. This work continues on `feat/otel-tracing`; MR #4 stays the vehicle.
+
+## Design decisions (from review, with evidence)
+
+- **Proxy CRs go in the `default` namespace, not the operator namespace.**
+  The squid pods the kubernetes provider creates carry no securityContext
+  (internal/provider/kubernetes/pod.go:33-49) and are created in the CR's
+  namespace; the operator ns is labeled `pod-security enforce=restricted`
+  (test/e2e/e2e_test.go:61-62) and would reject them. Pod RBAC is
+  cluster-scoped (config/rbac/role.yaml:10), so `default` works. All
+  kubectl calls use explicit `-n`.
+- **Anchor assertions on `provider.create`, not the root-span search.** A
+  `name="Reconcile Proxy"` hit may be the finalizer-add or a drift
+  reconcile (no provider call inside). TraceQL matches span names anywhere
+  in a trace, so search `{resource.test.run.id="<runID>" && name="provider.create"}`,
+  fetch *that* traceID, then assert companion span names.
+- **Run identification with zero code changes:** OTel resource attribute
+  `test.run.id=<runID>` via `OTEL_RESOURCE_ATTRIBUTES` (env wins —
+  `resource.WithFromEnv()` merges last, internal/tracing/tracing.go). Tempo
+  vParquet searches arbitrary resource attrs without config; fallback query
+  documented: `{resource.service.name="egress-proxies-operator" && name="provider.create"}`
+  + start/end window (Unix **seconds**).
+- **OTLP preflight from inside the cluster:** host-reachability of
+  192.168.0.30 doesn't prove pod-reachability from kind on macOS, and
+  export failures log only at V(1) (invisible) — without a preflight, a
+  broken path is a 2-minute opaque timeout. Reuse the suite's curl-pod
+  pattern (e2e_test.go:218-248, incl. the restricted-PSS overrides JSON)
+  to POST to `<OTLP_ENDPOINT>/v1/traces` and fail fast.
+- **No homelab defaults baked into the Makefile.** The spec Skips unless
+  `TEMPO_URL`/`OTLP_ENDPOINT` are set (otherwise everyone without that LAN
+  host inherits a 2-minute failure). The copy-pasteable invocation lives in
+  docs/testing.md.
+- **Ginkgo ordering:** top-level container order is randomized, so the
+  tracing Describe is fully self-contained (own ns create → make install →
+  make deploy → teardown), same shape as the Manager Describe. `kubectl
+  delete ns` blocks until termination, so no create/delete race between
+  the two Describes.
+
+## Steps
+
+### Step 1 — `test/e2e/tracing_test.go` (new; `//go:build e2e`, package e2e)
+
+`Describe("OTel tracing", Ordered)`:
+
+- **BeforeAll:**
+  1. Read `TEMPO_URL` + `OTLP_ENDPOINT`; `Skip("TEMPO_URL/OTLP_ENDPOINT not set")` when empty.
+  2. `runID := "e2e-" + strconv.FormatInt(time.Now().UnixNano(), 10)`;
+     print it to `GinkgoWriter` so the run is findable in Grafana by hand.
+  3. Create operator ns + restricted-PSS label; `make install`;
+     `make deploy IMG=<managerImage>` (same commands as e2e_test.go:54-75,
+     via `utils.Run`).
+  4. Pre-pull the proxy image to kill the biggest flake source:
+     `docker pull ubuntu/squid:6.6-24.04_edge` + `kind load docker-image`
+     (via `utils.Run`, honoring `KIND`/`KIND_CLUSTER` env like
+     utils.LoadImageToKindClusterWithName).
+  5. OTLP preflight: curl pod POSTing `{}` to `<OTLP_ENDPOINT>/v1/traces`,
+     assert HTTP 200 in its logs (fail message names the endpoint).
+  6. `kubectl set env deployment/egress-proxies-operator-controller-manager
+     -n <ns> OTEL_EXPORTER_OTLP_ENDPOINT=<OTLP_ENDPOINT>
+     "OTEL_RESOURCE_ATTRIBUTES=k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE),test.run.id=<runID>"`
+     — in-place update keeps the var after the downward-API vars ($(VAR)
+     expansion needs that ordering); `exec.Command` means no shell mangling
+     of `$()`. Then `kubectl rollout status`.
+  7. `Eventually`: newest controller pod (sorted by creationTimestamp — a
+     terminating old pod may otherwise be picked) logs contain
+     `"tracing enabled"`.
+- **It "creates proxies and reports reconcile traces":**
+  - Write 2 Proxy manifests (names `proxy-tracing-e2e-1/2`,
+    `provider: kubernetes`, modeled on config/samples/proxy_kubernetes.yaml)
+    to `GinkgoT().TempDir()` (absolute paths — `utils.Run` chdirs the
+    process); `kubectl apply -n default`.
+  - `Eventually` (5m — first squid start) `.status.phase == "Ready"`
+    (confirmed field, api/v1alpha1/proxy_types.go:41).
+  - `Eventually`: Tempo search
+    `{resource.test.run.id="<runID>" && name="provider.create"}` ≥1 hit
+    (BSP flushes every ~5s; 2m default timeout is ample).
+  - Fetch that traceID via `/api/traces/<id>`; assert span names include
+    `Reconcile Proxy`, `reconcile.managed`, `provider.create`,
+    `status.patch`, and resource attr `service.name=egress-proxies-operator`.
+- **It "traces deletion":**
+  - `kubectl delete proxy -n default …`; `Eventually` CRs gone (finalizer
+    → `provider.delete`, DeletionPoll 10s).
+  - `Eventually`: Tempo search finds `provider.delete` for the runID; fetch
+    and assert `reconcile.delete` in the same trace.
+- **AfterAll** (= the sketch's "shut down operator"): delete leftover
+  proxies in `default`; `make undeploy` (SIGTERM → `tracingShutdown`
+  flushes); `make uninstall`; delete operator ns — errors discarded
+  (`_, _ = utils.Run(...)`), mirroring e2e_test.go:79-95.
+- **Tempo client helpers** (same file, stdlib only — the test process runs
+  on the host, which reaches Tempo directly):
+  - `tempoSearch(traceql string)` → `GET {TEMPO_URL}/api/search?q=…&start=…&end=…`
+    (Unix seconds, window = suite start − 5m → now); minimal struct
+    `{Traces []struct{ TraceID string }}`.
+  - `tempoTrace(id string)` → `/api/traces/<id>`, decoded as **OTLP-JSON**
+    (`batches[].scopeSpans[].spans[].name`, resource attrs as
+    `{key, value:{stringValue}}`) — not Jaeger's shape. Helper flattens to
+    a span-name set + resource attr map.
+
+### Step 2 — Makefile
+
+- `test-e2e`: pass `TEMPO_URL`/`OTLP_ENDPOINT` through to `go test` env
+  (no defaults) and add `-timeout 30m` — the suite already runs
+  docker-build + kind-load in BeforeSuite plus 3–5m Eventuallys, and this
+  adds a second full deploy cycle + image pulls; the 10m default will be
+  exceeded on cold caches.
+
+### Step 3 — Docs
+
+- `docs/testing.md` (§ e2e, currently "scaffold… only asserts manager runs
+  and serves metrics"): document the tracing spec, its Skip gate, and the
+  invocation:
+  `TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`.
+- Execution-log entries per step in `docs/plans-executions/2026-08-24-1224-tracing-e2e.md`.
+- CHANGELOG entry only after the user confirms a green run (house rule).
+
+## Not doing (and why)
+
+- No dedicated flush assertion on shutdown — teardown exercises the flush
+  path, but attributing a specific late span to it is guesswork.
+- No gRPC (4317) variant — http/protobuf is the operator default and the
+  verified path; a variant run is a one-env-var change if ever wanted.
+- No changes to `internal/` — the whole test works through public surface
+  (env vars, kubectl, Tempo API), which is the point of an e2e test.
+
+## Verification
+
+1. `TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`
+   → tracing spec passes (both Its); without the env vars → spec reports
+   Skipped, remainder of suite unaffected.
+2. In Grafana: TraceQL `{resource.test.run.id="<runID printed in test log>"}`
+   shows the run's traces with the expected tree.
+3. `go vet -tags=e2e ./...` clean (documented routine).
+4. A deliberately wrong `OTLP_ENDPOINT` (e.g. port 9) fails fast in the
+   preflight step with a clear message, not a 2-minute search timeout.

docs/plans/2026-09-04-1404-kvm-provider.md

@@ -0,0 +1,313 @@
+# Plan: KVM/libvirt provider
+
+**Created:** 2026-09-04 14:04
+
+## Context
+
+The operator provisions egress proxies through named provider instances (today
+`gcp` and `kubernetes`). The user has several KVM hypervisors on the home
+network and wants proxy VMs created there, using stock Debian/Ubuntu cloud
+images so the VM stays a normal machine they can customise through their own
+cloud-init user-data. Longer term the operator may become a generic "run a VM
+or container with a set of scripts" tool; the KVM provider should not
+foreclose that.
+
+The provider contract (`internal/provider/provider.go`: `Create`/`Get`/`Delete`/
+`ListByTag`, keyed by `NameFromUID`, cloud-init passed verbatim, GC-by-tag) fits
+libvirt with **no CRD or reconciler changes**. All work is a new provider
+package plus config/wiring/docs, following the GCP provider's shape.
+
+Decisions taken with the user (2026-09-04):
+
+- **Transport:** SSH to each hypervisor, forwarding the libvirt unix socket
+  (`ssh.Client.Dial("unix", "/var/run/libvirt/libvirt-sock")` → `libvirt.New(conn)`).
+  Pure Go, no cgo, no libvirt libs in the image. Verified: `x/crypto/ssh`
+  `Client.Dial` has a `"unix"` case (`dialStreamLocal`), go-libvirt `New(net.Conn)`.
+- **Topology:** one provider instance can span several hosts from day one
+  (config has a `hosts` list; providerID encodes the host), so adding hosts
+  later is a config edit with no providerID migration. With one host configured
+  `placement.zone` may be omitted.
+- **Networking:** VM NIC on a host bridge to the LAN, IP from the router's
+  DHCP. IP discovery via `DomainInterfaceAddresses` (ARP source first, then
+  guest agent). Provider-injected `vendor-data` installs `qemu-guest-agent`
+  so agent discovery works on stock images without touching user-data.
+- **Base image:** a stock Debian/Ubuntu cloud image (qcow2, cloud-init
+  preinstalled) that **the operator fetches itself** on first use per host:
+  `placement.image` names an entry in the provider's `images:` map (url +
+  optional sha256) or is an `https://` URL directly. The provider streams the
+  download through libvirt's `StorageVolUpload` into the host's pool, so
+  nothing is pre-placed and the operator needs no filesystem access on the
+  host. Per-VM disk is a qcow2 overlay with the cached base as backing store.
+- **Seed:** NoCloud ISO (`user-data`, `meta-data`, `vendor-data`) built in
+  pure Go with `github.com/kdomanski/iso9660` (`WriteTo(w, "cidata")`) and
+  streamed to the pool with `StorageVolUpload(vol, io.Reader, ...)`. Nothing
+  to install on the hypervisors.
+
+## Design
+
+### Config (`internal/provider/config.go`)
+
+```yaml
+- name: kvm-home
+  type: kvm
+  kvm:
+    sshKeyFile: /etc/proxy-operator/kvm/id_ed25519        # required
+    knownHostsFile: /etc/proxy-operator/kvm/known_hosts   # required unless insecureIgnoreHostKey
+    insecureIgnoreHostKey: false
+    diskSizeGb: 10                    # overlay capacity; cloud-init growpart expands root fs
+    vendorData: |                     # optional override; default installs qemu-guest-agent
+      #cloud-config
+      ...
+    images:                           # optional; placement.image names a key (or is an https URL)
+      debian-13:
+        url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+        sha256: "…"                   # optional; verified after download, re-fetch when it changes
+        format: qcow2                 # default qcow2
+      ubuntu-24.04:
+        url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+    machineTypes:                     # required, ≥1; placement.machineType names a key
+      small:  {vcpus: 1, memoryMiB: 1024}
+      medium: {vcpus: 2, memoryMiB: 2048}
+    hosts:                            # required, ≥1, unique names
+      - name: nas1                    # placement.zone value
+        address: nas1.home.lan:22     # required
+        user: virt                    # required; must be in the libvirt group
+        socket: /var/run/libvirt/libvirt-sock   # default
+        pool: default                 # default "default"
+        bridge: br0                   # required
+```
+
+Add `KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage` structs; a `"kvm"`
+case in `validate()` (cross-block checks like the others, required fields
+above, unique host names, image URLs must be `http(s)`, sha256 if set must be
+64 hex chars). Tests in `config_test.go` extend the existing table.
+
+### Placement mapping
+
+| Placement field | KVM meaning |
+|---|---|
+| `zone` | host name from `hosts`; empty → the single configured host, else `ErrPermanent` |
+| `machineType` | key in `machineTypes`; unknown → `ErrPermanent` |
+| `image` | key in `images:`, or an `https://` URL; anything else → `ErrPermanent`. Fetched into the host's pool on first use (see *Image cache*) |
+| `region` | ignored |
+
+### providerID
+
+`hosts/<host>/domains/<name>` with `formatProviderID`/`parseProviderID`
+(same self-contained style as GCP's `zones/<zone>/instances/<name>`).
+
+### GC contract without labels
+
+Domain XML gets a `<metadata>` child in a private namespace, e.g.
+`xmlns="https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm"`,
+carrying `<managed>true</managed><uid>…</uid><createdAt>RFC3339</createdAt>`.
+`Get` reads it via `DomainGetMetadata(dom, DomainMetadataElement, uri)`;
+`ListByTag` does `ConnectListAllDomains` on every host and keeps domains where
+that call succeeds and `managed == "true"`. `CreatedAt` comes from the metadata
+(libvirt has no creation time). A host that is unreachable makes `ListByTag`
+return an error (GC skips the sweep that round rather than acting on a partial view).
+
+### Connection management (`internal/provider/kvm/conn.go`)
+
+Per-host lazily dialed `*ssh.Client` + `*libvirt.Libvirt`, cached under a
+mutex, re-dialed when `Disconnected()` is closed or a call fails with a
+transport error. SSH auth: private key from `sshKeyFile`; host key check via
+`knownhosts.New(knownHostsFile)` unless `insecureIgnoreHostKey`. Dial timeout
+from `ctx`. The provider calls libvirt through a narrow interface:
+
+```go
+type libvirtAPI interface {
+    DomainLookupByName(name string) (libvirt.Domain, error)
+    DomainDefineXML(xml string) (libvirt.Domain, error)
+    DomainCreate(d libvirt.Domain) error
+    DomainDestroy(d libvirt.Domain) error
+    DomainUndefineFlags(d libvirt.Domain, f libvirt.DomainUndefineFlagsValues) error
+    DomainGetState(d libvirt.Domain, flags uint32) (int32, int32, error)
+    DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, f libvirt.DomainModificationImpact) (string, error)
+    DomainInterfaceAddresses(d libvirt.Domain, src uint32, flags uint32) ([]libvirt.DomainInterface, error)
+    ConnectListAllDomains(need int32, f libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error)
+    StoragePoolLookupByName(name string) (libvirt.StoragePool, error)
+    StorageVolLookupByName(p libvirt.StoragePool, name string) (libvirt.StorageVol, error)
+    StorageVolGetPath(v libvirt.StorageVol) (string, error)
+    StorageVolCreateXML(p libvirt.StoragePool, xml string, f libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error)
+    StorageVolUpload(v libvirt.StorageVol, r io.Reader, off, n uint64, f libvirt.StorageVolUploadFlags) error
+    StorageVolDownload(v libvirt.StorageVol, w io.Writer, off, n uint64, f libvirt.StorageVolDownloadFlags) error
+    StorageVolDelete(v libvirt.StorageVol, f libvirt.StorageVolDeleteFlags) error
+    StoragePoolRefresh(p libvirt.StoragePool, flags uint32) error
+}
+```
+
+(All signatures verified against go-libvirt `main` on 2026-09-04.) A
+`hostDialer func(ctx, KVMHost) (libvirtAPI, io.Closer, error)` is the test seam;
+`newWithDialer(cfg, dialer)` mirrors GCP's `newWithAPI`.
+
+### Image cache (`internal/provider/kvm/image.go`)
+
+`ensureImage(ctx, host, api, img) (basePath string, err error)` makes the base
+image present in the host's pool, once per host per image:
+
+- Volume names: `base-<key>.<format>` for an `images:` key, or
+  `base-<sha256(url)[:16]>.<format>` for a URL given directly in
+  `placement.image`. Marker volume `<base>.sha256` (raw, 64 bytes) holds the
+  digest of the verified upload.
+- **Ready check:** base volume exists *and* marker exists *and* (no `sha256`
+  configured, or marker content == configured `sha256`). Read the marker with
+  `StorageVolDownload`. Ready → return `StorageVolGetPath(base)`.
+- **Not ready:** delete both volumes if present (partial upload from a crash,
+  or a changed checksum), then `GET url` with the ctx, require a
+  `Content-Length` (`ErrPermanent` otherwise), `StorageVolCreateXML` the base
+  with `<capacity unit="bytes">Content-Length</capacity>` and the configured
+  format, `StorageVolUpload(base, io.TeeReader(body, sha256), 0, 0, 0)` (length
+  0 = until EOF, so nothing is buffered in the operator), then compare the
+  digest with the configured one (mismatch → delete base, `ErrPermanent`),
+  `StoragePoolRefresh` so libvirt re-reads the real qcow2 capacity, create the
+  marker volume and upload the digest last.
+- HTTP 4xx → `ErrPermanent`; 5xx, network, or upload transport errors →
+  `ErrTransient` (partial volumes are cleaned on the next attempt).
+- A per-`(host, image)` `sync.Mutex` map serialises concurrent Creates for the
+  same image on the same host; the second caller finds it ready.
+- **Judgment call:** the fetch runs synchronously inside `Create`, so the first
+  Proxy for a given image on a given host holds a reconcile worker for the
+  download (LAN speed, a few hundred MB, once). This keeps `Create` idempotent
+  and needs no background state. Async fetch with a `Provisioning`-style
+  progress status is a possible later improvement, noted in the docs.
+
+### Create (idempotent, fire-and-forget after the image is cached)
+
+1. Resolve host, machine type, pool, and image source (`images:` key or URL;
+   else `ErrPermanent`). `ensureImage` → base path.
+2. Overlay volume `<name>.qcow2`: `StorageVolCreateXML` with
+   `<capacity unit="G">diskSizeGb</capacity>`, `<target><format type="qcow2"/></target>`,
+   `<backingStore><path>base</path><format type="qcow2"/></backingStore>`.
+   `ErrStorageVolExist` → reuse.
+3. Seed volume `<name>-seed.iso`: build ISO in memory (`user-data` = req.CloudInit,
+   `meta-data` = `instance-id: <name>\nlocal-hostname: <name>\n`, `vendor-data` =
+   cfg or default), `StorageVolCreateXML` with exact capacity + `<format type="raw"/>`,
+   then `StorageVolUpload`. Exists → skip upload (content is deterministic per spec hash;
+   spec changes recreate the whole instance anyway).
+4. `DomainLookupByName(name)`; if `ErrNoDomain` → `DomainDefineXML(buildDomainXML(...))`.
+5. `DomainGetState`; if not running → `DomainCreate`. Return providerID.
+
+`buildDomainXML` is a pure function over `encoding/xml` structs: `type="kvm"`,
+q35 machine, `host-passthrough` CPU, vcpus/memory from the profile, virtio disk
+(overlay) + sata cdrom (seed), `interface type="bridge"` with virtio model,
+`org.qemu.guest_agent.0` virtio channel, serial console, virtio rng, no
+graphics, and the `<metadata>` block. Unit-tested field by field like
+`buildInsertRequest`.
+
+### Get
+
+`parseProviderID` → host → `DomainLookupByName` (`ErrNoDomain` → `ErrNotFound`)
+→ `DomainGetState` + `DomainGetMetadata` + `DomainInterfaceAddresses`
+(ARP, then agent; first IPv4 wins; agent errors while the agent isn't up yet
+are swallowed). Mapping:
+
+| libvirt state | IP | Instance.State |
+|---|---|---|
+| running | yes | Running |
+| running | no | Provisioning |
+| shutoff / crashed / paused / pmsuspended | – | Stopped |
+| no domain | – | `ErrNotFound` |
+
+### Delete (idempotent)
+
+Destroy if running (`ErrOperationInvalid` on a stopped domain is fine) →
+`DomainUndefineFlags(managed-save | snapshots-metadata | nvram)` → delete
+`<name>.qcow2` and `<name>-seed.iso` (`ErrNoStorageVol` fine). Volumes are
+deleted by name even when the domain is already gone, so a crash between
+volume creation and define still gets cleaned up on the next reconcile.
+Known gap (documented, not solved now): volumes orphaned with no domain *and*
+no CR are invisible to `ListByTag`.
+
+### Errors (`internal/provider/kvm/errors.go`)
+
+`classify(err)` on `libvirt.Error.Code`: `ErrNoDomain`, `ErrNoStorageVol` →
+`ErrNotFound`; `ErrNoStoragePool`, `ErrXMLError`, `ErrConfigUnsupported`,
+`ErrNoSupport`, `ErrAuthFailed`, `ErrInvalidArg` → `ErrPermanent`; SSH dial /
+host-key / transport errors and everything else → `ErrTransient`. Image
+download: HTTP 4xx, missing Content-Length, checksum mismatch → `ErrPermanent`;
+5xx and network errors → `ErrTransient`. Quota is not detectable from libvirt;
+left as transient (revisit with a capacity check later).
+
+### Wiring & manifests
+
+- `cmd/main.go`: add `"kvm": kvm.New` to the builtins map.
+- `config/manager/manager.yaml`: mount Secret `kvm-ssh` (optional) at
+  `/etc/proxy-operator/kvm` (keys `id_ed25519`, `known_hosts`); document the
+  `kubectl create secret generic kvm-ssh --from-file=...` command.
+- `config/samples/providers-config.yaml`: add the `kvm-home` block with the
+  `images:` map (Debian 13 and Ubuntu 24.04 official cloud-image URLs);
+  `config/samples/proxy_kvm.yaml`: Proxy with `placement: {zone: nas1,
+  machineType: small, image: debian-13}` and the same squid cloud-init as the
+  GCP sample plus `ssh_authorized_keys`.
+- `hack/providers-dev.yaml` stays kubernetes-only (needs no secrets).
+- Deps: `github.com/digitalocean/go-libvirt` (latest pseudo-version),
+  `github.com/kdomanski/iso9660 v0.4.0`, promote `golang.org/x/crypto` to direct.
+
+### Docs
+
+- New `docs/kvm-provider.md`: hypervisor prerequisites (user in `libvirt`
+  group, a bridge to the LAN, a storage pool with enough space), the config
+  reference including `images:`, how the image cache works (`base-*` and
+  `*.sha256` volumes in the pool, when a re-download happens, how to evict
+  with `virsh vol-delete`), the placement table, how to log into a VM
+  (`ssh_authorized_keys` in user-data, `virsh console` on the host), and the
+  known gaps (no quota detection, orphan volumes without a domain,
+  synchronous first-time image fetch).
+- Update the provider tables in `README.md`, `docs/architecture.md` §5,
+  `docs/testing.md` (integration test section).
+
+### Towards the generic runner (not built now, just kept open)
+
+Provider knows nothing about squid. Future hooks: `vendor-data` for injecting an
+agent, `QEMUDomainAgentCommand` for reading exit status/output, and a future
+`Placement`-independent "job" mode in the CRD. None of this changes the KVM
+provider's shape.
+
+## Steps
+
+0. **Save plan + branch.** Copy this file to `docs/plans/<ts>-kvm-provider.md`,
+   commit alone, then `git checkout -b feat/kvm-provider` off `feat/proxy-operator`
+   (the working main for this project).
+1. **Config.** `KVMConfig` & friends, `validate()` case, table tests.
+2. **Pure builders.** `internal/provider/kvm/{domainxml.go,volume.go,seed.go,id.go,metadata.go}`
+   + tests: domain XML fields, volume XML, seed contents (unpack the ISO with
+   the same library's reader and assert the three files + label `cidata`),
+   providerID round-trip, metadata marshal/parse.
+3. **Transport + errors.** `conn.go` (SSH dial, host-key policy, per-host cache,
+   reconnect), `errors.go` + classification table test. SSH dialer tested
+   against an in-process `x/crypto/ssh` server that serves a unix socket
+   forward (cheap, keeps the dial code covered).
+3b. **Image cache.** `image.go` + `image_test.go`: `httptest.Server` serving a
+   small fake image over the `fakeAPI` (which records uploaded bytes);
+   cases: fresh fetch, already ready, marker missing (partial upload → redo),
+   checksum changed → redo, checksum mismatch → Permanent + cleanup, 404 →
+   Permanent, 503 → Transient, two concurrent callers → one download.
+4. **Provider methods.** `kvm.go`: `New`, `newWithDialer`, `Create/Get/Delete/ListByTag`
+   over `libvirtAPI`; `kvm_test.go` with a hand-written `fakeAPI` (domains and
+   volumes in maps, per-call error injection) covering idempotent re-Create
+   after each partial step, state/IP mapping, delete-when-gone, ListByTag
+   across two hosts and with one host down.
+5. **Wiring, manifests, samples, docs, CHANGELOG.**
+6. **Integration test** (`//go:build integration`, `kvm_integration_test.go`):
+   skipped unless `KVM_TEST_HOST`, `KVM_TEST_USER`, `KVM_TEST_SSH_KEY`,
+   `KVM_TEST_BRIDGE`, `KVM_TEST_IMAGE_URL` are set; fetches the image (or finds
+   it cached), creates, waits for Running+IP, deletes, asserts per-VM volumes
+   gone and the base image still cached. Deviation from the testcontainers convention,
+   called out in `docs/testing.md`: libvirt+KVM can't run inside a container
+   without `/dev/kvm`, so this targets a real hypervisor.
+7. **Manual validation** on one real hypervisor, results in
+   `docs/plans-executions/` (like `docs/gcp-vm-validation.md`), then MR via `tea`.
+
+## Verification
+
+- `make test` green (`-race`, envtest suite untouched but must still pass).
+- `go test -short ./internal/provider/...` covers builders, fake-API provider, errors, config.
+- Integration: `KVM_TEST_HOST=nas1.home.lan … go test -tags=integration -run TestKVM ./internal/provider/kvm/`.
+- End to end: deploy with the `kvm-ssh` Secret and `kvm-home` config against a
+  host with no cached image, apply `config/samples/proxy_kvm.yaml`, watch the
+  operator log the image fetch and `virsh vol-list` show `base-debian-13.qcow2`
+  plus its `.sha256` marker, watch `kubectl get px -w` reach `Ready` with a LAN
+  IP, apply a second Proxy and confirm no second download, confirm the VM shows up in `GET /v1/proxies` and works with
+  `hack/` demo egress-IP check, delete the Proxy and confirm `virsh list --all`
+  and `virsh vol-list default` are clean.

docs/plans/2026-09-10-0805-kvm-libvirt-tracing.md

@@ -0,0 +1,186 @@
+# Plan: Trace every libvirt call in the KVM provider
+**Created:** 2026-09-10 08:05
+
+## Context
+
+Step 7 of the KVM provider plan (live validation on `beelink`) has gone through
+three fixes found only on the real host, and the current failure (guest hangs at
+GRUB's "Booting" line) is being diagnosed blind: the provider logs at V(1)/V(2)
+but nothing shows them. The integration test never installs a logger, so
+controller-runtime prints "log.SetLogger(...) was never called" and swallows
+everything; and even with a logger, the provider only logs per *operation*
+(Create, Get…), not per libvirt RPC, so the sequence and timing of what was
+actually sent to the hypervisor is invisible.
+
+Goal: one log line per libvirt call at debug level (`--zap-log-level=debug`, logr
+V(1) in this repo's scheme: V(1) = outcome per operation, V(2) = curated detail,
+V(5) = raw wire, per the GCP verbose-logging plans), with method, host, domain or
+volume, duration and the libvirt error code on failure; and those lines visible in
+`make test-integration-kvm` output.
+
+Assumption: this lands on the current `feat/kvm-provider` branch, since it exists
+to finish that branch's validation, and the plan file plus execution log follow
+CLAUDE.md conventions (plan committed alone first, execution summary alongside
+each step). Say so if you want a separate branch instead.
+
+## Design
+
+- **Decorator, not call-site logging.** New `tracingAPI` implements the 17-method
+  `libvirtAPI` seam (`internal/provider/kvm/conn.go:26-44`), wrapping the real
+  client. Every method logs one V(1) line after the inner call returns. Applied in
+  `hostPool.call`, so every provider method, `image.go`'s cache and the
+  integration test's own `p.hosts.call` are traced with no per-site changes.
+- **Logger passed explicitly.** `call(ctx, log, host, fn)` and `get(ctx, log, host)`
+  gain a `logr.Logger` parameter, matching how this package already threads `log`
+  into every helper. The five call sites (`kvm.go:169, 322, 458, 530`,
+  `kvm_integration_test.go:220`) already hold the right logger with
+  provider/host/name values. `call` does `log = log.WithName("libvirt")`, so trace
+  lines are named `kvm.libvirt`.
+- **Wrap only when `log.V(1).Enabled()`.** Zero cost at info level; the
+  controller-runtime unfulfilled root sink reports disabled, so unit tests and
+  `conn_test.go`'s `hostClient{api: nil}` cases are unaffected.
+- **One line per RPC at every level.** At V(2) the same line gains extra keys
+  (`xml`, `metadata`, `addresses`, `names`) instead of a second line. Upload and
+  download log offset/length/duration only, never bytes (the seed ISO carries
+  cloud-init). Domain and volume XML contain no secrets, so V(2) is fine for them.
+- **Connection events traced too** in the pool: dial with duration, dial failure,
+  dead cached client redial, and both drop reasons (context cancelled, transport
+  error). `dialSSH`/`connectLibvirt`/`keepalive` stay untouched: changing
+  `hostDialer`'s signature would ripple into every test dialer, and the pool
+  already observes every event.
+- **Rendering.** Handles by name only (`domain`, `pool`, `volume`), never UUID/Key.
+  `libvirt.DomainState` and `DomainInterfaceAddressesSource` have stringers, pass
+  them typed. Bitmask flags don't stringify, log as integers only when non-zero.
+  `libvirt.OptString` → first element or `""` under `uri`.
+- **awaitOrCancel.** `done` runs inside fn's goroutine; on the normal path it logs
+  before `call` returns. On cancel, the abandoned RPC logs one late line when the
+  closed connection makes it fail: desirable in production, and in the integration
+  test only reachable in an already-failing timeout run (testr panics on a log after
+  the test ends; note that in the header, don't engineer around it).
+
+## Steps
+
+### Step 0 — Save plan
+Copy this file to `docs/plans/2026-09-10-0805-kvm-libvirt-tracing.md`, commit alone.
+
+### Step 1 — `errors.go`: shared error fields
+Factor `libvirtErrorFields(err error) []any` out of `logAPIError` (`errors.go:62-72`):
+returns `libvirtCode` (uint32), `libvirtCodeName` (`libvirt.ErrorNumber(code)`,
+e.g. `ErrNoDomain`), `libvirtMessage`; nil for non-libvirt errors; works through
+`%w` wrapping via `errors.As`. `logAPIError` keeps its message and gains
+`libvirtCodeName`. Test `TestLibvirtErrorFields` (table: libvirt.Error, wrapped,
+plain error) in `errors_test.go`.
+
+### Step 2 — new `trace.go`
+```go
+type tracingAPI struct { inner libvirtAPI; log logr.Logger }
+type tracedCall struct { t *tracingAPI; method string; start time.Time; kv []any }
+func (t *tracingAPI) begin(method string, kv ...any) *tracedCall
+func (c *tracedCall) detail(kv ...any) *tracedCall // appended only if V(2) enabled
+func (c *tracedCall) done(err error, kv ...any)    // one V(1) line
+```
+`done` emits `log.V(1).Info("libvirt call", ...)` with keys in order: `method`,
+begin args, result kv, `duration` (rounded to µs), then on error `error` plus
+`libvirtErrorFields`. Per-method keys:
+
+| method | V(1) args | V(1) results | V(2) extra |
+|---|---|---|---|
+| DomainLookupByName | domain | | |
+| DomainDefineXML | xmlBytes | domain | xml |
+| DomainCreate, DomainDestroy | domain | | |
+| DomainUndefineFlags | domain, flags | | |
+| DomainGetState | domain | state, reason | |
+| DomainGetMetadata | domain, type, uri | metadataBytes | metadata |
+| DomainInterfaceAddresses | domain, source | interfaces (count) | addresses |
+| ConnectListAllDomains | | count | names |
+| StoragePoolLookupByName, StoragePoolRefresh | pool | | |
+| StorageVolLookupByName | pool, volume | | |
+| StorageVolGetPath | pool, volume | path | |
+| StorageVolCreateXML | pool, xmlBytes | volume | xml |
+| StorageVolUpload, StorageVolDownload | pool, volume, offset, length | | (never content) |
+| StorageVolDelete | pool, volume | | |
+
+Return values pass through untouched.
+
+### Step 3 — `conn.go`: pool logging and wrapping
+- `get(ctx, log, host)`: dead cached client → `"libvirt connection lost, redialing"`;
+  time the dial; `"libvirt host connected"` with `address`, `duration`; failure →
+  `"libvirt host dial failed"` with `address`, `duration`, `error`.
+- `call(ctx, log, host, fn)`: `log = log.WithName("libvirt")`; wrap when
+  `log.V(1).Enabled()`; abort closure logs `"libvirt connection dropped"`
+  `reason="context cancelled"` before `drop`; transport-error branch logs the same
+  message with `reason="transport error"`. Update `call`'s doc comment.
+
+### Step 4 — call sites
+`kvm.go` Create/Get/Delete pass `log`, ListByTag passes `hlog`;
+`kvm_integration_test.go:220` passes `p.logger(ctx).WithValues("host", host.Name)`;
+`conn_test.go` (12 `p.call(` sites) gets `var discard = logr.Discard()`.
+
+### Step 5 — integration test output
+- `integrationLogger(t)`: `KVM_TEST_LOG_LEVEL` (default `1`, `strconv.Atoi`,
+  `t.Fatalf` on garbage) → `testr.NewWithOptions(t, testr.Options{LogTimestamp:
+  true, Verbosity: n})` (`github.com/go-logr/logr/testr`, present in logr v1.4.3,
+  no go.mod change); `logf.SetLogger(log)` once so nothing falls back to the
+  unfulfilled root.
+- `TestKVM_lifecycle`: `ctx = logr.NewContext(ctx, log)` after `WithTimeout`; the
+  cleanup ctx gets the same logger.
+- Header comment: document `KVM_TEST_LOG_LEVEL` (1 = one line per RPC, 2 adds
+  XML/metadata/addresses, 0 = only `t.Log`) and the late-line-after-timeout caveat.
+
+### Step 6 — unit tests
+- `kvm_test.go`: `captureContext(verbosity)` like `gcp_test.go:277-286` but with a
+  mutex around the slice; `TestLogging_verbosityTiers` (mirror `gcp_test.go:322-399`):
+  V0 → no lines; V1 → `"msg"="libvirt call"`, `"method"="DomainDefineXML"`,
+  `"method"="StorageVolUpload"`, `"msg"="libvirt host connected"`, provider/host/name
+  values, `"libvirtCodeName"="ErrNoDomain"` from the first lookup, no `"xml"=`;
+  V2 → `"xml"="<domain` and `"metadata"=`; a cloud-init sentinel absent at every tier.
+- `trace_test.go`: `TestTracingAPI_logsEveryMethod` (table of 17 calls over a
+  `readyHost()` fake, exactly one line each, plus
+  `reflect.TypeOf((*libvirtAPI)(nil)).Elem().NumMethod() == len(cases)` so a new
+  interface method without tracing fails); `TestTracingAPI_passesResultsThrough`;
+  `TestTracingAPI_errorFields` (libvirt error → code/name/message; `io.EOF` →
+  `"error"="EOF"`, no code); `TestTracingAPI_detailOnlyAtV2`;
+  `TestTracingAPI_uploadLogsSizeNotContent`.
+- `conn_test.go`: `fakeDialer` gains an `api` field; `TestHostPool_call_tracesAtV1`
+  (V1 → connected + method lines; V0 → no lines and fn receives the bare
+  `*fakeAPI`); `TestHostPool_call_logsDropReason` (transport error, cancel).
+
+### Step 7 — docs, execution log, CHANGELOG
+- `docs/kvm-provider.md`: `## Logging` section before "Known gaps":
+  `--zap-log-level=debug` = one `libvirt call` line per RPC plus connect/drop lines;
+  `--zap-log-level=2` adds XML, metadata, addresses; cloud-init never logged; no
+  V(5) wire tier because go-libvirt exposes no hook. Mention `KVM_TEST_LOG_LEVEL`
+  in its Testing paragraph.
+- `docs/testing.md`: `KVM_TEST_LOG_LEVEL` in the optional vars; `-v` now shows every
+  RPC; extend the `kvm_test.go` table row.
+- `docs/plans-executions/2026-09-10-0805-kvm-libvirt-tracing.md` with the checklist
+  and per-step entries; also a short cross-reference line in the KVM provider
+  execution log's Step 7 section.
+- `CHANGELOG.md` entry once the user confirms the live run shows the trace.
+
+## Verification
+
+```bash
+go build ./... && go vet ./... && go vet -tags=integration ./internal/provider/kvm/
+go test -race -count=1 ./internal/provider/kvm/
+go test -short ./...
+```
+Live, with the usual env: `make test-integration-kvm`. Expect, interleaved with
+`t.Log` lines and attributed to `trace.go:<line>`, lines like
+
+```
+kvm.libvirt: "level"=1 "msg"="libvirt call" "provider"="kvm-integration" "host"="kvm-test" "name"="proxy-…" "method"="DomainLookupByName" "domain"="proxy-…" "duration"="2.1ms" "error"="…" "libvirtCode"=42 "libvirtCodeName"="ErrNoDomain" "libvirtMessage"="…"
+```
+
+plus one `libvirt host connected` line with `duration`, a `StorageVolUpload` line
+whose duration is the seed upload, and no "log.SetLogger(...) was never called"
+text. `KVM_TEST_LOG_LEVEL=2` shows `"xml"="<domain` and `"addresses"=`;
+`KVM_TEST_LOG_LEVEL=0` shows only `t.Log` output. No line at any level contains
+`#cloud-config`.
+
+## Critical files
+- `internal/provider/kvm/conn.go`, `trace.go` (new), `trace_test.go` (new)
+- `internal/provider/kvm/errors.go`, `errors_test.go`
+- `internal/provider/kvm/kvm.go`, `kvm_test.go`, `conn_test.go`
+- `internal/provider/kvm/kvm_integration_test.go`
+- `docs/kvm-provider.md`, `docs/testing.md`

docs/plans/2026-09-10-2222-kvm-review-fixes.md

@@ -0,0 +1,195 @@
+# Plan: Fix the five code-review findings on `feat/kvm-provider`
+**Created:** 2026-09-10 22:22 UTC
+
+## Context
+
+PR #5 (`feat/kvm-provider` → `main`, 19 commits) is open and awaiting merge. A
+`/code-review high` pass over the branch returned five confirmed findings: two that can
+pin a Proxy's finalizer forever, one startup crash-loop, and two in the guest-agent merge
+committed earlier today. All five are fixed on the same branch so the PR updates in place,
+then re-validated against the real hypervisor before merging.
+
+Four decisions, made by the user during planning:
+
+1. **Wedged deletes** — attempt `Delete` even when `Get` fails permanently, make an
+   unconfigured KVM host a no-op delete, and add an annotation that abandons cleanup for
+   whatever still cannot finish.
+2. **Missing `kvm-ssh` Secret** — read the SSH material lazily, so only KVM Proxies fail.
+3. **Guest agent** — stop rewriting the user's cloud-config. Ship the bootstrap as a
+   vendor-data shell script instead.
+4. Validation is unit tests, the integration test against beelink, and the kind e2e. The
+   uncommitted dev-only files stay exactly as they are and out of the PR.
+
+## Why the guest-agent fix is a rewrite, not a patch
+
+Both cloud-init findings come from the same root cause: this operator edits the user's
+document. Reading cloud-init 26.1's sources settles that it never has to.
+
+Vendor-data is consumed by its **own** handler chain with separate script and config paths
+(`stages.py:_default_vendordata_handlers`). A vendor-data **shell script** is therefore
+written to the vendor script directory and run by `cc_scripts_vendor`, never passing
+through the config merger that drops colliding keys. On the final-stage module list it
+runs after `package_update_upgrade_install` and before the user's own `runcmd`, so the
+agent comes up early regardless of what the user's config does.
+
+That also closes the door the previous approach left open. The merge cannot be rescued
+from inside the documents: `merge_how` in the user-data governs only the first source, and
+`merge_how` in vendor-data is popped and discarded by `dict_extract_mergers` before the
+document is ever written. There is no supported way to make vendor-data cloud-config keys
+survive a user-data that defines the same key.
+
+Consequences: user-data reaches the seed byte for byte in any format, so Jinja templates,
+MIME archives, gzip payloads and shell scripts stop being special cases (finding 4), and
+nothing re-serialises YAML, so unquoted scalars keep their type (finding 5). The cost is
+that the script picks the package manager itself where cloud-init's `packages` key was
+distro-agnostic.
+
+## Why the delete fix is shaped this way
+
+KVM is the only provider that resolves a providerID segment against operator config. GCP
+and Kubernetes let the backend answer "not found", so retiring a location yields
+`ErrNotFound` and the finalizer drops. `Provider.host()`
+([kvm.go:118-139](internal/provider/kvm/kvm.go#L118-L139)) returns `ErrPermanent`, and
+`reconcileDelete` ([proxy_controller.go:281-291](internal/controller/proxy_controller.go#L281-L291))
+drops the finalizer only on `ErrNotFound`. `deletionFailure`
+([:327-336](internal/controller/proxy_controller.go#L327-L336)) says in its own comment
+that it retries "until an operator intervenes", and no intervention mechanism exists. The
+annotation is that mechanism, and it also rescues the unconfigured-provider wedge at
+[:276-279](internal/controller/proxy_controller.go#L276-L279).
+
+`Get` keeps returning `ErrPermanent` for an unknown host deliberately. Turning it into
+`ErrNotFound` would make a typo in the host list look like a missing VM on the live path,
+and the reconciler would recreate every affected Proxy elsewhere while the originals keep
+running where garbage collection cannot see them.
+
+## Steps
+
+One commit per step on `feat/kvm-provider`, each ending with
+`Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>`.
+
+### Step 0 — Save this plan into the repo
+Copy to `docs/plans/2026-09-10-2222-kvm-review-fixes.md` with the same header timestamp and
+commit it alone, before any code change. Start
+`docs/plans-executions/2026-09-10-2222-kvm-review-fixes.md` with the status checklist when
+Step 1 lands.
+
+### Step 1 — Unwedge deletion (finding 1)
+- `api/v1alpha1/proxy_types.go`: add `AnnotationSkipCleanup = "crawl.example.com/skip-cleanup"`
+  beside `AnnotationSpecHash` ([:59-63](api/v1alpha1/proxy_types.go#L59-L63)), documented as
+  abandoning the backend instance.
+- `internal/controller/proxy_controller.go`, `reconcileDelete`: honour the annotation after
+  the empty-providerID branch and **before** the provider lookup, so it also covers a
+  provider that is no longer configured. Log the abandoned providerID at Info.
+  `ProxyReconciler` has no `EventRecorder`; do not add one for this.
+- Same function: replace the two `if`s after `Get` with a switch. On
+  `provider.Class(err) == provider.ErrPermanent`, call `Delete` anyway, since it is
+  idempotent, and remove the finalizer when it succeeds rather than requeueing into a `Get`
+  that can never converge. On failure, `deletionFailure(errors.Join(getErr, delErr))`.
+  Guard on `err != nil`, since `Class(nil)` is nil.
+- `internal/provider/kvm/kvm.go`, `Delete`: an unconfigured host segment means there is
+  nothing this operator can clean, so log it and return success, mirroring `removeVolumes`
+  ([kvm.go:506-514](internal/provider/kvm/kvm.go#L506-L514)) for a missing pool. `Get` is
+  unchanged.
+- Tests: extend the deletion table in `internal/controller/reconcile_test.go`
+  ([:357-393](internal/controller/reconcile_test.go#L357-L393)) with permanent-Get plus
+  succeeding Delete, permanent-Get plus failing Delete, the annotation with the provider
+  never called, and the annotation on an unconfigured provider. In `kvm_test.go`, the
+  `"host not configured"` case at [:737](internal/provider/kvm/kvm_test.go#L737) flips from
+  `ErrPermanent` to success; add a named test beside `TestDelete_missingPoolIsNotAnError`
+  ([:715](internal/provider/kvm/kvm_test.go#L715)).
+- Docs: the annotation in `README.md` (~line 163) and `docs/architecture.md` (~line 487);
+  a *Known gaps* entry in `docs/kvm-provider.md` saying a domain on a host removed from
+  config can no longer be cleaned up by the operator, the Proxy still finishes deleting,
+  and the VM must be removed by hand because garbage collection only sweeps configured
+  hosts.
+
+### Step 2 — Validate names that become identifiers (finding 2)
+- `internal/provider/config_kvm.go`: add a name regexp beside `sha256Hex`
+  ([:6](internal/provider/config_kvm.go#L6)) allowing letters, digits, dot, dash and
+  underscore. Apply it to `hosts[].name` after the existing empty check
+  ([:129-131](internal/provider/config_kvm.go#L129-L131)) and to each `images` map key,
+  which becomes the cached volume name `base-<key>.<format>`
+  ([image.go:49-55](internal/provider/kvm/image.go#L49-L55)) and half of the image-cache
+  lock key ([image.go:96](internal/provider/kvm/image.go#L96)). `machineTypes` keys are only
+  compared, never embedded, so leave them.
+- Keep the file's first-error-wins style and `fmt.Sprintf` field paths. The host message
+  should name the consequence: the value becomes part of `hosts/<name>/domains/<vm>`, which
+  `parseProviderID` splits on `/` ([id.go:11-21](internal/provider/kvm/id.go#L11-L21)).
+- Tests: two entries in the `TestLoadConfig_kvmInvalid` table
+  ([config_kvm_test.go:76](internal/provider/config_kvm_test.go#L76)), existing
+  name/yaml/wantErrSub shape.
+
+### Step 3 — Read the SSH material lazily (finding 3)
+- `internal/provider/kvm/conn.go`: keep `newSSHAuth`, and add a lazy dialer that builds it
+  on first dial and caches it once it succeeds, so a `kvm-ssh` Secret created after the
+  manager starts heals on the next reconcile with no restart.
+- `internal/provider/kvm/kvm.go`, `New`: stop calling `newSSHAuth`; pass the lazy dialer.
+  Use the context parameter it currently ignores to log a warning when either file is
+  unreadable at startup, keeping the diagnostic without the exit.
+- Error classes: a missing or unreadable file stays unclassified, so `Class` defaults it to
+  transient and the Proxy retries until the Secret appears; an unparseable key or malformed
+  known_hosts wraps `ErrPermanent`. Distinguish with `errors.Is(err, fs.ErrNotExist)`.
+- Fix the two doc comments promising startup failure
+  ([kvm.go:79-81](internal/provider/kvm/kvm.go#L79-L81),
+  [conn.go:182-185](internal/provider/kvm/conn.go#L182-L185)) and the matching claim in
+  `docs/kvm-provider.md` (~lines 107-109). `DISCOVERY_TOKEN` already sets the house
+  convention for an optional mount: degrade with a loud warning.
+- Tests in `conn_test.go`, which already writes key files: `New` succeeds with no key
+  present, the first dial fails transiently, a dial after the file appears succeeds, and a
+  corrupt key is permanent.
+
+### Step 4 — Guest agent via vendor-data script (findings 4 and 5)
+- `internal/provider/kvm/seed.go`: delete `withGuestAgent`, `isCloudConfig`, `appendUnique`
+  and the `sigs.k8s.io/yaml` import. User-data goes onto the seed exactly as the Proxy
+  supplied it.
+- Add the bootstrap as a `#!/bin/sh` vendor-data script: skip when the agent is already
+  present, otherwise install `qemu-guest-agent` through whichever of `apt-get`, `dnf`,
+  `yum`, `zypper` or `apk` exists, then `systemctl enable --now`. Run `apt-get update`
+  before installing, since nothing else guarantees a fresh index when the user's config has
+  no `packages` key. Keep it idempotent and let it log to stdout, which lands in
+  `cloud-init-output.log`.
+- When the provider config also sets `vendorData`, wrap both into a MIME multipart
+  vendor-data: the user's document as one part, the script as another. Cloud-init detects
+  multipart by finding `mime-version:` in the first 4096 bytes, so the header must lead.
+- `internal/provider/config_kvm.go`: add `installGuestAgent`, defaulting to on. Off means
+  the seed carries only the user's own vendor-data and IP discovery falls back to the host
+  ARP table. Nothing fails permanently any more, so the error path in `writeSeed`
+  ([kvm.go:252-255](internal/provider/kvm/kvm.go#L252-L255)) narrows to MIME assembly.
+- Tests in `seed_test.go`: user-data byte-identical for cloud-config, Jinja, MIME, gzip and
+  script inputs; the script present in vendor-data; the opt-out; the multipart case
+  including the `mime-version:` position; keep the existing ISO tests.
+- Docs: rewrite the *What lands on the host* passage added earlier today, the `vendorData`
+  row and a new `installGuestAgent` row in the config table, and the ARP note in
+  *Known gaps*. Cite the cloud-init behaviour that forced the change.
+
+### Step 5 — Validate, record, push
+- `gofmt -l internal/`, `go vet ./...`, `go test -short ./...`.
+- `make test-integration-kvm` against beelink, environment as recorded in the KVM execution
+  log.
+- Rebuild the image, `kind load`, restart the manager, delete and recreate both Proxies,
+  curl a test URL through each.
+- `CHANGELOG.md` entry at the top with a real `date` timestamp, noting it supersedes the
+  merge approach described in today's earlier entry. Add a Run 8 section to
+  `docs/plans-executions/2026-09-04-1404-kvm-provider.md` plus this plan's own execution
+  file. Push with the token helper; PR #5 updates in place.
+
+## Verification
+- Unit: the controller cases prove a permanent `Get` no longer pins the finalizer and that
+  the annotation short-circuits before the provider is touched. `TestDelete_*` proves an
+  unconfigured host is a clean delete while `TestGet_*` stays permanent, which is what keeps
+  the live path from recreating VMs.
+- Seed: assert the uploaded user-data equals the Proxy's `cloudInit` byte for byte, which is
+  the property the old merge could not offer.
+- Integration on beelink: Create, IP via the guest agent, ListByTag, Delete. In the guest,
+  `cloud-init status --long` clean and `systemctl is-active qemu-guest-agent` active.
+- In-cluster: both Proxies reach Ready with an IP and return 204 through squid. Their
+  cloud-config defines its own `packages`, which is the exact case that failed in Run 7.
+- Negative check in kind: rename the `kvm-ssh` Secret away and restart. The pod must stay
+  up, the Kubernetes Proxies must stay Ready, and only the KVM Proxies degrade. Restore the
+  Secret and confirm they recover without a restart.
+
+## Out of scope
+- The two items the review noted but did not report: deleting a cached base image while
+  running VMs back onto it, and an ad-hoc image URL always being treated as qcow2. Both are
+  documented known gaps and neither is a regression from this branch.
+- The uncommitted dev-only files, by the user's decision.

docs/plans/2026-09-11-1526-provider-health-checks.md

@@ -0,0 +1,354 @@
+# Plan: Provider health checks — see whether the cluster can reach each provider
+**Created:** 2026-09-11 15:26 UTC
+
+## Context
+
+Today the operator can't say whether a provider's backend is reachable. The `Provider`
+interface ([provider.go:89-109](internal/provider/provider.go#L89-L109)) only has
+Create/Get/Delete/ListByTag, and `/healthz` and `/readyz` are bare `healthz.Ping`. A
+broken KVM path only shows up later: as a Proxy stuck in Provisioning, or as an orphan-GC
+sweep logging an error. Several failures stay invisible until a Proxy needs that host:
+
+- a missing `kvm-ssh` Secret;
+- a rotated host key;
+- a Tailscale route that is down;
+- a storage pool that is inactive or full.
+
+The goal is a periodic sanity check of every configured provider from inside the cluster,
+with results you can see through `kubectl` and Prometheus.
+
+Decisions made by the user during planning:
+
+1. **Surface:** a new operator-owned, read-only, cluster-scoped **`ProviderHealth`** CRD,
+   with one object per configured provider, a `Ready` condition and per-target detail.
+   Also Prometheus gauges, and a log line whenever a target goes up or down. No
+   EventRecorder, matching the earlier decision.
+2. **Scope:** real checks for all three providers.
+   - **kvm:** checked per host.
+   - **kubernetes:** confirms the API server and RBAC work.
+   - **gcp:** confirms credentials and the API work.
+3. **KVM depth:** reachability plus the storage pool.
+   - Over the existing pooled SSH+libvirt connection, call the libvirt version and
+     hostname RPCs.
+   - Confirm the configured pool is running and report its free space.
+   - No bridge check.
+
+Not affected: the Proxy reconcile behaviour, and the manager's `/healthz` and `/readyz`.
+A dead hypervisor must never make the manager unready.
+
+## Design
+
+### Contract: `internal/provider/check.go`
+
+```go
+type TargetStatus struct {
+    Name, Address   string
+    Ready           bool
+    Reason, Message string
+    Latency         time.Duration
+    Details         map[string]string
+}
+// Check never errors; each target carries its own verdict. Must return promptly once ctx ends.
+type Checker interface{ Check(ctx context.Context) []TargetStatus }
+```
+
+- **Shared reasons:** `Reachable`, `Timeout`, `Unreachable`, `Unauthorized`, `Forbidden`,
+  `APIError`.
+- **Compile-time checks:** each provider's `check.go` has `var _ provider.Checker = (*Provider)(nil)`.
+- **Bypass the wrappers.** `cmd/main.go:197-201` wraps every provider in
+  `WithTracing(WithMetrics(...))`, so the Checkers are collected from the **raw** providers
+  map, before that loop. As a result, checks don't inflate `provider_requests_total`.
+
+### The checks
+
+**kvm** (`internal/provider/kvm/check.go`)
+- **Concurrency:** one goroutine per `cfg.Hosts` entry (`wg.Go`), writing to `out[i]` so
+  results stay in config order.
+- **Inside `p.hosts.call`:** `ConnectGetLibVersion`, `ConnectGetHostname`,
+  `StoragePoolLookupByName(host.Pool)`, then `StoragePoolGetInfo`.
+- **Rules for `fn`:**
+  - It returns only raw RPC errors. `call` drops the shared connection on any other error
+    ([conn.go:160](internal/provider/kvm/conn.go#L160)), so "pool not running" is decided
+    after `call` returns, never returned from `fn`.
+  - It fills a captured result struct only after all four calls succeed. That struct is
+    read only when `call` returned nil, which avoids a race with an abandoned goroutine.
+- **Details:** `hostname`, `libvirtVersion` (decoded as major.minor.patch), `pool`,
+  `poolState`, `poolCapacity` and `poolAvailable` (in GiB).
+- **New RPCs:** add the three to `libvirtAPI` ([conn.go:29-47](internal/provider/kvm/conn.go#L29-L47)),
+  `tracingAPI` (`trace.go`) and `fakeAPI` (`fake_test.go`). `fakeAPI` gets a hostname, a
+  fixed version, and a per-pool state map that defaults to Running.
+- **Telling failure stages apart without matching strings:** add
+  `type dialError struct{ stage dialStage; err error }` in `conn.go`, with `Error()`
+  returning the inner message unchanged and `Unwrap()`. It wraps at four places:
+  - `lazySSH.dial`, when loading the key material fails (`ssh-material`)
+  - `dialSSH`, at the dial step (`tcp`)
+  - `dialSSH`, at the handshake (`ssh-handshake`)
+  - `connectLibvirt` (`libvirt-connect`)
+
+  Because `Unwrap` is kept, `classify` and `isClassified` still see `ErrPermanent` and
+  `fs.ErrNotExist`, so reconcile behaviour doesn't change.
+- **`checkReason(err)`, tested in order:**
+
+  | Condition | Reason |
+  |---|---|
+  | stage ssh-material | `SSHMaterialUnavailable` |
+  | stage tcp | `Unreachable` |
+  | stage ssh-handshake | `SSHHandshakeFailed` |
+  | stage libvirt-connect | `LibvirtConnectFailed` |
+  | ctx deadline or cancel | `Timeout` |
+  | `ErrNoStoragePool` | `PoolNotFound` |
+  | any other libvirt error | `LibvirtError` |
+  | anything else | `Unreachable` |
+  | success, pool not running | `PoolNotActive` |
+  | success | `Reachable` |
+- **Accepted behaviour:**
+  - A check that hits its timeout drops the shared connection. That already happens for
+    any cancelled reconcile context, and it only fires when the host isn't answering.
+  - The leader now keeps one SSH connection per host open from startup, with keepalives
+    every 15s. Before, the connection only opened with the first KVM reconcile.
+  - No forced periodic redial. `lazySSH` caches the key material forever, so a redial
+    would not pick up a rotated Secret anyway; that gap goes into the docs under
+    "Known gaps".
+
+**kubernetes** (`internal/provider/kubernetes/check.go`)
+- **Call:** `List` pods with the managed-label selector and `client.Limit(1)`,
+  cluster-wide. It uses the same uncached client, verb and RBAC as `ListByTag` and GC.
+  There is no namespace setting in the kubernetes config.
+- **Target:** one target named `apiserver`, with the address from a new `apiHost` field
+  set in `NewWithTransportWrapper`.
+- **Reasons:**
+  - `Timeout`: the context ended, or the error is `IsTimeout`/`IsServerTimeout`.
+  - `Unauthorized`, `Forbidden`.
+  - `APIError`: any other API status.
+  - `Unreachable`: anything else.
+
+**gcp** (`internal/provider/gcp/check.go`, plus a seam change in `gcp.go`)
+- **Why a new seam method:** the existing `AggregatedList` seam reads every page
+  ([gcp.go:63-78](internal/provider/gcp/gcp.go#L63-L78)), and MaxResults is only the page
+  size. So add `Probe(ctx, *computepb.AggregatedListInstancesRequest) error`, which makes
+  one `it.Next()` call and treats `iterator.Done` as nil.
+- **Request:** `Project`, `MaxResults: 1`, `ReturnPartialSuccess: true`, and the same
+  managed-label filter `ListByTag` uses.
+- **Target:** one per project, `Name: cfg.Project`, `Address: compute.googleapis.com`.
+  GCP config has no zones, and reading them from Proxy placements would tie the check to
+  the Proxy cache.
+- **Reasons:**
+  - `Timeout`: the context ended.
+  - For `*googleapi.Error`: 401 is `Unauthorized`, 403 is `Forbidden` (this also covers a
+    disabled API), anything else is `APIError`.
+  - `Unreachable`: anything else, including a failure to get a token.
+
+### The `ProviderHealth` CRD: `api/v1alpha1/providerhealth_types.go`
+
+- **Written by hand**, since kubebuilder isn't installed; `PROJECT` is edited by hand too.
+- **Scope and markers:** cluster-scoped, `shortName=ph`, status subresource, no spec.
+- **Scheme registration:** add the type to the existing `AddKnownTypes` call in
+  [proxy_types.go:362](api/v1alpha1/proxy_types.go#L362), not a second `init()`.
+- **Status:**
+  - `type`
+  - `conditions` (`Ready`; `ConditionReady` constant)
+  - `targets[]`, as a list-map keyed by `name`, each with: `name`, `address`, `ready`,
+    `reason`, `message` (at most 1024 characters), `latencyMillis`,
+    `lastTransitionTime`, `details`
+  - `targetsReady` (for example `"1/2"`, used by a printer column)
+  - `lastCheckTime`
+- **Printer columns:** Type, Ready, Targets, Last Check, Age.
+- **Viewer role:** `config/rbac/providerhealth_viewer_role.yaml`. No editor role, since the
+  operator owns the object.
+
+### Monitor: `internal/providerhealth`
+
+- **Fields:** `Monitor{Client, Reader, Targets []Target{Name, Type, Checker}, Interval, Timeout}`.
+- **Scheduling:**
+  - `NeedLeaderElection() true`.
+  - Checks once immediately, then on a ticker. Copy the ticker loop from
+    [gc.go:54-80](internal/gc/gc.go#L54-L80).
+- **Each cycle:**
+  - Runs every provider concurrently under `context.WithTimeout(Timeout)`, inside one
+    `tracing.Start` span.
+  - Stores the snapshot under a mutex; `Snapshot()` returns a deep copy.
+  - Logs changes. Going down logs `WARNING: provider target not ready` with provider,
+    target, address, reason and message. Coming back logs `Provider target ready`. A
+    target that is ready on first sight logs at V(1) only.
+- **Writes:**
+  - Get through `mgr.GetAPIReader()`, not the cache. That avoids a cluster-wide informer
+    and the watch permission it would need, whatever `--proxy-namespace` is set to.
+  - Create the object if it is missing, labelled
+    `app.kubernetes.io/managed-by=egress-proxies-operator`.
+  - Then `Status().Patch(MergeFrom(base))` when the status changed, following
+    [status.go:114-121](internal/controller/status.go#L114-L121).
+  - This means one write per provider per interval, because `lastCheckTime` moves each
+    time. That's acceptable given the 10s minimum interval.
+- **`buildStatus(prev, result, now)` is a pure function:**
+  - A target keeps its `lastTransitionTime` unless `ready` flipped.
+  - The `Ready` condition is set through `apimeta.SetStatusCondition`:
+    - `True/AllTargetsReady` when every target is ready
+    - `False/TargetsNotReady` otherwise, with the failing targets in the message
+    - `Unknown/ChecksNotImplemented` when the provider has no Checker
+    - `Unknown/NoTargets` when there are zero targets
+- **Cleanup, once at `Start`:** delete labelled objects whose name isn't in the config.
+  The config is loaded once per process, and unlabelled objects are never touched.
+- **Interval 0 (disabled):** run the cleanup with nothing kept, then return nil, so no
+  object is left showing a stale `Ready=True`.
+- **Invalid provider names:** provider names aren't validated as DNS-1123 names. For an
+  invalid name, warn once and skip the object writes; metrics are still emitted.
+- **`ValidateSchedule(interval, timeout)`** is called in `main.go` before the manager
+  starts. Interval must be 0 or at least 10s; timeout must be in (0, interval].
+- **RBAC markers on `Monitor`:**
+  - `providerhealths`: get, list, create, delete
+  - `providerhealths/status`: get, patch
+  - `make manifests` regenerates `config/rbac/role.yaml`.
+
+### Metrics and wiring
+
+- **New function:** `metrics.RegisterProviderTargets(reg, read func() []ProviderTarget) error`.
+  `Register`'s signature is left alone.
+- **Gauges:** a scrape-time collector, in the style of `constCollector`, reads the Monitor
+  snapshot and emits three gauges labelled `{provider,target}`:
+  - `proxy_operator_provider_target_up`
+  - `proxy_operator_provider_check_duration_seconds`
+  - `proxy_operator_provider_check_timestamp_seconds`
+- **Conversion closure:** the one that turns the Monitor snapshot into gauges lives in
+  `main.go`, next to `proxyPhaseCounts`. So `providerhealth` doesn't import `metrics`.
+- **No pool-bytes gauge:** that value is in status, and node-exporter is the right source
+  for disk alerts.
+- **`cmd/main.go`:**
+  - New flags `--provider-check-interval` (1m) and `--provider-check-timeout` (15s).
+  - Build the `Targets` from `cfg.Providers` in config order, with
+    `c, _ := raw[pc.Name].(provider.Checker)`.
+  - `mgr.Add(&providerhealth.Monitor{Client: mgr.GetClient(), Reader: mgr.GetAPIReader(), ...})`,
+    then register the collector.
+
+## Steps (one commit each, execution-log entry per step)
+
+**Step 0 — Save the plan.**
+- Create branch `feat/provider-health` off `main`.
+- Copy this plan to `docs/plans/2026-09-11-1526-provider-health-checks.md` and commit it
+  alone.
+- The uncommitted `.claude/settings.json` change stays out of every commit.
+
+**Step 1 — ProviderHealth API and CRD.**
+- `api/v1alpha1/providerhealth_types.go`
+- `zz_generated.deepcopy.go`, from `make generate`
+- `config/crd/bases/crawl.example.com_providerhealths.yaml`, from `make manifests`
+- the CRD and RBAC `kustomization.yaml` files, plus the viewer role
+- `PROJECT`
+
+**Step 2 — Contract plus the kubernetes and GCP checks.**
+- `internal/provider/check.go`
+- `internal/provider/kubernetes/{kubernetes.go,check.go,check_test.go}`
+- `internal/provider/gcp/{gcp.go,check.go,check_test.go,gcp_test.go}`
+
+**Step 3 — KVM check.**
+- `internal/provider/kvm/{conn.go,trace.go,check.go,check_test.go,fake_test.go,conn_test.go,trace_test.go,kvm_integration_test.go}`
+
+**Step 4 — Monitor.**
+- `internal/providerhealth/{monitor.go,status.go,monitor_test.go,status_test.go,envtest_test.go}`
+- `config/rbac/role.yaml`, from `make manifests`
+
+**Step 5 — Metrics and wiring.**
+- `internal/metrics/{metrics.go,metrics_test.go}`
+- `cmd/main.go`
+- `config/manager/manager.yaml`: a comment for the new flags
+- `test/e2e/e2e_test.go`
+
+**Step 6 — Docs.**
+- `docs/architecture.md`: a component row plus a provider-health section
+- `docs/kvm-provider.md`: the reason table, the shared-connection behaviour, and the
+  Secret-rotation gap under "Known gaps"
+- `docs/testing.md`: the new tests and the commands
+- `CLAUDE.md`: its Architecture and Gotchas sections still say "empty scaffold"; point
+  them at `docs/architecture.md`
+
+**Step 7 — Validate, record, push, open the PR.**
+- Run `go vet ./...`, `go test -short ./...`, `make test` and `make test-e2e`.
+- Hand the user the KVM commands below, and record their results in the execution log.
+- Add a CHANGELOG entry once the user confirms.
+- Push, using the token-file credential helper from memory.
+- Open the PR against `main` through the Gitea API (`tea` isn't installed).
+- Don't merge.
+
+## Tests
+
+**Provider checks** (table-driven, `Test<Func>_<scenario>`):
+- kubernetes:
+  - reachable
+  - list options: `Limit==1`, the label selector, no namespace
+  - failure reasons: forbidden, unauthorized, timeout, cancelled, internal, and a
+    network `*url.Error`
+- gcp:
+  - reachable, asserting the request fields
+  - failure reasons: 401, 403, 500, deadline, network
+  - `TestRealInstances_Probe_fetchesOnePage` against `httptest`: exactly one request with
+    `maxResults=1`, even though the page has a `nextPageToken`
+- kvm (using `hostFakes`/`newTestProvider` from `kvm_test.go`):
+  - reachable, with details filled in
+  - per-host verdicts, order preserved
+  - pool inactive, pool missing
+  - hosts checked concurrently (a sequential check would deadlock)
+  - a pool verdict keeps the connection (the dial count stays 1)
+  - a `checkReason` table
+  - `TestSSHAuth_dial_stages`, using the in-process SSH server in `conn_test.go`
+  - three new rows in the tracing table
+
+**Monitor:**
+- Setup: the fake client is built with a status subresource and a RESTMapper that
+  registers ProviderHealth as cluster-scoped.
+- Tests:
+  - `buildStatus` table
+  - objects are created with the label, and existing ones are patched
+  - the timeout bounds a checker that blocks
+  - a nil checker gives Unknown
+  - only transitions are logged (`funcr` logger)
+  - cleanup deletes only labelled objects that aren't configured
+  - the disabled path cleans up and returns
+  - an invalid name skips the writes
+  - `ValidateSchedule` table
+- An envtest round-trip, skipped under `-short`, which proves the generated schema is
+  accepted by a real apiserver.
+
+**Metrics:** `TestRegisterProviderTargets_scrape`, using a fresh registry and
+`GatherAndCompare`; an empty snapshot produces no series.
+
+**e2e (kind):** `providerhealth kubernetes` reaches `Ready=True`, which proves the
+generated RBAC works end to end.
+
+## Verification
+
+**Automated (Claude runs):** `go vet ./...`, `go test -short ./...`, `make test` (envtest
+with `-race`) and `make test-e2e`.
+
+**Against the real hypervisor (handed to the user, not run by Claude)**, with beelink as
+the host:
+
+```bash
+KVM_TEST_HOST=beelink.tail8c69c.ts.net:22 KVM_TEST_USER=novakj KVM_TEST_SSH_KEY=~/.ssh/id_ed25519 \
+KVM_TEST_KNOWN_HOSTS=~/.ssh/known_hosts KVM_TEST_BRIDGE=<bridge> KVM_TEST_IMAGE_URL=<image-url> \
+  go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/
+
+make install && go run ./cmd/main.go --providers-config <scratch copy of kvm-home config, local key paths> \
+  --metrics-bind-address :8080 --metrics-secure=false --provider-check-interval=15s
+kubectl get ph -w
+kubectl get ph kvm-home -o yaml
+curl -s localhost:8080/metrics | grep -E 'proxy_operator_provider_(target_up|check_)'
+```
+
+Each negative case is a one-line edit to the scratch config; nothing changes on the host.
+
+| Edit | Expected reason |
+|---|---|
+| `pool: nope` | `PoolNotFound` |
+| `address: …:2` | `Unreachable` |
+| `knownHostsFile: /dev/null` or `user: nobody` | `SSHHandshakeFailed` |
+| `sshKeyFile: /nonexistent` | `SSHMaterialUnavailable` |
+| `socket: /nonexistent` | `LibvirtConnectFailed` |
+
+## Out of scope
+
+- Gating Proxy creates on a host that is known to be down.
+- Tying `/readyz` to provider health.
+- Bridge checks.
+- SelfSubjectAccessReview checks of create/delete rights.
+- Hot-reloading the providers config.
+- Picking up a rotated `kvm-ssh` Secret without a restart. This is an existing gap; it
+  gets documented.

docs/plans/2026-09-16-0841-provider-blueprint.md

@@ -0,0 +1,119 @@
+# Plan: Provider blueprint (docs/providers.md)
+**Created:** 2026-09-16 08:41
+
+## Context
+
+The provider contract is spread across `docs/prompts/__initial-prompt.md` §5,
+the comments in `internal/provider/{provider,errors,check}.go`, `docs/architecture.md`
+§5 and §11, `docs/kvm-provider.md`, and plan Step 2/3/8. Many rules exist only in
+code: providerID must be self-contained, UID/CreatedAt must be read back, Create
+must survive a crash partway through, calls must stop when ctx is cancelled,
+decorators are wired in `cmd/main.go`, and so on. With "~5 providers soon", the
+next provider author needs one blueprint. It should say what a provider MUST,
+SHOULD or MAY do, show which features each existing provider (kubernetes, gcp,
+kvm) actually implements (with file:line evidence), give a checklist for adding
+one, and list the gaps.
+
+Decisions (user): store in `docs/providers.md`; scope = current state plus a
+separate "Gaps and proposed features" section (nothing implemented); wait for
+`feat/provider-health` to merge, then branch off `main`.
+
+## Step 0 — Save the plan; wait for the merge
+
+- Right after ExitPlanMode, copy this plan to `docs/plans/2026-09-16-0841-provider-blueprint.md`
+  and commit it on its own. The current branch is `feat/provider-health`, so the
+  plan reaches `main` with that merge. CLAUDE.md requires the plan commit to
+  come before any branching.
+- Stop and wait for the user to confirm that `feat/provider-health` is merged.
+  Then `git switch main && git pull`, and `git switch -c feat/provider-blueprint`.
+
+## Step 1 — Collect evidence (read-only, in parallel)
+
+Run 3 Explore agents at the same time, one each for `internal/provider/kubernetes`,
+`internal/provider/gcp` and `internal/provider/kvm`, including their tests. Each
+answers the **same questionnaire**, citing `file:line` for every answer and writing
+"not implemented" explicitly when that's the case:
+
+1. **Identity**: providerID format and parsing; what it carries (zone, host, namespace).
+2. **Create**: how it stays idempotent (name lookup, 409, DomainLookupByName);
+   whether it returns before the VM is up or waits; what happens when an earlier
+   attempt crashed partway; which Placement fields are used or ignored; how
+   cloud-init is delivered; how the port is used; how GC labels/UID are attached.
+3. **Get**: native state → InstanceState mapping; IP discovery rules; UID and
+   CreatedAt readback; how NotFound is detected.
+4. **Delete**: idempotency; what gets removed (disks, volumes, seeds); returns
+   before or after the resource is gone; unknown/malformed providerID behaviour.
+5. **ListByTag**: scope (namespaces, zones, hosts); partial-failure behaviour; how
+   managed resources are recognised.
+6. **Error classification**: table of backend error → NotFound/Quota/Transient/
+   Permanent; whether quota is detected at all.
+7. **Health check** (`Checker`): targets, provider-specific Reasons, Details keys.
+8. **Config**: YAML section, defaults, validation (`internal/provider/config*.go`),
+   credentials/Secrets, which settings are read once per process.
+9. **Connections and cancellation**: pooling, reconnects, timeouts, ctx honouring.
+10. **Observability**: log keys and verbosity levels, spans (`kvm/trace.go`, gcp
+    `wirelog.go`), anything beyond the shared metrics/tracing decorators.
+11. **Testing**: fakes, envtest/kind, `//go:build integration` tests, what is
+    not covered.
+12. **Known gaps**: TODOs, stale comments, documented limitations.
+
+Meanwhile, read the shared layer myself: `internal/provider/{provider,errors,check,
+name,config,metrics,tracing}.go`, `internal/provider/registry/registry.go`,
+`cmd/main.go` (registry wiring, decorator order, Checker collection), and the
+places that consume the contract, so every MUST rule traces to a caller that depends on it:
+`internal/controller` (error-class branching, ErrNotFound → replace/adopt),
+`internal/gc` (UID, CreatedAt age skip, ListByTag errors),
+`internal/providerhealth`. Then cross-check against the existing docs listed in Context.
+
+## Step 2 — Write `docs/providers.md`
+
+Sections:
+
+1. **What a provider is**: a few sentences plus links to architecture.md §5/§11.
+2. **The contract**: per method (Create/Get/Delete/ListByTag), MUST/SHOULD/MAY
+   rules, each followed by *why*: the reconciler or GC behaviour it protects,
+   linked to code.
+3. **Error taxonomy**: what each class makes the reconciler do; how to map backend
+   errors; "unclassified = transient" pitfall.
+4. **Identity and GC labels**: providerID guidance, `LabelManaged`/`LabelUID`,
+   CreatedAt, deterministic `NameFromUID`.
+5. **Optional interfaces**: `Checker` (shared reasons, never fails as a whole,
+   collected before decorators).
+6. **Feature matrix**: one row per questionnaire item, columns kubernetes/gcp/kvm,
+   cells `yes`/`partial`/`no` plus a short note. Put the file:line evidence in a
+   compact per-provider notes subsection under the table, not in the cells.
+7. **Adding a provider: checklist**: package layout, config section and validation,
+   registry constructor, `cmd/main.go` wiring, Checker, error mapping with tests,
+   `name`/label tests, integration test behind a build tag, provider doc page,
+   architecture.md rows, demo script (optional).
+8. **Gaps and proposed features**: current-state gaps per provider (e.g. no kvm
+   quota detection, orphan volumes invisible to GC, Secret rotation needs a
+   restart, ListByTag partial-failure inconsistency, stale `mock` references in
+   `provider.go` comments), plus proposals that are clearly marked as not
+   implemented. Only list them; don't design them.
+
+Also: link it from `docs/architecture.md` (Components table → provider contract row)
+and from the CLAUDE.md "Architecture" section. Leave per-provider docs where they are.
+
+Follow the plan-execution convention: create `docs/plans-executions/2026-09-16-0841-provider-blueprint.md`
+with the status checklist and step entries, and commit it with each step.
+
+## Step 3 — Verify, then open the MR
+
+- A verification agent (Explore) re-checks each matrix cell and each MUST rule
+  against the cited file:line and reports any mismatch. Fix what it finds.
+- Check that every relative link in the new and edited docs resolves (grep the
+  `](` targets and `ls` each one).
+- `go build ./... && go test -short ./...` as a sanity check (docs-only change,
+  expected no-op).
+- Push `feat/provider-blueprint` and open the MR against `main` via the Gitea API,
+  following the project's push/MR memory note. After the user confirms, add a CHANGELOG.md entry.
+
+## Critical files
+
+- New: `docs/providers.md`, `docs/plans/2026-09-16-0841-provider-blueprint.md`,
+  `docs/plans-executions/2026-09-16-0841-provider-blueprint.md`
+- Edited: `docs/architecture.md` (link), `CLAUDE.md` (link)
+- Read-only sources: `internal/provider/**`, `cmd/main.go`, `internal/controller`,
+  `internal/gc`, `internal/providerhealth`, `docs/prompts/__initial-prompt.md`,
+  `docs/kvm-provider.md`, `docs/gcp-vm-validation.md`

docs/plans/2026-09-16-1108-provider-docs-split.md

@@ -0,0 +1,99 @@
+# Plan: Split the provider blueprint into contract, appendix, template and index
+
+**Created:** 2026-09-16 11:08
+
+## Context
+
+`docs/providers.md` is one 731-line blueprint that mixes the generic provider contract with how kubernetes, gcp and kvm happen to meet it, plus gaps and proposals. The user wants to hand an agent a self-contained set of rules for building a new provider (Proxmox is next), so the generic part must stand alone, state explicitly where providers are allowed to differ (a pod is ready at once, a VM has a provisioning window, kvm blocks in Create, gcp does not), and leave implementation detail to a separate appendix. A third file, a fill-in spec template, lets the user write the new provider's specifics in the same vocabulary. `docs/providers.md` becomes a short index so every existing link keeps working.
+
+Decisions already made by the user: two new files plus a template; `providers.md` becomes an index; the contract is fully abstract (never names an existing provider); work continues on the checked-out branch `feat/provider-blueprint`.
+
+Conventions for all four files: code references are `path` plus function or symbol name, never line numbers; rule keywords MUST / SHOULD / MAY as today, and every MUST names the caller and what breaks; no em-dashes in prose. The contract cites only shared-layer files (`internal/provider/*.go`, `internal/provider/registry`, `internal/controller`, `internal/gc`, `internal/providerhealth`, `internal/tracing`, `cmd/main.go`, `config/manager`, `api/v1alpha1`) and never `internal/provider/{kubernetes,gcp,kvm}`.
+
+## Facts verified against the code that the new contract must carry
+
+- Callers put no deadline on the context. Only `Monitor.checkOnce` (`internal/providerhealth/monitor.go`) uses `WithTimeout`; the reconcile and GC contexts are cancel-only. `Sweeper.sweep` (`internal/gc/gc.go`) loops providers sequentially, so one hung `ListByTag` stalls orphan GC for every provider.
+- The reconciler's Stopped/Terminated branch in `reconcileManaged` (`internal/controller/proxy_controller.go`) deletes and requeues without clearing `ProviderID`, so a backend that reports a freshly created, not-yet-started instance as Stopped gets it deleted right after creation. kvm avoids this only by blocking `Create` through start.
+- `Sweeper.sweep` treats a zero `CreatedAt` as collectable now, skips `UID == ""`, and skips anything younger than `MinAge` (10m).
+- `cmd/main.go` type-asserts `provider.Checker` on the unwrapped provider before `WithTracing(WithMetrics(...))`; `registry.Build` fails the whole manager on a constructor error, so constructors must not dial.
+- Manager runtime: non-root, read-only rootfs, `/tmp` emptyDir 64Mi, 128Mi limit, distroless static image with CA roots (`config/manager/manager.yaml`, `Dockerfile`).
+- Corrections to earlier notes, for the appendix: kvm Checker `Details` also carry `user` and `pool`; the kubernetes provider has no seam interface (it injects `client.Client` via `newWithClient` and tests use the fake client); kvm `classify` has an `isClassified` pass-through for errors already wrapped with a class.
+
+## Files
+
+### 1. `docs/provider-contract.md` (new, the generic contract)
+
+Every behaviour axis gets a stable kebab-case code in backticks, used verbatim by the appendix table and the template so the three files can be cross-checked by grep.
+
+Sections:
+
+1. **Preamble**: audience, reading order (contract, template, appendix), rule keywords, "path plus symbol, never line numbers", where the rest lives.
+2. **What a provider is and who calls it**: the types in `provider.go`, `name.go`; callers and entry points (`reconcileManaged`, `replaceInstance`, `reconcileDelete`, `providerFailure`, `deletionFailure`, `placementFrom`, poll intervals, `MaxConcurrentReconciles`; `Sweeper.sweep` and `MinAge`; `Monitor.checkOnce`; decorators; `registry.Build`); what the instance is for (health probe CONNECT in `internal/health`, discovery handing out ip and port); leader-only vs every-replica; concurrency (MUST be safe, including two calls for one name); runtime envelope; time and cancellation (no caller deadline, MUST honour cancel, SHOULD bound each backend call, what breaks: a pinned worker, a stalled GC sweep, a wedged check round).
+3. **The contract, method by method**: current §2 rewritten abstractly, plus the new points. Create: idempotent by name, mark on create or within a documented window shorter than `MinAge`, self-contained ID, ctx, blocking vs submit, reject-early with `ErrPermanent` naming field and value, catalogue changes are outside the spec hash (`spechash.go`), MAY ignore Placement fields but MUST document, SHOULD deliver CloudInit, MUST NOT log it, and an eventually consistent backend MUST NOT let `Get` answer NotFound for an ID `Create` just returned. Get: NotFound only when gone, keep answering during delete, Running only with IP, Stopped/Terminated only when it will not recover, the "defined but not started" rule (block Create through start, or report Provisioning using written metadata or a task handle), behaviour during async operations and locks (last known state or Transient, never NotFound or Permanent), UID and CreatedAt read back, Permanent for an unresolvable ID. Delete: idempotent, re-entrant while deleting, removes secondary resources, MAY return before gone, SHOULD succeed where Get fails permanently. ListByTag: marked instances only with ID, UID, CreatedAt; error rather than an unvouched list; MAY omit unreadable parts; SHOULD cover the whole creatable scope; no installation identity.
+4. **Behaviour profiles**: one subsection per axis with the question, allowed profiles, what callers tolerate, obligations per profile. Axes: `readiness`, `create-return`, `delete-return`, `ip-source`, `gc-mark`, `created-at`, `list-partial`, `quota`, `not-found`, `primary-key`, `provisioning-state`, `async-ops`, `secondary-resources`, `credentials`, `client-kind`, `timeouts`, `logging`, `child-spans`, `backend-test`. Closing table: code, profiles, tolerated by which caller, template entry.
+5. **Error taxonomy**: `Wrap`, `Class`, `Error` fields, `Op` values, configured name not type; the class-to-reaction table; pitfalls; NotFound detection guidance (404, error code, list-and-miss, and why list-and-miss is forbidden with a partial list); the quota-vs-permission 403 case stated abstractly.
+6. **Identity, names, marks, CreatedAt**: `NameFromUID` shape, providerID strict parsing, marks and written metadata, provider name as DNS-1123 (`Monitor` skip, `spec.provider` MaxLength), primary key not the name.
+7. **Credentials, TLS and secret hygiene** (new): source and read time (constructor MUST NOT dial, SHOULD warn on a missing optional mount and fail at first use), never in logs, spans, error strings or `Error.Err`, TLS trust options and the naming of an insecure toggle, scratch only under `/tmp`.
+8. **Optional interfaces: Checker**: current §5 rules; `status.type` comment in `api/v1alpha1/providerhealth_types.go` must list the new type and the CRD be regenerated.
+9. **Observability**: the three layers, span and metric names, ctx and logger rules, child-span strategies by `client-kind`, no secrets or CloudInit in attributes, log tiers.
+10. **Configuration and wiring**: `ProviderConfig`, `setBlocks`, the allow-list in `Config.validate`, `UnmarshalStrict`, `config_<type>.go`, identifiers pasted into IDs, `registry.Constructor`, the map in `cmd/main.go`, decorator order, the duplicated type list.
+11. **Testing obligations**: fake behind a seam; the contract test list (current checklist item 5 plus Get during an async operation, the not-yet-started state, NotFound after Create is forbidden); classify table test; Checker tests; sentinel tests for CloudInit and credentials; span recorder test; integration test behind `//go:build integration` with `<TYPE>_TEST_*` variables, `make test-integration-<type>`, a section in `docs/testing.md`.
+12. **Checklist for adding a provider**: current §8 generalised, plus: Makefile target, testing.md section, `hack/providers-dev.yaml` decision, `ProviderHealth` type comment and `make manifests`, RBAC diff, CHANGELOG, appendix row and section, the filled template committed as `docs/<type>-provider-spec.md`, README row, architecture.md rows.
+13. **Known limits of the contract**: adoption unverified; Stopped and Terminated indistinguishable; config read once; no installation identity in GC; unconfigured provider latches; type list duplicated; Delete resubmitted every poll; no caller deadline; no conformance suite.
+14. **Related documents**.
+
+### 2. `docs/provider-spec-template.md` (new, the questionnaire)
+
+Preamble: copy to `docs/<type>-provider-spec.md`, fill every Answer line, hand it to the implementing agent with the contract and appendix; unanswered entries block implementation. Header block: type, example configured name, backend and API version, SDK or thin client, author, date.
+
+Fixed entry format: Question; Allowed answers (profile names linking to the contract §4 subsection); What each answer obliges; `Answer:` blank; `Evidence:` blank.
+
+Entries: identity and providerID shape; primary key (`primary-key`); placement mapping (Region, Zone, MachineType, Image, ignored fields); image source; cloud-init delivery; machine type mapping; network and IP discovery (`ip-source`, `readiness`); GC marks and CreatedAt (`gc-mark`, `created-at`); state mapping table incl. not yet started (`provisioning-state`); Create blocking and task tracking (`create-return`, `async-ops`); Delete and secondary resources (`delete-return`, `secondary-resources`); ListByTag scope and partial failure (`list-partial`); error classification table (`not-found`, `quota`); credentials and TLS (`credentials`); config block fields, defaults, validation; Checker targets, reasons, Details; SDK vs thin client and child spans (`client-kind`, `child-spans`); timeouts (`timeouts`); concurrency; logging tiers (`logging`); tests (`backend-test`); deployment (mounts, samples, Makefile helper, RBAC, dev config); docs; known limits accepted at design time.
+
+### 3. `docs/providers-existing.md` (new, the appendix)
+
+Sources: current `providers.md` §6 table, §7 matrix and per-provider notes, §9 per-provider gaps and proposed features, `docs/kvm-provider.md`, the gcp and kubernetes package facts. Links to architecture.md §5, §11 and Decisions, and to the README table, rather than duplicating them.
+
+Sections: preamble; **profile table** with exactly one row per contract axis code and one column per existing provider, each cell a profile name plus a symbol as evidence; **other differences** (matrix rows that are not axes: Placement used, Port used, CloudInit delivered, states produced, Error.Provider correctness, credential rotation, connection handling, Checker Details, fakes, proxy software, reachability, distinct egress, adopted-UID check); **observability status** (current §6 table); one section each for **kubernetes**, **gcp**, **kvm** with the same sub-headings: package layout and seam, config block, ID format, Create, Get and state mapping table, IP source, Delete, ListByTag, classify table, Check (targets, reasons, Details), logging, tracing layer, tests and fakes, deployment bits, known gaps; **gaps shared by the existing providers**; **proposed features** (current list, unchanged).
+
+### 4. `docs/providers.md` (rewritten in place as the index)
+
+One paragraph on the split; one line each for contract, template, appendix; the per-provider guides (`docs/kvm-provider.md`, `docs/gcp-vm-validation.md`, `docs/gcp-in-specific-project.md`); the architecture.md sections; reading order for a new provider; a note that the pre-split blueprint is in git history under this path.
+
+### 5. One-line link edits
+
+- `README.md` "### Adding a provider": point at the contract, appendix and template, with `docs/providers.md` named as the index. Keep the "Start there" sentence.
+- `CLAUDE.md` "## Architecture": same replacement for the providers.md sentence.
+- `docs/architecture.md` components row "Provider contract": rules in `provider-contract.md`, profile table in `providers-existing.md`.
+- Historical files under `docs/plans*` keep their links; they resolve to the index.
+
+## Steps (one commit per step, on `feat/provider-blueprint`)
+
+- **Step 0**: copy this plan to `docs/plans/<ts>-provider-docs-split.md` with the real timestamp and commit it alone, before anything else.
+- **Step 1**: write `docs/provider-contract.md`; freeze the axis code list. Create `docs/plans-executions/<ts>-provider-docs-split.md` with the status checklist.
+- **Step 2**: write `docs/provider-spec-template.md` from contract §4 and §12.
+- **Step 3**: write `docs/providers-existing.md`, replacing every `file:line` from the old blueprint with `file` plus symbol.
+- **Step 4**: rewrite `docs/providers.md` as the index; apply the three link edits.
+- **Step 5**: run the verification below; fix findings.
+- **Step 6**: final execution summary entry. CHANGELOG entry at the top only after the user confirms. Push and MR only when the user asks (per memory, push uses the token helper and the MR goes through the Gitea API).
+
+## Verification (read-only shell, from the repo root)
+
+1. Every MUST names a caller: `grep -n MUST docs/provider-contract.md` and confirm each bullet cites a shared-layer path or symbol and states what breaks.
+2. Every cited path exists: extract `(internal|cmd|config|docs|api|hack|test)/...` tokens from the four files and `test -e` each.
+3. Every cited symbol exists: extract backticked identifiers, drop paths, keywords and YAML keys, and `grep -rqw` each across `*.go`, `Makefile`, `*.yaml`.
+4. Contract is abstract: `grep -niE "kubernetes|gcp|kvm|libvirt|squid|google|compute engine|internal/provider/(kubernetes|gcp|kvm)" docs/provider-contract.md` is empty.
+5. Profiles line up 1:1: for each axis code from contract §4 headings, it appears in the appendix table and in the template; the appendix table has one row per code.
+6. Links resolve: extract `](target)` from the four files plus README, CLAUDE.md and architecture.md; check file targets exist and anchors match slugified headings.
+7. No em-dashes in the four files or in the added lines of the three edited files.
+8. No line references: `grep -nE '\.go:[0-9]+|:[0-9]+-[0-9]+' <four files>` is empty.
+9. Nothing lost: compare the gap and proposal bullets of `git show HEAD:docs/providers.md` against the appendix and contract §13 by keyword.
+10. `go build ./... && go test -short ./...` still pass (docs-only change).
+
+## Execution summary must record
+
+- Step 1: the frozen axis list; the two contract additions found in verification (sequential GC sweep; the Stopped branch re-deleting a not-yet-started instance); per-call timeout facts as verified for each existing provider.
+- Step 3: the fact corrections applied (kvm Details keys, kubernetes seam, kvm `isClassified`); matrix rows dropped in favour of the profile table.
+- Step 4: the exact link edits.
+- Step 5: the verification commands as run with trimmed output.
+- Worth noting: CHANGELOG headings keep the repo's own em-dash format; the stale `mock` comments in `provider.go`, `registry.go` and `proxy_types.go` stay listed as gaps and are not fixed here.

docs/provider-contract.md

@@ -0,0 +1,860 @@
+# The provider contract
+
+This document is the specification a provider must meet. It is written for
+whoever builds the next one, human or agent, and it deliberately says nothing
+about how any existing provider is built. Those details are in
+[providers-existing.md](providers-existing.md); the questionnaire a new
+provider's design must answer is [provider-spec-template.md](provider-spec-template.md).
+Read the three in that order: this contract, then the template, then the
+appendix for worked examples.
+
+Rule keywords: **MUST** means breaking the rule breaks a caller, and every MUST
+below names the caller and what breaks. **SHOULD** means every current provider
+does it and a new one needs a written reason not to. **MAY** means it is
+allowed and the choice must be recorded in the provider's spec.
+
+Code references are a file path plus a function or symbol name, never a line
+number. Where a rule is enforced by a caller, that caller is one of:
+
+- the **reconciler**, `internal/controller/proxy_controller.go`;
+- **orphan GC**, `internal/gc/gc.go`;
+- the **provider health monitor**, `internal/providerhealth/monitor.go`;
+- the **decorators**, `internal/provider/metrics.go` and
+  `internal/provider/tracing.go`;
+- the **composition root**, `cmd/main.go`.
+
+## 1. What a provider is and who calls it
+
+### 1.1 The types
+
+A provider is a Go type implementing `provider.Provider`
+(`internal/provider/provider.go`): four methods over one VM-like instance,
+identified by an opaque `providerID` string.
+
+- `Create(ctx, CreateRequest) (providerID, error)`
+- `Get(ctx, providerID) (*Instance, error)`
+- `Delete(ctx, providerID) error`
+- `ListByTag(ctx) ([]Instance, error)`
+
+`CreateRequest` carries `Name` (from `NameFromUID` in `internal/provider/name.go`,
+22 characters, `proxy-` plus base32), `UID` (the owning Proxy's UID),
+`Namespace`, `ProxyName`, `Placement` (`Region`, `Zone`, `MachineType`,
+`Image`), `CloudInit` (already resolved from inline text or a Secret) and
+`Port`.
+
+`Instance` carries `ID`, `IP`, `State` (one of `StateProvisioning`,
+`StateRunning`, `StateStopped`, `StateTerminated`), `UID` and `CreatedAt`.
+
+The GC marks are the constants `LabelManaged` (value `LabelManagedYes`) and
+`LabelUID`, also in `provider.go`.
+
+Providers are named instances of a type: two entries of one type with
+different config blocks are two providers. The configured name is what
+`Proxy.spec.provider` refers to, and it is immutable, so a Proxy never moves
+between providers.
+
+### 1.2 The callers
+
+Nothing above the provider watches backend resources. Everything the operator
+knows about an instance comes from polling `Get`.
+
+- **The reconciler** calls `Create`, `Get` and `Delete` from `reconcileManaged`,
+  `replaceInstance` and `reconcileDelete`, and branches only on the error class
+  (`providerFailure`, `deletionFailure`) and on `Instance.State`. It maps the
+  CRD's placement to `Placement` one to one in `placementFrom`. Its poll
+  intervals are set in `applyDefaults`: `ProvisioningPoll` and `DeletionPoll`
+  10s, `DriftPoll` 2m, `RequeueNow` 1s, `QuotaRetry` 5m. Up to 3 Proxies
+  reconcile at once (`MaxConcurrentReconciles` in `SetupWithManager`).
+- **Orphan GC** calls `ListByTag` and `Delete` from `Sweeper.sweep`, every
+  `Interval` (10m), and never deletes an instance younger than `MinAge` (10m).
+  It loops over providers one after another in a single goroutine.
+- **The provider health monitor** calls the optional `Checker` from
+  `Monitor.checkOnce`, every `--provider-check-interval`, all providers
+  concurrently under one timeout.
+- **The composition root** builds every provider through `registry.Build`
+  (`internal/provider/registry/registry.go`) before the manager starts, on
+  every replica, then wraps each one in `WithTracing(WithMetrics(...))`.
+
+**Where the calls come from.** The reconciler, GC, the health engine and the
+monitor run on the leader only. The constructors run on every replica at
+startup.
+
+**Concurrency.** Up to 3 reconciles, a GC sweep and a health-check round can
+call the same provider at the same time, and GC's `Delete` can overlap the
+reconciler's `Delete` for the same instance. A provider **MUST be safe for
+concurrent use, including two calls for the same instance name**. Nothing
+above the provider serialises them.
+
+### 1.3 What the instance is for
+
+The instance is a forward proxy with its own egress path. Two parties connect
+to `http://<Instance.IP>:<spec.port>`:
+
+- the health engine, from the operator pod, which sends a real `CONNECT`
+  through it (`probe` in `internal/health/probe.go`);
+- clients, which get `ip` and `port` from the discovery API (`viewOf` in
+  `internal/discovery/handlers.go`).
+
+So the IP a provider reports **MUST be reachable on that port from the
+operator's pod network and from the clients**. Whatever that takes on the
+backend (firewall rules, security groups, routing) is either done by the
+provider or documented as a prerequisite in the provider's guide.
+
+The provider does not know which proxy software runs. The proxy is installed
+and configured by the Proxy's `cloudInit`; building install logic into the
+operator is a stated non-goal (`docs/prompts/__initial-prompt.md`, non-goals).
+A backend that cannot take user-data (see `readiness` in §4) documents that
+it ignores the field and how the proxy gets configured instead.
+
+### 1.4 Where it runs
+
+The manager container (`config/manager/manager.yaml`) runs as non-root with a
+read-only root filesystem, a 64Mi `emptyDir` at `/tmp` and a 128Mi memory
+limit, from a distroless static image (`Dockerfile`) that carries the public
+CA roots and nothing else. So a provider:
+
+- **MUST stream large payloads** rather than buffering them in memory;
+- **MUST write scratch files only under `/tmp`**, and delete them;
+- gets credentials either as mounted files (the providers ConfigMap at
+  `/etc/proxy-operator`, optional Secret mounts) or from the environment or
+  the pod's own identity. See §7.
+
+### 1.5 Time and cancellation
+
+**No caller puts a deadline on the context.** The reconcile and GC contexts
+are cancelled on shutdown and on loss of leadership, and otherwise never.
+The only bounded context is the health monitor's (`Monitor.checkOnce`, one
+timeout per round).
+
+- **MUST honour `ctx`**: return promptly once it is cancelled, on every
+  method.
+- **SHOULD bound every backend call itself**: a dial timeout, a per-request
+  timeout, a cap on any wait loop. What breaks otherwise: a hung `Create`,
+  `Get` or `Delete` pins one of the 3 reconcile workers until the manager
+  stops; a hung `ListByTag` stalls orphan GC for every provider, because
+  `Sweeper.sweep` runs them sequentially; a `Check` that ignores its context
+  wedges a monitor round. Choose budgets against the polls: a `Get` that
+  routinely outlives `ProvisioningPoll` gains nothing.
+
+## 2. The contract, method by method
+
+### 2.1 Create(ctx, CreateRequest) (providerID, error)
+
+- **MUST be idempotent by `req.Name`.** A repeat call for a name that already
+  exists returns that instance's providerID with a nil error, creates no
+  duplicate and returns no error.
+  *Caller:* the reconciler records the ID in status only after `Create`
+  returns (`reconcileManaged`). A crash between the two, or a lost status
+  write, runs `Create` again for the same Proxy; `replaceInstance` recovers
+  from a crash between its two writes the same way.
+- **MUST mark the instance as managed**, with `LabelManaged=LabelManagedYes`
+  and `LabelUID=req.UID` or the backend's equivalent, in the same call that
+  creates it. When the backend's create call cannot carry the mark, the
+  window between creation and marking **MUST be far shorter than `MinAge`**
+  and documented in the provider's spec (`gc-mark` in §4).
+  *Caller:* orphan GC finds instances only through the mark and never deletes
+  one whose UID is empty (`Sweeper.sweep`). An instance that exists without
+  the mark leaks if the operator crashes before marking it.
+- **MUST return a providerID that is self-contained.** `Get` and `Delete`
+  receive only the ID, and the config or spec may have changed since
+  `Create` (a zone edit during a replacement). Put whatever `Get` needs to
+  find the instance into the ID, and validate config values that end up in
+  the ID so they cannot contain the separator. See `primary-key` in §4 for
+  backends that allocate their own identifier.
+- **MUST NOT let `Get` answer NotFound for an ID that `Create` just
+  returned.** A backend with eventual consistency between create and read
+  needs a read-after-write guarantee or a retry inside `Create`.
+  *Caller:* on NotFound the reconciler clears the ID and creates again
+  (`reconcileManaged`), so a lagging read makes a duplicate.
+- **MUST honour `ctx`** (§1.5).
+- **MAY return once the request is submitted, or block until the instance is
+  running** (`create-return` in §4). Submit-and-return is preferred: readiness
+  is discovered by polling `Get`, and a blocking `Create` holds one of the 3
+  workers for its whole duration. A provider that blocks MUST still honour
+  cancellation and SHOULD cap the wait.
+- **SHOULD reject a request it can never satisfy** with `ErrPermanent`
+  before touching the backend: a missing or unknown zone, machine type or
+  image, or any placement value not in its catalogue. The CRD cannot validate
+  `placement`, because it is provider-agnostic, so this error is the user's
+  only feedback. The message SHOULD name the offending field and value.
+- **Catalogue changes do not replace instances.** The spec hash
+  (`specHashInput` in `internal/controller/spechash.go`) covers `placement`,
+  the resolved `cloudInit` and `port`, not the provider config. When a
+  placement value names a config entry, changing what that entry means affects
+  only new instances. The provider's guide MUST say so and tell users to add a
+  new key rather than edit one in use.
+- **MAY ignore `Placement` fields that do not apply.** If a field is ignored,
+  the provider's guide MUST say so, because the field is still part of the
+  spec hash: changing it replaces the instance and has no other effect.
+- **SHOULD deliver `CloudInit` as user-data**, byte for byte, without parsing
+  or re-rendering it. A provider that cannot documents that it ignores the
+  field.
+- **MUST NOT log `CloudInit` contents**, put them in span attributes, or
+  include them in error messages. It can hold secrets. Log its byte count.
+  Test this with a sentinel string (§11).
+
+### 2.2 Get(ctx, providerID) (*Instance, error)
+
+- **MUST return `ErrNotFound` when, and only when, the instance is gone.**
+  *Caller:* `ErrNotFound` triggers three actions. In normal reconciling the
+  reconciler clears the ID and creates again (`reconcileManaged`). During
+  replacement it is the signal to create the new instance
+  (`replaceInstance`). During deletion it releases the finalizer
+  (`reconcileDelete`). A false NotFound during an outage creates a duplicate
+  or loses the instance. "Unreachable", "wrong credentials" and "cannot tell"
+  are Transient or Permanent, never NotFound.
+- **MUST keep returning the instance, not NotFound, while a delete is in
+  progress** and the name is still taken.
+  *Caller:* the replacement instance gets the same name. `replaceInstance`
+  waits for NotFound before creating it; a NotFound too early makes `Create`
+  adopt the dying instance.
+- **MUST report `StateRunning` only with a non-empty `IP`.** Running without
+  an address yet is `StateProvisioning`.
+  *Caller:* the IP from a Running answer goes straight into status and to the
+  health engine (`reconcileManaged`).
+- **MUST report `StateStopped` or `StateTerminated` only when the instance
+  will not recover by itself.** SHOULD map states it does not recognise to
+  `StateStopped`.
+  *Caller:* the reconciler reacts to both states identically: it calls
+  `Delete` and requeues after `DeletionPoll` without clearing the ID, then
+  polls `Get` and deletes again until NotFound, then creates
+  (`reconcileManaged`). Instances are disposable, so this is the safe answer
+  for anything unknown, but it makes the next rule critical.
+- **MUST NOT report a freshly created instance that has not been started yet
+  as `StateStopped`.** A backend with no provisioning state of its own, where
+  "defined but not yet started" and "stopped" look the same, MUST resolve this
+  in one of two ways (`provisioning-state` in §4): block `Create` until the
+  instance is started, or record enough at creation (written metadata, a task
+  handle) for `Get` to answer `StateProvisioning`.
+  *Caller:* the same branch of `reconcileManaged` would delete the instance
+  right after it was created, forever.
+- **During an asynchronous operation or a backend lock** (a clone or a delete
+  still running, an instance locked by the backend), `Get` MUST return either
+  the last known state or `ErrTransient`, never NotFound and never
+  Permanent (`async-ops` in §4).
+- **MUST fill `UID` and `CreatedAt` by reading them back from the backend**,
+  never from memory.
+  *Caller:* GC compares `UID` with the live Proxies and deletes anything whose
+  `CreatedAt` is older than `MinAge`. **A zero `CreatedAt` counts as very
+  old** (`Sweeper.sweep`), so a provider that forgets it makes instances still
+  being created collectable. An unparsable timestamp giving the zero time is
+  accepted: it never shields an orphan forever.
+- **SHOULD return `ErrPermanent` for an ID it can never resolve**: a malformed
+  ID, a location no longer in the config.
+  *Caller:* `reconcileDelete` then tries `Delete` anyway and releases the
+  finalizer if that works. If the provider returned Transient, deletion would
+  poll forever.
+
+### 2.3 Delete(ctx, providerID) error
+
+- **MUST be idempotent.** Deleting something already gone returns nil.
+  *Caller:* GC and the reconciler race, and a crash repeats the call.
+- **MUST accept being called again while an earlier delete is still in
+  progress.** Treat "already deleting" as success, and SHOULD keep that cheap.
+  *Caller:* while `Get` still returns the instance, `reconcileDelete` and the
+  Stopped branch of `reconcileManaged` call `Delete` again every
+  `DeletionPoll` until `Get` returns NotFound.
+- **MUST remove everything `Create` made**: disks, volumes, seeds, addresses,
+  uploaded files. Either delete them explicitly or rely on the backend
+  cascading (`secondary-resources` in §4).
+  *Caller:* GC sees only what `ListByTag` returns, so secondary resources
+  have no other cleanup path.
+- **MAY return before the instance is gone** (`delete-return` in §4), as long
+  as `Get` keeps showing it until it is.
+- **SHOULD succeed for an ID whose `Get` fails permanently** when there is
+  nothing left the provider could reach, and log that it did nothing.
+  *Caller:* on the deletion path a permanent error is not latched; it retries
+  forever (`deletionFailure`), and the Proxy is then only released through the
+  `AnnotationSkipCleanup` annotation (`api/v1alpha1/proxy_types.go`).
+
+### 2.4 ListByTag(ctx) ([]Instance, error)
+
+- **MUST return only instances that carry the managed mark**, each with an
+  `ID` that `Delete` accepts, plus `UID` and `CreatedAt`. `State` and `IP`
+  are not used by GC.
+  *Caller:* GC deletes every listed instance whose UID is not live and whose
+  age is past `MinAge` (`Sweeper.sweep`). Listing something without the mark
+  is how an operator deletes an instance it does not own.
+- **MUST return an error rather than a list it cannot vouch for.** An error
+  makes GC skip that provider for this sweep (`Sweeper.sweep`).
+- **MAY leave out instances it could not read.** Leaving something out is
+  always safe, because GC never deletes what is not listed. Pick a policy
+  (`list-partial` in §4) and document it.
+- **SHOULD cover the whole scope the provider can create in**: every
+  namespace, zone, node or host. Anything outside that scope is never
+  collected.
+- **Scope caveat:** GC protects instances by UID, not by installation. Two
+  operator installs sharing one backend delete each other's instances (§13).
+  Two provider entries of one install that share a scope are safe, because
+  they share the live-UID set.
+
+## 3. What a container-style backend and a VM-style backend have in common
+
+The contract above is the same for both, but the profiles in §4 differ.
+A backend that runs a container has an instance that is ready seconds after
+`Create`, gets its address from the platform, is deleted in one call and has
+no secondary resources; a backend that boots a VM has a provisioning window
+of minutes, may discover the address only after the guest reports it, may
+need several deletes, and leaves disks and seeds behind unless told
+otherwise. Neither is preferred. What matters is that the provider's spec
+names its profile on every axis, because the appendix table and the test list
+are keyed by those names.
+
+## 4. Behaviour profiles
+
+Each axis below has a stable code. The provider's spec (the template) answers
+every axis; the appendix maps each existing provider to its profile on every
+axis. A profile is not a permission to skip a MUST in §2; it says which of
+the allowed ways of meeting it the provider chose.
+
+### 4.1 `readiness`
+
+*When is an instance usable after `Create`?*
+
+- **immediate**: the platform starts it at once; `Get` reports Running
+  within a poll or two.
+- **provisioning-window**: minutes of booting; `Get` reports Provisioning
+  for that time.
+
+Callers tolerate both: the reconciler polls at `ProvisioningPoll` either
+way. An immediate provider still MUST report Provisioning until the IP is
+known.
+
+### 4.2 `create-return`
+
+*Does `Create` return once the request is submitted or once the instance is
+running?*
+
+- **submit**: preferred. Readiness comes from `Get`.
+- **blocking**: allowed. Costs one of 3 workers for the duration, and any
+  first-use setup (an image download) runs inside it. MUST honour `ctx`,
+  SHOULD cap the wait, and MUST keep every step idempotent so a cut-off
+  attempt resumes.
+
+### 4.3 `delete-return`
+
+*Does `Delete` return before the instance is gone?*
+
+- **returns-early**: `Get` keeps returning the instance until it is gone
+  (§2.2), and repeated `Delete` calls are cheap.
+- **waits**: `Delete` returns nil only when nothing is left. Bound the wait
+  and honour `ctx`.
+
+### 4.4 `ip-source`
+
+*Where does `Instance.IP` come from?*
+
+- **assigned-at-create**: the backend reports it with the instance.
+- **discovered**: an agent in the guest, a DHCP lease, a neighbour table.
+  Discovery failures mean "no address yet", never an error; `Get` returns
+  Provisioning. The provider's guide MUST say what the guest needs (an agent
+  package, outbound access to install it) and what happens without it.
+
+### 4.5 `gc-mark`
+
+*Where do `LabelManaged` and `LabelUID` live?*
+
+- **native-labels**: labels or tags the backend indexes; `ListByTag` can
+  filter server-side.
+- **written-metadata**: a description, an annotation or a metadata element
+  written by the provider; `ListByTag` lists everything and filters
+  client-side.
+- **mark-window**: the create call cannot carry the mark and a second call
+  writes it. Allowed only when the window is a few seconds, documented, and
+  the unmarked instance is cleaned by the same Proxy's next reconcile (§2.1).
+
+### 4.6 `created-at`
+
+*Where does `CreatedAt` come from?*
+
+- **backend-timestamp**: the backend's own creation time.
+- **written-at-create**: the provider writes the time into the mark
+  metadata in the create call. A backend with no timestamp MUST do this.
+
+### 4.7 `list-partial`
+
+*What does `ListByTag` do when part of its scope is unreachable?*
+
+- **fail-whole**: one unreachable part fails the list; GC skips the provider
+  this sweep. Nothing is collected until every part answers.
+- **omit-unreachable**: unreachable parts are left out and the rest is
+  returned. Orphans in the reachable parts are collected; the provider MUST
+  log what it left out.
+
+Both are safe (§2.4). A single unreadable instance MAY always be skipped.
+
+### 4.8 `quota`
+
+*Can the provider tell "no capacity" from other failures?*
+
+- **detected**: a recognisable quota, rate-limit or out-of-resources signal
+  maps to `ErrQuotaExceeded`.
+- **absent**: capacity errors map to `ErrTransient`. The guide MUST say so,
+  and the `Checker` SHOULD expose a capacity figure in `Details` instead.
+
+### 4.9 `not-found`
+
+*How does the provider know an instance is gone?*
+
+- **status-404**: a distinct not-found status or error code from the
+  backend.
+- **error-code**: a code or message the backend uses for "does not exist",
+  matched exactly and covered by the classify table test with real captured
+  responses.
+- **list-and-miss**: absence from a listing. Allowed only when the listing
+  can never be partial; otherwise a missing part reads as "gone" and §2.2 is
+  broken.
+
+### 4.10 `primary-key`
+
+*Is `req.Name` the backend's key?*
+
+- **name-is-key**: the backend addresses instances by name. `Create` looks
+  the name up, the ID is `<location>/<name>`.
+- **allocated-id**: the backend allocates an identifier and the name is a
+  label. `Create` MUST first find an existing instance by name (idempotency),
+  then allocate. Allocation MUST be safe against a concurrent `Create` for
+  another Proxy: a deterministic identifier derived from `req.UID`, or an
+  allocate-then-verify loop. The ID SHOULD embed the allocated key and the
+  name; `Get` and `Delete` use the allocated key. If instances can move
+  between locations (migration), the ID MUST NOT depend on the location.
+
+### 4.11 `provisioning-state`
+
+*Does the backend have a state for "being created"?*
+
+- **native**: the backend reports a provisioning or staging state, mapped to
+  `StateProvisioning`.
+- **create-blocks**: no such state; `Create` blocks until the instance is
+  started, so `Get` never sees "defined but not started".
+- **marker**: no such state and `Create` returns early; the provider records
+  at creation (metadata or a task handle) what lets `Get` answer
+  `StateProvisioning` for an instance that was never started, and
+  `StateStopped` for one that stopped afterwards.
+
+### 4.12 `async-ops`
+
+*Does the backend run create and delete as asynchronous tasks?*
+
+- **none**: calls complete synchronously.
+- **task-handle**: the backend returns a task the provider can poll. The
+  provider SHOULD keep the handle where `Get` can find it (metadata or the
+  ID) so a failed create can be reported instead of read as NotFound.
+- **backend-lock**: the backend locks the instance during a task. `Get`
+  returns the last known state or Transient while locked (§2.2); `Delete`
+  on a locked instance returns Transient, not Permanent.
+
+### 4.13 `secondary-resources`
+
+*What else does `Create` make, and who removes it?*
+
+- **none**.
+- **cascade**: the backend deletes them with the instance (auto-delete
+  flags). The provider MUST set those flags at create.
+- **explicit**: the provider deletes them by name in `Delete`, tolerating
+  their absence. Resources that can exist without an instance (a crash
+  between creating them and creating the instance) are a documented GC blind
+  spot unless `ListByTag` covers them.
+
+### 4.14 `credentials`
+
+*Where do credentials and trust material come from?*
+
+- **mounted-files**: paths in the config block, pointing into an optional
+  Secret mount (§7).
+- **environment**: standard SDK environment discovery.
+- **workload-identity**: the pod's own identity.
+
+TLS trust for an HTTPS backend with a private CA is one of **system-roots**,
+**ca-file** (a path in the config block) or **insecure** (a boolean whose
+name starts with `insecure`, warned about at startup).
+
+### 4.15 `client-kind`
+
+*What talks to the backend?*
+
+- **vendor-sdk**: put it behind a small interface (the seam) so a fake can
+  stand in.
+- **thin-client**: a `net/http` client written in the provider package.
+  Preferred when no maintained SDK exists (`CLAUDE.md`: stdlib over
+  third-party). It MUST set transport timeouts, MUST carry the backend's
+  status and body in one error type that `classify` reads, and SHOULD wrap
+  its transport for child spans (§9).
+- **rpc-over-ssh** or another non-HTTP transport: one pooled connection per
+  target, a keepalive, and a documented answer to "what does a cancelled call
+  do to the shared connection".
+
+### 4.16 `timeouts`
+
+*How is each backend call bounded?* (§1.5)
+
+- **per-call**: every request has its own deadline, independent of `ctx`.
+- **ctx-only**: the provider relies on `ctx`. Allowed only for backends
+  whose client already enforces a timeout; the spec MUST cite it.
+
+### 4.17 `logging`
+
+*Which log tiers does the provider produce?* The logger MUST come from
+`logf.FromContext(ctx)` at call time (§9).
+
+- **tiered**: V(1) one line per operation outcome and per failed backend
+  call; V(2) request and response detail; V(5) raw wire if there is one.
+- **outcome-only**: V(1) only.
+- **none**: allowed only for a backend whose client logs on its own.
+
+### 4.18 `child-spans`
+
+*How do backend calls appear under the operation span?* (§9)
+
+- **sdk-instrumentation**: the SDK emits spans itself.
+- **transport-wrapper**: a parent-gated `otelhttp` transport.
+- **span-decorator**: a decorator over the seam that starts one span per
+  call.
+- **none**: a documented gap, with log correlation through `traceID` only.
+
+### 4.19 `backend-test`
+
+*How is the provider proven against the real backend?*
+
+- **cluster-e2e**: the kind end-to-end suite.
+- **integration-tag**: a test behind `//go:build integration`, skipped
+  without its environment variables, with a `make test-integration-<type>`
+  target and a section in `docs/testing.md`.
+- **manual-runbook**: a documented by-hand run. Allowed only when the
+  backend cannot be reached from CI or a developer machine.
+
+### 4.20 Summary
+
+| Axis | Profiles | Who tolerates the difference | Template entry |
+|---|---|---|---|
+| `readiness` | immediate, provisioning-window | reconciler polls either way | 7 |
+| `create-return` | submit, blocking | reconciler; worker budget | 10 |
+| `delete-return` | returns-early, waits | reconciler re-calls Delete | 11 |
+| `ip-source` | assigned-at-create, discovered | reconciler gates Running on IP | 7 |
+| `gc-mark` | native-labels, written-metadata, mark-window | GC | 8 |
+| `created-at` | backend-timestamp, written-at-create | GC `MinAge` | 8 |
+| `list-partial` | fail-whole, omit-unreachable | GC | 12 |
+| `quota` | detected, absent | reconciler `QuotaRetry` | 13 |
+| `not-found` | status-404, error-code, list-and-miss | reconciler | 13 |
+| `primary-key` | name-is-key, allocated-id | reconciler idempotency | 2 |
+| `provisioning-state` | native, create-blocks, marker | reconciler Stopped branch | 9 |
+| `async-ops` | none, task-handle, backend-lock | reconciler | 10 |
+| `secondary-resources` | none, cascade, explicit | GC blind spots | 11 |
+| `credentials` | mounted-files, environment, workload-identity | deployment | 14 |
+| `client-kind` | vendor-sdk, thin-client, rpc-over-ssh | tests, tracing | 17 |
+| `timeouts` | per-call, ctx-only | worker and GC budget | 18 |
+| `logging` | tiered, outcome-only, none | operators | 20 |
+| `child-spans` | sdk-instrumentation, transport-wrapper, span-decorator, none | tracing | 17 |
+| `backend-test` | cluster-e2e, integration-tag, manual-runbook | CI | 21 |
+
+## 5. Error taxonomy
+
+Every error a provider method returns **MUST be built with
+`provider.Wrap(class, op, providerName, id, err)`** (`internal/provider/errors.go`).
+`op` is one of `create`, `get`, `delete` or `list`, and `providerName` is the
+**configured name** (`ProviderConfig.Name`), not the type.
+*Caller:* the decorators label metrics and spans with the class
+(`resultLabel`), and the reconciler branches on `provider.Class`.
+
+`Wrap` keeps both `errors.Is(err, class)` and `errors.As` to the backend's
+error type working (`Error.Unwrap` returns both).
+
+| Class | Use it for | During normal reconciling | While deleting |
+|---|---|---|---|
+| `ErrNotFound` | the instance does not exist | Get: recreate or advance (§2.2); other ops: returned as the error, so exponential backoff | Get: release the finalizer |
+| `ErrQuotaExceeded` | quota, rate limit, out of capacity | `Provisioned=False` with reason `QuotaExceeded`, requeue after `QuotaRetry`, not counted as an error (`providerFailure`) | requeue after `QuotaRetry` (`deletionFailure`) |
+| `ErrTransient` | network, 5xx, timeouts, cancellation, anything unknown | error returned, so workqueue exponential backoff | same |
+| `ErrPermanent` | bad config, bad argument, auth or permission denied, malformed ID | **latched**: `Provisioned=False` with reason `PermanentError`, no more provider calls until `metadata.generation` changes (`reconcileManaged`) | error returned and retried forever; a Permanent `Get` also triggers `Delete` (`reconcileDelete`) |
+
+Pitfalls:
+
+- **Unclassified means Transient.** `provider.Class` returns `ErrTransient`
+  for any error not built with `Wrap`. That is the safe default, but a
+  permanent failure that was never classified retries forever, and a quota
+  error backs off far too fast.
+- **Permanent is sticky.** Use it only for errors that editing the Proxy spec
+  or fixing the config can resolve. A permission-denied status that the
+  backend also uses for quota is the hard case: read the backend's reason if
+  it gives one, and if it does not, pick Permanent and document it.
+- **Do not return NotFound from `Create`.** The reconciler treats it as
+  Transient there, but it misleads the metrics and anyone reading logs.
+- **NotFound detection** follows the `not-found` profile (§4.9). A backend
+  that reports "does not exist" as a generic server error needs the exact
+  message or code matched, with real captured responses in the test table.
+- **Classify in one `classify` function** with a table-driven test covering
+  every class, including "not a backend error at all, so Transient".
+
+## 6. Identity, names, marks, CreatedAt
+
+- **Names:** `NameFromUID(uid)` gives `proxy-` plus 16 lowercase base32
+  characters. It is a valid DNS label. A backend with stricter rules SHOULD
+  map it deterministically rather than invent a new scheme, because
+  idempotency depends on the name being the same every time. A backend that
+  keys on an allocated identifier follows `primary-key` (§4.10).
+- **providerID:** opaque to everyone except the provider. Keep it
+  `<location>/<name>`-shaped or `<location>/<allocated-key>`-shaped, parse
+  it strictly, and return `ErrPermanent` for a malformed one (§2.2).
+- **Marks:** `LabelManaged` with value `LabelManagedYes`, and `LabelUID`.
+  Use native labels or tags where the backend has them; otherwise write them
+  into metadata in the create call (§4.5).
+- **`CreatedAt`:** the backend's creation timestamp, or written into the mark
+  metadata at creation (§4.6).
+- **Provider names:** a configured `name` MUST be a lowercase DNS-1123 label
+  of at most 63 characters (`spec.provider` in
+  `api/v1alpha1/proxy_types.go`). The `ProviderHealth` object is named after
+  the provider; `Monitor.Start` warns and `Monitor.publish` skips the object
+  for a name that is not a valid object name. Config loading does not check
+  this yet (§13).
+
+## 7. Credentials, TLS and secret hygiene
+
+- **The constructor MUST NOT dial the backend.** `registry.Build` aborts the
+  whole provider set on a constructor error, and the manager exits: one
+  unreachable backend must not take the other providers down. Fail in the
+  constructor only on config errors.
+- **Credential files SHOULD be read lazily.** A mount marked `optional: true`
+  is empty when the Secret is missing; the constructor logs a warning and the
+  first call fails Transient until the Secret lands. A file that exists but
+  does not parse is Permanent.
+- **Credentials MUST NOT appear in logs, span attributes, error strings or
+  `Error.Err`.** Backend errors that echo a request URL or header with a
+  token in it MUST be sanitised before wrapping. The same holds for
+  `CloudInit` (§2.1). Test both with a sentinel (§11).
+- **TLS trust** follows `credentials` (§4.14). An insecure toggle MUST be
+  named so that it reads as insecure, default off, and MUST be logged as a
+  warning at startup when on.
+- **Rotation.** State in the spec whether rotating the Secret takes effect
+  without a restart. Reading files on every dial makes rotation free; caching
+  them after first load needs a restart and must say so.
+- **Scratch files** only under `/tmp` (§1.4), removed when the call ends.
+
+## 8. Optional interfaces: `Checker`
+
+`provider.Checker` is `Check(ctx) []TargetStatus` (`internal/provider/check.go`).
+`cmd/main.go` type-asserts it on the **unwrapped** provider before adding the
+decorators, and the monitor treats a nil Checker as unsupported, not failed
+(`Monitor.run`).
+
+- **MUST never fail as a whole.** Every failure is a `TargetStatus` with
+  `Ready=false`.
+  *Caller:* `Monitor.checkOnce` has no error path per provider.
+- **MUST return promptly once `ctx` ends.** The monitor gives the whole round
+  one timeout.
+- **MUST return one entry per target, in a stable order.** Targets are what
+  can fail independently: an apiserver, a project, each host or node.
+  *Caller:* the monitor logs transitions per target (`logTransitions`) and
+  the gauges are labelled `{provider,target}`.
+- **MUST NOT disrupt reconciles.** A check that shares a connection with
+  provisioning must not tear that connection down when it times out.
+- **SHOULD use the shared reasons** (`ReasonReachable`, `ReasonTimeout`,
+  `ReasonUnreachable`, `ReasonUnauthorized`, `ReasonForbidden`,
+  `ReasonAPIError`). Add provider-specific reasons only for failures an admin
+  would fix differently, and document each one in the guide.
+- **SHOULD check with the same credentials, verb and scope that `ListByTag`
+  and `Create` use**, and be cheap: one page, one object.
+- **MAY fill `Details`** with facts useful when triaging: versions, capacity,
+  the account in use.
+- The `status.type` field of `ProviderHealth`
+  (`api/v1alpha1/providerhealth_types.go`) lists the known types in its doc
+  comment; add the new type there and run `make manifests`.
+
+## 9. Observability
+
+Tracing is OpenTelemetry, set up in `internal/tracing` from the standard
+`OTEL_*` environment variables. With no exporter configured, spans are no-ops
+and logs are unchanged ([architecture.md §10](architecture.md#10-tracing-internaltracing)).
+
+**Layer 1, operation spans and metrics: nothing to write.** `cmd/main.go`
+wraps every provider in `WithTracing(WithMetrics(...))`:
+
+- `WithTracing` opens one client span per call, named `provider.create`,
+  `provider.get`, `provider.delete` or `provider.list`, with
+  `provider.name`, `provider.id` and `provider.result` (the error class) as
+  attributes. `ErrNotFound` keeps the span status Ok, because polling to
+  NotFound is expected.
+- `WithMetrics` counts each call as
+  `proxy_operator_provider_requests_total{provider,op,result}`
+  (`internal/metrics/metrics.go`). A `Checker` feeds
+  `proxy_operator_provider_target_up`,
+  `proxy_operator_provider_check_duration_seconds` and
+  `proxy_operator_provider_check_timestamp_seconds`, labelled
+  `{provider,target}`.
+
+Rules:
+
+- **MUST NOT open its own per-operation span or count its own requests.**
+  They would duplicate the decorators.
+- **MUST pass the caller's `ctx` to every backend call and take the logger
+  from `logf.FromContext(ctx)` at call time.** `tracing.StartSpan`
+  (`internal/tracing/logger.go`) puts a logger carrying `traceID` and
+  `spanID` into `ctx`. A logger cached at construction, or a backend call
+  made with `context.Background()`, loses the trace.
+- **Health checks bypass the decorators.** A `Checker` runs under the
+  monitor's `providerhealth.check` span, and the same `ctx` rules apply.
+
+**Layer 2, child spans per backend call: SHOULD.** One operation span says
+that `Create` took 40 seconds; child spans say which backend call took them.
+Follow `child-spans` (§4.18):
+
+- HTTP backends: wrap the transport with `otelhttp`, gated on an existing
+  parent span as `tracing.RestConfigWrapper` does
+  (`internal/tracing/transport.go`), so background traffic (token refreshes,
+  keepalives) does not become root spans. Check first whether the SDK already
+  does this.
+- Non-HTTP backends: start child spans yourself with
+  `tracing.Start(ctx, "<type>.<call>")` around each call and around long
+  steps such as a dial or a download, in a decorator over the seam, not at
+  call sites.
+- **MUST NOT put secrets or CloudInit in span attributes.** Put identifiers,
+  sizes and error codes there.
+
+**Layer 3, logs.** Follow `logging` (§4.17). Log lines join a trace through
+the `traceID` key from layer 1.
+
+## 10. Configuration and wiring
+
+The providers file is parsed by `LoadConfig` (`internal/provider/config.go`)
+with `UnmarshalStrict`, so unknown keys already fail.
+
+- Add a `<Type>Config` struct and a `ProviderConfig` field for it.
+- Add the block to `setBlocks`, so a block that does not match `Type` is
+  rejected.
+- Add the type to the allow-list in `Config.validate`. The same list exists a
+  second time as the constructor map in `cmd/main.go`; both must change
+  (§13).
+- Add required-field validation in its own `config_<type>.go` when it is
+  more than a few lines, with rows in `config_test.go`.
+- **Validate every config value that is pasted into an identifier**: a
+  location name that becomes part of the providerID must not contain the
+  separator; a catalogue key that becomes a file or volume name must be a
+  safe filename.
+- The constructor has the signature `registry.Constructor`:
+  `func(ctx, provider.ProviderConfig) (provider.Provider, error)`. Register
+  it in the map in `cmd/main.go`. Flags a provider needs are defined there
+  too and passed through a `NewWith...` variant, never read from the
+  environment inside the package.
+- Decorators are applied by the composition root, in the order
+  `WithTracing(WithMetrics(p))`, after the Checker has been collected from
+  the unwrapped provider.
+
+## 11. Testing obligations
+
+`CLAUDE.md` applies: stdlib `testing`, table-driven, tests next to source,
+`t.Parallel()` where safe.
+
+- **A fake behind the seam.** Every unit test runs against an in-memory
+  fake of the backend, never the network.
+- **Contract tests**, each against the fake:
+  - repeat `Create` returns the same ID and creates nothing;
+  - `Get` after `Delete` returns `ErrNotFound`;
+  - `Delete` twice succeeds;
+  - malformed ID is `ErrPermanent` from `Get` and `Delete`;
+  - an unknown backend state maps to `StateStopped`;
+  - `StateRunning` requires an IP, and a running instance without one is
+    `StateProvisioning`;
+  - `UID` and `CreatedAt` are read back from the fake, not remembered;
+  - `ListByTag` excludes unmarked instances and instances with an empty UID
+    are still listed with `UID == ""`;
+  - `Get` during an in-progress delete still returns the instance;
+  - `Get` during an asynchronous operation or lock returns the last state or
+    Transient (`async-ops`);
+  - a created but not yet started instance is never `StateStopped`
+    (`provisioning-state`);
+  - a cancelled `ctx` makes every method return promptly.
+- **Classify**: one table test covering every class, every backend code the
+  provider matches, and "not a backend error at all, so Transient". Use
+  captured real responses for message matching.
+- **Checker**: a table test for the reasons, a test that `Check` returns
+  promptly with an expired `ctx`, and a test that it returns one entry per
+  target in config order.
+- **Sentinel tests**: a `CloudInit` containing a sentinel string and a
+  credential containing another never appear in any log line, span attribute
+  or error string.
+- **Tracing**: with an in-memory span recorder
+  (`sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracetest.NewSpanRecorder()))`,
+  as in `internal/provider/tracing_test.go`), the child spans are children
+  of the operation span.
+- **Real-backend test** per `backend-test` (§4.19). It runs the whole
+  lifecycle: create, create again, poll until Running, list, delete, delete
+  again, confirm it is gone and that no secondary resource remains. Keep it
+  compiling with `go vet -tags=integration ./...`.
+
+## 12. Checklist for adding a provider
+
+1. **Spec first.** Copy `docs/provider-spec-template.md` to
+   `docs/<type>-provider-spec.md` and answer every entry. Unanswered entries
+   block implementation.
+2. **Package** `internal/provider/<type>/` with the provider type, a
+   compile-time assertion for `provider.Provider` and one for
+   `provider.Checker`, the seam interface, and a logger built from
+   `logf.FromContext(ctx).WithName("<type>").WithValues("provider", name)`.
+3. **Config** per §10, including `config_test.go` rows.
+4. **Constructor** `New(ctx, provider.ProviderConfig)` registered in
+   `cmd/main.go`; no dialing.
+5. **Contract** per §2, tested per §11.
+6. **Errors** per §5, `classify` plus its table test, `ProviderConfig.Name`
+   in every `Wrap`.
+7. **Checker** per §8, with the `status.type` comment updated and
+   `make manifests` run.
+8. **Tracing and logging** per §9 and §4.17, §4.18.
+9. **Real-backend test** per §4.19, with a `make test-integration-<type>`
+   target and a section in `docs/testing.md`.
+10. **RBAC**: if the provider talks to the cluster's own API, add
+    `+kubebuilder:rbac` markers in their own comment block, never inside a
+    doc comment, then `make manifests` and check
+    `git diff config/rbac/role.yaml`.
+11. **Deployment**: an `optional: true` credentials mount in
+    `config/manager/manager.yaml`; a commented entry in
+    `config/manager/providers_config.yaml` and
+    `config/samples/providers-config.yaml`; a `config/samples/proxy_<type>.yaml`
+    whose cloud-init installs a proxy on the sample image; a Makefile helper
+    for the Secret if there is one; `hack/providers-dev.yaml` only if the
+    type needs no credentials.
+12. **Docs**: `docs/<type>-provider.md` (prerequisites, config reference,
+    placement mapping, what lands on the backend, the health check, logging,
+    known gaps); a row and a section in `providers-existing.md`; a row in the
+    `architecture.md` components table and a block in its §5 and §11
+    (§10 too if the provider adds spans); a row in the README "Implemented
+    providers" table; a `CHANGELOG.md` entry once the user has confirmed it
+    works.
+13. **Optional:** a demo script under `docs/demo/`.
+
+## 13. Known limits of the contract
+
+These are facts a new provider has to live with today. None is designed or
+scheduled.
+
+- **Adoption is not verified.** `Create` adopts any existing instance with
+  the right name without checking its `LabelUID`.
+- **`StateStopped` and `StateTerminated` are indistinguishable** to the
+  reconciler; both mean delete and recreate.
+- **Config is read once per process.** Adding a provider or a location, or
+  rotating credentials a provider caches, needs a manager restart.
+- **GC has no installation identity.** Two installs sharing one backend
+  delete each other's instances, because GC only knows its own live UIDs.
+- **An unconfigured provider latches the Proxy.** A Proxy naming a provider
+  missing from the config is latched `PermanentError`; adding the provider
+  and restarting does not change `metadata.generation`, and `spec.provider`
+  is immutable, so the Proxy stays latched until it is recreated.
+- **Provider names are not validated** as DNS-1123 at config load.
+- **The type list exists twice**: `Config.validate` and the constructor map
+  in `cmd/main.go`.
+- **`Delete` is resubmitted on every deletion poll** until `Get` says
+  NotFound.
+- **Callers set no deadline** (§1.5); each provider bounds itself.
+- **There is no conformance suite.** The §11 contract tests are written per
+  provider; `internal/provider/providertest` is a proposal in the appendix.
+- **Stale comments** in `internal/provider/provider.go`,
+  `internal/provider/registry/registry.go` and `api/v1alpha1/proxy_types.go`
+  still describe a `mock` provider that no longer exists.
+
+## 14. Related documents
+
+- [providers.md](providers.md): the index.
+- [provider-spec-template.md](provider-spec-template.md): the questionnaire.
+- [providers-existing.md](providers-existing.md): how the existing providers
+  meet this contract, and their gaps.
+- [architecture.md §5](architecture.md#5-what-provider-calls-do-in-the-outside-world),
+  [§10](architecture.md#10-tracing-internaltracing) and
+  [§11](architecture.md#11-provider-health-checks-internalproviderhealth).
+- [testing.md](testing.md).

docs/provider-spec-template.md

@@ -0,0 +1,455 @@
+# Provider spec template
+
+Copy this file to `docs/<type>-provider-spec.md`, fill in every **Answer**
+line, and hand the result to whoever implements the provider together with
+[provider-contract.md](provider-contract.md) and
+[providers-existing.md](providers-existing.md). An unanswered entry blocks
+implementation: it is a decision the contract leaves to the provider, and the
+implementer must not guess it.
+
+Where an entry names an axis in backticks, the allowed answers are the
+profiles of that axis in contract §4, and the entry number matches the last
+column of the contract's §4.20 table. **Evidence** is where the answer was
+confirmed: a link into the backend's API documentation, or the experiment
+that was run. "Verify against the target version" is a valid interim
+answer only for the Evidence line, never for the Answer.
+
+## Header
+
+- **Type** (the `type:` value in the providers config, lowercase):
+- **Example configured name**:
+- **Backend and API version this spec was written against**:
+- **Client** (`vendor-sdk` with the module path, or `thin-client`):
+- **Author and date**:
+
+## 1. Identity and providerID
+
+**Question.** What is the providerID's shape, which location parts does it
+embed, and what validates the config values pasted into it?
+
+**Allowed answers.** `<location>/<name>`, `<location>/<allocated-key>`, or a
+location-free `<allocated-key>` (contract §6; required when instances can
+migrate, see entry 2).
+
+**Obligations.** Strict parsing; `ErrPermanent` for a malformed ID from `Get`
+and `Delete`; a regexp on every config value that becomes an ID segment, in
+`config_<type>.go`.
+
+**Answer:**
+
+**Evidence:**
+
+## 2. Primary key (`primary-key`)
+
+**Question.** Is `req.Name` the backend's key, or does the backend allocate an
+identifier?
+
+**Allowed answers.** `name-is-key`; `allocated-id`.
+
+**Obligations.** `allocated-id`: `Create` finds an existing instance by name
+before allocating; allocation is safe against a concurrent `Create` for
+another Proxy (deterministic from `req.UID`, or allocate-then-verify); `Get`
+and `Delete` use the allocated key; state how a collision with an instance
+the operator does not own is detected and reported.
+
+**Answer:**
+
+**Evidence:**
+
+## 3. Placement mapping
+
+**Question.** What do `Placement.Region`, `Zone`, `MachineType` and `Image`
+mean for this backend, which are required, which are ignored, and what
+happens when a value is unknown?
+
+**Allowed answers.** Per field: a config catalogue key, a backend identifier
+passed through, or ignored (contract §2.1, MAY ignore with documentation).
+
+**Obligations.** Unknown values are rejected with `ErrPermanent` naming the
+field and value before any backend call; ignored fields are listed in the
+provider's guide because they are still in the spec hash; an empty value with
+a single-choice catalogue may default, and the guide says so.
+
+**Answer:**
+
+**Evidence:**
+
+## 4. Image source
+
+**Question.** Where does the boot image come from, and what does a catalogue
+edit do to running instances?
+
+**Allowed answers.** A catalogue in the config block (key to URL or backend
+image ID); a backend template or image referenced directly by
+`Placement.Image`; a URL fetched by the provider into the backend on first
+use.
+
+**Obligations.** If the provider fetches or converts images: streaming, no
+buffering (contract §1.4), a completeness marker so a cut-off fetch is redone,
+concurrency control so two Creates fetch once, and a written warning that
+replacing an image in use may corrupt instances that depend on it. If the
+backend owns the image: the prerequisite is documented in the guide.
+
+**Answer:**
+
+**Evidence:**
+
+## 5. Cloud-init delivery
+
+**Question.** How does `CreateRequest.CloudInit` reach the guest, byte for
+byte?
+
+**Allowed answers.** User-data metadata on the instance; a NoCloud seed
+built in-process and attached as media; a file placed on the backend and
+referenced from the instance config; not delivered (documented, with how the
+proxy gets configured instead).
+
+**Obligations.** Size limits of the channel stated; whether vendor-data or a
+second document can ride along; the sentinel test (contract §11); the channel
+must be fully reachable through the API the provider uses (no step that needs
+shell access on the backend unless the spec says the provider has it).
+
+**Answer:**
+
+**Evidence:**
+
+## 6. Machine type mapping
+
+**Question.** How does `Placement.MachineType` become CPU, memory and disk?
+
+**Allowed answers.** A catalogue of named profiles in the config block; a
+backend size name passed through.
+
+**Obligations.** A catalogue needs validation of each profile; disk size
+default and its config field named.
+
+**Answer:**
+
+**Evidence:**
+
+## 7. Network and IP discovery (`readiness`, `ip-source`)
+
+**Question.** Which network does the instance attach to, where does its
+address come from, how long until it is usable, and what makes it reachable
+from the operator pod and from clients on `spec.port`?
+
+**Allowed answers.** `readiness`: `immediate`; `provisioning-window`.
+`ip-source`: `assigned-at-create`; `discovered` (name the mechanism: guest
+agent, DHCP lease, neighbour table, and the order if several).
+
+**Obligations.** `discovered`: what the guest must run, how it gets
+installed, what happens without it; discovery failure is Provisioning, not an
+error; only routable IPv4 is reported (state the filter). Reachability:
+firewall or routing steps the provider performs, or the prerequisite the
+guide states. Whether `Port` is used by the provider at all.
+
+**Answer:**
+
+**Evidence:**
+
+## 8. GC marks and CreatedAt (`gc-mark`, `created-at`)
+
+**Question.** Where do `LabelManaged` and `LabelUID` live, where does
+`CreatedAt` come from, and which secondary resources are marked?
+
+**Allowed answers.** `gc-mark`: `native-labels`; `written-metadata`;
+`mark-window`. `created-at`: `backend-timestamp`; `written-at-create`.
+
+**Obligations.** `mark-window`: the window's length in seconds, why it is
+far below `MinAge`, and how the same Proxy's next reconcile cleans an
+unmarked instance. Any value-format constraint of the backend's labels
+(allowed characters, length) and how a Proxy UID fits it.
+
+**Answer:**
+
+**Evidence:**
+
+## 9. State mapping (`provisioning-state`)
+
+**Question.** How does every backend state map to `StateProvisioning`,
+`StateRunning`, `StateStopped`, `StateTerminated`, and how does `Get` tell
+"defined but never started" from "stopped"?
+
+**Allowed answers.** A table of backend state (plus lock or task flags) to
+provider state, with the unknown-state row mapping to `StateStopped`.
+`provisioning-state`: `native`; `create-blocks`; `marker`.
+
+**Obligations.** Running requires an IP; `marker` names what is recorded at
+creation and how `Get` reads it; `create-blocks` names the cap on the wait.
+
+**Answer:**
+
+| Backend state | Lock or task | Provider state |
+|---|---|---|
+| | | |
+
+**Evidence:**
+
+## 10. Create: blocking and task tracking (`create-return`, `async-ops`)
+
+**Question.** What does `Create` wait for before returning, and how are
+asynchronous backend tasks tracked?
+
+**Allowed answers.** `create-return`: `submit`; `blocking`. `async-ops`:
+`none`; `task-handle`; `backend-lock`.
+
+**Obligations.** `task-handle`: where the handle is kept and how a failed
+task surfaces (in `Get`, as which class, with which message). `backend-lock`:
+what `Get` and `Delete` return while locked. Every step of `Create` is
+idempotent so a cut-off attempt resumes; list the steps and what each finds
+on a retry.
+
+**Answer:**
+
+**Evidence:**
+
+## 11. Delete and secondary resources (`delete-return`, `secondary-resources`)
+
+**Question.** In what order does `Delete` tear things down, what does it do
+when called again mid-delete, and what does `Get` return meanwhile?
+
+**Allowed answers.** `delete-return`: `returns-early`; `waits`.
+`secondary-resources`: `none`; `cascade` (name the flags set at create);
+`explicit` (name each resource and how it is found by name).
+
+**Obligations.** Idempotent at every step; "already deleting" is success and
+cheap; a location no longer in the config is a logged no-op success; any
+resource that can outlive the instance without being listable is a documented
+GC blind spot.
+
+**Answer:**
+
+**Evidence:**
+
+## 12. ListByTag scope and partial failure (`list-partial`)
+
+**Question.** What is the full scope `ListByTag` enumerates, and what happens
+when part of it is unreachable?
+
+**Allowed answers.** `fail-whole`; `omit-unreachable`.
+
+**Obligations.** The scope covers everything the provider can create in;
+server-side filtering on the mark where the backend allows it; a single
+unreadable instance may be skipped with a log line; the cost of one list at
+fleet size.
+
+**Answer:**
+
+**Evidence:**
+
+## 13. Error classification (`not-found`, `quota`)
+
+**Question.** Which backend responses map to which of the four classes, and
+which signal means "does not exist"?
+
+**Allowed answers.** A table of backend status or code (with message pattern
+where needed) to class. `not-found`: `status-404`; `error-code`;
+`list-and-miss`. `quota`: `detected`; `absent`.
+
+**Obligations.** One `classify`, a table test with captured real responses
+for every matched message; the permission-denied-versus-quota case decided;
+authentication failures, host-key or certificate failures, and cancellation
+each assigned a class with a reason.
+
+**Answer:**
+
+| Backend response | Class | Why |
+|---|---|---|
+| | | |
+
+**Evidence:**
+
+## 14. Credentials and TLS (`credentials`)
+
+**Question.** What credential does the operator use, where does it come from,
+when is it read, and how is the backend's TLS certificate trusted?
+
+**Allowed answers.** `mounted-files` (Secret name, mount path, file names,
+`optional: true`); `environment`; `workload-identity`. TLS: `system-roots`;
+`ca-file`; `insecure` (the flag's exact name).
+
+**Obligations.** The minimum privilege set the credential needs, listed;
+rotation behaviour (free, or restart needed); the sentinel test for the
+credential (contract §7); the constructor never dials.
+
+**Answer:**
+
+**Evidence:**
+
+## 15. Config block
+
+**Question.** What are the fields of `<Type>Config`, their defaults, and their
+validation?
+
+**Allowed answers.** A table: field, required, default, validation, and which
+identifier or path the value ends up in.
+
+**Obligations.** Contract §10: `setBlocks`, the type allow-list in
+`Config.validate` and the constructor map in `cmd/main.go`, `config_<type>.go`
+with its tests, regexps for values pasted into IDs or file names.
+
+**Answer:**
+
+| Field | Required | Default | Validation | Ends up in |
+|---|---|---|---|---|
+| | | | | |
+
+**Evidence:**
+
+## 16. Checker
+
+**Question.** What are the check targets, what does one check do, which
+reasons can it report, and what goes into `Details`?
+
+**Allowed answers.** One target per thing that fails independently; the
+shared reasons from `internal/provider/check.go` first, provider-specific
+reasons only with a stated "what the admin does differently"; `Details` keys
+listed.
+
+**Obligations.** Same credential, verb and scope as `ListByTag` and `Create`;
+cheap (one page, one object); never disrupts a reconcile that shares a
+connection; returns promptly on `ctx`; the `status.type` comment in
+`api/v1alpha1/providerhealth_types.go` updated.
+
+**Answer:**
+
+**Evidence:**
+
+## 17. Client and child spans (`client-kind`, `child-spans`)
+
+**Question.** What talks to the backend, and how do its calls appear under the
+operation span?
+
+**Allowed answers.** `client-kind`: `vendor-sdk` (module path, and whether it
+is maintained); `thin-client`; `rpc-over-ssh`. `child-spans`:
+`sdk-instrumentation`; `transport-wrapper`; `span-decorator`; `none`.
+
+**Obligations.** The seam interface's name and the fake that implements it;
+`thin-client`: one error type carrying status and body, transport timeouts,
+parent-gated `otelhttp`; `none` is a documented gap; the span recorder test
+(contract §11).
+
+**Answer:**
+
+**Evidence:**
+
+## 18. Timeouts (`timeouts`)
+
+**Question.** What bounds each backend call, the dial, and any long step such
+as a download or a wait loop, and what does a cancelled `ctx` do to a shared
+connection?
+
+**Allowed answers.** `per-call` (the budgets, per operation); `ctx-only`
+(cite the client's own timeout).
+
+**Obligations.** Contract §1.5: no caller sets a deadline; a hung call pins a
+worker or stalls GC. Cancellation behaviour on a shared connection stated
+(abandon the call, or close the connection).
+
+**Answer:**
+
+**Evidence:**
+
+## 19. Concurrency
+
+**Question.** What is shared between concurrent calls (connections, caches,
+locks), and what happens when two calls for the same instance name overlap?
+
+**Allowed answers.** A description of the shared state and its locking.
+
+**Obligations.** Safe for 3 reconciles, a GC sweep and a check round at once;
+two `Delete` calls for one instance overlap safely; one fetch per image per
+location when several Creates race.
+
+**Answer:**
+
+**Evidence:**
+
+## 20. Logging (`logging`)
+
+**Question.** Which tiers does the provider log, with which keys on every
+line?
+
+**Allowed answers.** `tiered`; `outcome-only`; `none` (cite the client's own
+logging).
+
+**Obligations.** Logger from `logf.FromContext(ctx)` on every call; never
+`CloudInit` or credentials; `provider` and the instance name on every line
+that concerns an instance.
+
+**Answer:**
+
+**Evidence:**
+
+## 21. Tests (`backend-test`)
+
+**Question.** What is the fake, which contract tests exist, and how is the
+provider proven against the real backend?
+
+**Allowed answers.** `backend-test`: `cluster-e2e`; `integration-tag` (the
+`<TYPE>_TEST_*` variables, the `make test-integration-<type>` target); `manual-runbook`.
+
+**Obligations.** Every item of the contract §11 list, ticked here:
+
+- [ ] repeat `Create`
+- [ ] `Get` after `Delete`
+- [ ] `Delete` twice
+- [ ] malformed ID
+- [ ] unknown state maps to `StateStopped`
+- [ ] `StateRunning` requires an IP
+- [ ] `UID` and `CreatedAt` read back
+- [ ] `ListByTag` excludes unmarked instances
+- [ ] `Get` during an in-progress delete
+- [ ] `Get` during an asynchronous operation or lock
+- [ ] never started is never `StateStopped`
+- [ ] cancelled `ctx` returns promptly
+- [ ] `classify` table with captured responses
+- [ ] Checker reasons, expired `ctx`, stable order
+- [ ] sentinel tests for `CloudInit` and the credential
+- [ ] child spans under the operation span
+- [ ] real-backend lifecycle test
+
+**Answer:**
+
+**Evidence:**
+
+## 22. Deployment
+
+**Question.** What changes in `config/` and the Makefile?
+
+**Allowed answers.** The list: the `optional: true` mount in
+`config/manager/manager.yaml`; the commented entries in
+`config/manager/providers_config.yaml` and
+`config/samples/providers-config.yaml`; `config/samples/proxy_<type>.yaml`
+with a cloud-init that installs a proxy on the sample image; a Makefile
+helper for the Secret; RBAC markers if the provider talks to the cluster's
+own API; whether `hack/providers-dev.yaml` gets an entry.
+
+**Answer:**
+
+**Evidence:**
+
+## 23. Docs
+
+**Question.** Which documents are written or updated?
+
+**Allowed answers.** `docs/<type>-provider.md` with its section list; the row
+and section in `providers-existing.md`; the `architecture.md` rows (components
+table, §5, §11, and §10 if spans are added); the README row; the
+`docs/testing.md` section; the `CHANGELOG.md` entry after confirmation.
+
+**Answer:**
+
+**Evidence:**
+
+## 24. Known limits accepted at design time
+
+**Question.** Which of the contract's §13 limits bite this provider in
+practice, and which new gaps does this design accept?
+
+**Allowed answers.** A list, each with the consequence for an operator and
+the workaround.
+
+**Answer:**
+
+**Evidence:**

docs/providers-existing.md

@@ -0,0 +1,392 @@
+# The existing providers
+
+This is the appendix to [provider-contract.md](provider-contract.md): how the
+three providers that exist today (kubernetes, gcp, kvm) meet the contract,
+which profile each one chose on every axis, and what is still missing. It is
+reference material for whoever writes the next provider, not a user guide;
+the user guides are [kvm-provider.md](kvm-provider.md),
+[gcp-vm-validation.md](gcp-vm-validation.md) and
+[gcp-in-specific-project.md](gcp-in-specific-project.md). The README's
+"Implemented providers" table stays the short public summary.
+
+The per-provider call flows are drawn in
+[architecture.md §5](architecture.md#5-what-provider-calls-do-in-the-outside-world),
+the health checks are tabulated in
+[§11](architecture.md#11-provider-health-checks-internalproviderhealth), and the
+judgment calls behind several rows below are recorded under its "Decisions"
+list. They are linked, not repeated.
+
+Code references are a file path plus a function or symbol name.
+
+## 1. Profile table
+
+One row per axis of contract §4, in the same order, with the profile name
+and the symbol that shows it.
+
+| Axis | kubernetes | gcp | kvm |
+|---|---|---|---|
+| `readiness` | immediate: a Pod is scheduled within seconds (`stateFromPod`) | provisioning-window: minutes of boot (`mapState`) | provisioning-window: boot plus, on first use, an image download (`imageCache.ensure`) |
+| `create-return` | submit: returns after `client.Create` | submit: `Insert` is never awaited, `Operation.Wait` is not called | **blocking**: waits through `DomainCreate` (`ensureRunning`), and the first image fetch runs inside (`imageCache.fetch`) |
+| `delete-return` | returns-early: `client.Delete`, Pod terminates on its own | returns-early: `Delete` submits an operation each time | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) |
+| `ip-source` | assigned-at-create: `PodIP` (`instanceFromPod`) | assigned-at-create: the external NAT IP of the first NIC (`toInstance`) | discovered: ARP table first, then the guest agent, first routable IPv4 (`address`, `firstIPv4`) |
+| `gc-mark` | native-labels: Pod labels (`buildPod`) | native-labels: instance labels (`buildInsertRequest`) | written-metadata: a namespaced `<metadata>` element written in `DomainDefineXML` (`metadata.go`) |
+| `created-at` | backend-timestamp: `CreationTimestamp` | backend-timestamp: `CreationTimestamp`, parse error gives zero | written-at-create: `createdAt` in the metadata element (`newInstanceMetadata`) |
+| `list-partial` | n/a: one call, all namespaces | omit-unreachable: `ReturnPartialSuccess` on `AggregatedList` | fail-whole per host, one unreadable domain skipped (`ListByTag`, `managedInstance`) |
+| `quota` | absent: a quota 403 is Permanent (`classify`) | detected: 429, and 403 with reason `quotaExceeded` or `rateLimitExceeded` (`classify`, `hasReason`) | absent (`classify`) |
+| `not-found` | status-404: `apierrors.IsNotFound` | status-404: `isNotFound` | error-code: `ErrNoDomain`, `ErrNoStorageVol` (`hasCode`) |
+| `primary-key` | name-is-key: `namespace/name` | name-is-key: `zones/<zone>/instances/<name>` | name-is-key: `hosts/<host>/domains/<name>` |
+| `provisioning-state` | native: `PodPending` | native: `PROVISIONING`, `STAGING`, `REPAIRING` | create-blocks: a domain is defined and started in one `Create`; `nostate` maps to Provisioning |
+| `async-ops` | none | task-handle, but the handle is discarded: an insert that fails after acceptance is never read | none |
+| `secondary-resources` | none | cascade: `AutoDelete` on the boot disk (`buildInsertRequest`) | explicit: overlay and seed removed by name (`removeVolumes`); base image kept |
+| `credentials` | workload-identity: in-cluster config or kubeconfig (`NewWithTransportWrapper`) | environment: Application Default Credentials | mounted-files: SSH key and known_hosts from the optional `kvm-ssh` Secret, read lazily (`newLazySSH`) |
+| `client-kind` | vendor-sdk: controller-runtime `client.Client`, no seam interface | vendor-sdk: `cloud.google.com/go/compute` behind `instancesAPI` | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind `libvirtAPI` (`conn.go`) |
+| `timeouts` | ctx-only: no `rest.Config.Timeout` is set | ctx-only: SDK defaults, with SDK retries on Get and List | per-call for the dial only (`defaultHandshakeTimeout` in `dialSSH`); RPCs are ctx-only and a cancelled `call` closes the connection |
+| `logging` | none | tiered: V(1) outcomes and `logAPIError`, V(2) detail, V(5) wire log (`wirelog.go`) | tiered: V(1) outcomes and one `libvirt call` line per RPC (`trace.go`), V(2) XML and addresses; no wire tier |
+| `child-spans` | transport-wrapper: `tracing.RestConfigWrapper()` on its own `rest.Config`, parent-gated | sdk-instrumentation: the SDK adds `otelhttp` itself; not parent-gated, but every call site has a parent | **none**: `trace.go` is a log decorator, not spans |
+| `backend-test` | cluster-e2e: kind (`test/e2e`) | manual-runbook: [gcp-vm-validation.md](gcp-vm-validation.md) | integration-tag: `kvm_integration_test.go`, `make test-integration-kvm` |
+
+## 2. Other differences
+
+Rows that are not profile axes but that a new provider author compares
+against.
+
+| Feature | kubernetes | gcp | kvm |
+|---|---|---|---|
+| Create verifies the adopted instance's UID | no | no | no |
+| Placement used | none | Zone, MachineType, Image | Zone (host), MachineType, Image |
+| CloudInit delivered | **no**: ignored | yes: `user-data` metadata | yes: NoCloud seed ISO built in-process (`buildSeedISO`) |
+| Port used | yes: Squid config and containerPort (`squidConf`) | no: firewall is manual | no |
+| States produced | Provisioning, Running, Terminated | all four | Provisioning, Running, Stopped |
+| Deleting instance still reports its state | yes: a Pod being deleted reports its phase | n/a | n/a |
+| Error.Provider is the configured name | **no**: hardcoded `"kubernetes"` | yes | yes |
+| Checker target | `apiserver` | the project | each host, concurrently |
+| Checker Details | none | none | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacity`, `poolAvailable` |
+| Credential rotation without restart | n/a | via ADC | no: cached after first successful load |
+| Connection handling | client-go | SDK client, never closed | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostPool`) |
+| Unit-test double | controller-runtime fake client | `fakeAPI` behind `instancesAPI`, plus `httptest` for the real SDK client in `check_test.go` | `fakeAPI` behind `libvirtAPI`, plus an in-process `x/crypto/ssh` server in `conn_test.go` |
+| Proxy software | generated Squid config (`pod.go`) | from `cloudInit`; the image must run cloud-init | from `cloudInit`, plus a vendor-data script that installs the guest agent (`seed.go`) |
+| IP reachable from the operator | Pod network, so in-cluster only (not from `make run-dev` on a laptop) | external IP, once a firewall rule allows the cluster's egress IP | LAN address on the host's bridge; the operator needs a route to that LAN |
+| Distinct egress IP per proxy | no: the cluster's egress | yes: ephemeral external IP | whatever the host's LAN routes through; not per VM by itself |
+
+## 3. Observability status
+
+| Layer | kubernetes | gcp | kvm |
+|---|---|---|---|
+| 1. Operation spans and metrics | yes: decorators | yes: decorators | yes: decorators |
+| 2. Child spans per backend call | yes: `tracing.RestConfigWrapper()`, parent-gated (`cmd/main.go`, `NewWithTransportWrapper`) | yes: the SDK's own `otelhttp`; not parent-gated | **no** |
+| 3. Logs carry `traceID` and `spanID` | n/a: the package does not log | yes, including the V(5) wire log | yes: every `libvirt call` line at V(1) |
+| Tested against a real collector | yes: the kind e2e checks `provider.create` and `provider.delete` in Tempo (`test/e2e/tracing_test.go`), operation spans only | no | no |
+
+## 4. kubernetes (`internal/provider/kubernetes`)
+
+- **Package and seam.** `kubernetes.go`, `pod.go`, `check.go`. There is no
+  seam interface: `newWithClient` injects a controller-runtime
+  `client.Client`, and tests use the fake client. `New` calls
+  `NewWithTransportWrapper`, which builds its own uncached client from
+  `ctrl.GetConfig()` with an optional `transport.WrapperFunc` for tracing.
+- **Config.** `KubernetesConfig.Image` only, defaulting to the Squid image
+  constant in `kubernetes.go`.
+- **ID.** `providerID(namespace, name)` gives `namespace/name`;
+  `parseProviderID` uses `cache.SplitMetaNamespaceKey` and rejects an empty
+  namespace with `ErrPermanent`. An empty name part is not rejected.
+- **Create.** `buildPod` (pure) then `client.Create`; `IsAlreadyExists` is
+  success. Only `Name`, `Namespace`, `UID` and `Port` are used. The Squid
+  config is generated by `squidConf` from the port and passed through the
+  `SQUID_CONF` environment variable, so there is exactly one object per
+  proxy.
+- **Get and state.** `stateFromPod`:
+
+  | Pod phase | PodIP | Provider state |
+  |---|---|---|
+  | Pending | any | Provisioning |
+  | Running | empty | Provisioning |
+  | Running | set | Running |
+  | Succeeded, Failed, Unknown | any | Terminated |
+
+  `instanceFromPod` sets `IP` only when the state is Running. A Pod with a
+  deletion timestamp still reports its phase. `UID` from the `LabelUID`
+  label, `CreatedAt` from `CreationTimestamp`.
+- **Delete.** `client.Delete`, `IsNotFound` ignored.
+- **ListByTag.** One `client.List` across all namespaces with
+  `MatchingLabels{LabelManaged: LabelManagedYes}`; this is what the
+  cluster-wide Pod RBAC is for.
+- **Errors.** `classify`: `IsNotFound` NotFound; `IsTooManyRequests`,
+  `IsServerTimeout`, `IsTimeout` Transient; `IsForbidden`, `IsInvalid`,
+  `IsBadRequest`, `IsUnauthorized` Permanent; default Transient. Nothing is
+  Quota, because a quota 403 and an RBAC 403 cannot be told apart.
+- **Check.** Lists at most one managed Pod cluster-wide (`client.Limit(1)`),
+  proving read and list access, not create. Target `apiserver`, address the
+  rest config host. `checkReason`: context errors and timeouts
+  `ReasonTimeout`; `IsUnauthorized`; `IsForbidden`; any `APIStatus`
+  `ReasonAPIError`; else `ReasonUnreachable`.
+- **Logging and tracing.** No log lines in the package. Child spans come
+  from the parent-gated transport wrapper.
+- **Tests.** `kubernetes_test.go`, `pod_test.go`, `check_test.go` over the
+  fake client; the kind e2e (`test/e2e`) is the real-backend run.
+- **Deployment.** RBAC markers for Pods live next to the reconciler's in
+  `internal/controller/proxy_controller.go`. `hack/providers-dev.yaml` holds
+  only this provider, for `make run-dev`.
+- **Known gaps.**
+  - `Error.Provider` is hardcoded to `"kubernetes"`, so errors and logs show
+    the wrong name when the configured name differs.
+  - `CloudInit` and all of `Placement` are ignored.
+  - No quota class: a quota 403 is Permanent.
+  - A Pod being deleted still reports its phase, so it can read as Running.
+  - No securityContext, no resources, and `http_access allow all` in
+    `pod.go`.
+  - No logging in the package, and no Checker `Details`.
+  - No `OwnerReference` on Pods: a bypassed finalizer leaves the Pod behind.
+  - The Pod RBAC is always installed, even when no kubernetes provider is
+    configured.
+
+## 5. gcp (`internal/provider/gcp`)
+
+- **Package and seam.** `gcp.go`, `insert.go`, `errors.go`, `check.go`,
+  `wirelog.go`. The seam is `instancesAPI` (`Insert`, `Get`, `Delete`,
+  `AggregatedList`, `Probe`); `realInstances` adapts the SDK client and
+  `newWithAPI` is the test constructor. `New` calls `NewWithWireOptions`,
+  which takes the `--gcp-wire-log-full-payloads` flag from `cmd/main.go`.
+- **Config.** `GCPConfig`: `project` required; `network`, `networkTag`,
+  `diskSizeGb` defaulted in `withDefaults`.
+- **ID.** `formatProviderID(zone, name)` gives
+  `zones/<zone>/instances/<name>`; `parseProviderID` rejects anything else
+  with `ErrPermanent`. The project comes from the config.
+- **Create.** Validates `Zone`, `MachineType` and `Image` up front and
+  returns `ErrPermanent` naming the missing field. `buildInsertRequest`
+  (pure) builds one NIC with an ephemeral external IP (`ONE_TO_ONE_NAT`) and
+  a boot disk with `AutoDelete`, labelled with both marks. `Insert` is
+  submitted and never awaited; 409 (`isAlreadyExists`) is success.
+- **Get and state.** `mapState(status, ip)`:
+
+  | Instance status | NAT IP | Provider state |
+  |---|---|---|
+  | PROVISIONING, STAGING, REPAIRING | any | Provisioning |
+  | RUNNING | empty | Provisioning |
+  | RUNNING | set | Running |
+  | STOPPING, STOPPED, SUSPENDING, SUSPENDED | any | Stopped |
+  | TERMINATED | any | Terminated |
+  | anything else | any | Stopped |
+
+  `toInstance` reads the NAT IP of the first access config of the first
+  NIC, `UID` from the label, and `CreatedAt` from `CreationTimestamp` with
+  the parse error dropped (zero time on failure).
+- **Delete.** Submits a delete operation and returns; 404 (`isNotFound`) is
+  success. Every deletion poll submits a new operation.
+- **ListByTag.** `AggregatedList` with `managedFilter` on the managed label
+  only and `ReturnPartialSuccess`, so an unreachable zone is left out
+  silently; the zone is recovered from the URL by `lastPathSegment`.
+- **Errors.** `classify`: non-`googleapi` errors Transient; 404 NotFound;
+  429 Quota; 403 with reason `quotaExceeded` or `rateLimitExceeded`
+  (`hasReason`) Quota; 400, 401 and other 403s Permanent; other codes
+  Transient. `logAPIError` records the HTTP status and reasons at V(1)
+  because `classify` collapses them.
+- **Check.** `Probe` makes one `AggregatedList` page with `maxResults=1`
+  and the managed filter: credentials, the Compute API and
+  `compute.instances.list` in one call. Target is the project, address
+  `compute.googleapis.com`. `checkReason`: context errors `ReasonTimeout`;
+  non-`googleapi` errors `ReasonUnreachable`; 401; 403; else
+  `ReasonAPIError`.
+- **Logging.** `logger(ctx)` from the context, named `gcp` with
+  `provider`. V(2) for the curated insert request (never the user-data),
+  V(1) per operation outcome, V(2) per listed instance. The wire log in
+  `wirelog.go` runs at V(5) with payloads elided unless the flag is set.
+- **Tracing.** The SDK adds `otelhttp` itself unless telemetry is disabled;
+  not parent-gated.
+- **Tests.** `insert_test.go` (pure builder), `errors_test.go` (code
+  table), `gcp_test.go` (IDs, 409, state mapping, list filter and partial
+  success) over `fakeAPI`; `check_test.go` also drives the real SDK client
+  against `httptest` to prove exactly one request with `maxResults=1`.
+  Real backend: the manual runbook in [gcp-vm-validation.md](gcp-vm-validation.md).
+- **Deployment.** Credentials by ADC; the runbook mounts a service-account
+  key and points `GOOGLE_APPLICATION_CREDENTIALS` at it. A commented entry
+  in both providers configs, `config/samples/proxy_gcp.yaml`, and demo
+  scripts under `docs/demo/`.
+- **Known gaps.**
+  - Asynchronous insert failures are never read: an insert the API accepts
+    but that fails later (quota, bad image) is seen only as NotFound on the
+    next `Get`, and the reconciler creates again, so the reason never
+    reaches status. [gcp-vm-validation.md](gcp-vm-validation.md#gotchas)
+    still says such errors do reach status.
+  - External IP only: a VM without an external NAT IP stays Provisioning
+    forever.
+  - No firewall management: the rule for the proxy port is a manual
+    prerequisite, and `req.Port` is unused.
+  - The SDK client is never closed, and `Placement.Region` is unused.
+  - Wrong label value in the docs: `gcp-vm-validation.md` filters on
+    `labels.proxy-operator-managed=yes`; the value is `true`.
+  - The sample cannot work: `config/samples/proxy_gcp.yaml` uses a
+    `debian-cloud` image, which has no cloud-init, so its Squid install never
+    runs ([gcp-in-specific-project.md](gcp-in-specific-project.md) uses
+    Ubuntu for that reason).
+
+## 6. kvm (`internal/provider/kvm`)
+
+The full user guide is [kvm-provider.md](kvm-provider.md); this section is
+the author's view.
+
+- **Package and seam.** `kvm.go` (the four methods), `conn.go` (SSH dial,
+  `hostPool`, `libvirtAPI`), `image.go` (`imageCache`), `seed.go` (NoCloud
+  ISO and vendor-data), `domainxml.go`, `volume.go`, `metadata.go`, `id.go`,
+  `errors.go`, `check.go`, `trace.go`. The seam is `libvirtAPI`, the twenty
+  go-libvirt calls the provider uses; `fake_test.go` implements it in memory
+  and `newWithDialer` injects the dialer and the HTTP client.
+- **Config.** `KVMConfig` in `internal/provider/config_kvm.go`:
+  `sshKeyFile`, `knownHostsFile` or `insecureIgnoreHostKey`, `diskSizeGb`,
+  `vendorData`, `installGuestAgent`, `images` (`url`, `sha256`, `format`,
+  `firmware`), `machineTypes` (`vcpus`, `memoryMiB`), `hosts` (`name`,
+  `address`, `user`, `socket`, `pool`, `bridge`). `validateKVM` restricts
+  image keys and host names with `kvmIdentifier` because they become a volume
+  name and an ID segment.
+- **ID.** `formatProviderID(host, name)` gives `hosts/<host>/domains/<name>`.
+- **Create.** `host` resolves `Placement.Zone` (optional with one host),
+  the machine type must be a catalogue key, `resolveImage` accepts a
+  catalogue key or an http(s) URL. Then, inside one `hostPool.call`:
+  `DomainLookupByName`, and if found `ensureRunning`; otherwise
+  `imageCache.ensure` (download on first use, streamed through the operator
+  into the pool with a sha256 marker volume), `ensureOverlay` (reused if it
+  exists), `writeSeed` (always rebuilt: a cut-off upload must not be
+  trusted), `defineDomain` with the metadata element, `ensureRunning`.
+  `Create` returns only when the domain is running.
+- **Get and state.** `mapState(state, ip)`:
+
+  | libvirt state | IP | Provider state |
+  |---|---|---|
+  | nostate | any | Provisioning |
+  | running, blocked | empty | Provisioning |
+  | running, blocked | set | Running |
+  | paused, shutdown, shutoff, crashed, pmsuspended | any | Stopped |
+
+  `address` asks the ARP table first, then the guest agent, and
+  `firstIPv4` drops loopback, link-local and unspecified addresses. Metadata
+  is read back by `metadata` and parsed by `parseInstanceMetadata`; a domain
+  without the element reports an empty UID.
+- **Delete.** `removeDomain` (destroy, tolerating not running or gone; then
+  `DomainUndefineFlags` with NVRAM, managed save and snapshot metadata) and
+  `removeVolumes` (overlay and seed by name, a missing pool is a no-op). A
+  host no longer in the config is a logged no-op success.
+- **ListByTag.** `ConnectListAllDomains` on every host; `managedInstance`
+  keeps domains whose metadata says managed and skips a domain whose
+  metadata or state cannot be read. One unreachable host fails the whole
+  list.
+- **Errors.** `classify`: `isClassified` errors keep their class;
+  `ErrNoDomain`, `ErrNoStorageVol` NotFound; XML, unsupported,
+  invalid-argument, auth and access errors, and a missing pool, Permanent;
+  everything else Transient, including SSH handshake and auth failures.
+  Nothing is Quota. `logAPIError` adds `libvirtErrorFields`.
+- **Check.** `checkHost` per host concurrently through `hostPool.probe`,
+  not `call`, so a check that runs out of time abandons its RPCs instead of
+  closing the connection a reconcile may be using. Two cheap RPCs
+  (`ConnectGetLibVersion`, `ConnectGetHostname`) then the pool must exist
+  and be running. Reasons: the shared ones plus `SSHMaterialUnavailable`,
+  `SSHHandshakeFailed`, `LibvirtConnectFailed`, `LibvirtError`,
+  `PoolNotFound`, `PoolNotActive`, chosen by `checkReason` from the
+  `dialError` stage. Details: `user`, `pool`, `hostname`, `libvirtVersion`,
+  `poolState`, `poolCapacity`, `poolAvailable`.
+- **Connections.** `hostPool` keeps one SSH plus libvirt connection per
+  host, dialled lazily (`lazySSH.dial`, `sshAuth.dial`, `dialSSH`,
+  `connectLibvirt`), with a 15s `keepalive`. `hostEntry.acquire` is a
+  one-slot semaphore so waiting can end with the caller's context. A
+  cancelled `call` closes the shared connection to unblock the RPC;
+  `probe` abandons it and leaves the connection open. The SSH material is
+  read once by `lazySSH.load` and cached.
+- **Logging.** Logger from the context, named `kvm`, with `provider`,
+  `host` and `name`. V(1): one line per operation outcome and one
+  `libvirt call` line per RPC from the `tracingAPI` decorator in `trace.go`
+  (rpc name, identifiers, duration, error fields). V(2): domain and volume
+  XML, metadata, addresses. Upload and download bodies are never logged.
+- **Tracing.** No child spans; `trace.go` is a log decorator.
+- **Tests.** Pure builders (`domainxml_test.go`, `volume_test.go`,
+  `seed_test.go`, `id_test.go`, `metadata_test.go`); `errors_test.go`,
+  `conn_test.go` (pool behaviour and a real in-process SSH server);
+  `image_test.go`, `kvm_test.go` (Create resumes after every partial step,
+  every Delete already-gone case, ListByTag policies, logging tiers, a
+  cloud-init sentinel never logged); `trace_test.go`; `check_test.go`. Real
+  backend: `kvm_integration_test.go` behind `//go:build integration`, driven
+  by `KVM_TEST_*` variables and `make test-integration-kvm`, described in
+  [testing.md](testing.md#the-kvm-integration-test-real-hypervisor).
+- **Deployment.** The optional `kvm-ssh` Secret mounted at
+  `/etc/proxy-operator/kvm` (`config/manager/manager.yaml`),
+  `make kvm-ssh-secret`, a commented entry in both providers configs,
+  `config/samples/proxy_kvm.yaml`.
+- **Known gaps** (see also [kvm-provider.md § Known gaps](kvm-provider.md#known-gaps)).
+  - No quota or capacity detection.
+  - `Create` blocks, including the first image download on each host.
+  - GC blind spots: volumes without a domain are invisible to GC, and so are
+    VMs on a host removed from the config, which `Delete` treats as done.
+  - SSH material is cached after the first successful load, so rotating
+    keys needs a restart.
+  - Cancelling one operation cuts off the others on that host: a cancelled
+    `call` closes the shared per-host connection.
+  - SSH auth failures retry forever: SSH authentication and host-key
+    failures are Transient, while libvirt auth errors are Permanent.
+  - `Create` can return `ErrNotFound` when a volume or domain disappears
+    during the call.
+  - No reused-overlay check: an existing overlay is reused without checking
+    its backing image or size.
+  - Sample mismatch: `config/samples/proxy_kvm.yaml` places on host `nas1`,
+    while both providers configs define only `beelink`.
+  - Doc drift in kvm-provider.md: it says ARP is the fallback, but the code
+    tries ARP first; it says `host` and `name` are on every log line, but
+    the check and list paths lack them.
+
+## 7. Gaps shared by the existing providers
+
+- **Adoption is not verified.** None of the three checks `LabelUID` when
+  `Create` finds an existing instance with the right name.
+- **The Create comment does not match kvm.** The interface comment in
+  `internal/provider/provider.go` says `Create` returns as soon as the
+  request is submitted; kvm blocks until the domain has started.
+- **`Stopped` and `Terminated` are produced inconsistently.** kubernetes
+  never produces Stopped, kvm never produces Terminated, and the reconciler
+  treats them the same.
+- **Different `ListByTag` partial-failure policies.** gcp leaves failed
+  parts out, kvm fails the whole list. Both are safe.
+- **Stale `mock` comments** in `internal/provider/provider.go`,
+  `internal/provider/registry/registry.go` and `api/v1alpha1/proxy_types.go`
+  (and so the generated CRD's `spec.provider` description).
+- **GC does not know about installations.** Two clusters sharing one project
+  or hypervisor delete each other's instances. This was a merge-blocker in
+  the 2026-08-10 PR review
+  ([docs/reviews](reviews/2026-08-10-1134-proxy-operator-pr-review.md)) and
+  is still open.
+- **Tracing.** kvm has no child spans, so a kvm trace shows `provider.create`
+  and nothing under it; correlation works only through `traceID` on its V(1)
+  log lines. Only kubernetes is tested against a real collector, and that
+  test checks only the operation spans; nothing checks that gcp's SDK spans
+  or the kubernetes API spans nest under `provider.*`.
+
+## 8. Proposed features
+
+Listed only. Nothing below is designed or implemented.
+
+- **Child spans in kvm:** extend the `tracingAPI` decorator in
+  `kvm/trace.go`, which already wraps every libvirt call, to also start a
+  span per RPC, and add spans for the SSH dial and the image download. The
+  seam's methods take no `ctx` today (`hostPool.call` hands `fn` only the
+  API), so the span context has to be plumbed in.
+- **Tempo e2e checks for child spans**, and for gcp and kvm through their
+  real backend runbooks.
+- **A shared conformance suite:** for example `internal/provider/providertest`,
+  which runs the contract §11 tests against any provider plus a fake
+  backend, so a new provider gets the contract tests for free.
+- **Verify the UID when `Create` adopts an instance.** Return `ErrPermanent`
+  on a mismatch instead of adopting an instance that belongs to someone else.
+- **Report asynchronous create failures:** gcp keeps the operation name, and
+  `Get` reports the failure instead of NotFound.
+- **Capacity or quota signals** for kvm, such as pool free space and host
+  memory, mapped to `ErrQuotaExceeded`.
+- **GC for secondary resources:** mark volumes and similar resources, and
+  let `ListByTag` or a second listing method cover them.
+- **Reload credentials and config without a restart.**
+- **Checker `Details`** for kubernetes (server version) and gcp (quota
+  headroom).
+- **Internal-IP mode** for gcp, and a firewall rule managed per provider.
+- **An installation-identity mark**, set on create and filtered on in
+  `ListByTag`, so installations sharing a backend do not collect each
+  other's instances.
+- **A permanent-failure latch that sees provider availability**, so adding
+  a missing provider releases the Proxies waiting on it.
+- **Validate provider names at load**, and derive the config type
+  allow-list from the constructor map.
+- **Per-provider RBAC** as an optional kustomize component, and an
+  `OwnerReference` on kubernetes Pods.

docs/providers.md

@@ -0,0 +1,39 @@
+# Providers
+
+The provider documentation is split into three files so that a new provider
+can be specified and built from the generic rules alone, without reading how
+the existing ones happen to work.
+
+- **[provider-contract.md](provider-contract.md)**: what every provider MUST,
+  SHOULD and MAY do, which caller depends on each rule, the behaviour profiles
+  where providers are allowed to differ, the error taxonomy, credential
+  hygiene, testing obligations, the checklist for adding a provider, and the
+  contract's known limits. It never names an existing provider.
+- **[provider-spec-template.md](provider-spec-template.md)**: the questionnaire
+  a new provider's design answers, one entry per decision the contract leaves
+  open. Copy it to `docs/<type>-provider-spec.md`, fill it in, and hand it to
+  the implementer with the other two files.
+- **[providers-existing.md](providers-existing.md)**: how kubernetes, gcp and
+  kvm meet the contract, a table mapping each of them to every profile, their
+  per-provider reference notes, shared gaps and proposed features.
+
+Per-provider user guides: [kvm-provider.md](kvm-provider.md),
+[gcp-vm-validation.md](gcp-vm-validation.md) and
+[gcp-in-specific-project.md](gcp-in-specific-project.md).
+
+Related sections of [architecture.md](architecture.md): the components table,
+[§5 what provider calls do](architecture.md#5-what-provider-calls-do-in-the-outside-world),
+[§10 tracing](architecture.md#10-tracing-internaltracing),
+[§11 provider health checks](architecture.md#11-provider-health-checks-internalproviderhealth)
+and the "Decisions" list. The test layout is in [testing.md](testing.md).
+
+## Reading order for a new provider
+
+1. The contract, end to end.
+2. The template, filled in for the new backend; every unanswered entry is a
+   design decision still to make.
+3. The appendix, for a worked example of each profile the spec chose.
+4. The user guide of the existing provider closest to the new one.
+
+The single-file blueprint that preceded this split is in the git history of
+this path.

docs/testing.md

@@ -53,13 +53,23 @@ manual runs too.
 | | `config_test.go` | providers-config parsing + fail-fast validation | pure |
 | | `metrics_test.go` | `WithMetrics` decorator: classified result labels, error passthrough | fake recorder + static provider |
 | `internal/provider/kubernetes` | `kubernetes_test.go`, `pod_test.go` | Create/Get/Delete/ListByTag over real Pod objects; pure `buildPod`/`squidConf` incl. the `max_filedescriptors` OOM-regression assertion | `client/fake` (real `corev1.Pod`s, no kubelet) |
+| | `check_test.go` | health check: a single cluster-wide list with `Limit=1` and the managed-label selector; the reason for every API error class | `client/fake` + interceptor |
 | `internal/provider/gcp` | `insert_test.go`, `errors_test.go`, `gcp_test.go` | field-by-field `buildInsertRequest`; HTTP-code classification table; zone-qualified IDs, 409-is-success, 9-row state mapping, ListByTag filter + `ReturnPartialSuccess` | pure builder needs **no fake**; the rest uses the flattened `instancesAPI` seam |
+| | `check_test.go` | health check: request shape and the reason per HTTP code or network error; the **real SDK client** makes exactly one request with `maxResults=1`, even when a next page exists | fake seam + `httptest` |
+| `internal/provider/kvm` | `domainxml_test.go`, `volume_test.go`, `seed_test.go`, `id_test.go`, `metadata_test.go` | pure builders: domain/volume XML unmarshalled back and checked field by field, seed ISO reopened with the same library, providerID round-trip, metadata in both namespace spellings | pure |
+| | `errors_test.go`, `conn_test.go` | libvirt-code classification table; connection pool (reuse, redial, drop on transport error, cancel releases a blocked RPC, one dial per host under concurrency); SSH dial, host-key policy, keepalive | fake dialer; a **real in-process `x/crypto/ssh` server** forwarding a unix socket |
+| | `image_test.go`, `kvm_test.go` | image cache (fresh fetch, ready, marker missing, checksum change/mismatch, 4xx/5xx, concurrent callers); Create resume after every partial step, leftover truncated seed, state/IP mapping, every Delete already-gone case, ListByTag across hosts / one down / unreadable domain; logging tiers (V0 silent, V1 one line per RPC, V2 XML) with cloud-init never logged | `httptest` image server + in-memory `fakeAPI` behind the `libvirtAPI` seam |
+| | `trace_test.go`, `conn_test.go`, `errors_test.go` | every `libvirtAPI` method traced exactly once (guarded by reflection on the interface), results and errors pass through, libvirt error fields, detail keys only at V2, upload content never logged; pool connect/drop lines, no wrapping below V1 | `fakeAPI` + `funcr` capture |
+| | `check_test.go`, `conn_test.go` | health check: details, per-host verdicts in config order, hosts checked concurrently, a timeout that keeps the connection and refuses a second check while the first still waits, no waiting behind a reconcile's dial, the reason table; `probe` abandons without closing, but still drops on a transport error; waiting for a host's slot ends with the caller's deadline; an inactive or missing pool and daemon errors keep the connection; every dial stage tagged through the real SSH server, with classification unchanged | `fakeAPI`, fake dialers, in-process SSH server |
+| | `kvm_integration_test.go` (`-tags=integration`) | one real VM: image fetch, Create, idempotent re-Create, Running with IP, ListByTag, Delete, pool state afterwards; `TestKVMCheck_realHost` runs the health check against the host and creates nothing | a **real hypervisor** — see below |
 | `internal/health` | `probe_test.go` | probes through a **real CONNECT-capable proxy stub** to a real TLS server: tunnel success, refused CONNECT, wrong status, dead proxy, plain-http forward | `httptest` + hijacked bidirectional tunnel |
 | | `engine_test.go` | thresholds, first-verdict, latency suppression matrix, dropped-event retry, stale-probe UID guard, tick scheduling/seeding/pruning, `Start` end-to-end | fake `client.Reader` + injected `probeFn` |
 | `internal/lease` | `store_test.go` | capacity, `MaxLeases=0`, all three selection tie-breaks, target-scoped vs global cooldowns, expiry via fake clock, report-on-retained-lease, idempotent release, **40 concurrent acquires never exceeding capacity** | injectable clock; the concurrency case is why `-race` matters |
 | `internal/discovery` | `server_test.go` | auth matrix (incl. `/healthz` exemption), list filtering, grant shape, full 409 arithmetic, invalid TTL/body/result, idempotent release, report→cooldown→409 round trip, `Start`/shutdown | `httptest` over the real handler chain, fake cache reader, **real** `lease.Store` |
 | `internal/gc` | `gc_test.go` | the true-orphan matrix (live/deleting/young/unlabelled all kept), broken-provider isolation, list-failure skips sweep, namespace guard, sweep loop | fake reader + canned-list provider |
-| `internal/metrics` | `metrics_test.go` | scrape-time collectors (`GatherAndCompare`), per-proxy series cleanup via `ForgetProxy`, label counts, fresh-registry-per-test property | `prometheus/client_golang/testutil` |
+| `internal/providerhealth` | `status_test.go`, `monitor_test.go` | `buildStatus` (the Ready condition, per-target transition times, truncation); objects created with the label and patched, the round timeout, no-checker Unknown, invalid names, transition-only logging, cleanup of labelled unconfigured objects, the checks-off path, `ValidateSchedule` | fake client with a cluster-scoped RESTMapper + `funcr` capture |
+| | `envtest_test.go` | the generated `ProviderHealth` schema against a real API server: cluster scope, status subresource, required fields, list-map keys, the 1024-character message limit | envtest (skipped under `-short`) |
+| `internal/metrics` | `metrics_test.go` | scrape-time collectors (`GatherAndCompare`) incl. the provider-target gauges, per-proxy series cleanup via `ForgetProxy`, label counts, fresh-registry-per-test property | `prometheus/client_golang/testutil` |
 
 Conventions (from `CLAUDE.md`): stdlib `testing`, table-driven by default,
 `t.Parallel()` where safe, tests next to source, ginkgo only in the
@@ -83,17 +93,117 @@ envtest suite the scaffold generated.
   bug (below).
 - **Live GCP**: no test talks to Google. The seam boundary
   (`buildInsertRequest` + classification) is tested exhaustively instead.
+- **Live libvirt, by default**: the KVM unit tests run over an in-memory
+  fake of the 20 libvirt calls the provider uses. Whether libvirt accepts
+  the domain XML, whether a raw qcow2 upload plus pool refresh yields a
+  usable backing image, and whether the guest actually boots and reports
+  an address are provable only on a real host — that is the opt-in
+  integration test below.
+
+## The KVM integration test (real hypervisor)
+
+`internal/provider/kvm/kvm_integration_test.go` compiles only under
+`-tags=integration` and skips itself unless the environment names a
+host. It deviates from the "testcontainers for real infra" convention on
+purpose: libvirt needs `/dev/kvm` and a bridge to a DHCP network, and a
+container has neither. So it targets whichever hypervisor you point it
+at and leaves that host's pool with one more cached base image than it
+had.
+
+```sh
+export KVM_TEST_HOST=nas1.home.lan:22          # SSH endpoint
+export KVM_TEST_USER=virt                      # must be in the libvirt group
+export KVM_TEST_SSH_KEY=$HOME/.ssh/id_ed25519  # private key for that user
+export KVM_TEST_KNOWN_HOSTS=$HOME/.ssh/kvm_known_hosts  # the hypervisors' host keys
+export KVM_TEST_BRIDGE=br0                     # bridge to a network with DHCP
+export KVM_TEST_IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+make test-integration-kvm
+```
+
+`KVM_TEST_KNOWN_HOSTS` is technically optional, but leaving it out turns
+host-key verification **off** for the run (the log says so); keep a
+`kvm_known_hosts` holding just the hypervisors rather than pointing the
+test at your everyday `known_hosts`, so a re-keyed hypervisor is a
+one-line fix in a file nothing else uses.
+
+Also optional: `KVM_TEST_IMAGE_SHA256`, `KVM_TEST_POOL`
+(default `default`), `KVM_TEST_SOCKET`, `KVM_TEST_TIMEOUT` (default
+`20m`, must cover the first-time image download), `KVM_TEST_KEEP`
+to leave the VM behind for `virsh console` when something goes wrong,
+and `KVM_TEST_LOG_LEVEL` (default `1`) for how much of the provider's
+logging the `-v` output shows: at `1` every libvirt RPC is one compact
+line (`HH:MM:SS.mmm libvirt virDomainLookupByName domain=... duration=...
+error=... libvirtError=ErrNoDomain`), interleaved with the test's own
+`t.Log` lines; `2` adds the domain and volume XML, metadata and
+addresses; `0` leaves only `t.Log`. The per-run constants (provider,
+host, domain name) are left off each line. See the Logging section of
+[kvm-provider.md](kvm-provider.md#logging).
+
+What one run asserts, in order: `New` dials the host; `Create` returns a
+`hosts/<host>/domains/<name>` ID (downloading the image on a cold host —
+the log prints how long it took); a second `Create` returns the same ID;
+`Get` reaches `Running` with an IPv4 within the budget, and the UID and
+`CreatedAt` survive the metadata round-trip through libvirt; `ListByTag`
+lists it; `Delete` removes it, `Get` says `ErrNotFound`, a second
+`Delete` is a no-op, and the pool holds the base image and its marker
+but no per-VM overlay or seed. It also tries a TCP connect to port 22 on
+the guest, but only logs the outcome — the machine running the test may
+not route to the bridge network.
+
+`TestKVMCheck_realHost` runs the provider health check
+([kvm-provider.md](kvm-provider.md#health-check)) against the same host.
+It must find the host reachable and the pool running, and it creates
+nothing. It needs the same variables; to run only this test:
+
+```sh
+go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/
+```
+
+The test is not part of `make test` and not run in CI. Keep it compiling
+with `go vet -tags=integration ./...` when the provider's internals move.
 
 ## The scaffolded `make test-e2e` suite
 
 `test/e2e/` is the kubebuilder-generated smoke suite (build image → kind
-cluster → deploy → assert the manager pod runs and serves metrics). It
+cluster → deploy → assert the manager pod runs, publishes a Ready
+`ProviderHealth` for the `kubernetes` provider, and serves metrics that
+include that provider's `proxy_operator_provider_target_up` series). It
 compiles only under `-tags=e2e`, manages its own kind cluster
 (`make test-e2e` / `make cleanup-test-e2e`), and has been kept compiling
 (`go vet -tags=e2e ./...` is part of the routine) but is **not part of
 `make test` and was not used for the release verification** — the manual
-kind run below covers strictly more. Treat it as scaffold to grow into if
-CI wants an automated cluster smoke test.
+kind run below covers strictly more.
+
+### The OTel tracing spec (Tempo-gated)
+
+`test/e2e/tracing_test.go` proves the full tracing pipeline against a
+**real Tempo**: deploy in kind with tracing enabled, create two
+kubernetes-provider proxies, delete them, and assert in Tempo that the
+traces exist with the documented span topology (`Reconcile Proxy` →
+`reconcile.managed` / `provider.create` / `status.patch`, and
+`reconcile.delete` / `provider.delete` on the way out).
+
+It **skips unless both env vars are set** (so the rest of the suite runs
+anywhere). Homelab invocation:
+
+```bash
+TEMPO_URL=http://192.168.0.30:3200 \
+OTLP_ENDPOINT=http://192.168.0.30:4318 \
+make test-e2e
+```
+
+Worth knowing:
+
+- Every span of a run carries the resource attribute
+  `test.run.id=e2e-<nanos>` (injected via `OTEL_RESOURCE_ATTRIBUTES`, no
+  code involved); the run ID is printed in the test log, and
+  `{resource.test.run.id="<id>"}` in Grafana shows exactly that run.
+- The spec preflights the OTLP endpoint **from inside the cluster** with a
+  curl pod and fails fast with a clear message if it's unreachable —
+  export failures are otherwise only visible at `-zap-log-level=1`.
+- Proxy CRs are created in `default`, not the operator namespace: the
+  squid pods carry no securityContext and the operator namespace enforces
+  restricted PSS.
 
 ## The kind verification run (the real end-to-end)
 

go.mod

@@ -4,10 +4,20 @@ go 1.26.0
 
 require (
 	cloud.google.com/go/compute v1.65.0
-	github.com/go-logr/logr v1.4.3
+	github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858
+	github.com/go-logr/logr v1.4.4
+	github.com/kdomanski/iso9660 v0.4.0
 	github.com/onsi/ginkgo/v2 v2.27.4
 	github.com/onsi/gomega v1.39.0
 	github.com/prometheus/client_golang v1.23.2
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
+	go.opentelemetry.io/otel v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0
+	go.opentelemetry.io/otel/sdk v1.45.0
+	go.opentelemetry.io/otel/trace v1.45.0
+	golang.org/x/crypto v0.54.0
 	google.golang.org/api v0.292.0
 	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
 	k8s.io/api v0.36.0
@@ -31,7 +41,7 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/felixge/httpsnoop v1.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -48,7 +58,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect
 	github.com/googleapis/gax-go/v2 v2.23.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -66,19 +76,13 @@ require (
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
-	go.opentelemetry.io/otel v1.44.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.44.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.44.0 // indirect
-	go.opentelemetry.io/otel/trace v1.44.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
+	go.opentelemetry.io/otel/metric v1.45.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.54.0 // indirect
 	golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
 	golang.org/x/mod v0.37.0 // indirect
 	golang.org/x/net v0.57.0 // indirect
@@ -91,7 +95,7 @@ require (
 	golang.org/x/tools v0.47.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
 	google.golang.org/grpc v1.83.0 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect

go.sum

@@ -28,14 +28,16 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858 h1:8xCFt73OddCD5WXxoYidJBWp3bJMC+CpE7Zwf5tcwk8=
+github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858/go.mod h1:qb0Ofa71d3oXARQf633h2tNaeBxLsVxuDp+jcsVO2+4=
 github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
 github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
 github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
 github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
-github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
-github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
+github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
@@ -47,8 +49,8 @@ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6O
 github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
 github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
-github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
+github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
@@ -86,8 +88,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.19 h1:mMOE7DN2+p76/EdIrm
 github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k=
 github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE=
 github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -96,6 +98,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE
 github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg=
+github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -170,24 +174,28 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
-go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
-go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
-go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
-go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
-go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
-go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
-go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
-go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
-go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
-go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
-go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
-go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
+go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
+go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=
+go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
+go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
+go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
+go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
+go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
+go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
+go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
+go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
+go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
+go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -228,8 +236,8 @@ google.golang.org/api v0.292.0 h1:Ewiwo/GTtiaPZSNAZQUcWLh8AYDEoPmIXyJfeoTSMHU=
 google.golang.org/api v0.292.0/go.mod h1:07kjmMnFGm2RQuCza2EZM/5N68G/fVvFb1xKjWqoFA0=
 google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
 google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
-google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU=
-google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=

internal/controller/proxy_controller.go

@@ -41,6 +41,7 @@ import (
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // HealthSnapshotter provides the current probe verdict for a proxy. The
@@ -96,9 +97,14 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 	}
 	base := p.DeepCopy()
 	defer func() {
+		// Runs inside the root reconcile span (this defer fires before the
+		// tracing.NewReconciler wrapper sees the return), and its error is
+		// folded into err, which that wrapper records.
+		pctx, span := tracing.Start(ctx, "status.patch")
+		defer span.End()
 		// NotFound is expected when this reconcile just removed the last
 		// finalizer and the object is already gone.
-		if perr := r.patchStatusIfChanged(ctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
+		if perr := r.patchStatusIfChanged(pctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
 			err = errors.Join(err, perr)
 		}
 	}()
@@ -114,6 +120,8 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 }
 
 func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.managed")
+	defer span.End()
 	log := logf.FromContext(ctx)
 
 	if controllerutil.AddFinalizer(p, crawlv1alpha1.FinalizerName) {
@@ -224,6 +232,8 @@ func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1
 // exists" — hence: delete, poll to NotFound, only then advance the hash and
 // let the create branch run.
 func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.Proxy, prov provider.Provider, hash string) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.replaceInstance")
+	defer span.End()
 	_, err := prov.Get(ctx, p.Status.ProviderID)
 	if provider.Class(err) == provider.ErrNotFound {
 		// Old instance is gone. The Update inside setSpecHash refreshes p
@@ -252,14 +262,22 @@ func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.
 }
 
 func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.delete")
+	defer span.End()
 	if !controllerutil.ContainsFinalizer(p, crawlv1alpha1.FinalizerName) {
 		return ctrl.Result{}, nil
 	}
 	if p.Status.ProviderID == "" {
 		// Nothing was ever recorded as created; orphan GC reaps any stray
 		// instance a crashed create might have left behind.
-		controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
-		return ctrl.Result{}, r.Update(ctx, p)
+		return r.releaseFinalizer(ctx, p)
+	}
+	if p.Annotations[crawlv1alpha1.AnnotationSkipCleanup] == "true" {
+		// Checked before the provider lookup so it also releases a Proxy whose
+		// provider is gone from the config. The instance is left running.
+		logf.FromContext(ctx).Info("skip-cleanup annotation set; abandoning the instance",
+			"providerID", p.Status.ProviderID, "provider", p.Spec.Provider)
+		return r.releaseFinalizer(ctx, p)
 	}
 	prov, ok := r.Providers[p.Spec.Provider]
 	if !ok {
@@ -267,11 +285,22 @@ func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.
 			"provider %q is not configured; cannot clean up instance %s", p.Spec.Provider, p.Status.ProviderID)
 	}
 	_, err := prov.Get(ctx, p.Status.ProviderID)
-	if provider.Class(err) == provider.ErrNotFound {
-		controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
-		return ctrl.Result{}, r.Update(ctx, p)
-	}
-	if err != nil {
+	switch {
+	case err == nil:
+	case provider.Class(err) == provider.ErrNotFound:
+		return r.releaseFinalizer(ctx, p)
+	case provider.Class(err) == provider.ErrPermanent:
+		// Get will never answer for this instance — a host retired from the
+		// config, a providerID the provider can no longer resolve. Delete is
+		// idempotent and may still finish, so try it instead of polling a Get
+		// that cannot converge, and release the object when it does.
+		if delErr := prov.Delete(ctx, p.Status.ProviderID); delErr != nil {
+			return r.deletionFailure(errors.Join(err, delErr))
+		}
+		logf.FromContext(ctx).Info("instance unreachable but cleanup succeeded; removing the finalizer",
+			"providerID", p.Status.ProviderID, "error", err.Error())
+		return r.releaseFinalizer(ctx, p)
+	default:
 		return r.deletionFailure(err)
 	}
 	if err := prov.Delete(ctx, p.Status.ProviderID); err != nil {
@@ -282,6 +311,11 @@ func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.
 	return ctrl.Result{RequeueAfter: r.DeletionPoll}, nil
 }
 
+func (r *ProxyReconciler) releaseFinalizer(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
+	controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
+	return ctrl.Result{}, r.Update(ctx, p)
+}
+
 func (r *ProxyReconciler) reconcileExternal(_ context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
 	if p.Spec.Endpoint == nil {
 		// CEL guarantees an endpoint on any object that went through the API
@@ -409,7 +443,10 @@ func (r *ProxyReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	if r.HealthEvents != nil {
 		b = b.WatchesRawSource(source.Channel(r.HealthEvents, &handler.EnqueueRequestForObject{}))
 	}
-	return b.Complete(r)
+	// Root span per reconcile; sub-reconcilers and the status flush hang
+	// their spans off it. Tests calling r.Reconcile directly bypass the
+	// wrapper and see no-op spans — the global tracer is never set there.
+	return b.Complete(tracing.NewReconciler("Proxy", r))
 }
 
 func (r *ProxyReconciler) applyDefaults() {

internal/controller/reconcile_test.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"context"
+	"slices"
 	"testing"
 	"time"
 
@@ -78,6 +79,10 @@ func notFoundErr() error {
 	return provider.Wrap(provider.ErrNotFound, "get", "stub", "some-id", nil)
 }
 
+func permanentErr() error {
+	return provider.Wrap(provider.ErrPermanent, "get", "stub", "some-id", nil)
+}
+
 func testScheme(t *testing.T) *runtime.Scheme {
 	t.Helper()
 	s := runtime.NewScheme()
@@ -125,6 +130,15 @@ func withSpecHashAnnotation(hash string) func(*crawlv1alpha1.Proxy) {
 	}
 }
 
+func withSkipCleanup() func(*crawlv1alpha1.Proxy) {
+	return func(p *crawlv1alpha1.Proxy) {
+		if p.Annotations == nil {
+			p.Annotations = map[string]string{}
+		}
+		p.Annotations[crawlv1alpha1.AnnotationSkipCleanup] = "true"
+	}
+}
+
 func deleting() func(*crawlv1alpha1.Proxy) {
 	return func(p *crawlv1alpha1.Proxy) {
 		now := metav1.Now()
@@ -375,6 +389,59 @@ func TestReconcile_actionTable(t *testing.T) {
 				assertProxyGone(t, r)
 			},
 		},
+		{
+			// A permanent Get (a retired KVM host, a providerID the provider
+			// cannot resolve) used to retry forever with the finalizer on.
+			name:       "deletion with an unreachable instance still deletes and releases",
+			proxy:      managedProxy(deleting(), withProviderID("stub-id-1")),
+			stub:       &stubProvider{getErr: permanentErr()},
+			wantResult: ctrl.Result{},
+			verify: func(t *testing.T, r *ProxyReconciler, stub *stubProvider) {
+				assertProxyGone(t, r)
+				if stub.deleteCalls != 1 {
+					t.Errorf("deleteCalls = %d, want 1 — Delete must be tried when Get cannot answer", stub.deleteCalls)
+				}
+			},
+		},
+		{
+			name:  "deletion keeps the finalizer when cleanup itself fails",
+			proxy: managedProxy(deleting(), withProviderID("stub-id-1")),
+			stub: &stubProvider{
+				getErr:    permanentErr(),
+				deleteErr: provider.Wrap(provider.ErrPermanent, "delete", "stub", "stub-id-1", nil),
+			},
+			wantResult: ctrl.Result{},
+			wantErr:    true,
+			verify: func(t *testing.T, r *ProxyReconciler, _ *stubProvider) {
+				// getProxy fails the test if the object is gone, which is what
+				// finalizer removal would have caused.
+				if p := getProxy(t, r); !slices.Contains(p.Finalizers, crawlv1alpha1.FinalizerName) {
+					t.Error("finalizer removed although cleanup failed; the instance would leak silently")
+				}
+			},
+		},
+		{
+			name:       "skip-cleanup annotation abandons the instance",
+			proxy:      managedProxy(deleting(), withProviderID("stub-id-1"), withSkipCleanup()),
+			stub:       &stubProvider{getInst: &provider.Instance{ID: "stub-id-1", State: provider.StateRunning}},
+			wantResult: ctrl.Result{},
+			verify: func(t *testing.T, r *ProxyReconciler, stub *stubProvider) {
+				assertProxyGone(t, r)
+				if stub.getCalls != 0 || stub.deleteCalls != 0 {
+					t.Errorf("getCalls/deleteCalls = %d/%d, want 0/0 — the provider must not be asked", stub.getCalls, stub.deleteCalls)
+				}
+			},
+		},
+		{
+			name: "skip-cleanup annotation releases a proxy whose provider is gone",
+			proxy: managedProxy(deleting(), withProviderID("stub-id-1"), withSkipCleanup(),
+				func(p *crawlv1alpha1.Proxy) { p.Spec.Provider = "retired" }),
+			stub:       &stubProvider{},
+			wantResult: ctrl.Result{},
+			verify: func(t *testing.T, r *ProxyReconciler, _ *stubProvider) {
+				assertProxyGone(t, r)
+			},
+		},
 		{
 			name:  "deletion deletes the instance and polls",
 			proxy: managedProxy(deleting(), withProviderID("stub-id-1")),

internal/discovery/handlers.go

@@ -11,6 +11,7 @@ import (
 
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
@@ -82,7 +83,10 @@ func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
 
 	var list crawlv1alpha1.ProxyList
 	if err := s.Reader.List(r.Context(), &list); err != nil {
-		s.log.Error(err, "listing proxies")
+		// The ctx logger is s.log enriched with this request's
+		// traceID/spanID by the tracing middleware (logr sinks can't read
+		// ctx at log time, so per-request values ride on the logger).
+		logf.FromContext(r.Context()).Error(err, "listing proxies")
 		writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
 		return
 	}
@@ -138,7 +142,7 @@ func (s *Server) handleAcquireLease(w http.ResponseWriter, r *http.Request) {
 
 	var list crawlv1alpha1.ProxyList
 	if err := s.Reader.List(r.Context(), &list); err != nil {
-		s.log.Error(err, "listing proxies for lease")
+		logf.FromContext(r.Context()).Error(err, "listing proxies for lease")
 		writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
 		return
 	}
@@ -231,7 +235,7 @@ func (s *Server) handleReportLease(w http.ResponseWriter, r *http.Request) {
 			writeError(w, http.StatusNotFound, "unknown_lease", "no such lease")
 			return
 		}
-		s.log.Error(err, "reporting lease", "leaseID", r.PathValue("id"))
+		logf.FromContext(r.Context()).Error(err, "reporting lease", "leaseID", r.PathValue("id"))
 		writeError(w, http.StatusInternalServerError, "internal", "report failed")
 		return
 	}

internal/discovery/server.go

@@ -20,6 +20,7 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // LeaseStore is what the handlers need from a lease backend. Defined here,
@@ -136,7 +137,9 @@ func (s *Server) Start(ctx context.Context) error {
 }
 
 // handler assembles the mux and the middleware chain, outermost first:
-// recover → request-log → body-size cap → bearer auth.
+// recover → tracing (server span + request logger) → request-log →
+// body-size cap → bearer auth. Everything inside the tracing layer logs via
+// logf.FromContext(r.Context()) and so carries traceID/spanID.
 func (s *Server) handler() http.Handler {
 	mux := http.NewServeMux()
 	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
@@ -152,6 +155,7 @@ func (s *Server) handler() http.Handler {
 	h = s.authMiddleware(h)
 	h = maxBytesMiddleware(h)
 	h = s.logMiddleware(h)
+	h = tracing.HTTPMiddleware("discovery", s.log)(h)
 	h = s.recoverMiddleware(h)
 	return h
 }
@@ -188,7 +192,7 @@ func (s *Server) logMiddleware(next http.Handler) http.Handler {
 		rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
 		start := time.Now()
 		next.ServeHTTP(rec, r)
-		s.log.Info("request",
+		logf.FromContext(r.Context()).Info("request",
 			"method", r.Method, "path", r.URL.Path,
 			"status", rec.status, "duration", time.Since(start).String())
 	})

internal/gc/gc.go

@@ -12,8 +12,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
+	"go.opentelemetry.io/otel/attribute"
+
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // Sweeper is the manager Runnable running the sweep loop.
@@ -81,6 +84,8 @@ func (s *Sweeper) Start(ctx context.Context) error {
 // that deletion, and GC racing it would double-delete. A UID becomes
 // orphan-eligible only once the object is fully gone.
 func (s *Sweeper) sweep(ctx context.Context) {
+	ctx, span := tracing.Start(ctx, "gc.sweep")
+	defer span.End()
 	log := logf.FromContext(ctx).WithName("orphan-gc")
 
 	var list crawlv1alpha1.ProxyList
@@ -95,6 +100,7 @@ func (s *Sweeper) sweep(ctx context.Context) {
 		live[string(list.Items[i].UID)] = true
 	}
 
+	deleted := 0
 	for name, prov := range s.Providers {
 		instances, err := prov.ListByTag(ctx)
 		if err != nil {
@@ -119,7 +125,13 @@ func (s *Sweeper) sweep(ctx context.Context) {
 			if err := prov.Delete(ctx, inst.ID); err != nil {
 				log.Error(err, "deleting orphaned instance",
 					"provider", name, "providerID", inst.ID, "uid", inst.UID)
+				continue
 			}
+			deleted++
 		}
 	}
+	span.SetAttributes(
+		attribute.Int("gc.providers", len(s.Providers)),
+		attribute.Int("gc.deleted", deleted),
+	)
 }

internal/health/engine.go

@@ -10,6 +10,9 @@ import (
 	"sync"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -18,6 +21,7 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // Snapshot is the engine's current verdict for one proxy, read by the
@@ -99,6 +103,10 @@ type Engine struct {
 	// transition-only by design, so metrics are where high-frequency
 	// signal (true probe recency, every latency sample) lives.
 	Metrics ProbeMetrics
+	// TraceProbes emits one root span per probe (--trace-health-probes).
+	// Off by default: probes run about once per second per proxy and would
+	// dominate trace volume.
+	TraceProbes bool
 
 	probeFn func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult
 
@@ -155,8 +163,7 @@ func (e *Engine) Start(ctx context.Context) error {
 				case <-ctx.Done():
 					return
 				case job := <-jobs:
-					res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
-					e.record(job, res, time.Now())
+					e.record(job, e.runProbe(ctx, job), time.Now())
 				}
 			}
 		})
@@ -264,6 +271,29 @@ func newState(p *crawlv1alpha1.Proxy, now time.Time, interval time.Duration) *st
 	return st
 }
 
+// runProbe executes one probe, inside its own root span when TraceProbes is
+// on. The span ends before record folds the result (record stays ctx-free),
+// and the probe transport is deliberately uninstrumented — no traceparent
+// must ever leak through a proxy toward external targets.
+func (e *Engine) runProbe(ctx context.Context, job probeJob) probeResult {
+	if !e.TraceProbes {
+		return e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
+	}
+	ctx, span := tracing.Start(ctx, "health.probe",
+		trace.WithSpanKind(trace.SpanKindClient),
+		trace.WithAttributes(attribute.String("proxy", job.key.String())))
+	defer span.End()
+	res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
+	span.SetAttributes(
+		attribute.Bool("probe.ok", res.ok),
+		attribute.Int64("probe.latency_ms", res.latency.Milliseconds()),
+	)
+	if !res.ok {
+		span.SetStatus(codes.Error, res.err.Error())
+	}
+	return res
+}
+
 // record folds one probe result into the proxy's threshold state and emits
 // an event when the result is status-affecting: a first-ever verdict, a
 // threshold-crossing flip, or a material latency change (beyond

internal/metrics/metrics.go

@@ -117,3 +117,51 @@ func (c *constCollector) Collect(ch chan<- prometheus.Metric) {
 		ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(value), label)
 	}
 }
+
+// ProviderTarget is one provider health-check target as the gauges see it.
+type ProviderTarget struct {
+	Provider, Target string
+	Up               bool
+	Duration         time.Duration
+	CheckedAt        time.Time
+}
+
+// RegisterProviderTargets registers the provider-check gauges. Like the
+// other scrape-time collectors they are read from the source at every
+// scrape, so a target that goes away stops being reported rather than
+// leaving a stale series behind.
+func RegisterProviderTargets(reg prometheus.Registerer, read func() []ProviderTarget) error {
+	labels := []string{"provider", "target"}
+	return reg.Register(&providerTargetCollector{
+		up: prometheus.NewDesc("proxy_operator_provider_target_up",
+			"Whether the target passed its latest provider health check (1) or not (0).", labels, nil),
+		duration: prometheus.NewDesc("proxy_operator_provider_check_duration_seconds",
+			"Duration of the target's latest provider health check.", labels, nil),
+		checked: prometheus.NewDesc("proxy_operator_provider_check_timestamp_seconds",
+			"Unix time of the target's latest provider health check.", labels, nil),
+		read: read,
+	})
+}
+
+type providerTargetCollector struct {
+	up, duration, checked *prometheus.Desc
+	read                  func() []ProviderTarget
+}
+
+func (c *providerTargetCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.up
+	ch <- c.duration
+	ch <- c.checked
+}
+
+func (c *providerTargetCollector) Collect(ch chan<- prometheus.Metric) {
+	for _, t := range c.read() {
+		up := 0.0
+		if t.Up {
+			up = 1
+		}
+		ch <- prometheus.MustNewConstMetric(c.up, prometheus.GaugeValue, up, t.Provider, t.Target)
+		ch <- prometheus.MustNewConstMetric(c.duration, prometheus.GaugeValue, t.Duration.Seconds(), t.Provider, t.Target)
+		ch <- prometheus.MustNewConstMetric(c.checked, prometheus.GaugeValue, float64(t.CheckedAt.UnixNano())/1e9, t.Provider, t.Target)
+	}
+}

internal/metrics/metrics_test.go

@@ -111,3 +111,45 @@ func TestRegister_freshRegistryPerTest(t *testing.T) {
 	register(t, m1, nil, 0)
 	register(t, m2, nil, 0)
 }
+
+func TestRegisterProviderTargets_scrape(t *testing.T) {
+	t.Parallel()
+	checked := time.Unix(1757602800, 0)
+	targets := []ProviderTarget{
+		{Provider: "kvm-home", Target: "nas1", Up: true, Duration: 84 * time.Millisecond, CheckedAt: checked},
+		{Provider: "kvm-home", Target: "nas2", Up: false, Duration: 15 * time.Second, CheckedAt: checked},
+	}
+	reg := prometheus.NewRegistry()
+	if err := RegisterProviderTargets(reg, func() []ProviderTarget { return targets }); err != nil {
+		t.Fatalf("RegisterProviderTargets: %v", err)
+	}
+
+	expected := `
+# HELP proxy_operator_provider_check_duration_seconds Duration of the target's latest provider health check.
+# TYPE proxy_operator_provider_check_duration_seconds gauge
+proxy_operator_provider_check_duration_seconds{provider="kvm-home",target="nas1"} 0.084
+proxy_operator_provider_check_duration_seconds{provider="kvm-home",target="nas2"} 15
+# HELP proxy_operator_provider_check_timestamp_seconds Unix time of the target's latest provider health check.
+# TYPE proxy_operator_provider_check_timestamp_seconds gauge
+proxy_operator_provider_check_timestamp_seconds{provider="kvm-home",target="nas1"} 1.7576028e+09
+proxy_operator_provider_check_timestamp_seconds{provider="kvm-home",target="nas2"} 1.7576028e+09
+# HELP proxy_operator_provider_target_up Whether the target passed its latest provider health check (1) or not (0).
+# TYPE proxy_operator_provider_target_up gauge
+proxy_operator_provider_target_up{provider="kvm-home",target="nas1"} 1
+proxy_operator_provider_target_up{provider="kvm-home",target="nas2"} 0
+`
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(expected)); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestRegisterProviderTargets_emptySnapshotHasNoSeries(t *testing.T) {
+	t.Parallel()
+	reg := prometheus.NewRegistry()
+	if err := RegisterProviderTargets(reg, func() []ProviderTarget { return nil }); err != nil {
+		t.Fatalf("RegisterProviderTargets: %v", err)
+	}
+	if n, err := testutil.GatherAndCount(reg); err != nil || n != 0 {
+		t.Errorf("series before the first check = %d (%v), want 0", n, err)
+	}
+}

internal/provider/check.go

@@ -0,0 +1,46 @@
+package provider
+
+import (
+	"context"
+	"time"
+)
+
+// Reasons shared by every Checker, so a listing reads the same across
+// providers. Providers add their own for failures only they can tell
+// apart (KVM's SSHHandshakeFailed or PoolNotActive, say).
+const (
+	ReasonReachable    = "Reachable"
+	ReasonTimeout      = "Timeout"
+	ReasonUnreachable  = "Unreachable"
+	ReasonUnauthorized = "Unauthorized"
+	ReasonForbidden    = "Forbidden"
+	ReasonAPIError     = "APIError"
+)
+
+// TargetStatus is one target's verdict from a Checker.
+type TargetStatus struct {
+	// Name identifies the target within its provider: a KVM host name,
+	// "apiserver", a GCP project.
+	Name string
+	// Address is what the check connected to, for humans.
+	Address string
+	Ready   bool
+	// Reason is ReasonReachable when Ready, otherwise why not.
+	Reason string
+	// Message is human-readable detail, usually the error.
+	Message string
+	Latency time.Duration
+	// Details are provider-specific facts gathered by the check.
+	Details map[string]string
+}
+
+// Checker is the optional interface of a provider that can verify its
+// backend is reachable and usable without touching any instance. The
+// provider health monitor type-asserts for it on the unwrapped providers,
+// so it stays out of the Provider contract.
+type Checker interface {
+	// Check returns one verdict per target, in a stable order. It never
+	// fails as a whole — a failure is a verdict — and must return promptly
+	// once ctx ends.
+	Check(ctx context.Context) []TargetStatus
+}

internal/provider/config.go

@@ -21,6 +21,23 @@ type ProviderConfig struct {
 	Type       string            `json:"type"`
 	Kubernetes *KubernetesConfig `json:"kubernetes,omitempty"`
 	GCP        *GCPConfig        `json:"gcp,omitempty"`
+	KVM        *KVMConfig        `json:"kvm,omitempty"`
+}
+
+// setBlocks names the type-specific blocks that are non-nil, so validate can
+// reject a block that doesn't match Type without a pairwise check per type.
+func (p ProviderConfig) setBlocks() []string {
+	var out []string
+	if p.Kubernetes != nil {
+		out = append(out, "kubernetes")
+	}
+	if p.GCP != nil {
+		out = append(out, "gcp")
+	}
+	if p.KVM != nil {
+		out = append(out, "kvm")
+	}
+	return out
 }
 
 // KubernetesConfig configures the kubernetes-pod provider, which creates
@@ -81,22 +98,28 @@ func (c *Config) validate() error {
 		}
 		seen[p.Name] = true
 
+		prefix := fmt.Sprintf("providers[%d] %q", i, p.Name)
 		switch p.Type {
-		case "kubernetes":
-			if p.GCP != nil {
-				return fmt.Errorf("providers[%d] %q: type is kubernetes but a gcp block is set", i, p.Name)
-			}
-		case "gcp":
-			if p.Kubernetes != nil {
-				return fmt.Errorf("providers[%d] %q: type is gcp but a kubernetes block is set", i, p.Name)
-			}
-			if p.GCP == nil || p.GCP.Project == "" {
-				return fmt.Errorf("providers[%d] %q: gcp.project is required", i, p.Name)
-			}
+		case "kubernetes", "gcp", "kvm":
 		case "":
-			return fmt.Errorf("providers[%d] %q: type is required", i, p.Name)
+			return fmt.Errorf("%s: type is required", prefix)
 		default:
-			return fmt.Errorf("providers[%d] %q: unknown provider type %q", i, p.Name, p.Type)
+			return fmt.Errorf("%s: unknown provider type %q", prefix, p.Type)
+		}
+		for _, block := range p.setBlocks() {
+			if block != p.Type {
+				return fmt.Errorf("%s: type is %s but a %s block is set", prefix, p.Type, block)
+			}
+		}
+		switch p.Type {
+		case "gcp":
+			if p.GCP == nil || p.GCP.Project == "" {
+				return fmt.Errorf("%s: gcp.project is required", prefix)
+			}
+		case "kvm":
+			if err := validateKVM(prefix, p.KVM); err != nil {
+				return err
+			}
 		}
 	}
 	return nil

internal/provider/config_kvm.go

@@ -0,0 +1,184 @@
+package provider
+
+import (
+	"fmt"
+	"net/url"
+	"regexp"
+)
+
+// KVMConfig configures a kvm provider instance: one or more libvirt/KVM
+// hypervisors reached over SSH (the libvirt unix socket is forwarded through
+// the SSH connection, so hosts need no libvirtd TCP/TLS listener).
+// placement.zone on a Proxy selects a host by name; with a single host it may
+// be omitted.
+type KVMConfig struct {
+	// SSHKeyFile is the path to the private key used for every host. Required.
+	SSHKeyFile string `json:"sshKeyFile"`
+	// KnownHostsFile is an OpenSSH known_hosts file used to verify host keys.
+	// Required unless InsecureIgnoreHostKey is set.
+	KnownHostsFile string `json:"knownHostsFile,omitempty"`
+	// InsecureIgnoreHostKey skips host-key verification entirely.
+	InsecureIgnoreHostKey bool `json:"insecureIgnoreHostKey,omitempty"`
+	// DiskSizeGB is the capacity of each VM's overlay disk. Default 10. The
+	// guest's cloud-init growpart expands the root filesystem into it.
+	DiskSizeGB int64 `json:"diskSizeGb,omitempty"`
+	// VendorData is optional cloud-init vendor-data placed on every seed next
+	// to the Proxy's own user-data. Note that cloud-init drops any top-level
+	// key of a vendor-data cloud-config that the user-data also defines.
+	// When InstallGuestAgent is on, this document and the agent script are
+	// carried as separate parts of a MIME multipart vendor-data.
+	VendorData string `json:"vendorData,omitempty"`
+	// InstallGuestAgent adds a vendor-data script that installs and starts
+	// qemu-guest-agent, which is how the provider learns a VM's address
+	// before the host's ARP table has seen traffic from it. Default true.
+	// Turn it off for images that already ship the agent, or that cannot
+	// reach a package mirror; discovery then depends on ARP alone.
+	InstallGuestAgent *bool `json:"installGuestAgent,omitempty"`
+	// Images are named base images; placement.image names a key here (or is
+	// an http(s) URL directly). The provider downloads each image into a
+	// host's storage pool on first use and caches it there.
+	Images map[string]KVMImage `json:"images,omitempty"`
+	// MachineTypes are named size profiles; placement.machineType names a
+	// key here. Required, at least one.
+	MachineTypes map[string]KVMMachineType `json:"machineTypes"`
+	// Hosts are the hypervisors. Required, at least one, unique names.
+	Hosts []KVMHost `json:"hosts"`
+}
+
+// KVMImage is one downloadable base image (a distro cloud image with
+// cloud-init preinstalled, e.g. Debian's generic qcow2).
+type KVMImage struct {
+	// URL is where to fetch the image from. Required, http or https.
+	URL string `json:"url"`
+	// SHA256 is the expected hex digest of the file. Optional; when set it is
+	// verified after download and a later change triggers a re-download.
+	SHA256 string `json:"sha256,omitempty"`
+	// Format is the image's disk format, "qcow2" (default) or "raw".
+	Format string `json:"format,omitempty"`
+	// Firmware is how VMs from this image boot: "uefi" (default; OVMF with
+	// secure boot off) or "bios" (SeaBIOS) for images with no EFI system
+	// partition. Current Debian, Ubuntu, Fedora and Rocky cloud images all
+	// boot UEFI, and Debian 13's BIOS path was found not to boot at all.
+	Firmware string `json:"firmware,omitempty"`
+}
+
+// KVMMachineType is a VM size profile.
+type KVMMachineType struct {
+	VCPUs     int32 `json:"vcpus"`
+	MemoryMiB int64 `json:"memoryMiB"`
+}
+
+// KVMHost is one hypervisor.
+type KVMHost struct {
+	// Name is the placement.zone value that selects this host. Required.
+	Name string `json:"name"`
+	// Address is the SSH endpoint, host:port. Required.
+	Address string `json:"address"`
+	// User is the SSH user; it must be allowed to use the libvirt socket
+	// (typically a member of the libvirt group). Required.
+	User string `json:"user"`
+	// Socket is the libvirt unix socket path on the host.
+	// Default /var/run/libvirt/libvirt-sock.
+	Socket string `json:"socket,omitempty"`
+	// Pool is the libvirt storage pool for base images, overlay disks and
+	// seed ISOs. Default "default".
+	Pool string `json:"pool,omitempty"`
+	// Bridge is the host bridge the VM's NIC attaches to; it must lead to a
+	// network the cluster can reach and that hands out DHCP leases. Required.
+	Bridge string `json:"bridge"`
+}
+
+// Accepted KVMImage.Format and KVMImage.Firmware values.
+const (
+	KVMFormatQcow2  = "qcow2"
+	KVMFormatRaw    = "raw"
+	KVMFirmwareUEFI = "uefi"
+	KVMFirmwareBIOS = "bios"
+)
+
+// IsHTTPURL reports whether s is an absolute http(s) URL with a host, the
+// only kind of image source the kvm provider can fetch.
+func IsHTTPURL(s string) bool {
+	u, err := url.Parse(s)
+	return err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != ""
+}
+
+var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
+
+// kvmIdentifier constrains the config names that are pasted into
+// identifiers elsewhere: a host name goes into the providerID
+// hosts/<name>/domains/<vm>, which is split on "/" to parse, and an image
+// key becomes the file name of the cached base volume. A slash in either
+// produces an ID or a path that never resolves again, so the VM leaks with
+// nothing able to address it.
+var kvmIdentifier = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`)
+
+func validateKVM(prefix string, c *KVMConfig) error {
+	if c == nil {
+		return fmt.Errorf("%s: kvm block is required", prefix)
+	}
+	if c.SSHKeyFile == "" {
+		return fmt.Errorf("%s: kvm.sshKeyFile is required", prefix)
+	}
+	if c.KnownHostsFile == "" && !c.InsecureIgnoreHostKey {
+		return fmt.Errorf("%s: kvm.knownHostsFile is required unless kvm.insecureIgnoreHostKey is true", prefix)
+	}
+	if c.DiskSizeGB < 0 {
+		return fmt.Errorf("%s: kvm.diskSizeGb must not be negative", prefix)
+	}
+	for name, img := range c.Images {
+		p := fmt.Sprintf("%s: kvm.images[%q]", prefix, name)
+		if !kvmIdentifier.MatchString(name) {
+			return fmt.Errorf("%s: key must be letters, digits, dots, dashes or underscores"+
+				" — it becomes the cached volume name base-<key>.<format>", p)
+		}
+		if !IsHTTPURL(img.URL) {
+			return fmt.Errorf("%s.url must be an http(s) URL (got %q)", p, img.URL)
+		}
+		if img.SHA256 != "" && !sha256Hex.MatchString(img.SHA256) {
+			return fmt.Errorf("%s.sha256 must be 64 hex characters", p)
+		}
+		if img.Format != "" && img.Format != KVMFormatQcow2 && img.Format != KVMFormatRaw {
+			return fmt.Errorf("%s.format must be %s or %s (got %q)", p, KVMFormatQcow2, KVMFormatRaw, img.Format)
+		}
+		if img.Firmware != "" && img.Firmware != KVMFirmwareUEFI && img.Firmware != KVMFirmwareBIOS {
+			return fmt.Errorf("%s.firmware must be %s or %s (got %q)", p, KVMFirmwareUEFI, KVMFirmwareBIOS, img.Firmware)
+		}
+	}
+	if len(c.MachineTypes) == 0 {
+		return fmt.Errorf("%s: kvm.machineTypes must define at least one profile", prefix)
+	}
+	for name, mt := range c.MachineTypes {
+		if mt.VCPUs <= 0 || mt.MemoryMiB <= 0 {
+			return fmt.Errorf("%s: kvm.machineTypes[%q] needs vcpus > 0 and memoryMiB > 0", prefix, name)
+		}
+	}
+	if len(c.Hosts) == 0 {
+		return fmt.Errorf("%s: kvm.hosts must list at least one host", prefix)
+	}
+	seen := make(map[string]bool, len(c.Hosts))
+	for i, h := range c.Hosts {
+		p := fmt.Sprintf("%s: kvm.hosts[%d]", prefix, i)
+		if h.Name == "" {
+			return fmt.Errorf("%s: name is required", p)
+		}
+		if !kvmIdentifier.MatchString(h.Name) {
+			return fmt.Errorf("%s: name %q must be letters, digits, dots, dashes or underscores"+
+				" — it becomes part of the providerID hosts/<name>/domains/<vm>", p, h.Name)
+		}
+		if seen[h.Name] {
+			return fmt.Errorf("%s: duplicate host name %q", p, h.Name)
+		}
+		seen[h.Name] = true
+		if h.Address == "" {
+			return fmt.Errorf("%s %q: address is required", p, h.Name)
+		}
+		if h.User == "" {
+			return fmt.Errorf("%s %q: user is required", p, h.Name)
+		}
+		if h.Bridge == "" {
+			return fmt.Errorf("%s %q: bridge is required", p, h.Name)
+		}
+	}
+	return nil
+}

internal/provider/config_kvm_test.go

@@ -0,0 +1,226 @@
+package provider
+
+import (
+	"strings"
+	"testing"
+)
+
+const validKVMYAML = `
+providers:
+  - name: kvm-home
+    type: kvm
+    kvm:
+      sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+      knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+      diskSizeGb: 20
+      images:
+        debian-13:
+          url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+          sha256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+        ubuntu:
+          url: http://mirror.local/noble.img
+          format: raw
+      machineTypes:
+        small: {vcpus: 1, memoryMiB: 1024}
+        medium: {vcpus: 2, memoryMiB: 2048}
+      hosts:
+        - name: nas1
+          address: nas1.home.lan:22
+          user: virt
+          bridge: br0
+        - name: lab2
+          address: 10.0.0.2:2222
+          user: root
+          socket: /run/libvirt/virtqemud-sock
+          pool: fast
+          bridge: br1
+`
+
+func TestLoadConfig_kvmValid(t *testing.T) {
+	t.Parallel()
+	cfg, err := LoadConfig([]byte(validKVMYAML))
+	if err != nil {
+		t.Fatalf("LoadConfig() error = %v, want nil", err)
+	}
+	k := cfg.Providers[0].KVM
+	if k == nil {
+		t.Fatal("providers[0].kvm = nil")
+	}
+	if k.SSHKeyFile != "/etc/proxy-operator/kvm/id_ed25519" || k.DiskSizeGB != 20 {
+		t.Errorf("kvm = %+v, want sshKeyFile and diskSizeGb parsed", k)
+	}
+	if got := k.Images["debian-13"].SHA256; !strings.HasPrefix(got, "0123") {
+		t.Errorf("images[debian-13].sha256 = %q", got)
+	}
+	if k.Images["ubuntu"].Format != "raw" {
+		t.Errorf("images[ubuntu].format = %q, want raw", k.Images["ubuntu"].Format)
+	}
+	if mt := k.MachineTypes["medium"]; mt.VCPUs != 2 || mt.MemoryMiB != 2048 {
+		t.Errorf("machineTypes[medium] = %+v, want {2 2048}", mt)
+	}
+	if len(k.Hosts) != 2 || k.Hosts[1].Socket != "/run/libvirt/virtqemud-sock" || k.Hosts[1].Pool != "fast" {
+		t.Errorf("hosts = %+v, want two hosts with lab2's socket and pool parsed", k.Hosts)
+	}
+}
+
+func TestLoadConfig_kvmInsecureHostKeyNeedsNoKnownHosts(t *testing.T) {
+	t.Parallel()
+	yaml := strings.Replace(validKVMYAML,
+		"knownHostsFile: /etc/proxy-operator/kvm/known_hosts",
+		"insecureIgnoreHostKey: true", 1)
+	if _, err := LoadConfig([]byte(yaml)); err != nil {
+		t.Fatalf("LoadConfig() error = %v, want nil", err)
+	}
+}
+
+func TestLoadConfig_kvmInvalid(t *testing.T) {
+	t.Parallel()
+	minimal := func(kvmBody string) string {
+		return "providers:\n  - name: kvm-home\n    type: kvm\n    kvm:\n" + indent(kvmBody, 6)
+	}
+	hostsAndTypes := `
+machineTypes:
+  small: {vcpus: 1, memoryMiB: 1024}
+hosts:
+  - {name: nas1, address: nas1:22, user: virt, bridge: br0}
+`
+	tests := []struct {
+		name       string
+		yaml       string
+		wantErrSub string
+	}{
+		{
+			name:       "missing kvm block",
+			yaml:       "providers:\n  - name: kvm-home\n    type: kvm\n",
+			wantErrSub: "kvm block is required",
+		},
+		{
+			name:       "missing sshKeyFile",
+			yaml:       minimal("knownHostsFile: /kh\n" + hostsAndTypes),
+			wantErrSub: "kvm.sshKeyFile is required",
+		},
+		{
+			name:       "missing knownHostsFile without insecure flag",
+			yaml:       minimal("sshKeyFile: /key\n" + hostsAndTypes),
+			wantErrSub: "kvm.knownHostsFile is required unless",
+		},
+		{
+			name:       "negative disk size",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\ndiskSizeGb: -1\n" + hostsAndTypes),
+			wantErrSub: "diskSizeGb must not be negative",
+		},
+		{
+			name:       "no machine types",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nhosts:\n  - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
+			wantErrSub: "machineTypes must define at least one",
+		},
+		{
+			name:       "machine type with zero memory",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1}\nhosts:\n  - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `machineTypes["small"] needs vcpus > 0 and memoryMiB > 0`,
+		},
+		{
+			name:       "no hosts",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\n"),
+			wantErrSub: "kvm.hosts must list at least one host",
+		},
+		{
+			name:       "duplicate host name",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, user: virt, bridge: br0}\n  - {name: nas1, address: b:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `duplicate host name "nas1"`,
+		},
+		{
+			// A slash would produce hosts/dc1/rack2/domains/<vm>, which
+			// parseProviderID can never split back, so the VM would be
+			// unreachable for Get, Delete and GC alike.
+			name:       "host name with a slash",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: dc1/rack2, address: a:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `name "dc1/rack2" must be letters, digits, dots, dashes or underscores`,
+		},
+		{
+			name:       "image key with a slash",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  ../etc/d: {url: https://x/y.qcow2}\n" + hostsAndTypes),
+			wantErrSub: `images["../etc/d"]: key must be letters, digits, dots, dashes or underscores`,
+		},
+		{
+			name:       "host missing address",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, user: virt, bridge: br0}\n"),
+			wantErrSub: "address is required",
+		},
+		{
+			name:       "host missing user",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, bridge: br0}\n"),
+			wantErrSub: "user is required",
+		},
+		{
+			name:       "host missing bridge",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, user: virt}\n"),
+			wantErrSub: "bridge is required",
+		},
+		{
+			name:       "image with non-http url",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: file:///tmp/x.qcow2}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].url must be an http(s) URL`,
+		},
+		{
+			name:       "image with bad sha256",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.qcow2, sha256: abc}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].sha256 must be 64 hex characters`,
+		},
+		{
+			name:       "image with unknown firmware",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.qcow2, firmware: coreboot}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].firmware must be uefi or bios`,
+		},
+		{
+			name:       "image with unknown format",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.vmdk, format: vmdk}\n" + hostsAndTypes),
+			wantErrSub: `images["d"].format must be qcow2 or raw`,
+		},
+		{
+			name: "kvm type with gcp block",
+			yaml: `
+providers:
+  - name: p1
+    type: kvm
+    gcp:
+      project: my-project`,
+			wantErrSub: "type is kvm but a gcp block is set",
+		},
+		{
+			name: "gcp type with kvm block",
+			yaml: `
+providers:
+  - name: p1
+    type: gcp
+    gcp:
+      project: my-project
+    kvm:
+      sshKeyFile: /key`,
+			wantErrSub: "type is gcp but a kvm block is set",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			_, err := LoadConfig([]byte(tc.yaml))
+			if err == nil {
+				t.Fatalf("LoadConfig() error = nil, want error containing %q", tc.wantErrSub)
+			}
+			if !strings.Contains(err.Error(), tc.wantErrSub) {
+				t.Errorf("LoadConfig() error = %q, want substring %q", err.Error(), tc.wantErrSub)
+			}
+		})
+	}
+}
+
+func indent(s string, n int) string {
+	pad := strings.Repeat(" ", n)
+	lines := strings.Split(strings.TrimLeft(s, "\n"), "\n")
+	for i, l := range lines {
+		if l != "" {
+			lines[i] = pad + l
+		}
+	}
+	return strings.Join(lines, "\n")
+}

internal/provider/gcp/check.go

@@ -0,0 +1,78 @@
+package gcp
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"time"
+
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/iterator"
+	"google.golang.org/protobuf/proto"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+var _ provider.Checker = (*Provider)(nil)
+
+// computeAddress is the endpoint the SDK talks to, shown only to humans.
+const computeAddress = "compute.googleapis.com"
+
+// managedFilter selects the instances this operator created.
+const managedFilter = "labels." + provider.LabelManaged + " = " + provider.LabelManagedYes
+
+// Check asks the project for one page holding at most one managed
+// instance. That exercises the credentials, the Compute API and the
+// compute.instances.list permission orphan GC depends on, in a single
+// request whatever the project's size. The target is the project: the
+// provider config names no zones.
+func (p *Provider) Check(ctx context.Context) []provider.TargetStatus {
+	start := time.Now()
+	err := p.api.Probe(ctx, &computepb.AggregatedListInstancesRequest{
+		Project:              p.cfg.Project,
+		Filter:               proto.String(managedFilter),
+		MaxResults:           proto.Uint32(1),
+		ReturnPartialSuccess: proto.Bool(true),
+	})
+	ts := provider.TargetStatus{Name: p.cfg.Project, Address: computeAddress, Latency: time.Since(start)}
+	if err != nil {
+		logAPIError(p.logger(ctx), "check", err)
+		ts.Reason = checkReason(ctx, err)
+		ts.Message = err.Error()
+		return []provider.TargetStatus{ts}
+	}
+	ts.Ready = true
+	ts.Reason = provider.ReasonReachable
+	return []provider.TargetStatus{ts}
+}
+
+// Probe fetches the first page of an aggregated list and discards it. The
+// SDK treats MaxResults as the page size, so AggregatedList, which drains
+// the iterator, would still walk every page of the project.
+func (r *realInstances) Probe(ctx context.Context, req *computepb.AggregatedListInstancesRequest) error {
+	_, err := r.client.AggregatedList(ctx, req).Next()
+	if err == iterator.Done {
+		return nil
+	}
+	return err
+}
+
+// checkReason maps a failed probe to a reason. A token that cannot be
+// obtained never reaches the API, so it has no status code and lands on
+// Unreachable, with the detail in the message.
+func checkReason(ctx context.Context, err error) string {
+	var gerr *googleapi.Error
+	switch {
+	case ctx.Err() != nil, errors.Is(err, context.DeadlineExceeded):
+		return provider.ReasonTimeout
+	case !errors.As(err, &gerr):
+		return provider.ReasonUnreachable
+	case gerr.Code == http.StatusUnauthorized:
+		return provider.ReasonUnauthorized
+	case gerr.Code == http.StatusForbidden:
+		return provider.ReasonForbidden
+	default:
+		return provider.ReasonAPIError
+	}
+}

internal/provider/gcp/check_test.go

@@ -0,0 +1,165 @@
+package gcp
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"sync"
+	"syscall"
+	"testing"
+
+	compute "cloud.google.com/go/compute/apiv1"
+	"cloud.google.com/go/compute/apiv1/computepb"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
+	"google.golang.org/protobuf/proto"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func TestCheck_reachable(t *testing.T) {
+	t.Parallel()
+	api := &fakeAPI{}
+	p := newTestProvider(api)
+
+	got := p.Check(context.Background())
+
+	if len(got) != 1 {
+		t.Fatalf("Check() returned %d targets, want 1", len(got))
+	}
+	ts := got[0]
+	if !ts.Ready || ts.Reason != provider.ReasonReachable || ts.Message != "" {
+		t.Errorf("Check() = %+v, want ready and Reachable", ts)
+	}
+	if ts.Name != "my-project" || ts.Address != "compute.googleapis.com" {
+		t.Errorf("target = %q at %q, want my-project at compute.googleapis.com", ts.Name, ts.Address)
+	}
+	want := &computepb.AggregatedListInstancesRequest{
+		Project:              "my-project",
+		Filter:               proto.String("labels.proxy-operator-managed = true"),
+		MaxResults:           proto.Uint32(1),
+		ReturnPartialSuccess: proto.Bool(true),
+	}
+	if !proto.Equal(api.probeReq, want) {
+		t.Errorf("probe request = %v, want %v", api.probeReq, want)
+	}
+}
+
+func TestCheck_failureReasons(t *testing.T) {
+	t.Parallel()
+	ended, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	tests := []struct {
+		name string
+		ctx  context.Context
+		err  error
+		want string
+	}{
+		{"unauthorized", context.Background(), gerr(401), provider.ReasonUnauthorized},
+		{"forbidden", context.Background(), gerr(403, "forbidden"), provider.ReasonForbidden},
+		{"api disabled", context.Background(), gerr(403, "accessNotConfigured"), provider.ReasonForbidden},
+		{"server error", context.Background(), gerr(500), provider.ReasonAPIError},
+		{"deadline", context.Background(), &url.Error{
+			Op: "Get", URL: "https://compute.googleapis.com/", Err: context.DeadlineExceeded,
+		}, provider.ReasonTimeout},
+		{"context ended", ended, context.Canceled, provider.ReasonTimeout},
+		{"connection refused", context.Background(), &url.Error{
+			Op: "Get", URL: "https://compute.googleapis.com/", Err: syscall.ECONNREFUSED,
+		}, provider.ReasonUnreachable},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			p := newTestProvider(&fakeAPI{probeErr: tc.err})
+
+			got := p.Check(tc.ctx)
+
+			if len(got) != 1 {
+				t.Fatalf("Check() returned %d targets, want 1", len(got))
+			}
+			if got[0].Ready || got[0].Reason != tc.want || got[0].Message != tc.err.Error() {
+				t.Errorf("Check() = %+v, want not ready, reason %s, message %q", got[0], tc.want, tc.err.Error())
+			}
+		})
+	}
+}
+
+// TestRealInstances_Probe runs the real SDK client against a local server,
+// since the point of Probe is how many requests the SDK makes.
+func TestRealInstances_Probe(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name     string
+		status   int
+		body     string
+		wantCode int
+	}{
+		{
+			name:   "fetches only the first page",
+			status: http.StatusOK,
+			body:   `{"items":{"zones/europe-west1-b":{"instances":[{"name":"proxy-a"}]}},"nextPageToken":"page-2"}`,
+		},
+		{
+			name:   "empty project",
+			status: http.StatusOK,
+			body:   `{}`,
+		},
+		{
+			name:     "api error keeps its status",
+			status:   http.StatusForbidden,
+			body:     `{"error":{"code":403,"message":"Compute Engine API has not been used","errors":[{"reason":"accessNotConfigured"}]}}`,
+			wantCode: http.StatusForbidden,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			var (
+				mu   sync.Mutex
+				reqs []*url.URL
+			)
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				mu.Lock()
+				reqs = append(reqs, r.URL)
+				mu.Unlock()
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(tc.status)
+				_, _ = w.Write([]byte(tc.body))
+			}))
+			t.Cleanup(srv.Close)
+			ctx := context.Background()
+			c, err := compute.NewInstancesRESTClient(ctx, option.WithEndpoint(srv.URL), option.WithoutAuthentication())
+			if err != nil {
+				t.Fatalf("NewInstancesRESTClient: %v", err)
+			}
+			t.Cleanup(func() { _ = c.Close() })
+
+			err = (&realInstances{client: c}).Probe(ctx, &computepb.AggregatedListInstancesRequest{
+				Project:    "my-project",
+				MaxResults: proto.Uint32(1),
+			})
+
+			var gerr *googleapi.Error
+			switch {
+			case tc.wantCode == 0 && err != nil:
+				t.Fatalf("Probe() = %v, want nil", err)
+			case tc.wantCode != 0 && (!errors.As(err, &gerr) || gerr.Code != tc.wantCode):
+				t.Fatalf("Probe() = %v, want a googleapi error with code %d", err, tc.wantCode)
+			}
+			mu.Lock()
+			defer mu.Unlock()
+			if len(reqs) != 1 {
+				t.Fatalf("server saw %d requests, want 1", len(reqs))
+			}
+			if got := reqs[0].Query().Get("maxResults"); got != "1" {
+				t.Errorf("maxResults = %q, want 1", got)
+			}
+			if want := "/compute/v1/projects/my-project/aggregated/instances"; reqs[0].Path != want {
+				t.Errorf("path = %q, want %q", reqs[0].Path, want)
+			}
+		})
+	}
+}

internal/provider/gcp/gcp.go

@@ -33,6 +33,8 @@ type instancesAPI interface {
 	Get(ctx context.Context, req *computepb.GetInstanceRequest) (*computepb.Instance, error)
 	Delete(ctx context.Context, req *computepb.DeleteInstanceRequest) (opName string, err error)
 	AggregatedList(ctx context.Context, req *computepb.AggregatedListInstancesRequest) ([]*computepb.Instance, error)
+	// Probe fetches only the first page of an aggregated list; see Check.
+	Probe(ctx context.Context, req *computepb.AggregatedListInstancesRequest) error
 }
 
 // realInstances adapts *compute.InstancesClient to the seam.
@@ -221,17 +223,16 @@ func (p *Provider) Delete(ctx context.Context, providerID string) error {
 // entire GC sweep.
 func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
 	log := p.logger(ctx)
-	filter := fmt.Sprintf("labels.%s = %s", provider.LabelManaged, provider.LabelManagedYes)
 	instances, err := p.api.AggregatedList(ctx, &computepb.AggregatedListInstancesRequest{
 		Project:              p.cfg.Project,
-		Filter:               proto.String(filter),
+		Filter:               proto.String(managedFilter),
 		ReturnPartialSuccess: proto.Bool(true),
 	})
 	if err != nil {
 		logAPIError(log, "list", err)
 		return nil, p.wrapErr("list", "", err)
 	}
-	log.V(1).Info("GCP instances listed", "filter", filter, "count", len(instances))
+	log.V(1).Info("GCP instances listed", "filter", managedFilter, "count", len(instances))
 	out := make([]provider.Instance, 0, len(instances))
 	for _, inst := range instances {
 		conv := toInstance(inst, lastPathSegment(inst.GetZone()))

internal/provider/gcp/gcp_test.go

@@ -31,6 +31,14 @@ type fakeAPI struct {
 	listReq   *computepb.AggregatedListInstancesRequest
 	listInsts []*computepb.Instance
 	listErr   error
+
+	probeReq *computepb.AggregatedListInstancesRequest
+	probeErr error
+}
+
+func (f *fakeAPI) Probe(_ context.Context, req *computepb.AggregatedListInstancesRequest) error {
+	f.probeReq = req
+	return f.probeErr
 }
 
 func (f *fakeAPI) Insert(_ context.Context, req *computepb.InsertInstanceRequest) (string, error) {

internal/provider/gcp/wirelog.go

@@ -6,6 +6,7 @@ import (
 	"log/slog"
 
 	"github.com/go-logr/logr"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // wireLogMaxFieldBytes is the elision threshold for string fields in wire
@@ -47,6 +48,15 @@ func (h *wireFilterHandler) Handle(ctx context.Context, rec slog.Record) error {
 	if rec.Level <= slog.LevelDebug && rec.Message != "api request" && rec.Message != "api response" {
 		return nil
 	}
+	// The SDK logs with the request ctx, so wire records can carry the
+	// surrounding provider span — the one place slog's ctx-aware handlers
+	// beat logr, and the same keys the logr enrichment uses.
+	if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
+		rec = rec.Clone()
+		rec.AddAttrs(
+			slog.String("traceID", sc.TraceID().String()),
+			slog.String("spanID", sc.SpanID().String()))
+	}
 	if h.fullPayloads {
 		return h.inner.Handle(ctx, rec)
 	}

internal/provider/gcp/wirelog_test.go

@@ -0,0 +1,50 @@
+package gcp
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/go-logr/logr/funcr"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+func TestWireFilterHandler_addsTraceContext(t *testing.T) {
+	t.Parallel()
+
+	var lines []string
+	base := funcr.New(func(prefix, args string) {
+		lines = append(lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: 5})
+	log := wireLogger(base, WireLogOptions{})
+
+	tp := sdktrace.NewTracerProvider()
+	t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
+	ctx, span := tp.Tracer("test").Start(context.Background(), "provider.create")
+	defer span.End()
+
+	log.DebugContext(ctx, "api request", "url", "https://compute.googleapis.com/x")
+	if len(lines) != 1 {
+		t.Fatalf("got %d lines, want 1: %v", len(lines), lines)
+	}
+	traceID := span.SpanContext().TraceID().String()
+	if !strings.Contains(lines[0], "traceID") || !strings.Contains(lines[0], traceID) {
+		t.Errorf("wire record missing trace context %s: %s", traceID, lines[0])
+	}
+
+	// The security filter must still win: non-wire Debug records are
+	// dropped even when a span is present.
+	log.DebugContext(ctx, "token exchange", "assertion", "secret-jwt")
+	if len(lines) != 1 {
+		t.Fatalf("filtered record leaked: %v", lines[1:])
+	}
+
+	// No span in ctx: record passes through without trace keys.
+	log.DebugContext(context.Background(), "api response", "status", 200)
+	if len(lines) != 2 {
+		t.Fatalf("got %d lines, want 2", len(lines))
+	}
+	if strings.Contains(lines[1], "traceID") {
+		t.Errorf("spanless record must not carry traceID: %s", lines[1])
+	}
+}

internal/provider/kubernetes/check.go

@@ -0,0 +1,53 @@
+package kubernetes
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+var _ provider.Checker = (*Provider)(nil)
+
+// Check lists at most one managed Pod across the cluster: the same client,
+// verb and scope ListByTag and orphan GC rely on, so a pass means the API
+// server answers and the operator's RBAC still covers them. It does not
+// prove create or delete rights.
+func (p *Provider) Check(ctx context.Context) []provider.TargetStatus {
+	start := time.Now()
+	var pods corev1.PodList
+	err := p.client.List(ctx, &pods, client.MatchingLabels{
+		provider.LabelManaged: provider.LabelManagedYes,
+	}, client.Limit(1))
+	ts := provider.TargetStatus{Name: "apiserver", Address: p.apiHost, Latency: time.Since(start)}
+	if err != nil {
+		ts.Reason = checkReason(ctx, err)
+		ts.Message = err.Error()
+		return []provider.TargetStatus{ts}
+	}
+	ts.Ready = true
+	ts.Reason = provider.ReasonReachable
+	return []provider.TargetStatus{ts}
+}
+
+func checkReason(ctx context.Context, err error) string {
+	var status apierrors.APIStatus
+	switch {
+	case ctx.Err() != nil, errors.Is(err, context.DeadlineExceeded),
+		apierrors.IsTimeout(err), apierrors.IsServerTimeout(err):
+		return provider.ReasonTimeout
+	case apierrors.IsUnauthorized(err):
+		return provider.ReasonUnauthorized
+	case apierrors.IsForbidden(err):
+		return provider.ReasonForbidden
+	case errors.As(err, &status):
+		return provider.ReasonAPIError
+	default:
+		return provider.ReasonUnreachable
+	}
+}

internal/provider/kubernetes/check_test.go

@@ -0,0 +1,114 @@
+package kubernetes
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"syscall"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+type listFunc func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error
+
+func newCheckProvider(list listFunc) *Provider {
+	scheme := runtime.NewScheme()
+	if err := corev1.AddToScheme(scheme); err != nil {
+		panic(err)
+	}
+	c := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{List: list}).Build()
+	p := newWithClient(c, provider.ProviderConfig{Name: "kubernetes"})
+	p.apiHost = "https://10.96.0.1:443"
+	return p
+}
+
+func TestCheck_reachable(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(testPod("ns", "a", "uid-a", corev1.PodRunning, "10.0.0.1"))
+	p.apiHost = "https://10.96.0.1:443"
+
+	got := p.Check(context.Background())
+
+	if len(got) != 1 {
+		t.Fatalf("Check() returned %d targets, want 1", len(got))
+	}
+	ts := got[0]
+	if !ts.Ready || ts.Reason != provider.ReasonReachable || ts.Message != "" {
+		t.Errorf("Check() = %+v, want ready and Reachable", ts)
+	}
+	if ts.Name != "apiserver" || ts.Address != "https://10.96.0.1:443" {
+		t.Errorf("target = %q at %q, want apiserver at https://10.96.0.1:443", ts.Name, ts.Address)
+	}
+}
+
+func TestCheck_listOptions(t *testing.T) {
+	t.Parallel()
+	var got client.ListOptions
+	p := newCheckProvider(func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
+		got.ApplyOptions(opts)
+		return c.List(ctx, list, opts...)
+	})
+
+	p.Check(context.Background())
+
+	if got.Limit != 1 {
+		t.Errorf("Limit = %d, want 1", got.Limit)
+	}
+	if got.Namespace != "" {
+		t.Errorf("Namespace = %q, want cluster-wide", got.Namespace)
+	}
+	want := provider.LabelManaged + "=" + provider.LabelManagedYes
+	if got.LabelSelector == nil || got.LabelSelector.String() != want {
+		t.Errorf("LabelSelector = %v, want %s", got.LabelSelector, want)
+	}
+}
+
+func TestCheck_failureReasons(t *testing.T) {
+	t.Parallel()
+	pods := schema.GroupResource{Resource: "pods"}
+	ended, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	tests := []struct {
+		name string
+		ctx  context.Context
+		err  error
+		want string
+	}{
+		{"forbidden", context.Background(), apierrors.NewForbidden(pods, "", errors.New("rbac denied")), provider.ReasonForbidden},
+		{"unauthorized", context.Background(), apierrors.NewUnauthorized("token expired"), provider.ReasonUnauthorized},
+		{"api timeout", context.Background(), apierrors.NewTimeoutError("slow etcd", 1), provider.ReasonTimeout},
+		{"server timeout", context.Background(), apierrors.NewServerTimeout(pods, "list", 1), provider.ReasonTimeout},
+		{"context ended", ended, context.Canceled, provider.ReasonTimeout},
+		{"internal error", context.Background(), apierrors.NewInternalError(errors.New("etcd down")), provider.ReasonAPIError},
+		{"connection refused", context.Background(), &url.Error{
+			Op: "Get", URL: "https://10.96.0.1:443/api/v1/pods", Err: syscall.ECONNREFUSED,
+		}, provider.ReasonUnreachable},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			p := newCheckProvider(func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error {
+				return tc.err
+			})
+
+			got := p.Check(tc.ctx)
+
+			if len(got) != 1 {
+				t.Fatalf("Check() returned %d targets, want 1", len(got))
+			}
+			if got[0].Ready || got[0].Reason != tc.want || got[0].Message != tc.err.Error() {
+				t.Errorf("Check() = %+v, want not ready, reason %s, message %q", got[0], tc.want, tc.err.Error())
+			}
+		})
+	}
+}

internal/provider/kubernetes/kubernetes.go

@@ -14,6 +14,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/transport"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -35,15 +36,28 @@ const defaultImage = "ubuntu/squid:6.6-24.04_edge"
 type Provider struct {
 	client client.Client
 	image  string
+	// apiHost is the API server URL, shown by Check; empty in tests.
+	apiHost string
 }
 
 // New builds a kubernetes Provider from its config block. Satisfies
 // registry.Constructor.
-func New(_ context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
+func New(ctx context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
+	return NewWithTransportWrapper(ctx, cfg, nil)
+}
+
+// NewWithTransportWrapper is New with an optional transport wrapper applied
+// to the provider's own rest.Config (this client is built independently of
+// the manager's, so the composition root must wrap it separately for
+// tracing). A nil wrapper means a plain client.
+func NewWithTransportWrapper(_ context.Context, cfg provider.ProviderConfig, wrap transport.WrapperFunc) (provider.Provider, error) {
 	restCfg, err := ctrl.GetConfig()
 	if err != nil {
 		return nil, fmt.Errorf("kubernetes provider %q: loading kubeconfig: %w", cfg.Name, err)
 	}
+	if wrap != nil {
+		restCfg.Wrap(wrap)
+	}
 	scheme := runtime.NewScheme()
 	if err := corev1.AddToScheme(scheme); err != nil {
 		return nil, fmt.Errorf("kubernetes provider %q: %w", cfg.Name, err)
@@ -52,7 +66,9 @@ func New(_ context.Context, cfg provider.ProviderConfig) (provider.Provider, err
 	if err != nil {
 		return nil, fmt.Errorf("kubernetes provider %q: building client: %w", cfg.Name, err)
 	}
-	return newWithClient(c, cfg), nil
+	p := newWithClient(c, cfg)
+	p.apiHost = restCfg.Host
+	return p, nil
 }
 
 // newWithClient builds a Provider around an already-constructed client,

internal/provider/kvm/check.go

@@ -0,0 +1,166 @@
+package kvm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+var _ provider.Checker = (*Provider)(nil)
+
+// Reasons only the KVM check can tell apart; the rest are provider-wide.
+const (
+	reasonSSHMaterialUnavailable = "SSHMaterialUnavailable"
+	reasonSSHHandshakeFailed     = "SSHHandshakeFailed"
+	reasonLibvirtConnectFailed   = "LibvirtConnectFailed"
+	reasonLibvirtError           = "LibvirtError"
+	reasonPoolNotFound           = "PoolNotFound"
+	reasonPoolNotActive          = "PoolNotActive"
+)
+
+// hostFacts is what a successful check learns about a host.
+type hostFacts struct {
+	version   uint64
+	hostname  string
+	poolState libvirt.StoragePoolState
+	capacity  uint64
+	available uint64
+}
+
+// Check verifies every configured host concurrently, in kvm.hosts order.
+// It goes through the same pooled connection Create and Get use, so the
+// SSH and libvirt handshakes only happen on the first check or after the
+// link dropped; then two cheap RPCs prove the daemon answers, and the
+// storage pool VMs are created in must exist and be running.
+func (p *Provider) Check(ctx context.Context) []provider.TargetStatus {
+	out := make([]provider.TargetStatus, len(p.cfg.Hosts))
+	var wg sync.WaitGroup
+	for i, h := range p.cfg.Hosts {
+		wg.Go(func() { out[i] = p.checkHost(ctx, h) })
+	}
+	wg.Wait()
+	return out
+}
+
+// checkHost goes through probe, so a check that runs out of time abandons
+// its RPCs rather than closing the connection the reconciles share. fn
+// returns the RPC errors untouched, since a transport error still drops the
+// link; a pool that exists but is stopped says nothing about the link, so
+// that verdict is reached afterwards. facts is read only after probe
+// returned nil, since an abandoned fn may still write it.
+func (p *Provider) checkHost(ctx context.Context, host provider.KVMHost) provider.TargetStatus {
+	ts := provider.TargetStatus{
+		Name:    host.Name,
+		Address: host.Address,
+		Details: map[string]string{"user": host.User, "pool": host.Pool},
+	}
+	start := time.Now()
+	var facts hostFacts
+	err := p.hosts.probe(ctx, p.logger(ctx), host, func(api libvirtAPI) error {
+		version, err := api.ConnectGetLibVersion()
+		if err != nil {
+			return err
+		}
+		hostname, err := api.ConnectGetHostname()
+		if err != nil {
+			return err
+		}
+		pool, err := api.StoragePoolLookupByName(host.Pool)
+		if err != nil {
+			return err
+		}
+		state, capacity, _, available, err := api.StoragePoolGetInfo(pool)
+		if err != nil {
+			return err
+		}
+		facts = hostFacts{
+			version:   version,
+			hostname:  hostname,
+			poolState: libvirt.StoragePoolState(state),
+			capacity:  capacity,
+			available: available,
+		}
+		return nil
+	})
+	ts.Latency = time.Since(start)
+	if err != nil {
+		ts.Reason = checkReason(err)
+		ts.Message = err.Error()
+		return ts
+	}
+	ts.Details["hostname"] = facts.hostname
+	ts.Details["libvirtVersion"] = formatLibVersion(facts.version)
+	ts.Details["poolState"] = poolStateName(facts.poolState)
+	ts.Details["poolCapacity"] = formatGiB(facts.capacity)
+	ts.Details["poolAvailable"] = formatGiB(facts.available)
+	if facts.poolState != libvirt.StoragePoolRunning {
+		ts.Reason = reasonPoolNotActive
+		ts.Message = fmt.Sprintf("storage pool %q is %s", host.Pool, poolStateName(facts.poolState))
+		return ts
+	}
+	ts.Ready = true
+	ts.Reason = provider.ReasonReachable
+	return ts
+}
+
+// checkReason names where a failed check broke. The dial stage wins over
+// a timeout: "the SSH handshake did not finish" says more than "the check
+// ran out of time", and the message still carries the deadline.
+func checkReason(err error) string {
+	var de *dialError
+	if errors.As(err, &de) {
+		switch de.stage {
+		case stageSSHMaterial:
+			return reasonSSHMaterialUnavailable
+		case stageSSHHandshake:
+			return reasonSSHHandshakeFailed
+		case stageLibvirtConnect:
+			return reasonLibvirtConnectFailed
+		default:
+			return provider.ReasonUnreachable
+		}
+	}
+	switch {
+	case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled), errors.Is(err, errProbeBusy):
+		return provider.ReasonTimeout
+	case hasCode(err, libvirt.ErrNoStoragePool):
+		return reasonPoolNotFound
+	case isLibvirtError(err):
+		return reasonLibvirtError
+	default:
+		return provider.ReasonUnreachable
+	}
+}
+
+// formatLibVersion decodes libvirt's major*1e6 + minor*1e3 + micro.
+func formatLibVersion(v uint64) string {
+	return fmt.Sprintf("%d.%d.%d", v/1_000_000, v/1_000%1_000, v%1_000)
+}
+
+// poolStateName uses virsh's words for a pool state.
+func poolStateName(s libvirt.StoragePoolState) string {
+	switch s {
+	case libvirt.StoragePoolInactive:
+		return "inactive"
+	case libvirt.StoragePoolBuilding:
+		return "building"
+	case libvirt.StoragePoolRunning:
+		return "running"
+	case libvirt.StoragePoolDegraded:
+		return "degraded"
+	case libvirt.StoragePoolInaccessible:
+		return "inaccessible"
+	default:
+		return fmt.Sprintf("state %d", s)
+	}
+}
+
+func formatGiB(bytes uint64) string {
+	return fmt.Sprintf("%.1fGiB", float64(bytes)/(1<<30))
+}

internal/provider/kvm/check_test.go

@@ -0,0 +1,319 @@
+package kvm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"maps"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// withoutLatency zeroes the one field a test cannot predict.
+func withoutLatency(in []provider.TargetStatus) []provider.TargetStatus {
+	out := make([]provider.TargetStatus, len(in))
+	for i, ts := range in {
+		ts.Latency = 0
+		out[i] = ts
+	}
+	return out
+}
+
+func TestCheck_reachable(t *testing.T) {
+	t.Parallel()
+	api := newFakeAPI("default")
+	api.poolCapacity = 10 << 30
+	api.pools["default"]["disk.qcow2"] = &fakeVol{capacity: 4 << 30}
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	got := withoutLatency(p.Check(context.Background()))
+
+	want := provider.TargetStatus{
+		Name:    "nas1",
+		Address: "nas1:22",
+		Ready:   true,
+		Reason:  provider.ReasonReachable,
+		Details: map[string]string{
+			"user":           "virt",
+			"pool":           "default",
+			"hostname":       "fake-host",
+			"libvirtVersion": "10.2.0",
+			"poolState":      "running",
+			"poolCapacity":   "10.0GiB",
+			"poolAvailable":  "6.0GiB",
+		},
+	}
+	if len(got) != 1 {
+		t.Fatalf("Check() returned %d targets, want 1", len(got))
+	}
+	if got[0].Name != want.Name || got[0].Address != want.Address || got[0].Ready != want.Ready ||
+		got[0].Reason != want.Reason || got[0].Message != "" || !maps.Equal(got[0].Details, want.Details) {
+		t.Errorf("Check() = %+v, want %+v", got[0], want)
+	}
+}
+
+func TestCheck_perHostVerdicts(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(hostFakes{"nas1": newFakeAPI("default"), "nas3": newFakeAPI("default")}, "nas1", "nas2", "nas3")
+
+	got := p.Check(context.Background())
+
+	if len(got) != 3 {
+		t.Fatalf("Check() returned %d targets, want 3", len(got))
+	}
+	for i, want := range []struct {
+		name  string
+		ready bool
+	}{{"nas1", true}, {"nas2", false}, {"nas3", true}} {
+		if got[i].Name != want.name || got[i].Ready != want.ready {
+			t.Errorf("target %d = %s ready=%v, want %s ready=%v", i, got[i].Name, got[i].Ready, want.name, want.ready)
+		}
+	}
+	if got[1].Reason != provider.ReasonUnreachable || !strings.Contains(got[1].Message, errHostOff.Error()) {
+		t.Errorf("unreachable host = %s %q, want Unreachable carrying %q", got[1].Reason, got[1].Message, errHostOff)
+	}
+	if got[1].Details["pool"] != "default" || got[1].Details["hostname"] != "" {
+		t.Errorf("unreachable host details = %v, want the configured pool and nothing learned", got[1].Details)
+	}
+}
+
+// TestCheck_libvirtVerdicts also checks twice per case: none of these
+// verdicts is evidence against the link, so the connection must survive.
+func TestCheck_libvirtVerdicts(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name        string
+		setup       func(api *fakeAPI)
+		pools       []string
+		wantReason  string
+		wantMessage string
+	}{
+		{
+			name:        "pool inactive",
+			pools:       []string{"default"},
+			setup:       func(api *fakeAPI) { api.poolStates["default"] = libvirt.StoragePoolInactive },
+			wantReason:  reasonPoolNotActive,
+			wantMessage: `storage pool "default" is inactive`,
+		},
+		{
+			name:        "pool missing",
+			pools:       []string{"other"},
+			wantReason:  reasonPoolNotFound,
+			wantMessage: "no storage pool with matching name 'default'",
+		},
+		{
+			name:        "daemon error",
+			pools:       []string{"default"},
+			setup:       func(api *fakeAPI) { api.fail["ConnectGetHostname"] = lvError(libvirt.ErrInternalError, "boom") },
+			wantReason:  reasonLibvirtError,
+			wantMessage: "boom",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			api := newFakeAPI(tc.pools...)
+			if tc.setup != nil {
+				tc.setup(api)
+			}
+			var dials atomic.Int32
+			dial := func(ctx context.Context, h provider.KVMHost) (*hostClient, error) {
+				dials.Add(1)
+				return hostFakes{"nas1": api}.dial(ctx, h)
+			}
+			pc := testConfig("nas1")
+			p := newWithDialer(pc.Name, *pc.KVM, dial, nil)
+
+			for range 2 {
+				got := p.Check(context.Background())
+				if got[0].Ready || got[0].Reason != tc.wantReason || !strings.Contains(got[0].Message, tc.wantMessage) {
+					t.Errorf("Check() = %+v, want not ready, %s, message containing %q", got[0], tc.wantReason, tc.wantMessage)
+				}
+			}
+			if n := dials.Load(); n != 1 {
+				t.Errorf("dialed %d times over two checks, want 1", n)
+			}
+		})
+	}
+}
+
+func TestCheck_hostsCheckedConcurrently(t *testing.T) {
+	t.Parallel()
+	nas2Dialing := make(chan struct{})
+	dial := func(ctx context.Context, h provider.KVMHost) (*hostClient, error) {
+		switch h.Name {
+		case "nas1":
+			select {
+			case <-nas2Dialing:
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			}
+		case "nas2":
+			close(nas2Dialing)
+		}
+		return &hostClient{api: newFakeAPI("default"), close: func() {}}, nil
+	}
+	pc := testConfig("nas1", "nas2")
+	p := newWithDialer(pc.Name, *pc.KVM, dial, nil)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	got := p.Check(ctx)
+
+	for _, ts := range got {
+		if !ts.Ready {
+			t.Errorf("%s = %s %q, want ready: a sequential check would time out nas1", ts.Name, ts.Reason, ts.Message)
+		}
+	}
+}
+
+// A daemon that stopped answering times the check out, but the connection
+// the reconciles share stays up, and the next check does not stack a
+// second hung RPC on top of the first.
+func TestCheck_timeoutKeepsTheConnection(t *testing.T) {
+	t.Parallel()
+	api := newFakeAPI("default")
+	block := make(chan struct{})
+	t.Cleanup(func() { close(block) })
+	blocked := &blockingAPI{fakeAPI: api, block: block}
+	var closed atomic.Bool
+	dial := func(context.Context, provider.KVMHost) (*hostClient, error) {
+		return &hostClient{api: blocked, close: func() { closed.Store(true) }}, nil
+	}
+	pc := testConfig("nas1")
+	p := newWithDialer(pc.Name, *pc.KVM, dial, nil)
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+
+	got := p.Check(ctx)
+
+	if got[0].Ready || got[0].Reason != provider.ReasonTimeout {
+		t.Errorf("Check() = %+v, want Timeout", got[0])
+	}
+	if closed.Load() {
+		t.Error("the timed-out check closed the connection the reconciles share")
+	}
+	again := p.Check(context.Background())
+	if again[0].Reason != provider.ReasonTimeout || !strings.Contains(again[0].Message, "still waiting") {
+		t.Errorf("second Check() = %+v, want an immediate Timeout while the first still waits", again[0])
+	}
+}
+
+// A reconcile redialing a dead host holds the host's slot for minutes; the
+// check must give up at its own deadline rather than wait that out.
+func TestCheck_doesNotWaitBehindAReconcileDial(t *testing.T) {
+	t.Parallel()
+	release := make(chan struct{})
+	dialing := make(chan struct{})
+	var once sync.Once
+	dial := func(context.Context, provider.KVMHost) (*hostClient, error) {
+		once.Do(func() { close(dialing) })
+		<-release
+		return nil, errHostOff
+	}
+	pc := testConfig("nas1")
+	p := newWithDialer(pc.Name, *pc.KVM, dial, nil)
+	reconcile := make(chan error, 1)
+	go func() {
+		_, err := p.Get(context.Background(), formatProviderID("nas1", testName))
+		reconcile <- err
+	}()
+	<-dialing
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+
+	got := p.Check(ctx)
+
+	if waited := time.Since(start); waited > time.Second {
+		t.Errorf("Check() took %s behind the reconcile's dial, want about 50ms", waited)
+	}
+	if got[0].Reason != provider.ReasonTimeout {
+		t.Errorf("Check() = %+v, want Timeout", got[0])
+	}
+	close(release)
+	<-reconcile
+}
+
+// blockingAPI hangs in ConnectGetLibVersion until block closes, like a
+// daemon that stopped answering.
+type blockingAPI struct {
+	*fakeAPI
+	block chan struct{}
+}
+
+func (b *blockingAPI) ConnectGetLibVersion() (uint64, error) {
+	<-b.block
+	return 0, io.EOF
+}
+
+func TestCheckReason(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		err  error
+		want string
+	}{
+		{"ssh material", &dialError{stage: stageSSHMaterial, err: fmt.Errorf("%w: parsing key", provider.ErrPermanent)}, reasonSSHMaterialUnavailable},
+		{"tcp", &dialError{stage: stageTCP, err: errors.New("ssh dial: connection refused")}, provider.ReasonUnreachable},
+		{"ssh handshake", &dialError{stage: stageSSHHandshake, err: errors.New("ssh handshake: unable to authenticate")}, reasonSSHHandshakeFailed},
+		{"libvirt connect", &dialError{stage: stageLibvirtConnect, err: errors.New("libvirt connect via /sock: EOF")}, reasonLibvirtConnectFailed},
+		{"stage survives the pool's wrapping", fmt.Errorf("connecting to host nas1 (nas1:22): %w",
+			&dialError{stage: stageSSHHandshake, err: context.DeadlineExceeded}), reasonSSHHandshakeFailed},
+		{"deadline", context.DeadlineExceeded, provider.ReasonTimeout},
+		{"cancelled", context.Canceled, provider.ReasonTimeout},
+		{"previous check still waiting", errProbeBusy, provider.ReasonTimeout},
+		{"no pool", lvError(libvirt.ErrNoStoragePool, "no pool"), reasonPoolNotFound},
+		{"other libvirt error", lvError(libvirt.ErrInternalError, "boom"), reasonLibvirtError},
+		{"link died mid-call", io.EOF, provider.ReasonUnreachable},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := checkReason(tc.err); got != tc.want {
+				t.Errorf("checkReason(%v) = %s, want %s", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestDialError_keepsClassification(t *testing.T) {
+	t.Parallel()
+	err := fmt.Errorf("connecting to host nas1: %w",
+		&dialError{stage: stageSSHMaterial, err: fmt.Errorf("%w: parsing kvm.sshKeyFile: bad", provider.ErrPermanent)})
+
+	if got := classify(err); !errors.Is(got, provider.ErrPermanent) {
+		t.Errorf("classify() = %v, want ErrPermanent through the dialError", got)
+	}
+	if !isClassified(err) {
+		t.Error("isClassified() = false, want true")
+	}
+	if want := "connecting to host nas1: permanent: parsing kvm.sshKeyFile: bad"; !strings.HasSuffix(err.Error(), "parsing kvm.sshKeyFile: bad") {
+		t.Errorf("message = %q, want it unchanged, like %q", err.Error(), want)
+	}
+}
+
+func TestFormatLibVersion(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		in   uint64
+		want string
+	}{
+		{10002000, "10.2.0"},
+		{11006001, "11.6.1"},
+		{9000000, "9.0.0"},
+	}
+	for _, tc := range tests {
+		if got := formatLibVersion(tc.in); got != tc.want {
+			t.Errorf("formatLibVersion(%d) = %s, want %s", tc.in, got, tc.want)
+		}
+	}
+}

internal/provider/kvm/conn.go

@@ -0,0 +1,464 @@
+package kvm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net"
+	"os"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+const (
+	defaultHandshakeTimeout = 30 * time.Second
+	keepaliveInterval       = 15 * time.Second
+)
+
+// libvirtAPI is the slice of *libvirt.Libvirt the provider uses, and the
+// test seam: the fake in kvm_test.go implements exactly this.
+type libvirtAPI interface {
+	DomainLookupByName(name string) (libvirt.Domain, error)
+	DomainDefineXML(xml string) (libvirt.Domain, error)
+	DomainCreate(d libvirt.Domain) error
+	DomainDestroy(d libvirt.Domain) error
+	DomainUndefineFlags(d libvirt.Domain, flags libvirt.DomainUndefineFlagsValues) error
+	DomainGetState(d libvirt.Domain, flags uint32) (state int32, reason int32, err error)
+	DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, flags libvirt.DomainModificationImpact) (string, error)
+	DomainInterfaceAddresses(d libvirt.Domain, source uint32, flags uint32) ([]libvirt.DomainInterface, error)
+	ConnectListAllDomains(needResults int32, flags libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error)
+	StoragePoolLookupByName(name string) (libvirt.StoragePool, error)
+	StoragePoolRefresh(pool libvirt.StoragePool, flags uint32) error
+	StorageVolLookupByName(pool libvirt.StoragePool, name string) (libvirt.StorageVol, error)
+	StorageVolGetPath(vol libvirt.StorageVol) (string, error)
+	StorageVolCreateXML(pool libvirt.StoragePool, xml string, flags libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error)
+	StorageVolUpload(vol libvirt.StorageVol, r io.Reader, offset, length uint64, flags libvirt.StorageVolUploadFlags) error
+	StorageVolDownload(vol libvirt.StorageVol, w io.Writer, offset, length uint64, flags libvirt.StorageVolDownloadFlags) error
+	StorageVolDelete(vol libvirt.StorageVol, flags libvirt.StorageVolDeleteFlags) error
+	ConnectGetLibVersion() (uint64, error)
+	ConnectGetHostname() (string, error)
+	StoragePoolGetInfo(pool libvirt.StoragePool) (state uint8, capacity, allocation, available uint64, err error)
+}
+
+// hostClient is one live connection to a hypervisor.
+type hostClient struct {
+	api libvirtAPI
+	// done is closed once the connection is known to be lost. A nil channel
+	// never fires, which suits fakes.
+	done  <-chan struct{}
+	close func()
+}
+
+func (c *hostClient) alive() bool {
+	select {
+	case <-c.done:
+		return false
+	default:
+		return true
+	}
+}
+
+// hostDialer opens a connection to one host. The SSH implementation is the
+// real one; tests inject their own.
+type hostDialer func(ctx context.Context, host provider.KVMHost) (*hostClient, error)
+
+// dialStage names the step of a connection attempt that failed, so the
+// health check can say why a host is unreachable without parsing messages.
+type dialStage string
+
+const (
+	stageSSHMaterial    dialStage = "ssh-material"
+	stageTCP            dialStage = "tcp"
+	stageSSHHandshake   dialStage = "ssh-handshake"
+	stageLibvirtConnect dialStage = "libvirt-connect"
+)
+
+// dialError tags a dial failure with its stage. It keeps the message and
+// unwraps to the cause, so classify sees exactly what it saw before.
+type dialError struct {
+	stage dialStage
+	err   error
+}
+
+func (e *dialError) Error() string { return e.err.Error() }
+func (e *dialError) Unwrap() error { return e.err }
+
+// hostPool caches one connection per host, dialing lazily and redialing
+// after the connection drops. Dials for different hosts don't block each
+// other; concurrent callers for the same host share a single dial.
+type hostPool struct {
+	dial hostDialer
+
+	mu      sync.Mutex
+	entries map[string]*hostEntry
+}
+
+// hostEntry is one host's slot in the pool. lock is a one-slot semaphore
+// rather than a mutex so that waiting for it can end with the caller's
+// context: whoever holds it may be dialing a dead host for as long as the
+// dial takes.
+type hostEntry struct {
+	lock   chan struct{}
+	client *hostClient
+	// probing is set while a probe's fn runs, including one whose caller
+	// has already given up on it.
+	probing atomic.Bool
+}
+
+func newHostPool(dial hostDialer) *hostPool {
+	return &hostPool{dial: dial, entries: make(map[string]*hostEntry)}
+}
+
+func (p *hostPool) entry(name string) *hostEntry {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	e, ok := p.entries[name]
+	if !ok {
+		e = &hostEntry{lock: make(chan struct{}, 1)}
+		p.entries[name] = e
+	}
+	return e
+}
+
+// acquire takes e.lock. Only waiting is bounded by ctx: a free slot is
+// taken even when ctx has already ended, so an uncontended caller behaves
+// as it did with a mutex, and a contended one gives up with ctx's error.
+func (e *hostEntry) acquire(ctx context.Context) error {
+	select {
+	case e.lock <- struct{}{}:
+		return nil
+	default:
+	}
+	select {
+	case e.lock <- struct{}{}:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (e *hostEntry) release() { <-e.lock }
+
+func (p *hostPool) get(ctx context.Context, log logr.Logger, host provider.KVMHost) (*hostClient, error) {
+	e := p.entry(host.Name)
+	if err := e.acquire(ctx); err != nil {
+		return nil, fmt.Errorf("waiting for the connection to host %s (%s): %w", host.Name, host.Address, err)
+	}
+	defer e.release()
+	if e.client != nil {
+		if e.client.alive() {
+			return e.client, nil
+		}
+		log.V(1).Info("libvirt connection lost, redialing", "address", host.Address)
+		e.client.close()
+		e.client = nil
+	}
+	start := time.Now()
+	c, err := p.dial(ctx, host)
+	if err != nil {
+		log.V(1).Info("libvirt host dial failed", "address", host.Address, "duration", time.Since(start).Round(time.Millisecond), "error", err.Error())
+		return nil, fmt.Errorf("connecting to host %s (%s): %w", host.Name, host.Address, err)
+	}
+	log.V(1).Info("libvirt host connected", "address", host.Address, "duration", time.Since(start).Round(time.Millisecond))
+	e.client = c
+	return c, nil
+}
+
+// drop closes and forgets c if it is still the cached client for host, so a
+// connection that was already replaced by a redial is left alone.
+func (p *hostPool) drop(host string, c *hostClient) {
+	e := p.entry(host)
+	e.lock <- struct{}{}
+	defer e.release()
+	if e.client == c {
+		e.client = nil
+	}
+	c.close()
+}
+
+// call runs fn against host's connection. go-libvirt RPCs take no context
+// and block until the daemon answers, so when ctx ends first the connection
+// is closed — which unblocks fn with an error — and ctx.Err() is returned.
+// A transport error also drops the cached connection so the next call
+// redials instead of reusing a dead link. "Transport error" means anything
+// that is neither a libvirt.Error nor an error fn already classified with a
+// provider sentinel (a failed seed build, a bad placement): those are local
+// verdicts, not evidence about the link. When log has V(1) enabled, every
+// RPC fn makes is traced through it (see tracingAPI), as are connection
+// events.
+func (p *hostPool) call(ctx context.Context, log logr.Logger, host provider.KVMHost, fn func(api libvirtAPI) error) error {
+	log = log.WithName("libvirt")
+	c, err := p.get(ctx, log, host)
+	if err != nil {
+		return err
+	}
+	err = awaitOrCancel(ctx, func() error { return fn(traced(c, log)) }, func() {
+		log.V(1).Info("libvirt connection dropped", "reason", "context cancelled", "error", ctx.Err().Error())
+		p.drop(host.Name, c)
+	})
+	p.dropOnTransportError(ctx, log, host, c, err)
+	return err
+}
+
+// errProbeBusy refuses a probe while the previous one on the same host has
+// not come back yet.
+var errProbeBusy = errors.New("the previous check of this host is still waiting for libvirt to answer")
+
+// probe is call for the health check, which must never cost the reconciles
+// their connection: when ctx ends first, fn is abandoned and the connection
+// left alone. A link that is really dead is closed by the keepalive, which
+// ends fn too. At most one probe per host runs at a time, an abandoned one
+// included, so a daemon that stopped answering cannot pile up goroutines;
+// until it answers, further probes get errProbeBusy.
+func (p *hostPool) probe(ctx context.Context, log logr.Logger, host provider.KVMHost, fn func(api libvirtAPI) error) error {
+	log = log.WithName("libvirt")
+	e := p.entry(host.Name)
+	if !e.probing.CompareAndSwap(false, true) {
+		return errProbeBusy
+	}
+	c, err := p.get(ctx, log, host)
+	if err != nil {
+		e.probing.Store(false)
+		return err
+	}
+	err = awaitOrCancel(ctx, func() error {
+		defer e.probing.Store(false)
+		return fn(traced(c, log))
+	}, func() {
+		log.V(1).Info("libvirt probe abandoned, connection kept", "error", ctx.Err().Error())
+	})
+	p.dropOnTransportError(ctx, log, host, c, err)
+	return err
+}
+
+// traced returns c's API, logging every RPC when log has V(1) enabled.
+func traced(c *hostClient, log logr.Logger) libvirtAPI {
+	if log.V(1).Enabled() {
+		return &tracingAPI{inner: c.api, log: log}
+	}
+	return c.api
+}
+
+// dropOnTransportError drops c when err is evidence against the link:
+// neither a libvirt error, nor a verdict already classified, nor the
+// caller's own context ending.
+func (p *hostPool) dropOnTransportError(ctx context.Context, log logr.Logger, host provider.KVMHost, c *hostClient, err error) {
+	if err != nil && !isLibvirtError(err) && !isClassified(err) && !errors.Is(err, ctx.Err()) {
+		log.V(1).Info("libvirt connection dropped", "reason", "transport error", "error", err.Error())
+		p.drop(host.Name, c)
+	}
+}
+
+// awaitOrCancel runs fn in the background and returns its result, unless ctx
+// ends first: then abort is invoked (it must make fn return eventually, e.g.
+// by closing the connection fn is blocked on) and ctx.Err() is returned. The
+// abandoned fn's result is discarded.
+func awaitOrCancel(ctx context.Context, fn func() error, abort func()) error {
+	errc := make(chan error, 1)
+	go func() { errc <- fn() }()
+	select {
+	case err := <-errc:
+		return err
+	case <-ctx.Done():
+		abort()
+		return ctx.Err()
+	}
+}
+
+// sshAuth holds the pieces of the SSH client config shared by every host:
+// the private key and the host-key policy. Both are loaded on the first
+// connection rather than at startup — see lazySSH.
+type sshAuth struct {
+	signer  ssh.Signer
+	hostKey ssh.HostKeyCallback
+}
+
+// newSSHAuth loads the key material. A file that is not there yet stays
+// unclassified, so the provider treats it as transient and heals when the
+// Secret appears; content that will never parse is permanent, since
+// retrying cannot fix it.
+func newSSHAuth(cfg provider.KVMConfig) (*sshAuth, error) {
+	key, err := os.ReadFile(cfg.SSHKeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("reading kvm.sshKeyFile: %w", err)
+	}
+	signer, err := ssh.ParsePrivateKey(key)
+	if err != nil {
+		return nil, fmt.Errorf("%w: parsing kvm.sshKeyFile: %v", provider.ErrPermanent, err)
+	}
+	hostKey := ssh.InsecureIgnoreHostKey() //nolint:gosec // opt-in via config, documented as insecure
+	if !cfg.InsecureIgnoreHostKey {
+		hostKey, err = knownhosts.New(cfg.KnownHostsFile)
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, fmt.Errorf("loading kvm.knownHostsFile: %w", err)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("%w: loading kvm.knownHostsFile: %v", provider.ErrPermanent, err)
+		}
+	}
+	return &sshAuth{signer: signer, hostKey: hostKey}, nil
+}
+
+// lazySSH defers loading the SSH material until it is first needed and
+// caches it once it works. The kvm-ssh Secret is an optional mount: the
+// manager has to start without it and keep every other provider running,
+// and a Secret created afterwards has to heal the KVM proxies on the next
+// reconcile rather than needing a restart.
+type lazySSH struct {
+	cfg provider.KVMConfig
+
+	mu   sync.Mutex
+	auth *sshAuth
+}
+
+// newLazySSH keeps only the SSH fields of cfg, since the loader lives as
+// long as the Provider.
+func newLazySSH(cfg provider.KVMConfig) *lazySSH {
+	return &lazySSH{cfg: provider.KVMConfig{
+		SSHKeyFile:            cfg.SSHKeyFile,
+		KnownHostsFile:        cfg.KnownHostsFile,
+		InsecureIgnoreHostKey: cfg.InsecureIgnoreHostKey,
+	}}
+}
+
+func (l *lazySSH) load() (*sshAuth, error) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.auth != nil {
+		return l.auth, nil
+	}
+	auth, err := newSSHAuth(l.cfg)
+	if err != nil {
+		return nil, err
+	}
+	l.auth = auth
+	return auth, nil
+}
+
+// dial is the production hostDialer.
+func (l *lazySSH) dial(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
+	a, err := l.load()
+	if err != nil {
+		return nil, &dialError{stage: stageSSHMaterial, err: err}
+	}
+	return a.dial(ctx, host)
+}
+
+// dial SSHes to the host, forwards its libvirt unix socket through the
+// connection, and opens a libvirt session over that.
+func (a *sshAuth) dial(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
+	client, err := a.dialSSH(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+	hc, err := connectLibvirt(ctx, client, host.Socket)
+	if err != nil {
+		client.Close()
+		return nil, err
+	}
+	return hc, nil
+}
+
+// dialSSH establishes the SSH connection. The handshake is bounded by ctx's
+// deadline, or defaultHandshakeTimeout when ctx has none, since a
+// half-open hypervisor must not pin a reconcile worker indefinitely.
+func (a *sshAuth) dialSSH(ctx context.Context, host provider.KVMHost) (*ssh.Client, error) {
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		deadline = time.Now().Add(defaultHandshakeTimeout)
+	}
+	var d net.Dialer
+	nc, err := d.DialContext(ctx, "tcp", host.Address)
+	if err != nil {
+		return nil, &dialError{stage: stageTCP, err: fmt.Errorf("ssh dial: %w", err)}
+	}
+	_ = nc.SetDeadline(deadline)
+	conn, chans, reqs, err := ssh.NewClientConn(nc, host.Address, &ssh.ClientConfig{
+		User:            host.User,
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(a.signer)},
+		HostKeyCallback: a.hostKey,
+	})
+	if err != nil {
+		nc.Close()
+		return nil, &dialError{stage: stageSSHHandshake, err: fmt.Errorf("ssh handshake: %w", err)}
+	}
+	_ = nc.SetDeadline(time.Time{})
+	return ssh.NewClient(conn, chans, reqs), nil
+}
+
+// unixForward adapts an SSH client to go-libvirt's socket.Dialer: each Dial
+// opens a direct-streamlocal channel to the libvirt socket on the host.
+type unixForward struct {
+	client *ssh.Client
+	path   string
+}
+
+func (u unixForward) Dial() (net.Conn, error) {
+	return u.client.Dial("unix", u.path)
+}
+
+// connectLibvirt opens the libvirt session over the SSH client and starts a
+// keepalive loop so a dead link is noticed within roughly two intervals
+// rather than at the TCP stack's leisure. Closing the returned client tears
+// down the SSH connection, which in turn ends the libvirt session.
+func connectLibvirt(ctx context.Context, client *ssh.Client, socket string) (*hostClient, error) {
+	lv := libvirt.NewWithDialer(unixForward{client: client, path: socket})
+	if err := awaitOrCancel(ctx, lv.Connect, func() { client.Close() }); err != nil {
+		return nil, &dialError{stage: stageLibvirtConnect, err: fmt.Errorf("libvirt connect via %s: %w", socket, err)}
+	}
+	stop := make(chan struct{})
+	var once sync.Once
+	go keepalive(client, keepaliveInterval, stop)
+	return &hostClient{
+		api:  lv,
+		done: lv.Disconnected(),
+		close: func() {
+			once.Do(func() {
+				close(stop)
+				client.Close()
+			})
+		},
+	}, nil
+}
+
+// keepalive sends OpenSSH keepalive requests until stop closes; a failed or
+// unanswered request closes the client, which surfaces as Disconnected on
+// the libvirt side and makes the pool redial.
+func keepalive(client *ssh.Client, interval time.Duration, stop <-chan struct{}) {
+	t := time.NewTicker(interval)
+	defer t.Stop()
+	for {
+		select {
+		case <-stop:
+			return
+		case <-t.C:
+		}
+		replied := make(chan error, 1)
+		go func() {
+			_, _, err := client.SendRequest("keepalive@openssh.com", true, nil)
+			replied <- err
+		}()
+		select {
+		case <-stop:
+			return
+		case err := <-replied:
+			if err != nil {
+				client.Close()
+				return
+			}
+		case <-time.After(interval):
+			client.Close()
+			return
+		}
+	}
+}

internal/provider/kvm/conn_test.go

@@ -0,0 +1,893 @@
+package kvm
+
+import (
+	"bytes"
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// --- hostPool over a fake dialer ---
+
+type fakeClient struct {
+	done   chan struct{}
+	closed atomic.Bool
+}
+
+func (c *fakeClient) close() {
+	if c.closed.CompareAndSwap(false, true) {
+		close(c.done)
+	}
+}
+
+type fakeDialer struct {
+	mu      sync.Mutex
+	dials   int
+	err     error
+	delay   time.Duration
+	clients []*fakeClient
+	// api is what the dialled client exposes; nil for pool-only tests.
+	api libvirtAPI
+}
+
+func (d *fakeDialer) dial(ctx context.Context, _ provider.KVMHost) (*hostClient, error) {
+	if d.delay > 0 {
+		select {
+		case <-time.After(d.delay):
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.dials++
+	if d.err != nil {
+		return nil, d.err
+	}
+	fc := &fakeClient{done: make(chan struct{})}
+	d.clients = append(d.clients, fc)
+	return &hostClient{api: d.api, done: fc.done, close: fc.close}, nil
+}
+
+func (d *fakeDialer) dialCount() int {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.dials
+}
+
+var testHost = provider.KVMHost{Name: "nas1", Address: "nas1:22", User: "virt", Socket: defaultSocket, Pool: "default", Bridge: "br0"}
+
+var discard = logr.Discard()
+
+func noop(libvirtAPI) error { return nil }
+
+func TestHostPool_call_reusesConnection(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	for range 3 {
+		if err := p.call(context.Background(), discard, testHost, noop); err != nil {
+			t.Fatalf("call() error = %v", err)
+		}
+	}
+	if d.dialCount() != 1 {
+		t.Errorf("dials = %d, want 1", d.dialCount())
+	}
+}
+
+func TestHostPool_call_redialsAfterDisconnect(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	if err := p.call(context.Background(), discard, testHost, noop); err != nil {
+		t.Fatalf("call() error = %v", err)
+	}
+	d.clients[0].close()
+	if err := p.call(context.Background(), discard, testHost, noop); err != nil {
+		t.Fatalf("second call() error = %v", err)
+	}
+	if d.dialCount() != 2 {
+		t.Errorf("dials = %d, want 2 after the first connection dropped", d.dialCount())
+	}
+}
+
+// Only a transport error is evidence about the link. A daemon-reported
+// libvirt error, or a verdict fn reached locally, must not cost the
+// connection.
+func TestHostPool_call_dropsConnectionOnlyOnTransportError(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name        string
+		err         error
+		wantDropped bool
+	}{
+		{"transport error", io.ErrUnexpectedEOF, true},
+		{"libvirt error", lvErr(libvirt.ErrNoDomain), false},
+		{"locally classified error", fmt.Errorf("%w: creating iso writer: read-only file system", provider.ErrPermanent), false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			d := &fakeDialer{}
+			p := newHostPool(d.dial)
+			log, snapshot := captureLogger(1)
+			err := p.call(context.Background(), log, testHost, func(libvirtAPI) error { return tc.err })
+			if !errors.Is(err, tc.err) {
+				t.Fatalf("call() error = %v, want %v passed through", err, tc.err)
+			}
+			if got := d.clients[0].closed.Load(); got != tc.wantDropped {
+				t.Errorf("connection closed = %v, want %v", got, tc.wantDropped)
+			}
+			joined := strings.Join(snapshot(), "\n")
+			if logged := strings.Contains(joined, `"msg"="libvirt connection dropped"`) && strings.Contains(joined, `"reason"="transport error"`); logged != tc.wantDropped {
+				t.Errorf("transport drop logged = %v, want %v:\n%s", logged, tc.wantDropped, joined)
+			}
+			if err := p.call(context.Background(), discard, testHost, noop); err != nil {
+				t.Fatalf("next call() error = %v", err)
+			}
+			wantDials := 1
+			if tc.wantDropped {
+				wantDials = 2
+			}
+			if d.dialCount() != wantDials {
+				t.Errorf("dials = %d, want %d", d.dialCount(), wantDials)
+			}
+		})
+	}
+}
+
+func TestHostPool_call_dialErrorPropagates(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{err: errors.New("ssh: handshake failed")}
+	p := newHostPool(d.dial)
+	err := p.call(context.Background(), discard, testHost, noop)
+	if err == nil || !strings.Contains(err.Error(), "nas1") || !strings.Contains(err.Error(), "handshake failed") {
+		t.Errorf("call() error = %v, want the dial error naming the host", err)
+	}
+}
+
+func TestHostPool_call_cancelAbandonsBlockedCall(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	ctx, cancel := context.WithCancel(context.Background())
+	released := make(chan struct{})
+	go func() {
+		time.Sleep(20 * time.Millisecond)
+		cancel()
+	}()
+	err := p.call(ctx, discard, testHost, func(libvirtAPI) error {
+		// Simulates an RPC stuck on a hung hypervisor: it only returns once
+		// the pool closes the connection out from under it.
+		<-d.clients[0].done
+		close(released)
+		return io.EOF
+	})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("call() error = %v, want context.Canceled", err)
+	}
+	select {
+	case <-released:
+	case <-time.After(time.Second):
+		t.Fatal("blocked call was not released by closing the connection")
+	}
+	if err := p.call(context.Background(), discard, testHost, noop); err != nil {
+		t.Fatalf("call() after cancel = %v", err)
+	}
+	if d.dialCount() != 2 {
+		t.Errorf("dials = %d, want 2", d.dialCount())
+	}
+}
+
+func TestHostPool_call_concurrentCallersShareOneDial(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{delay: 30 * time.Millisecond}
+	p := newHostPool(d.dial)
+	var wg sync.WaitGroup
+	for range 8 {
+		wg.Go(func() {
+			if err := p.call(context.Background(), discard, testHost, noop); err != nil {
+				t.Errorf("call() error = %v", err)
+			}
+		})
+	}
+	wg.Wait()
+	if d.dialCount() != 1 {
+		t.Errorf("dials = %d, want 1", d.dialCount())
+	}
+}
+
+func TestHostPool_call_hostsAreIndependent(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	other := testHost
+	other.Name = "lab2"
+	for _, h := range []provider.KVMHost{testHost, other, testHost, other} {
+		if err := p.call(context.Background(), discard, h, noop); err != nil {
+			t.Fatalf("call(%s) error = %v", h.Name, err)
+		}
+	}
+	if d.dialCount() != 2 {
+		t.Errorf("dials = %d, want one per host", d.dialCount())
+	}
+}
+
+// A dial to a dead host holds the host's slot for as long as it takes; a
+// caller with a deadline must not wait past it.
+func TestHostPool_get_waitingRespectsContext(t *testing.T) {
+	t.Parallel()
+	release := make(chan struct{})
+	dialing := make(chan struct{})
+	var once sync.Once
+	dial := func(context.Context, provider.KVMHost) (*hostClient, error) {
+		once.Do(func() { close(dialing) })
+		<-release
+		return nil, errors.New("ssh dial: connection timed out")
+	}
+	p := newHostPool(dial)
+	slow := make(chan error, 1)
+	go func() { slow <- p.call(context.Background(), discard, testHost, noop) }()
+	<-dialing
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+
+	err := p.call(ctx, discard, testHost, noop)
+
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("call() error = %v, want the caller's deadline", err)
+	}
+	if waited := time.Since(start); waited > time.Second {
+		t.Errorf("call() waited %s behind another caller's dial, want about 50ms", waited)
+	}
+	close(release)
+	if err := <-slow; err == nil {
+		t.Error("the blocked dial succeeded, want its error")
+	}
+}
+
+func TestHostPool_probe_abandonsWithoutClosing(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	unblock := make(chan struct{})
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+
+	err := p.probe(ctx, discard, testHost, func(libvirtAPI) error {
+		<-unblock // a daemon that stopped answering
+		return nil
+	})
+
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("probe() error = %v, want DeadlineExceeded", err)
+	}
+	if d.clients[0].closed.Load() {
+		t.Error("probe closed the shared connection when it gave up")
+	}
+	if err := p.probe(context.Background(), discard, testHost, noop); !errors.Is(err, errProbeBusy) {
+		t.Errorf("probe() while the abandoned one still waits = %v, want errProbeBusy", err)
+	}
+	if err := p.call(context.Background(), discard, testHost, noop); err != nil {
+		t.Errorf("call() on the kept connection = %v", err)
+	}
+	close(unblock)
+	deadline := time.Now().Add(time.Second)
+	for {
+		err := p.probe(context.Background(), discard, testHost, noop)
+		if err == nil {
+			break
+		}
+		if !errors.Is(err, errProbeBusy) || time.Now().After(deadline) {
+			t.Fatalf("probe() after the abandoned one returned = %v, want nil", err)
+		}
+		time.Sleep(time.Millisecond)
+	}
+	if d.dialCount() != 1 {
+		t.Errorf("dials = %d, want 1: nothing should have redialed", d.dialCount())
+	}
+}
+
+func TestHostPool_probe_dropsConnectionOnTransportError(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+
+	err := p.probe(context.Background(), discard, testHost, func(libvirtAPI) error { return io.ErrUnexpectedEOF })
+
+	if !errors.Is(err, io.ErrUnexpectedEOF) {
+		t.Fatalf("probe() error = %v, want io.ErrUnexpectedEOF passed through", err)
+	}
+	if !d.clients[0].closed.Load() {
+		t.Error("a transport error during a probe left the connection cached")
+	}
+}
+
+// --- SSH dial against an in-process server ---
+
+func newKeyPair(t *testing.T) (ssh.Signer, []byte) {
+	t.Helper()
+	_, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	signer, err := ssh.NewSignerFromKey(priv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	block, err := ssh.MarshalPrivateKey(priv, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return signer, pem.EncodeToMemory(block)
+}
+
+// testSSHServer accepts one public key and serves
+// direct-streamlocal@openssh.com channels by dialing the requested unix
+// path, which is exactly what OpenSSH does for ssh.Client.Dial("unix").
+type testSSHServer struct {
+	addr    string
+	hostKey ssh.PublicKey
+
+	mu    sync.Mutex
+	paths []string
+}
+
+func startSSHServer(t *testing.T, clientKey ssh.PublicKey, answerGlobalRequests bool) *testSSHServer {
+	t.Helper()
+	hostSigner, _ := newKeyPair(t)
+	cfg := &ssh.ServerConfig{
+		PublicKeyCallback: func(_ ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
+			if bytes.Equal(key.Marshal(), clientKey.Marshal()) {
+				return nil, nil
+			}
+			return nil, errors.New("unknown public key")
+		},
+	}
+	cfg.AddHostKey(hostSigner)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { ln.Close() })
+	s := &testSSHServer{addr: ln.Addr().String(), hostKey: hostSigner.PublicKey()}
+	go func() {
+		for {
+			nc, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			go s.serve(nc, cfg, answerGlobalRequests)
+		}
+	}()
+	return s
+}
+
+func (s *testSSHServer) serve(nc net.Conn, cfg *ssh.ServerConfig, answerGlobalRequests bool) {
+	_, chans, reqs, err := ssh.NewServerConn(nc, cfg)
+	if err != nil {
+		return
+	}
+	if answerGlobalRequests {
+		go ssh.DiscardRequests(reqs)
+	} else {
+		go func() {
+			for range reqs { //nolint:revive // deliberately never replies, to stall keepalives
+			}
+		}()
+	}
+	for newCh := range chans {
+		if newCh.ChannelType() != "direct-streamlocal@openssh.com" {
+			_ = newCh.Reject(ssh.UnknownChannelType, "test server only forwards unix sockets")
+			continue
+		}
+		var payload struct {
+			Path      string
+			Reserved0 string
+			Reserved1 uint32
+		}
+		if err := ssh.Unmarshal(newCh.ExtraData(), &payload); err != nil {
+			_ = newCh.Reject(ssh.ConnectionFailed, err.Error())
+			continue
+		}
+		s.mu.Lock()
+		s.paths = append(s.paths, payload.Path)
+		s.mu.Unlock()
+		target, err := net.Dial("unix", payload.Path)
+		if err != nil {
+			_ = newCh.Reject(ssh.ConnectionFailed, err.Error())
+			continue
+		}
+		ch, chReqs, err := newCh.Accept()
+		if err != nil {
+			target.Close()
+			continue
+		}
+		go ssh.DiscardRequests(chReqs)
+		go func() { _, _ = io.Copy(ch, target); ch.Close() }()
+		go func() { _, _ = io.Copy(target, ch); target.Close() }()
+	}
+}
+
+func (s *testSSHServer) requestedPaths() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.paths...)
+}
+
+// startEchoSocket serves a unix socket that echoes everything back, standing
+// in for libvirt-sock.
+func startEchoSocket(t *testing.T) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "libvirt-sock")
+	ln, err := net.Listen("unix", path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { ln.Close() })
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			go func() { _, _ = io.Copy(c, c); c.Close() }()
+		}
+	}()
+	return path
+}
+
+type sshFixture struct {
+	server  *testSSHServer
+	cfg     provider.KVMConfig
+	host    provider.KVMHost
+	keyFile string
+}
+
+func newSSHFixture(t *testing.T, answerGlobalRequests bool) *sshFixture {
+	t.Helper()
+	clientSigner, clientPEM := newKeyPair(t)
+	server := startSSHServer(t, clientSigner.PublicKey(), answerGlobalRequests)
+	dir := t.TempDir()
+	keyFile := filepath.Join(dir, "id_ed25519")
+	if err := os.WriteFile(keyFile, clientPEM, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	khFile := filepath.Join(dir, "known_hosts")
+	line := knownhosts.Line([]string{server.addr}, server.hostKey)
+	if err := os.WriteFile(khFile, []byte(line+"\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	return &sshFixture{
+		server:  server,
+		keyFile: keyFile,
+		cfg:     provider.KVMConfig{SSHKeyFile: keyFile, KnownHostsFile: khFile},
+		host:    provider.KVMHost{Name: "nas1", Address: server.addr, User: "virt", Socket: startEchoSocket(t)},
+	}
+}
+
+func mustSSHAuth(t *testing.T, cfg provider.KVMConfig) *sshAuth {
+	t.Helper()
+	auth, err := newSSHAuth(cfg)
+	if err != nil {
+		t.Fatalf("newSSHAuth() error = %v", err)
+	}
+	return auth
+}
+
+func TestSSHAuth_dialSSH_forwardsUnixSocket(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	auth := mustSSHAuth(t, f.cfg)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	client, err := auth.dialSSH(ctx, f.host)
+	if err != nil {
+		t.Fatalf("dialSSH() error = %v", err)
+	}
+	defer client.Close()
+
+	conn, err := unixForward{client: client, path: f.host.Socket}.Dial()
+	if err != nil {
+		t.Fatalf("unixForward.Dial() error = %v", err)
+	}
+	defer conn.Close()
+	if _, err := conn.Write([]byte("ping")); err != nil {
+		t.Fatal(err)
+	}
+	buf := make([]byte, 4)
+	if _, err := io.ReadFull(conn, buf); err != nil || string(buf) != "ping" {
+		t.Fatalf("echo through the forward = %q, %v; want ping", buf, err)
+	}
+	if paths := f.server.requestedPaths(); len(paths) != 1 || paths[0] != f.host.Socket {
+		t.Errorf("server saw forward requests %v, want [%s]", paths, f.host.Socket)
+	}
+}
+
+func TestSSHAuth_dialSSH_rejectsUnknownHostKey(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	impostor, _ := newKeyPair(t)
+	line := knownhosts.Line([]string{f.server.addr}, impostor.PublicKey())
+	if err := os.WriteFile(f.cfg.KnownHostsFile, []byte(line+"\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	auth := mustSSHAuth(t, f.cfg)
+	client, err := auth.dialSSH(context.Background(), f.host)
+	if err == nil {
+		client.Close()
+		t.Fatal("dialSSH() succeeded against a host whose key is not in known_hosts")
+	}
+	var keyErr *knownhosts.KeyError
+	if !errors.As(err, &keyErr) {
+		t.Errorf("dialSSH() error = %v, want a knownhosts.KeyError", err)
+	}
+}
+
+func TestSSHAuth_dialSSH_insecureIgnoreHostKey(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	f.cfg.KnownHostsFile = ""
+	f.cfg.InsecureIgnoreHostKey = true
+	client, err := mustSSHAuth(t, f.cfg).dialSSH(context.Background(), f.host)
+	if err != nil {
+		t.Fatalf("dialSSH() error = %v", err)
+	}
+	client.Close()
+}
+
+func TestSSHAuth_dialSSH_rejectsWrongClientKey(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	_, otherPEM := newKeyPair(t)
+	if err := os.WriteFile(f.keyFile, otherPEM, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	client, err := mustSSHAuth(t, f.cfg).dialSSH(context.Background(), f.host)
+	if err == nil {
+		client.Close()
+		t.Fatal("dialSSH() succeeded with a key the server does not know")
+	}
+	if !strings.Contains(err.Error(), "ssh handshake") {
+		t.Errorf("dialSSH() error = %v, want an auth failure from the handshake", err)
+	}
+}
+
+func TestSSHAuth_dialSSH_cancelledContext(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if client, err := mustSSHAuth(t, f.cfg).dialSSH(ctx, f.host); err == nil {
+		client.Close()
+		t.Fatal("dialSSH() succeeded with an already-cancelled context")
+	}
+}
+
+func TestNewSSHAuth_missingFiles(t *testing.T) {
+	t.Parallel()
+	_, keyPEM := newKeyPair(t)
+	keyFile := filepath.Join(t.TempDir(), "key")
+	if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	tests := []struct {
+		name    string
+		cfg     provider.KVMConfig
+		wantSub string
+	}{
+		{"key file missing", provider.KVMConfig{SSHKeyFile: "/nonexistent/key", InsecureIgnoreHostKey: true}, "sshKeyFile"},
+		{"key file garbage", provider.KVMConfig{SSHKeyFile: keyFile + ".bad", InsecureIgnoreHostKey: true}, "sshKeyFile"},
+		{"known_hosts missing", provider.KVMConfig{SSHKeyFile: keyFile, KnownHostsFile: "/nonexistent/kh"}, "knownHostsFile"},
+	}
+	if err := os.WriteFile(keyFile+".bad", []byte("not a key"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			_, err := newSSHAuth(tc.cfg)
+			if err == nil || !strings.Contains(err.Error(), tc.wantSub) {
+				t.Errorf("newSSHAuth() error = %v, want one mentioning %s", err, tc.wantSub)
+			}
+		})
+	}
+}
+
+// The kvm-ssh Secret is an optional mount, so the provider has to be
+// constructible without it and has to start working once it appears — the
+// alternative was a constructor error that crash-looped the whole manager.
+func TestLazySSH_load_healsWhenTheKeyAppears(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	keyPEM, err := os.ReadFile(f.keyFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Remove(f.keyFile); err != nil {
+		t.Fatal(err)
+	}
+	auth := newLazySSH(f.cfg).load
+
+	if _, err := auth(); err == nil {
+		t.Fatal("loading with no key file succeeded, want an error")
+	} else if class := classify(err); class != provider.ErrTransient {
+		t.Errorf("class = %v, want ErrTransient so the proxy retries once the Secret lands", class)
+	}
+	if err := os.WriteFile(f.keyFile, keyPEM, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	loaded, err := auth()
+	if err != nil {
+		t.Fatalf("loading after the key appeared: %v", err)
+	}
+	// Cached from here on: the file may vanish again (a Secret rotated out)
+	// without breaking connections that already work.
+	if err := os.Remove(f.keyFile); err != nil {
+		t.Fatal(err)
+	}
+	if again, err := auth(); err != nil || again != loaded {
+		t.Errorf("auth() = %v, %v; want the cached value reused", again, err)
+	}
+}
+
+func TestLazySSH_load_unparseableKeyIsPermanent(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	if err := os.WriteFile(f.keyFile, []byte("not a key"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	_, err := newLazySSH(f.cfg).load()
+	if classify(err) != provider.ErrPermanent {
+		t.Errorf("class = %v, want ErrPermanent: retrying cannot fix a key that does not parse", classify(err))
+	}
+}
+
+// The dialer must authenticate with the material the lazy loader produced,
+// not a stale or empty one. The echo socket is not a libvirt server, so the
+// call gets as far as the libvirt handshake and no further; what matters is
+// that it is past SSH by then.
+func TestLazySSH_dial_authenticatesOverSSH(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	_, err := newLazySSH(f.cfg).dial(ctx, f.host)
+	if err == nil {
+		t.Fatal("dial to an echo socket succeeded, want a libvirt handshake error")
+	}
+	if strings.Contains(err.Error(), "sshKeyFile") || strings.Contains(err.Error(), "ssh handshake") {
+		t.Errorf("dial failed before libvirt: %v", err)
+	}
+	if paths := f.server.requestedPaths(); len(paths) != 1 || paths[0] != f.host.Socket {
+		t.Errorf("server saw forward requests %v, want [%s]", paths, f.host.Socket)
+	}
+}
+
+// New must not fail on missing key material, or one misconfigured provider
+// aborts registry.Build and the manager exits before any controller starts.
+func TestNew_missingSSHMaterialStillBuilds(t *testing.T) {
+	t.Parallel()
+	p, err := New(context.Background(), provider.ProviderConfig{
+		Name: "kvm-home", Type: "kvm",
+		KVM: &provider.KVMConfig{
+			SSHKeyFile:     filepath.Join(t.TempDir(), "absent"),
+			KnownHostsFile: filepath.Join(t.TempDir(), "absent"),
+			Hosts:          []provider.KVMHost{{Name: "nas1", Address: "nas1:22", User: "virt", Bridge: "br0"}},
+		},
+	})
+	if err != nil || p == nil {
+		t.Fatalf("New() = %v, %v; want a usable provider", p, err)
+	}
+	if _, err := p.Get(context.Background(), "hosts/nas1/domains/proxy-x"); provider.Class(err) != provider.ErrTransient {
+		t.Errorf("Get class = %v, want ErrTransient", provider.Class(err))
+	}
+}
+
+// The health check reports where a connection broke, so every stage of a
+// real dial must come back tagged.
+func TestLazySSH_dial_tagsTheFailingStage(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name       string
+		mutate     func(t *testing.T, f *sshFixture)
+		want       dialStage
+		wantReason string
+	}{
+		{
+			name:       "key material missing",
+			mutate:     func(t *testing.T, f *sshFixture) { f.cfg.SSHKeyFile = filepath.Join(t.TempDir(), "absent") },
+			want:       stageSSHMaterial,
+			wantReason: reasonSSHMaterialUnavailable,
+		},
+		{
+			name: "nothing listening",
+			mutate: func(t *testing.T, f *sshFixture) {
+				ln, err := net.Listen("tcp", "127.0.0.1:0")
+				if err != nil {
+					t.Fatal(err)
+				}
+				f.host.Address = ln.Addr().String()
+				ln.Close()
+			},
+			want:       stageTCP,
+			wantReason: provider.ReasonUnreachable,
+		},
+		{
+			name: "host key not in known_hosts",
+			mutate: func(t *testing.T, f *sshFixture) {
+				impostor, _ := newKeyPair(t)
+				line := knownhosts.Line([]string{f.server.addr}, impostor.PublicKey())
+				if err := os.WriteFile(f.cfg.KnownHostsFile, []byte(line+"\n"), 0o600); err != nil {
+					t.Fatal(err)
+				}
+			},
+			want:       stageSSHHandshake,
+			wantReason: reasonSSHHandshakeFailed,
+		},
+		{
+			name: "client key rejected",
+			mutate: func(t *testing.T, f *sshFixture) {
+				_, otherPEM := newKeyPair(t)
+				if err := os.WriteFile(f.keyFile, otherPEM, 0o600); err != nil {
+					t.Fatal(err)
+				}
+			},
+			want:       stageSSHHandshake,
+			wantReason: reasonSSHHandshakeFailed,
+		},
+		{
+			name:       "libvirt socket missing",
+			mutate:     func(_ *testing.T, f *sshFixture) { f.host.Socket = "/nonexistent/libvirt-sock" },
+			want:       stageLibvirtConnect,
+			wantReason: reasonLibvirtConnectFailed,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			f := newSSHFixture(t, true)
+			tc.mutate(t, f)
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+
+			_, err := newLazySSH(f.cfg).dial(ctx, f.host)
+
+			var de *dialError
+			if !errors.As(err, &de) || de.stage != tc.want {
+				t.Fatalf("dial() error = %v, want a %s dialError", err, tc.want)
+			}
+			if got := checkReason(err); got != tc.wantReason {
+				t.Errorf("checkReason() = %s, want %s", got, tc.wantReason)
+			}
+		})
+	}
+}
+
+func TestKeepalive_closesClientWhenServerStalls(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, false)
+	client, err := mustSSHAuth(t, f.cfg).dialSSH(context.Background(), f.host)
+	if err != nil {
+		t.Fatalf("dialSSH() error = %v", err)
+	}
+	defer client.Close()
+	stop := make(chan struct{})
+	defer close(stop)
+	go keepalive(client, 20*time.Millisecond, stop)
+
+	closed := make(chan struct{})
+	go func() { _ = client.Wait(); close(closed) }()
+	select {
+	case <-closed:
+	case <-time.After(2 * time.Second):
+		t.Fatal("client still open after keepalives went unanswered")
+	}
+}
+
+func TestKeepalive_keepsHealthyClientOpen(t *testing.T) {
+	t.Parallel()
+	f := newSSHFixture(t, true)
+	client, err := mustSSHAuth(t, f.cfg).dialSSH(context.Background(), f.host)
+	if err != nil {
+		t.Fatalf("dialSSH() error = %v", err)
+	}
+	defer client.Close()
+	stop := make(chan struct{})
+	defer close(stop)
+	go keepalive(client, 10*time.Millisecond, stop)
+	time.Sleep(60 * time.Millisecond)
+	fwd := unixForward{client: client, path: f.host.Socket}
+	conn, err := fwd.Dial()
+	if err != nil {
+		t.Fatalf("forward after several keepalives failed: %v", err)
+	}
+	conn.Close()
+}
+
+func TestHostPool_call_tracesAtV1(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{api: readyHost()}
+	p := newHostPool(d.dial)
+	log, snapshot := captureLogger(1)
+	err := p.call(context.Background(), log, testHost, func(api libvirtAPI) error {
+		_, err := api.StoragePoolLookupByName("default")
+		return err
+	})
+	if err != nil {
+		t.Fatalf("call() error = %v", err)
+	}
+	joined := strings.Join(snapshot(), "\n")
+	for _, want := range []string{`"msg"="libvirt host connected"`, `"address"="nas1:22"`, `"rpc"="virStoragePoolLookupByName"`, `"pool"="default"`, "libvirt "} {
+		if !strings.Contains(joined, want) {
+			t.Errorf("output missing %s:\n%s", want, joined)
+		}
+	}
+}
+
+func TestHostPool_call_noTracingBelowV1(t *testing.T) {
+	t.Parallel()
+	inner := readyHost()
+	d := &fakeDialer{api: inner}
+	p := newHostPool(d.dial)
+	log, snapshot := captureLogger(0)
+	err := p.call(context.Background(), log, testHost, func(api libvirtAPI) error {
+		if api != libvirtAPI(inner) {
+			t.Errorf("fn received %T, want the bare fake when V(1) is off", api)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("call() error = %v", err)
+	}
+	if lines := snapshot(); len(lines) != 0 {
+		t.Errorf("verbosity 0 logged %d lines:\n%s", len(lines), strings.Join(lines, "\n"))
+	}
+}
+
+func TestHostPool_call_logsCancelDropReason(t *testing.T) {
+	t.Parallel()
+	d := &fakeDialer{}
+	p := newHostPool(d.dial)
+	log, snapshot := captureLogger(1)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	released := make(chan struct{})
+	err := p.call(ctx, log, testHost, func(libvirtAPI) error {
+		<-d.clients[0].done
+		close(released)
+		return io.EOF
+	})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("call() error = %v, want context.Canceled", err)
+	}
+	<-released
+	joined := strings.Join(snapshot(), "\n")
+	if !strings.Contains(joined, `"reason"="context cancelled"`) {
+		t.Errorf("output missing the cancel drop line:\n%s", joined)
+	}
+	if strings.Contains(joined, `"reason"="transport error"`) {
+		t.Errorf("cancel was also reported as a transport error:\n%s", joined)
+	}
+}

internal/provider/kvm/domainxml.go

@@ -0,0 +1,253 @@
+package kvm
+
+import (
+	"encoding/xml"
+	"fmt"
+	"time"
+)
+
+const guestAgentChannel = "org.qemu.guest_agent.0"
+
+// domainParams is everything buildDomainXML needs, already resolved from the
+// config and placement so the builder stays a pure function. DiskPath and
+// SeedPath are the volumes' host paths as reported by libvirt; Firmware is
+// firmwareUEFI or firmwareBIOS.
+type domainParams struct {
+	Name      string
+	UID       string
+	CreatedAt time.Time
+	VCPUs     int32
+	MemoryMiB int64
+	Firmware  string
+	Bridge    string
+	DiskPath  string
+	SeedPath  string
+}
+
+// buildDomainXML renders a q35 KVM guest: host-passthrough CPU, a virtio
+// overlay disk and a SATA cdrom for the seed, one virtio NIC on the host
+// bridge, a serial console, the guest-agent channel, a virtio RNG, no
+// graphics, and the operator's metadata element.
+//
+// Disks are type="file" with host paths, not type="volume" with pool/volume
+// names: libvirt's AppArmor helper (virt-aa-helper) only grants QEMU access
+// to sources that carry a path, so a volume-referenced disk fails to start
+// with "Permission denied" on any AppArmor-confined host (stock Debian and
+// Ubuntu). The paths cost one StorageVolGetPath round-trip each per Create.
+//
+// UEFI is expressed as <os firmware="efi"> with secure boot off, and libvirt
+// picks the OVMF binary and creates the NVRAM file itself (removed again by
+// the nvram undefine flag). Under SeaBIOS, Debian 13's GRUB hangs on a
+// headless domain with a serial console (Debian bug #1111240:
+// "terminal_output gfxterm serial" with no video device); the same image
+// boots under UEFI with the serial console intact.
+func buildDomainXML(p domainParams) (string, error) {
+	os := osXML{
+		Type: osTypeXML{Arch: "x86_64", Machine: "q35", Value: "hvm"},
+		Boot: bootXML{Dev: "hd"},
+	}
+	if p.Firmware == firmwareUEFI {
+		os.Firmware = "efi"
+		os.FirmwareFeatures = &firmwareXML{Features: []firmwareFeatureXML{{Enabled: "no", Name: "secure-boot"}}}
+	}
+	d := domainXML{
+		Type:       "kvm",
+		Name:       p.Name,
+		Metadata:   domainMetadataXML{Instance: newInstanceMetadata(p.UID, p.CreatedAt)},
+		Memory:     sizeXML{Unit: "MiB", Value: p.MemoryMiB},
+		VCPU:       p.VCPUs,
+		OS:         os,
+		Features:   featuresXML{ACPI: &struct{}{}, APIC: &struct{}{}},
+		CPU:        cpuXML{Mode: "host-passthrough"},
+		Clock:      clockXML{Offset: "utc"},
+		OnPoweroff: "destroy",
+		OnReboot:   "restart",
+		OnCrash:    "destroy",
+		Devices: devicesXML{
+			Disks: []diskXML{
+				{
+					Type:   "file",
+					Device: "disk",
+					Driver: driverXML{Name: "qemu", Type: formatQcow2},
+					Source: diskSourceXML{File: p.DiskPath},
+					Target: targetXML{Dev: "vda", Bus: "virtio"},
+				},
+				{
+					Type:     "file",
+					Device:   "cdrom",
+					Driver:   driverXML{Name: "qemu", Type: formatRaw},
+					Source:   diskSourceXML{File: p.SeedPath},
+					Target:   targetXML{Dev: "sda", Bus: "sata"},
+					ReadOnly: &struct{}{},
+				},
+			},
+			Interfaces: []interfaceXML{{
+				Type:   "bridge",
+				Source: ifaceSourceXML{Bridge: p.Bridge},
+				Model:  modelXML{Type: "virtio"},
+			}},
+			Serials:  []serialXML{{Type: "pty", Target: portXML{Port: 0}}},
+			Consoles: []consoleXML{{Type: "pty", Target: consoleTargetXML{Type: "serial", Port: 0}}},
+			Channels: []channelXML{{
+				Type:   "unix",
+				Target: channelTargetXML{Type: "virtio", Name: guestAgentChannel},
+			}},
+			RNG: rngXML{Model: "virtio", Backend: rngBackendXML{Model: "random", Value: "/dev/urandom"}},
+		},
+	}
+	out, err := xml.Marshal(d)
+	if err != nil {
+		return "", fmt.Errorf("marshalling domain xml: %w", err)
+	}
+	return string(out), nil
+}
+
+type domainXML struct {
+	XMLName    xml.Name          `xml:"domain"`
+	Type       string            `xml:"type,attr"`
+	Name       string            `xml:"name"`
+	Metadata   domainMetadataXML `xml:"metadata"`
+	Memory     sizeXML           `xml:"memory"`
+	VCPU       int32             `xml:"vcpu"`
+	OS         osXML             `xml:"os"`
+	Features   featuresXML       `xml:"features"`
+	CPU        cpuXML            `xml:"cpu"`
+	Clock      clockXML          `xml:"clock"`
+	OnPoweroff string            `xml:"on_poweroff"`
+	OnReboot   string            `xml:"on_reboot"`
+	OnCrash    string            `xml:"on_crash"`
+	Devices    devicesXML        `xml:"devices"`
+}
+
+type domainMetadataXML struct {
+	Instance instanceMetadata
+}
+
+type sizeXML struct {
+	Unit  string `xml:"unit,attr"`
+	Value int64  `xml:",chardata"`
+}
+
+type osXML struct {
+	Firmware         string       `xml:"firmware,attr,omitempty"`
+	Type             osTypeXML    `xml:"type"`
+	FirmwareFeatures *firmwareXML `xml:"firmware"`
+	Boot             bootXML      `xml:"boot"`
+}
+
+type firmwareXML struct {
+	Features []firmwareFeatureXML `xml:"feature"`
+}
+
+type firmwareFeatureXML struct {
+	Enabled string `xml:"enabled,attr"`
+	Name    string `xml:"name,attr"`
+}
+
+type osTypeXML struct {
+	Arch    string `xml:"arch,attr"`
+	Machine string `xml:"machine,attr"`
+	Value   string `xml:",chardata"`
+}
+
+type bootXML struct {
+	Dev string `xml:"dev,attr"`
+}
+
+type featuresXML struct {
+	ACPI *struct{} `xml:"acpi"`
+	APIC *struct{} `xml:"apic"`
+}
+
+type cpuXML struct {
+	Mode string `xml:"mode,attr"`
+}
+
+type clockXML struct {
+	Offset string `xml:"offset,attr"`
+}
+
+type devicesXML struct {
+	Disks      []diskXML      `xml:"disk"`
+	Interfaces []interfaceXML `xml:"interface"`
+	Serials    []serialXML    `xml:"serial"`
+	Consoles   []consoleXML   `xml:"console"`
+	Channels   []channelXML   `xml:"channel"`
+	RNG        rngXML         `xml:"rng"`
+}
+
+type diskXML struct {
+	Type     string        `xml:"type,attr"`
+	Device   string        `xml:"device,attr"`
+	Driver   driverXML     `xml:"driver"`
+	Source   diskSourceXML `xml:"source"`
+	Target   targetXML     `xml:"target"`
+	ReadOnly *struct{}     `xml:"readonly"`
+}
+
+type driverXML struct {
+	Name string `xml:"name,attr"`
+	Type string `xml:"type,attr"`
+}
+
+type diskSourceXML struct {
+	File string `xml:"file,attr"`
+}
+
+type targetXML struct {
+	Dev string `xml:"dev,attr"`
+	Bus string `xml:"bus,attr"`
+}
+
+type interfaceXML struct {
+	Type   string         `xml:"type,attr"`
+	Source ifaceSourceXML `xml:"source"`
+	Model  modelXML       `xml:"model"`
+}
+
+type ifaceSourceXML struct {
+	Bridge string `xml:"bridge,attr"`
+}
+
+type modelXML struct {
+	Type string `xml:"type,attr"`
+}
+
+type serialXML struct {
+	Type   string  `xml:"type,attr"`
+	Target portXML `xml:"target"`
+}
+
+type portXML struct {
+	Port int `xml:"port,attr"`
+}
+
+type consoleXML struct {
+	Type   string           `xml:"type,attr"`
+	Target consoleTargetXML `xml:"target"`
+}
+
+type consoleTargetXML struct {
+	Type string `xml:"type,attr"`
+	Port int    `xml:"port,attr"`
+}
+
+type channelXML struct {
+	Type   string           `xml:"type,attr"`
+	Target channelTargetXML `xml:"target"`
+}
+
+type channelTargetXML struct {
+	Type string `xml:"type,attr"`
+	Name string `xml:"name,attr"`
+}
+
+type rngXML struct {
+	Model   string        `xml:"model,attr"`
+	Backend rngBackendXML `xml:"backend"`
+}
+
+type rngBackendXML struct {
+	Model string `xml:"model,attr"`
+	Value string `xml:",chardata"`
+}

internal/provider/kvm/domainxml_test.go

@@ -0,0 +1,137 @@
+package kvm
+
+import (
+	"encoding/xml"
+	"strings"
+	"testing"
+	"time"
+)
+
+func testDomainParams() domainParams {
+	return domainParams{
+		Name:      "proxy-abc123def456ghij",
+		UID:       "11111111-2222-3333-4444-555555555555",
+		CreatedAt: time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC),
+		VCPUs:     2,
+		MemoryMiB: 2048,
+		Firmware:  firmwareUEFI,
+		Bridge:    "br0",
+		DiskPath:  "/var/lib/libvirt/images/proxy-abc123def456ghij.qcow2",
+		SeedPath:  "/var/lib/libvirt/images/proxy-abc123def456ghij-seed.iso",
+	}
+}
+
+func TestBuildDomainXML_fieldByField(t *testing.T) {
+	t.Parallel()
+	raw, err := buildDomainXML(testDomainParams())
+	if err != nil {
+		t.Fatalf("buildDomainXML() error = %v", err)
+	}
+	var d domainXML
+	if err := xml.Unmarshal([]byte(raw), &d); err != nil {
+		t.Fatalf("output is not well-formed XML: %v\n%s", err, raw)
+	}
+
+	if d.Type != "kvm" || d.Name != "proxy-abc123def456ghij" {
+		t.Errorf("type/name = %s/%s", d.Type, d.Name)
+	}
+	if d.VCPU != 2 || d.Memory.Value != 2048 || d.Memory.Unit != "MiB" {
+		t.Errorf("vcpu/memory = %d/%d %s, want 2/2048 MiB", d.VCPU, d.Memory.Value, d.Memory.Unit)
+	}
+	if d.OS.Type.Value != "hvm" || d.OS.Type.Arch != "x86_64" || d.OS.Type.Machine != "q35" || d.OS.Boot.Dev != "hd" {
+		t.Errorf("os = %+v", d.OS)
+	}
+	if d.OS.Firmware != "efi" || d.OS.FirmwareFeatures == nil || len(d.OS.FirmwareFeatures.Features) != 1 ||
+		d.OS.FirmwareFeatures.Features[0] != (firmwareFeatureXML{Enabled: "no", Name: "secure-boot"}) {
+		t.Errorf("os firmware = %q %+v, want efi with secure-boot disabled", d.OS.Firmware, d.OS.FirmwareFeatures)
+	}
+	if d.Features.ACPI == nil || d.Features.APIC == nil {
+		t.Errorf("features = %+v, want acpi and apic", d.Features)
+	}
+	if d.CPU.Mode != "host-passthrough" {
+		t.Errorf("cpu mode = %q", d.CPU.Mode)
+	}
+	if d.OnPoweroff != "destroy" || d.OnReboot != "restart" || d.OnCrash != "destroy" {
+		t.Errorf("lifecycle = %s/%s/%s", d.OnPoweroff, d.OnReboot, d.OnCrash)
+	}
+
+	m := d.Metadata.Instance
+	if !m.managed() || m.UID != "11111111-2222-3333-4444-555555555555" || m.CreatedAt != "2026-09-04T12:00:00Z" {
+		t.Errorf("metadata = %+v", m)
+	}
+	if !strings.Contains(raw, `xmlns="`+metadataURI+`"`) {
+		t.Errorf("metadata element is not namespaced; libvirt rejects that:\n%s", raw)
+	}
+
+	if len(d.Devices.Disks) != 2 {
+		t.Fatalf("disks = %d, want 2", len(d.Devices.Disks))
+	}
+	disk, seed := d.Devices.Disks[0], d.Devices.Disks[1]
+	if disk.Type != "file" || disk.Device != "disk" || disk.Driver.Type != "qcow2" ||
+		disk.Source.File != "/var/lib/libvirt/images/proxy-abc123def456ghij.qcow2" ||
+		disk.Target.Dev != "vda" || disk.Target.Bus != "virtio" || disk.ReadOnly != nil {
+		t.Errorf("overlay disk = %+v", disk)
+	}
+	if seed.Type != "file" || seed.Device != "cdrom" || seed.Driver.Type != "raw" ||
+		seed.Source.File != "/var/lib/libvirt/images/proxy-abc123def456ghij-seed.iso" ||
+		seed.Target.Bus != "sata" || seed.ReadOnly == nil {
+		t.Errorf("seed cdrom = %+v", seed)
+	}
+	if strings.Contains(raw, `type="volume"`) || strings.Contains(raw, " pool=") {
+		t.Errorf("disks reference pool volumes; virt-aa-helper cannot grant QEMU access to those:\n%s", raw)
+	}
+
+	if len(d.Devices.Interfaces) != 1 {
+		t.Fatalf("interfaces = %d, want 1", len(d.Devices.Interfaces))
+	}
+	if nic := d.Devices.Interfaces[0]; nic.Type != "bridge" || nic.Source.Bridge != "br0" || nic.Model.Type != "virtio" {
+		t.Errorf("interface = %+v", nic)
+	}
+
+	if len(d.Devices.Channels) != 1 || d.Devices.Channels[0].Target.Name != guestAgentChannel || d.Devices.Channels[0].Target.Type != "virtio" {
+		t.Errorf("channels = %+v, want one virtio %s channel", d.Devices.Channels, guestAgentChannel)
+	}
+	if len(d.Devices.Serials) != 1 || len(d.Devices.Consoles) != 1 || d.Devices.Consoles[0].Target.Type != "serial" {
+		t.Errorf("serial/console = %+v/%+v", d.Devices.Serials, d.Devices.Consoles)
+	}
+	if d.Devices.RNG.Model != "virtio" || d.Devices.RNG.Backend.Value != "/dev/urandom" {
+		t.Errorf("rng = %+v", d.Devices.RNG)
+	}
+	if strings.Contains(raw, "<graphics") {
+		t.Errorf("domain has a graphics device; headless VMs must not:\n%s", raw)
+	}
+}
+
+func TestBuildDomainXML_escapesUserValues(t *testing.T) {
+	t.Parallel()
+	p := testDomainParams()
+	p.Bridge = `br0" onload="x`
+	raw, err := buildDomainXML(p)
+	if err != nil {
+		t.Fatalf("buildDomainXML() error = %v", err)
+	}
+	var d domainXML
+	if err := xml.Unmarshal([]byte(raw), &d); err != nil {
+		t.Fatalf("output is not well-formed XML: %v", err)
+	}
+	if d.Devices.Interfaces[0].Source.Bridge != p.Bridge {
+		t.Errorf("bridge = %q, want the raw value back", d.Devices.Interfaces[0].Source.Bridge)
+	}
+}
+
+func TestBuildDomainXML_biosFirmware(t *testing.T) {
+	t.Parallel()
+	p := testDomainParams()
+	p.Firmware = firmwareBIOS
+	raw, err := buildDomainXML(p)
+	if err != nil {
+		t.Fatalf("buildDomainXML() error = %v", err)
+	}
+	var d domainXML
+	if err := xml.Unmarshal([]byte(raw), &d); err != nil {
+		t.Fatalf("output is not well-formed XML: %v", err)
+	}
+	if d.OS.Firmware != "" || d.OS.FirmwareFeatures != nil || strings.Contains(raw, "firmware") {
+		t.Errorf("bios domain still carries firmware settings:\n%s", raw)
+	}
+}

internal/provider/kvm/errors.go

@@ -0,0 +1,90 @@
+package kvm
+
+import (
+	"errors"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// classify maps an error onto the provider taxonomy. Errors the package
+// already tagged with a sentinel (image download failures, placement
+// errors) keep their class. For libvirt errors: missing domain or volume →
+// NotFound; anything the host will keep rejecting (bad XML, unsupported
+// config, denied access, missing pool) → Permanent; every other libvirt
+// code, plus SSH/transport failures and cancellation, → Transient, because
+// retrying is safer than latching Failed on an error nobody taught this
+// function to recognise. libvirt exposes no quota signal, so nothing maps
+// to QuotaExceeded.
+func classify(err error) error {
+	if isClassified(err) {
+		return provider.Class(err)
+	}
+	var lerr libvirt.Error
+	if !errors.As(err, &lerr) {
+		return provider.ErrTransient
+	}
+	switch libvirt.ErrorNumber(lerr.Code) {
+	case libvirt.ErrNoDomain, libvirt.ErrNoStorageVol:
+		return provider.ErrNotFound
+	case libvirt.ErrNoStoragePool,
+		libvirt.ErrXMLError, libvirt.ErrXMLDetail, libvirt.ErrXMLInvalidSchema,
+		libvirt.ErrConfigUnsupported, libvirt.ErrNoSupport,
+		libvirt.ErrOperationUnsupported, libvirt.ErrArgumentUnsupported,
+		libvirt.ErrInvalidArg,
+		libvirt.ErrAuthFailed, libvirt.ErrAccessDenied, libvirt.ErrOperationDenied:
+		return provider.ErrPermanent
+	default:
+		return provider.ErrTransient
+	}
+}
+
+// isLibvirtError reports whether err came back from the libvirt daemon (as
+// opposed to the SSH or RPC transport), which is what decides whether the
+// cached connection is still trustworthy.
+func isLibvirtError(err error) bool {
+	var lerr libvirt.Error
+	return errors.As(err, &lerr)
+}
+
+// isClassified reports whether err already carries one of the provider's
+// sentinels, i.e. the package reached a verdict itself rather than hitting
+// a transport failure.
+func isClassified(err error) bool {
+	for _, class := range []error{provider.ErrNotFound, provider.ErrQuotaExceeded, provider.ErrPermanent, provider.ErrTransient} {
+		if errors.Is(err, class) {
+			return true
+		}
+	}
+	return false
+}
+
+func hasCode(err error, code libvirt.ErrorNumber) bool {
+	var lerr libvirt.Error
+	return errors.As(err, &lerr) && libvirt.ErrorNumber(lerr.Code) == code
+}
+
+// logAPIError records the raw libvirt code and message at V(1); classify
+// collapses them onto the coarser taxonomy, so this is the only place the
+// original detail survives.
+func logAPIError(log logr.Logger, op string, err error) {
+	if !log.V(1).Enabled() {
+		return
+	}
+	kv := append([]any{"op", op, "error", err.Error()}, libvirtErrorFields(err)...)
+	log.V(1).Info("libvirt call failed", kv...)
+}
+
+// libvirtErrorFields renders a daemon-reported error as log key/values:
+// its constant name (ErrNoDomain, ...) and numeric code. The daemon's
+// message is already the error text. Nil for transport and other
+// non-libvirt errors.
+func libvirtErrorFields(err error) []any {
+	var lerr libvirt.Error
+	if !errors.As(err, &lerr) {
+		return nil
+	}
+	return []any{"libvirtError", libvirt.ErrorNumber(lerr.Code).String(), "libvirtCode", lerr.Code}
+}

internal/provider/kvm/errors_test.go

@@ -0,0 +1,93 @@
+package kvm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"reflect"
+	"testing"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func lvErr(code libvirt.ErrorNumber) error {
+	return libvirt.Error{Code: uint32(code), Message: code.String()}
+}
+
+func TestClassify(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		err  error
+		want error
+	}{
+		{"no domain", lvErr(libvirt.ErrNoDomain), provider.ErrNotFound},
+		{"no storage vol", lvErr(libvirt.ErrNoStorageVol), provider.ErrNotFound},
+		{"no storage pool", lvErr(libvirt.ErrNoStoragePool), provider.ErrPermanent},
+		{"xml error", lvErr(libvirt.ErrXMLError), provider.ErrPermanent},
+		{"xml detail", lvErr(libvirt.ErrXMLDetail), provider.ErrPermanent},
+		{"config unsupported", lvErr(libvirt.ErrConfigUnsupported), provider.ErrPermanent},
+		{"no support", lvErr(libvirt.ErrNoSupport), provider.ErrPermanent},
+		{"invalid arg", lvErr(libvirt.ErrInvalidArg), provider.ErrPermanent},
+		{"auth failed", lvErr(libvirt.ErrAuthFailed), provider.ErrPermanent},
+		{"access denied", lvErr(libvirt.ErrAccessDenied), provider.ErrPermanent},
+		{"operation invalid", lvErr(libvirt.ErrOperationInvalid), provider.ErrTransient},
+		{"operation failed", lvErr(libvirt.ErrOperationFailed), provider.ErrTransient},
+		{"internal error", lvErr(libvirt.ErrInternalError), provider.ErrTransient},
+		{"storage vol exists", lvErr(libvirt.ErrStorageVolExist), provider.ErrTransient},
+		{"wrapped libvirt error", fmt.Errorf("defining: %w", lvErr(libvirt.ErrNoDomain)), provider.ErrNotFound},
+		{"pre-tagged permanent", fmt.Errorf("%w: image returned HTTP 404", provider.ErrPermanent), provider.ErrPermanent},
+		{"pre-tagged transient wrapping libvirt not-found", fmt.Errorf("%w: %w", provider.ErrTransient, lvErr(libvirt.ErrNoDomain)), provider.ErrTransient},
+		{"transport eof", io.EOF, provider.ErrTransient},
+		{"context cancelled", context.Canceled, provider.ErrTransient},
+		{"plain error", errors.New("ssh: handshake failed"), provider.ErrTransient},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := classify(tc.err); !errors.Is(got, tc.want) {
+				t.Errorf("classify(%v) = %v, want %v", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestHasCode(t *testing.T) {
+	t.Parallel()
+	err := fmt.Errorf("lookup: %w", lvErr(libvirt.ErrNoDomain))
+	if !hasCode(err, libvirt.ErrNoDomain) {
+		t.Error("hasCode(ErrNoDomain) = false through a wrap")
+	}
+	if hasCode(err, libvirt.ErrNoStorageVol) {
+		t.Error("hasCode(ErrNoStorageVol) = true for an ErrNoDomain error")
+	}
+	if hasCode(io.EOF, libvirt.ErrNoDomain) || isLibvirtError(io.EOF) {
+		t.Error("io.EOF classified as a libvirt error")
+	}
+}
+
+func TestLibvirtErrorFields(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		err  error
+		want []any
+	}{
+		{"libvirt error", lvErr(libvirt.ErrNoDomain), []any{"libvirtError", "ErrNoDomain", "libvirtCode", uint32(42)}},
+		{"wrapped libvirt error", fmt.Errorf("looking up: %w", lvErr(libvirt.ErrNoDomain)), []any{"libvirtError", "ErrNoDomain", "libvirtCode", uint32(42)}},
+		{"transport error", io.EOF, nil},
+		{"cancelled", context.Canceled, nil},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got := libvirtErrorFields(tc.err)
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("libvirtErrorFields() = %#v, want %#v", got, tc.want)
+			}
+		})
+	}
+}

internal/provider/kvm/fake_test.go

@@ -0,0 +1,465 @@
+package kvm
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"slices"
+	"strings"
+	"sync"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+)
+
+// fakeAPI is an in-memory libvirtAPI: pools of volumes and a set of
+// domains, with per-method error injection. It models just enough libvirt
+// semantics for the provider's decisions to be observable — ErrStorageVolExist
+// on duplicate create, ErrNoDomain/ErrNoStorageVol on missing lookups,
+// ErrOperationInvalid on destroying a stopped domain, and so on.
+type fakeAPI struct {
+	mu       sync.Mutex
+	poolPath string
+	pools    map[string]map[string]*fakeVol
+	domains  map[string]*fakeDomain
+	calls    []string
+	fail     map[string]error
+	failOnce map[string]error
+	// uploadHook, when set, sees every StorageVolUpload before the fake
+	// reads the stream; it can return an error to simulate a broken upload.
+	uploadHook func(vol string, r io.Reader) error
+	// hostname, libVersion, poolCapacity and poolStates answer the health
+	// check's RPCs. A pool missing from poolStates is running.
+	hostname     string
+	libVersion   uint64
+	poolCapacity uint64
+	poolStates   map[string]libvirt.StoragePoolState
+}
+
+type fakeVol struct {
+	format   string
+	capacity int64
+	backing  string
+	data     []byte
+}
+
+type fakeDomain struct {
+	name     string
+	xml      string
+	metadata string
+	running  bool
+	// addrs is what DomainInterfaceAddresses returns per source.
+	addrs map[uint32][]libvirt.DomainInterface
+}
+
+func newFakeAPI(pools ...string) *fakeAPI {
+	f := &fakeAPI{
+		poolPath: "/var/lib/libvirt/images",
+		pools:    map[string]map[string]*fakeVol{},
+		domains:  map[string]*fakeDomain{},
+		fail:     map[string]error{},
+		failOnce: map[string]error{},
+
+		hostname:     "fake-host",
+		libVersion:   10002000,
+		poolCapacity: 100 << 30,
+		poolStates:   map[string]libvirt.StoragePoolState{},
+	}
+	for _, p := range pools {
+		f.pools[p] = map[string]*fakeVol{}
+	}
+	return f
+}
+
+// enter records the call and returns any injected error for method.
+func (f *fakeAPI) enter(method string) error {
+	f.calls = append(f.calls, method)
+	if err, ok := f.failOnce[method]; ok {
+		delete(f.failOnce, method)
+		return err
+	}
+	return f.fail[method]
+}
+
+func (f *fakeAPI) callCount(method string) int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	n := 0
+	for _, c := range f.calls {
+		if c == method {
+			n++
+		}
+	}
+	return n
+}
+
+func (f *fakeAPI) hasCall(method string) bool { return f.callCount(method) > 0 }
+
+func (f *fakeAPI) vol(pool, name string) *fakeVol {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.pools[pool][name]
+}
+
+func (f *fakeAPI) putVol(pool, name, format string, data []byte) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.pools[pool][name] = &fakeVol{format: format, capacity: int64(len(data)), data: data}
+}
+
+func (f *fakeAPI) volNames(pool string) []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []string
+	for n := range f.pools[pool] {
+		out = append(out, n)
+	}
+	slices.Sort(out)
+	return out
+}
+
+func (f *fakeAPI) domain(name string) *fakeDomain {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.domains[name]
+}
+
+func (f *fakeAPI) putDomain(d *fakeDomain) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.domains[d.name] = d
+}
+
+func lvError(code libvirt.ErrorNumber, format string, args ...any) error {
+	return libvirt.Error{Code: uint32(code), Message: fmt.Sprintf(format, args...)}
+}
+
+// --- domains ---
+
+func (f *fakeAPI) DomainLookupByName(name string) (libvirt.Domain, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainLookupByName"); err != nil {
+		return libvirt.Domain{}, err
+	}
+	if _, ok := f.domains[name]; !ok {
+		return libvirt.Domain{}, lvError(libvirt.ErrNoDomain, "Domain not found: no domain with matching name '%s'", name)
+	}
+	return libvirt.Domain{Name: name}, nil
+}
+
+func (f *fakeAPI) DomainDefineXML(raw string) (libvirt.Domain, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainDefineXML"); err != nil {
+		return libvirt.Domain{}, err
+	}
+	var d domainXML
+	if err := xml.Unmarshal([]byte(raw), &d); err != nil {
+		return libvirt.Domain{}, lvError(libvirt.ErrXMLError, "%v", err)
+	}
+	meta, err := xml.Marshal(d.Metadata.Instance)
+	if err != nil {
+		return libvirt.Domain{}, lvError(libvirt.ErrXMLError, "%v", err)
+	}
+	// libvirt hands the element back without its namespace declaration
+	// (seen on 11.6); mimic that so Get/ListByTag parse the real shape.
+	stored := strings.Replace(string(meta), ` xmlns="`+metadataURI+`"`, "", 1)
+	existing := f.domains[d.Name]
+	nd := &fakeDomain{name: d.Name, xml: raw, metadata: stored}
+	if existing != nil {
+		nd.running = existing.running
+		nd.addrs = existing.addrs
+	}
+	f.domains[d.Name] = nd
+	return libvirt.Domain{Name: d.Name}, nil
+}
+
+func (f *fakeAPI) DomainCreate(d libvirt.Domain) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainCreate"); err != nil {
+		return err
+	}
+	dom, ok := f.domains[d.Name]
+	if !ok {
+		return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	if dom.running {
+		return lvError(libvirt.ErrOperationInvalid, "domain is already running")
+	}
+	dom.running = true
+	return nil
+}
+
+func (f *fakeAPI) DomainDestroy(d libvirt.Domain) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainDestroy"); err != nil {
+		return err
+	}
+	dom, ok := f.domains[d.Name]
+	if !ok {
+		return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	if !dom.running {
+		return lvError(libvirt.ErrOperationInvalid, "domain is not running")
+	}
+	dom.running = false
+	return nil
+}
+
+func (f *fakeAPI) DomainUndefineFlags(d libvirt.Domain, _ libvirt.DomainUndefineFlagsValues) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainUndefineFlags"); err != nil {
+		return err
+	}
+	if _, ok := f.domains[d.Name]; !ok {
+		return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	delete(f.domains, d.Name)
+	return nil
+}
+
+func (f *fakeAPI) DomainGetState(d libvirt.Domain, _ uint32) (int32, int32, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainGetState"); err != nil {
+		return 0, 0, err
+	}
+	dom, ok := f.domains[d.Name]
+	if !ok {
+		return 0, 0, lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	if dom.running {
+		return int32(libvirt.DomainRunning), 0, nil
+	}
+	return int32(libvirt.DomainShutoff), 0, nil
+}
+
+func (f *fakeAPI) DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, _ libvirt.DomainModificationImpact) (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainGetMetadata"); err != nil {
+		return "", err
+	}
+	dom, ok := f.domains[d.Name]
+	if !ok {
+		return "", lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	if typ != int32(libvirt.DomainMetadataElement) || len(uri) != 1 || uri[0] != metadataURI || dom.metadata == "" {
+		return "", lvError(libvirt.ErrNoDomainMetadata, "Requested metadata element is not present")
+	}
+	return dom.metadata, nil
+}
+
+func (f *fakeAPI) DomainInterfaceAddresses(d libvirt.Domain, source uint32, _ uint32) ([]libvirt.DomainInterface, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("DomainInterfaceAddresses"); err != nil {
+		return nil, err
+	}
+	dom, ok := f.domains[d.Name]
+	if !ok {
+		return nil, lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
+	}
+	if !dom.running {
+		return nil, lvError(libvirt.ErrOperationInvalid, "domain is not running")
+	}
+	if source == uint32(libvirt.DomainInterfaceAddressesSrcAgent) && dom.addrs[source] == nil {
+		return nil, lvError(libvirt.ErrAgentUnresponsive, "Guest agent is not responding")
+	}
+	return dom.addrs[source], nil
+}
+
+func (f *fakeAPI) ConnectListAllDomains(_ int32, _ libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("ConnectListAllDomains"); err != nil {
+		return nil, 0, err
+	}
+	names := make([]string, 0, len(f.domains))
+	for n := range f.domains {
+		names = append(names, n)
+	}
+	slices.Sort(names)
+	out := make([]libvirt.Domain, 0, len(names))
+	for _, n := range names {
+		out = append(out, libvirt.Domain{Name: n})
+	}
+	return out, uint32(len(out)), nil
+}
+
+// --- storage ---
+
+func (f *fakeAPI) StoragePoolLookupByName(name string) (libvirt.StoragePool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StoragePoolLookupByName"); err != nil {
+		return libvirt.StoragePool{}, err
+	}
+	if _, ok := f.pools[name]; !ok {
+		return libvirt.StoragePool{}, lvError(libvirt.ErrNoStoragePool, "Storage pool not found: no storage pool with matching name '%s'", name)
+	}
+	return libvirt.StoragePool{Name: name}, nil
+}
+
+func (f *fakeAPI) StoragePoolRefresh(pool libvirt.StoragePool, _ uint32) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StoragePoolRefresh"); err != nil {
+		return err
+	}
+	if _, ok := f.pools[pool.Name]; !ok {
+		return lvError(libvirt.ErrNoStoragePool, "no pool %s", pool.Name)
+	}
+	return nil
+}
+
+func (f *fakeAPI) StorageVolLookupByName(pool libvirt.StoragePool, name string) (libvirt.StorageVol, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StorageVolLookupByName"); err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	if _, ok := f.pools[pool.Name][name]; !ok {
+		return libvirt.StorageVol{}, lvError(libvirt.ErrNoStorageVol, "Storage volume not found: no storage vol with matching name '%s'", name)
+	}
+	return libvirt.StorageVol{Pool: pool.Name, Name: name, Key: f.poolPath + "/" + name}, nil
+}
+
+func (f *fakeAPI) StorageVolGetPath(vol libvirt.StorageVol) (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StorageVolGetPath"); err != nil {
+		return "", err
+	}
+	if _, ok := f.pools[vol.Pool][vol.Name]; !ok {
+		return "", lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
+	}
+	return f.poolPath + "/" + vol.Name, nil
+}
+
+func (f *fakeAPI) StorageVolCreateXML(pool libvirt.StoragePool, raw string, _ libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StorageVolCreateXML"); err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	vols, ok := f.pools[pool.Name]
+	if !ok {
+		return libvirt.StorageVol{}, lvError(libvirt.ErrNoStoragePool, "no pool %s", pool.Name)
+	}
+	var v volumeXML
+	if err := xml.Unmarshal([]byte(raw), &v); err != nil {
+		return libvirt.StorageVol{}, lvError(libvirt.ErrXMLError, "%v", err)
+	}
+	if _, exists := vols[v.Name]; exists {
+		return libvirt.StorageVol{}, lvError(libvirt.ErrStorageVolExist, "storage volume '%s' exists already", v.Name)
+	}
+	fv := &fakeVol{format: v.Target.Format.Type, capacity: v.Capacity.Value}
+	if v.BackingStore != nil {
+		fv.backing = v.BackingStore.Path
+	}
+	vols[v.Name] = fv
+	return libvirt.StorageVol{Pool: pool.Name, Name: v.Name, Key: f.poolPath + "/" + v.Name}, nil
+}
+
+func (f *fakeAPI) StorageVolUpload(vol libvirt.StorageVol, r io.Reader, _, _ uint64, _ libvirt.StorageVolUploadFlags) error {
+	f.mu.Lock()
+	if err := f.enter("StorageVolUpload"); err != nil {
+		f.mu.Unlock()
+		return err
+	}
+	fv, ok := f.pools[vol.Pool][vol.Name]
+	hook := f.uploadHook
+	f.mu.Unlock()
+	if !ok {
+		return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
+	}
+	if hook != nil {
+		if err := hook(vol.Name, r); err != nil {
+			return err
+		}
+	}
+	// Read outside the lock: the stream may be a slow HTTP body.
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return fmt.Errorf("reading upload stream: %w", err)
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	fv.data = data
+	return nil
+}
+
+func (f *fakeAPI) StorageVolDownload(vol libvirt.StorageVol, w io.Writer, _, _ uint64, _ libvirt.StorageVolDownloadFlags) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StorageVolDownload"); err != nil {
+		return err
+	}
+	fv, ok := f.pools[vol.Pool][vol.Name]
+	if !ok {
+		return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
+	}
+	_, err := io.Copy(w, bytes.NewReader(fv.data))
+	return err
+}
+
+func (f *fakeAPI) StorageVolDelete(vol libvirt.StorageVol, _ libvirt.StorageVolDeleteFlags) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StorageVolDelete"); err != nil {
+		return err
+	}
+	if _, ok := f.pools[vol.Pool][vol.Name]; !ok {
+		return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
+	}
+	delete(f.pools[vol.Pool], vol.Name)
+	return nil
+}
+
+// --- host ---
+
+func (f *fakeAPI) ConnectGetLibVersion() (uint64, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("ConnectGetLibVersion"); err != nil {
+		return 0, err
+	}
+	return f.libVersion, nil
+}
+
+func (f *fakeAPI) ConnectGetHostname() (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("ConnectGetHostname"); err != nil {
+		return "", err
+	}
+	return f.hostname, nil
+}
+
+// StoragePoolGetInfo reports the pool's state and poolCapacity minus the
+// volumes' capacities as available space.
+func (f *fakeAPI) StoragePoolGetInfo(pool libvirt.StoragePool) (uint8, uint64, uint64, uint64, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.enter("StoragePoolGetInfo"); err != nil {
+		return 0, 0, 0, 0, err
+	}
+	vols, ok := f.pools[pool.Name]
+	if !ok {
+		return 0, 0, 0, 0, lvError(libvirt.ErrNoStoragePool, "no pool %s", pool.Name)
+	}
+	var used uint64
+	for _, v := range vols {
+		used += uint64(v.capacity)
+	}
+	state := libvirt.StoragePoolRunning
+	if s, ok := f.poolStates[pool.Name]; ok {
+		state = s
+	}
+	return uint8(state), f.poolCapacity, used, f.poolCapacity - used, nil
+}

internal/provider/kvm/id.go

@@ -0,0 +1,21 @@
+package kvm
+
+import (
+	"fmt"
+	"strings"
+)
+
+// The providerID carries the host, so Get/Delete stay correct after a
+// placement.zone edit mid-replacement — re-reading the spec then would look
+// at the wrong hypervisor.
+func formatProviderID(host, name string) string {
+	return fmt.Sprintf("hosts/%s/domains/%s", host, name)
+}
+
+func parseProviderID(id string) (host, name string, err error) {
+	parts := strings.Split(id, "/")
+	if len(parts) != 4 || parts[0] != "hosts" || parts[2] != "domains" || parts[1] == "" || parts[3] == "" {
+		return "", "", fmt.Errorf("malformed kvm providerID %q, want hosts/<host>/domains/<name>", id)
+	}
+	return parts[1], parts[3], nil
+}

internal/provider/kvm/id_test.go

@@ -0,0 +1,41 @@
+package kvm
+
+import "testing"
+
+func TestParseProviderID_roundTrip(t *testing.T) {
+	t.Parallel()
+	id := formatProviderID("nas1", "proxy-abc123def456ghij")
+	if id != "hosts/nas1/domains/proxy-abc123def456ghij" {
+		t.Fatalf("formatProviderID() = %q", id)
+	}
+	host, name, err := parseProviderID(id)
+	if err != nil {
+		t.Fatalf("parseProviderID(%q) error = %v", id, err)
+	}
+	if host != "nas1" || name != "proxy-abc123def456ghij" {
+		t.Errorf("parseProviderID(%q) = %q, %q", id, host, name)
+	}
+}
+
+func TestParseProviderID_malformed(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		id   string
+	}{
+		{"empty", ""},
+		{"gcp style", "zones/europe-west1-b/instances/proxy-x"},
+		{"missing name", "hosts/nas1/domains/"},
+		{"missing host", "hosts//domains/proxy-x"},
+		{"too many segments", "hosts/nas1/domains/proxy-x/extra"},
+		{"bare name", "proxy-x"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if _, _, err := parseProviderID(tc.id); err == nil {
+				t.Errorf("parseProviderID(%q) error = nil, want error", tc.id)
+			}
+		})
+	}
+}

internal/provider/kvm/image.go

@@ -0,0 +1,228 @@
+package kvm
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// imageSource is a resolved placement.image: either an entry from the
+// provider's images map (Key set) or a URL given directly.
+type imageSource struct {
+	Key string
+	provider.KVMImage
+}
+
+// resolveImage maps placement.image to a source. Anything that is neither a
+// configured key nor an http(s) URL is a spec error the reconciler should
+// surface, not retry.
+func resolveImage(cfg provider.KVMConfig, image string) (imageSource, error) {
+	if img, ok := cfg.Images[image]; ok {
+		return imageSource{Key: image, KVMImage: img}, nil
+	}
+	if provider.IsHTTPURL(image) {
+		return imageSource{KVMImage: imageDefaults(provider.KVMImage{URL: image})}, nil
+	}
+	return imageSource{}, fmt.Errorf("%w: placement.image %q is neither a key in kvm.images nor an http(s) URL",
+		provider.ErrPermanent, image)
+}
+
+// volumeName is the cached base image's name in the pool: readable for
+// configured keys, a URL hash for ad-hoc URLs.
+func (s imageSource) volumeName() string {
+	if s.Key != "" {
+		return "base-" + s.Key + "." + s.Format
+	}
+	sum := sha256.Sum256([]byte(s.URL))
+	return "base-" + hex.EncodeToString(sum[:8]) + "." + s.Format
+}
+
+// markerName is the sidecar volume holding the sha256 of the verified
+// upload. Its presence is what says "the base volume is complete".
+func (s imageSource) markerName() string {
+	return s.volumeName() + ".sha256"
+}
+
+// imageCache fetches base images into a host's pool on first use. It keeps
+// no state about what is cached — the pool is the source of truth, checked
+// on every call — only a per-(host, image) lock so concurrent Creates for
+// the same image on the same host result in one download.
+type imageCache struct {
+	http *http.Client
+
+	mu    sync.Mutex
+	locks map[string]*sync.Mutex
+}
+
+func newImageCache(client *http.Client) *imageCache {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	return &imageCache{http: client, locks: make(map[string]*sync.Mutex)}
+}
+
+func (c *imageCache) lockFor(key string) *sync.Mutex {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	m, ok := c.locks[key]
+	if !ok {
+		m = &sync.Mutex{}
+		c.locks[key] = m
+	}
+	return m
+}
+
+// ensure makes src present in pool and returns the base volume's path on
+// the host. The image is downloaded synchronously through the libvirt
+// stream when the pool does not already hold a verified copy.
+func (c *imageCache) ensure(ctx context.Context, log logr.Logger, api libvirtAPI, hostName string, pool libvirt.StoragePool, src imageSource) (string, error) {
+	m := c.lockFor(hostName + "/" + src.volumeName())
+	m.Lock()
+	defer m.Unlock()
+
+	base, ready, err := c.lookup(api, pool, src)
+	if err != nil {
+		return "", err
+	}
+	if ready {
+		log.V(1).Info("base image already cached", "volume", src.volumeName())
+		return api.StorageVolGetPath(base)
+	}
+	if err := deleteIfExists(api, pool, src.markerName()); err != nil {
+		return "", err
+	}
+	if err := deleteIfExists(api, pool, src.volumeName()); err != nil {
+		return "", err
+	}
+	base, err = c.fetch(ctx, log, api, hostName, pool, src)
+	if err != nil {
+		return "", err
+	}
+	return api.StorageVolGetPath(base)
+}
+
+// lookup reports whether the pool holds a complete, verified base volume:
+// the volume and its marker both exist and, when a checksum is configured,
+// the marker records that checksum.
+func (c *imageCache) lookup(api libvirtAPI, pool libvirt.StoragePool, src imageSource) (libvirt.StorageVol, bool, error) {
+	base, err := api.StorageVolLookupByName(pool, src.volumeName())
+	if hasCode(err, libvirt.ErrNoStorageVol) {
+		return libvirt.StorageVol{}, false, nil
+	}
+	if err != nil {
+		return libvirt.StorageVol{}, false, fmt.Errorf("looking up base volume: %w", err)
+	}
+	marker, err := api.StorageVolLookupByName(pool, src.markerName())
+	if hasCode(err, libvirt.ErrNoStorageVol) {
+		return base, false, nil
+	}
+	if err != nil {
+		return libvirt.StorageVol{}, false, fmt.Errorf("looking up image marker: %w", err)
+	}
+	if src.SHA256 == "" {
+		return base, true, nil
+	}
+	var buf bytes.Buffer
+	if err := api.StorageVolDownload(marker, &buf, 0, 0, 0); err != nil {
+		return libvirt.StorageVol{}, false, fmt.Errorf("reading image marker: %w", err)
+	}
+	return base, strings.EqualFold(strings.TrimSpace(buf.String()), src.SHA256), nil
+}
+
+// fetch streams the image from its URL straight into a new base volume,
+// hashing on the way, then writes the marker. Nothing is buffered in the
+// operator beyond the HTTP body's read buffer.
+func (c *imageCache) fetch(ctx context.Context, log logr.Logger, api libvirtAPI, hostName string, pool libvirt.StoragePool, src imageSource) (libvirt.StorageVol, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, src.URL, nil)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("%w: building image request: %v", provider.ErrPermanent, err)
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("%w: fetching image %s: %v", provider.ErrTransient, src.URL, err)
+	}
+	defer resp.Body.Close()
+	switch {
+	case resp.StatusCode >= 200 && resp.StatusCode < 300:
+	case resp.StatusCode >= 400 && resp.StatusCode < 500:
+		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s returned HTTP %d", provider.ErrPermanent, src.URL, resp.StatusCode)
+	default:
+		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s returned HTTP %d", provider.ErrTransient, src.URL, resp.StatusCode)
+	}
+	if resp.ContentLength < 0 {
+		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s has no Content-Length; the base volume needs a known size", provider.ErrPermanent, src.URL)
+	}
+
+	log.Info("fetching base image into host pool",
+		"pool", pool.Name, "volume", src.volumeName(),
+		"url", src.URL, "bytes", resp.ContentLength)
+	start := time.Now()
+
+	baseXML, err := buildVolumeXML(volumeSpec{Name: src.volumeName(), SizeBytes: resp.ContentLength, Format: src.Format})
+	if err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	base, err := api.StorageVolCreateXML(pool, baseXML, 0)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("creating base volume: %w", err)
+	}
+	hasher := sha256.New()
+	if err := api.StorageVolUpload(base, io.TeeReader(resp.Body, hasher), 0, 0, 0); err != nil {
+		_ = api.StorageVolDelete(base, 0)
+		return libvirt.StorageVol{}, fmt.Errorf("uploading image to base volume: %w", err)
+	}
+	digest := hex.EncodeToString(hasher.Sum(nil))
+	if src.SHA256 != "" && !strings.EqualFold(digest, src.SHA256) {
+		_ = api.StorageVolDelete(base, 0)
+		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s sha256 is %s, config expects %s",
+			provider.ErrPermanent, src.URL, digest, src.SHA256)
+	}
+	// libvirt sized the volume from Content-Length; a refresh makes it
+	// re-read the real virtual size from the uploaded qcow2 header.
+	if err := api.StoragePoolRefresh(pool, 0); err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("refreshing pool after upload: %w", err)
+	}
+
+	markerXML, err := buildVolumeXML(rawVolume(src.markerName(), int64(len(digest))))
+	if err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	marker, err := api.StorageVolCreateXML(pool, markerXML, 0)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("creating image marker: %w", err)
+	}
+	if err := api.StorageVolUpload(marker, strings.NewReader(digest), 0, 0, 0); err != nil {
+		_ = api.StorageVolDelete(marker, 0)
+		return libvirt.StorageVol{}, fmt.Errorf("writing image marker: %w", err)
+	}
+	log.Info("base image cached",
+		"volume", src.volumeName(), "sha256", digest,
+		"bytes", resp.ContentLength, "duration", time.Since(start).Round(time.Second))
+	return base, nil
+}
+
+func deleteIfExists(api libvirtAPI, pool libvirt.StoragePool, name string) error {
+	vol, err := api.StorageVolLookupByName(pool, name)
+	if hasCode(err, libvirt.ErrNoStorageVol) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("looking up volume %s: %w", name, err)
+	}
+	if err := api.StorageVolDelete(vol, 0); err != nil && !hasCode(err, libvirt.ErrNoStorageVol) {
+		return fmt.Errorf("deleting volume %s: %w", name, err)
+	}
+	return nil
+}

internal/provider/kvm/image_test.go

@@ -0,0 +1,371 @@
+package kvm
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/go-logr/logr"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+var testImage = bytes.Repeat([]byte("QFI\xfb fake qcow2 payload "), 1024)
+
+func testImageSHA() string {
+	sum := sha256.Sum256(testImage)
+	return hex.EncodeToString(sum[:])
+}
+
+// imageServer serves testImage and counts hits. Status overrides the
+// response, noLength forces chunked encoding (no Content-Length), delay
+// slows each response so concurrent callers overlap.
+type imageServer struct {
+	*httptest.Server
+	hits     atomic.Int32
+	status   int
+	noLength bool
+	delay    time.Duration
+}
+
+func newImageServer(t *testing.T) *imageServer {
+	t.Helper()
+	s := &imageServer{status: http.StatusOK}
+	s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		s.hits.Add(1)
+		time.Sleep(s.delay)
+		if s.status != http.StatusOK {
+			w.WriteHeader(s.status)
+			return
+		}
+		// Bodies larger than the server's write buffer would otherwise go
+		// out chunked without a Content-Length, unlike real image mirrors.
+		if s.noLength {
+			w.(http.Flusher).Flush()
+		} else {
+			w.Header().Set("Content-Length", strconv.Itoa(len(testImage)))
+		}
+		_, _ = w.Write(testImage)
+	}))
+	t.Cleanup(s.Close)
+	return s
+}
+
+func testSource(srv *imageServer, sha string) imageSource {
+	return imageSource{Key: "debian-13", KVMImage: provider.KVMImage{URL: srv.URL + "/debian.qcow2", SHA256: sha, Format: formatQcow2}}
+}
+
+func ensureTest(t *testing.T, api *fakeAPI, src imageSource) (string, error) {
+	t.Helper()
+	c := newImageCache(nil)
+	pool, err := api.StoragePoolLookupByName("default")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, src)
+}
+
+func TestResolveImage(t *testing.T) {
+	t.Parallel()
+	cfg := withDefaults(provider.KVMConfig{Images: map[string]provider.KVMImage{
+		"debian-13": {URL: "https://cloud.debian.org/d.qcow2", SHA256: "abc"},
+	}})
+	tests := []struct {
+		name    string
+		image   string
+		want    imageSource
+		wantErr bool
+	}{
+		{"configured key", "debian-13", imageSource{Key: "debian-13", KVMImage: provider.KVMImage{URL: "https://cloud.debian.org/d.qcow2", SHA256: "abc", Format: "qcow2", Firmware: "uefi"}}, false},
+		{"direct https url", "https://x.example/img.qcow2", imageSource{KVMImage: provider.KVMImage{URL: "https://x.example/img.qcow2", Format: "qcow2", Firmware: "uefi"}}, false},
+		{"direct http url", "http://mirror/img.qcow2", imageSource{KVMImage: provider.KVMImage{URL: "http://mirror/img.qcow2", Format: "qcow2", Firmware: "uefi"}}, false},
+		{"unknown key", "ubuntu", imageSource{}, true},
+		{"file url", "file:///tmp/x.qcow2", imageSource{}, true},
+		{"empty", "", imageSource{}, true},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := resolveImage(cfg, tc.image)
+			if tc.wantErr {
+				wantClass(t, err, provider.ErrPermanent)
+				return
+			}
+			if err != nil {
+				t.Fatalf("resolveImage(%q) error = %v", tc.image, err)
+			}
+			if got != tc.want {
+				t.Errorf("resolveImage(%q) = %+v, want %+v", tc.image, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestImageSource_volumeNames(t *testing.T) {
+	t.Parallel()
+	keyed := imageSource{Key: "debian-13", KVMImage: provider.KVMImage{URL: "https://a/b.qcow2", Format: "qcow2"}}
+	if keyed.volumeName() != "base-debian-13.qcow2" || keyed.markerName() != "base-debian-13.qcow2.sha256" {
+		t.Errorf("keyed names = %s / %s", keyed.volumeName(), keyed.markerName())
+	}
+	direct := imageSource{KVMImage: provider.KVMImage{URL: "https://a/b.qcow2", Format: "qcow2"}}
+	name := direct.volumeName()
+	if !strings.HasPrefix(name, "base-") || !strings.HasSuffix(name, ".qcow2") || len(name) != len("base-")+16+len(".qcow2") {
+		t.Errorf("direct-url name = %q, want base-<16 hex>.qcow2", name)
+	}
+	other := imageSource{KVMImage: provider.KVMImage{URL: "https://a/c.qcow2", Format: "qcow2"}}
+	if other.volumeName() == name {
+		t.Error("different URLs produced the same volume name")
+	}
+}
+
+func TestImageCache_ensure_freshFetch(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	path, err := ensureTest(t, api, testSource(srv, testImageSHA()))
+	if err != nil {
+		t.Fatalf("ensure() error = %v", err)
+	}
+	if path != "/var/lib/libvirt/images/base-debian-13.qcow2" {
+		t.Errorf("path = %q", path)
+	}
+	base := api.vol("default", "base-debian-13.qcow2")
+	if base == nil || !bytes.Equal(base.data, testImage) {
+		t.Fatalf("base volume missing or content differs")
+	}
+	if base.format != "qcow2" || base.capacity != int64(len(testImage)) {
+		t.Errorf("base volume format/capacity = %s/%d, want qcow2/%d", base.format, base.capacity, len(testImage))
+	}
+	marker := api.vol("default", "base-debian-13.qcow2.sha256")
+	if marker == nil || string(marker.data) != testImageSHA() {
+		t.Fatalf("marker missing or wrong: %+v", marker)
+	}
+	if !api.hasCall("StoragePoolRefresh") {
+		t.Error("pool was not refreshed after the upload")
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("http hits = %d, want 1", srv.hits.Load())
+	}
+}
+
+func TestImageCache_ensure_noChecksumConfigured(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	if _, err := ensureTest(t, api, testSource(srv, "")); err != nil {
+		t.Fatalf("ensure() error = %v", err)
+	}
+	if marker := api.vol("default", "base-debian-13.qcow2.sha256"); marker == nil || string(marker.data) != testImageSHA() {
+		t.Errorf("marker should still record the computed digest, got %+v", marker)
+	}
+	// Second call: ready without a checksum to compare, so no marker read.
+	if _, err := ensureTest(t, api, testSource(srv, "")); err != nil {
+		t.Fatalf("second ensure() error = %v", err)
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("http hits = %d, want 1", srv.hits.Load())
+	}
+}
+
+func TestImageCache_ensure_existingVolumes(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name     string
+		base     []byte
+		marker   []byte
+		wantHits int32
+	}{
+		{"already cached", testImage, []byte(testImageSHA()), 0},
+		{"marker missing is a partial upload", []byte("half an image"), nil, 1},
+		{"configured checksum changed", []byte("an older release"), []byte(strings.Repeat("0", 64)), 1},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			srv := newImageServer(t)
+			api := newFakeAPI("default")
+			api.putVol("default", "base-debian-13.qcow2", "qcow2", tc.base)
+			if tc.marker != nil {
+				api.putVol("default", "base-debian-13.qcow2.sha256", "raw", tc.marker)
+			}
+			path, err := ensureTest(t, api, testSource(srv, testImageSHA()))
+			if err != nil {
+				t.Fatalf("ensure() error = %v", err)
+			}
+			if path != "/var/lib/libvirt/images/base-debian-13.qcow2" {
+				t.Errorf("path = %q", path)
+			}
+			if srv.hits.Load() != tc.wantHits {
+				t.Errorf("http hits = %d, want %d", srv.hits.Load(), tc.wantHits)
+			}
+			if touched := api.hasCall("StorageVolCreateXML") || api.hasCall("StorageVolDelete"); touched != (tc.wantHits > 0) {
+				t.Errorf("volumes touched = %v, want %v: calls = %v", touched, tc.wantHits > 0, api.calls)
+			}
+			if base := api.vol("default", "base-debian-13.qcow2"); base == nil || !bytes.Equal(base.data, testImage) {
+				t.Error("base volume does not hold the verified image")
+			}
+			if marker := api.vol("default", "base-debian-13.qcow2.sha256"); marker == nil || string(marker.data) != testImageSHA() {
+				t.Errorf("marker = %+v, want the verified digest", marker)
+			}
+		})
+	}
+}
+
+func TestImageCache_ensure_checksumMismatchIsPermanent(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	_, err := ensureTest(t, api, testSource(srv, strings.Repeat("a", 64)))
+	wantClass(t, err, provider.ErrPermanent)
+	if names := api.volNames("default"); len(names) != 0 {
+		t.Errorf("volumes left behind after a checksum mismatch: %v", names)
+	}
+}
+
+func TestImageCache_ensure_httpErrors(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name   string
+		status int
+		want   error
+	}{
+		{"404 is permanent", http.StatusNotFound, provider.ErrPermanent},
+		{"403 is permanent", http.StatusForbidden, provider.ErrPermanent},
+		{"503 is transient", http.StatusServiceUnavailable, provider.ErrTransient},
+		{"500 is transient", http.StatusInternalServerError, provider.ErrTransient},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			srv := newImageServer(t)
+			srv.status = tc.status
+			api := newFakeAPI("default")
+			_, err := ensureTest(t, api, testSource(srv, ""))
+			if !errors.Is(err, tc.want) {
+				t.Fatalf("ensure() error = %v, want %v", err, tc.want)
+			}
+			if names := api.volNames("default"); len(names) != 0 {
+				t.Errorf("volumes created despite HTTP %d: %v", tc.status, names)
+			}
+		})
+	}
+}
+
+func TestImageCache_ensure_noContentLengthIsPermanent(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	srv.noLength = true
+	api := newFakeAPI("default")
+	_, err := ensureTest(t, api, testSource(srv, ""))
+	wantClass(t, err, provider.ErrPermanent)
+	if !strings.Contains(err.Error(), "Content-Length") {
+		t.Errorf("ensure() error = %v, want one about Content-Length", err)
+	}
+}
+
+func TestImageCache_ensure_uploadFailureCleansUp(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	api.uploadHook = func(_ string, r io.Reader) error {
+		_, _ = io.CopyN(io.Discard, r, 100)
+		return io.ErrUnexpectedEOF
+	}
+	_, err := ensureTest(t, api, testSource(srv, testImageSHA()))
+	if !errors.Is(err, provider.ErrTransient) && classify(err) != provider.ErrTransient {
+		t.Fatalf("ensure() error = %v, want a transient failure", err)
+	}
+	if names := api.volNames("default"); len(names) != 0 {
+		t.Errorf("volumes left behind after a failed upload: %v", names)
+	}
+	// Next attempt starts clean and succeeds.
+	api.uploadHook = nil
+	if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
+		t.Fatalf("retry ensure() error = %v", err)
+	}
+}
+
+func TestImageCache_ensure_missingPoolIsPermanent(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	c := newImageCache(nil)
+	pool, _ := api.StoragePoolLookupByName("default")
+	delete(api.pools, "default")
+	_, err := c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, testSource(srv, ""))
+	if classify(err) != provider.ErrPermanent {
+		t.Fatalf("ensure() error = %v, want one classifying as Permanent", err)
+	}
+}
+
+func TestImageCache_ensure_concurrentCallersDownloadOnce(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	srv.delay = 50 * time.Millisecond
+	api := newFakeAPI("default")
+	c := newImageCache(nil)
+	pool, _ := api.StoragePoolLookupByName("default")
+	src := testSource(srv, testImageSHA())
+	var wg sync.WaitGroup
+	paths := make([]string, 4)
+	errs := make([]error, 4)
+	for i := range 4 {
+		wg.Go(func() {
+			paths[i], errs[i] = c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, src)
+		})
+	}
+	wg.Wait()
+	for i := range 4 {
+		if errs[i] != nil || paths[i] != "/var/lib/libvirt/images/base-debian-13.qcow2" {
+			t.Errorf("caller %d: path=%q err=%v", i, paths[i], errs[i])
+		}
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("http hits = %d, want 1 for four concurrent callers", srv.hits.Load())
+	}
+}
+
+func TestImageCache_ensure_differentHostsFetchIndependently(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	c := newImageCache(nil)
+	src := testSource(srv, "")
+	for _, host := range []string{"nas1", "lab2"} {
+		api := newFakeAPI("default")
+		pool, _ := api.StoragePoolLookupByName("default")
+		if _, err := c.ensure(context.Background(), logr.Discard(), api, host, pool, src); err != nil {
+			t.Fatalf("ensure(%s) error = %v", host, err)
+		}
+	}
+	if srv.hits.Load() != 2 {
+		t.Errorf("http hits = %d, want one per host", srv.hits.Load())
+	}
+}
+
+func TestImageCache_ensure_cancelledContextIsTransient(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	c := newImageCache(nil)
+	pool, _ := api.StoragePoolLookupByName("default")
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	_, err := c.ensure(ctx, logr.Discard(), api, "nas1", pool, testSource(srv, ""))
+	if classify(err) != provider.ErrTransient {
+		t.Fatalf("ensure() error = %v, want transient", err)
+	}
+	if names := api.volNames("default"); len(names) != 0 {
+		t.Errorf("volumes created despite cancelled context: %v", names)
+	}
+}

internal/provider/kvm/kvm.go

@@ -0,0 +1,596 @@
+// Package kvm implements the provider contract on libvirt/KVM hypervisors.
+// Each configured host is reached over SSH with the libvirt unix socket
+// forwarded through the connection, so the hypervisors need no libvirtd TCP
+// or TLS listener and the operator image needs no libvirt libraries. VMs
+// boot a stock cloud image (fetched into the host's storage pool on first
+// use) through a qcow2 overlay, with the Proxy's cloud-init delivered on a
+// NoCloud seed ISO built in-process.
+package kvm
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+const (
+	defaultDiskSizeGB = 10
+	defaultSocket     = "/var/run/libvirt/libvirt-sock"
+	defaultPool       = "default"
+	formatQcow2       = provider.KVMFormatQcow2
+	formatRaw         = provider.KVMFormatRaw
+	firmwareUEFI      = provider.KVMFirmwareUEFI
+	firmwareBIOS      = provider.KVMFirmwareBIOS
+
+	// undefineFlags removes everything libvirt may keep alongside a domain
+	// definition, so an undefine never fails on leftover state.
+	undefineFlags = libvirt.DomainUndefineManagedSave |
+		libvirt.DomainUndefineSnapshotsMetadata |
+		libvirt.DomainUndefineNvram
+)
+
+func withDefaults(cfg provider.KVMConfig) provider.KVMConfig {
+	if cfg.DiskSizeGB == 0 {
+		cfg.DiskSizeGB = defaultDiskSizeGB
+	}
+	images := make(map[string]provider.KVMImage, len(cfg.Images))
+	for k, img := range cfg.Images {
+		images[k] = imageDefaults(img)
+	}
+	cfg.Images = images
+	hosts := make([]provider.KVMHost, len(cfg.Hosts))
+	for i, h := range cfg.Hosts {
+		if h.Socket == "" {
+			h.Socket = defaultSocket
+		}
+		if h.Pool == "" {
+			h.Pool = defaultPool
+		}
+		hosts[i] = h
+	}
+	cfg.Hosts = hosts
+	return cfg
+}
+
+// Provider implements provider.Provider on one or more libvirt hosts.
+type Provider struct {
+	name   string
+	cfg    provider.KVMConfig
+	hosts  *hostPool
+	images *imageCache
+	now    func() time.Time
+}
+
+func imageDefaults(img provider.KVMImage) provider.KVMImage {
+	if img.Format == "" {
+		img.Format = formatQcow2
+	}
+	if img.Firmware == "" {
+		img.Firmware = firmwareUEFI
+	}
+	return img
+}
+
+// New builds a Provider that reaches its hosts over SSH. The key and
+// known_hosts files are tried once here and kept when usable, but a failure
+// is only a warning, retried on the first connection: they come from an
+// optional Secret mount, and a constructor that failed without it would
+// abort the whole provider set and crash-loop the manager, taking the
+// kubernetes and gcp providers down with it. Satisfies registry.Constructor.
+func New(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+	cfg := provider.KVMConfig{}
+	if pc.KVM != nil {
+		cfg = *pc.KVM
+	}
+	loader := newLazySSH(cfg)
+	if _, err := loader.load(); err != nil {
+		logf.FromContext(ctx).WithName("kvm").WithValues("provider", pc.Name).Info(
+			"WARNING: kvm ssh material is not usable; proxies on this provider will fail until it is",
+			"error", err.Error())
+	}
+	return newWithDialer(pc.Name, cfg, loader.dial, nil), nil
+}
+
+// newWithDialer is the test seam: the dialer decides what libvirtAPI each
+// host resolves to, and the HTTP client is what image downloads go through.
+func newWithDialer(name string, cfg provider.KVMConfig, dial hostDialer, httpClient *http.Client) *Provider {
+	return &Provider{
+		name:   name,
+		cfg:    withDefaults(cfg),
+		hosts:  newHostPool(dial),
+		images: newImageCache(httpClient),
+		now:    time.Now,
+	}
+}
+
+func (p *Provider) logger(ctx context.Context) logr.Logger {
+	return logf.FromContext(ctx).WithName("kvm").WithValues("provider", p.name)
+}
+
+func (p *Provider) wrapErr(op, id string, err error) error {
+	return provider.Wrap(classify(err), op, p.name, id, err)
+}
+
+// host resolves placement.zone (or a providerID's host segment) to a
+// configured host. An empty zone is allowed only when exactly one host is
+// configured; anything else is a spec error.
+func (p *Provider) host(zone string) (provider.KVMHost, error) {
+	if zone == "" {
+		switch len(p.cfg.Hosts) {
+		case 0:
+			return provider.KVMHost{}, fmt.Errorf("%w: kvm.hosts is empty", provider.ErrPermanent)
+		case 1:
+			return p.cfg.Hosts[0], nil
+		default:
+			return provider.KVMHost{}, fmt.Errorf("%w: placement.zone is required when kvm.hosts lists more than one host",
+				provider.ErrPermanent)
+		}
+	}
+	for _, h := range p.cfg.Hosts {
+		if h.Name == zone {
+			return h, nil
+		}
+	}
+	return provider.KVMHost{}, fmt.Errorf("%w: host %q is not in kvm.hosts", provider.ErrPermanent, zone)
+}
+
+// Create makes the VM exist and running on the placed host, and is
+// idempotent: a defined domain is only started if needed, its volumes
+// untouched. Before the domain exists nothing can be using the volumes,
+// so an existing overlay is reused (its creation is atomic) but the seed
+// is rebuilt from scratch — a previous attempt may have been cut off mid
+// upload, and a truncated seed would boot a VM without cloud-init. The
+// first Create for an image on a host also downloads that image into the
+// host's pool, synchronously.
+func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
+	pl := req.Placement
+	host, err := p.host(pl.Zone)
+	if err != nil {
+		return "", p.wrapErr("create", "", err)
+	}
+	mt, ok := p.cfg.MachineTypes[pl.MachineType]
+	if !ok {
+		return "", p.wrapErr("create", "", fmt.Errorf("%w: placement.machineType %q is not in kvm.machineTypes",
+			provider.ErrPermanent, pl.MachineType))
+	}
+	src, err := resolveImage(p.cfg, pl.Image)
+	if err != nil {
+		return "", p.wrapErr("create", "", err)
+	}
+	id := formatProviderID(host.Name, req.Name)
+	log := p.logger(ctx).WithValues("host", host.Name, "name", req.Name)
+	log.V(2).Info("KVM create resolved",
+		"pool", host.Pool, "bridge", host.Bridge, "machineType", pl.MachineType,
+		"vcpus", mt.VCPUs, "memoryMiB", mt.MemoryMiB, "diskSizeGb", p.cfg.DiskSizeGB,
+		"image", src.volumeName(), "cloudInitBytes", len(req.CloudInit))
+
+	err = p.hosts.call(ctx, log, host, func(api libvirtAPI) error {
+		dom, err := api.DomainLookupByName(req.Name)
+		if err == nil {
+			log.V(1).Info("domain already defined")
+			return p.ensureRunning(log, api, dom)
+		}
+		if !hasCode(err, libvirt.ErrNoDomain) {
+			return fmt.Errorf("looking up domain: %w", err)
+		}
+		pool, err := api.StoragePoolLookupByName(host.Pool)
+		if err != nil {
+			return fmt.Errorf("looking up pool %s: %w", host.Pool, err)
+		}
+		basePath, err := p.images.ensure(ctx, log, api, host.Name, pool, src)
+		if err != nil {
+			return err
+		}
+		overlay, err := p.ensureOverlay(log, api, pool, req.Name, basePath, src.Format)
+		if err != nil {
+			return err
+		}
+		seed, err := p.writeSeed(log, api, pool, req.Name, req.CloudInit)
+		if err != nil {
+			return err
+		}
+		diskPath, err := api.StorageVolGetPath(overlay)
+		if err != nil {
+			return fmt.Errorf("resolving overlay path: %w", err)
+		}
+		seedPath, err := api.StorageVolGetPath(seed)
+		if err != nil {
+			return fmt.Errorf("resolving seed path: %w", err)
+		}
+		dom, err = p.defineDomain(log, api, host, req, mt, src.Firmware, diskPath, seedPath)
+		if err != nil {
+			return err
+		}
+		return p.ensureRunning(log, api, dom)
+	})
+	if err != nil {
+		logAPIError(log, "create", err)
+		return "", p.wrapErr("create", id, err)
+	}
+	log.V(1).Info("KVM domain ensured running", "id", id)
+	return id, nil
+}
+
+// ensureOverlay creates the per-VM overlay volume, or returns the existing
+// one: creation is atomic, so an overlay that exists is complete.
+func (p *Provider) ensureOverlay(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, basePath, baseFormat string) (libvirt.StorageVol, error) {
+	xml, err := buildVolumeXML(overlayVolume(name, p.cfg.DiskSizeGB, basePath, baseFormat))
+	if err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	vol, err := api.StorageVolCreateXML(pool, xml, 0)
+	switch {
+	case err == nil:
+		log.V(1).Info("overlay volume created", "volume", overlayVolumeName(name), "base", basePath)
+		return vol, nil
+	case hasCode(err, libvirt.ErrStorageVolExist):
+		log.V(1).Info("overlay volume already exists", "volume", overlayVolumeName(name))
+		vol, err := api.StorageVolLookupByName(pool, overlayVolumeName(name))
+		if err != nil {
+			return libvirt.StorageVol{}, fmt.Errorf("looking up existing overlay volume: %w", err)
+		}
+		return vol, nil
+	default:
+		return libvirt.StorageVol{}, fmt.Errorf("creating overlay volume: %w", err)
+	}
+}
+
+// writeSeed replaces the seed volume with a freshly built NoCloud ISO.
+// Only called while no domain references it.
+func (p *Provider) writeSeed(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, userData string) (libvirt.StorageVol, error) {
+	seedName := seedVolumeName(name)
+	if err := deleteIfExists(api, pool, seedName); err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	vendorData, err := buildVendorData(p.cfg.VendorData, p.cfg.InstallGuestAgent == nil || *p.cfg.InstallGuestAgent)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
+	}
+	iso, err := buildSeedISO(name, userData, vendorData)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
+	}
+	xml, err := buildVolumeXML(rawVolume(seedName, int64(len(iso))))
+	if err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	vol, err := api.StorageVolCreateXML(pool, xml, 0)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("creating seed volume: %w", err)
+	}
+	if err := api.StorageVolUpload(vol, bytes.NewReader(iso), 0, 0, 0); err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("uploading seed volume: %w", err)
+	}
+	log.V(1).Info("seed volume uploaded", "volume", seedName, "bytes", len(iso))
+	return vol, nil
+}
+
+func (p *Provider) defineDomain(log logr.Logger, api libvirtAPI, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType, firmware, diskPath, seedPath string) (libvirt.Domain, error) {
+	xml, err := buildDomainXML(domainParams{
+		Name:      req.Name,
+		UID:       req.UID,
+		CreatedAt: p.now(),
+		VCPUs:     mt.VCPUs,
+		MemoryMiB: mt.MemoryMiB,
+		Firmware:  firmware,
+		Bridge:    host.Bridge,
+		DiskPath:  diskPath,
+		SeedPath:  seedPath,
+	})
+	if err != nil {
+		return libvirt.Domain{}, err
+	}
+	dom, err := api.DomainDefineXML(xml)
+	if err != nil {
+		return libvirt.Domain{}, fmt.Errorf("defining domain: %w", err)
+	}
+	log.V(1).Info("domain defined", "uid", req.UID, "firmware", firmware)
+	return dom, nil
+}
+
+func (p *Provider) ensureRunning(log logr.Logger, api libvirtAPI, dom libvirt.Domain) error {
+	state, _, err := api.DomainGetState(dom, 0)
+	if err != nil {
+		return fmt.Errorf("reading domain state: %w", err)
+	}
+	if isActive(libvirt.DomainState(state)) {
+		return nil
+	}
+	err = api.DomainCreate(dom)
+	switch {
+	case err == nil:
+		log.V(1).Info("domain started")
+	case hasCode(err, libvirt.ErrOperationInvalid):
+		log.V(1).Info("domain already running, start treated as success")
+	default:
+		return fmt.Errorf("starting domain: %w", err)
+	}
+	return nil
+}
+
+// Get reads the domain's state, metadata and addresses. The providerID
+// carries the host, so this looks at the right hypervisor even after a
+// placement.zone edit.
+func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instance, error) {
+	hostName, name, err := parseProviderID(providerID)
+	if err != nil {
+		return nil, provider.Wrap(provider.ErrPermanent, "get", p.name, providerID, err)
+	}
+	host, err := p.host(hostName)
+	if err != nil {
+		return nil, p.wrapErr("get", providerID, err)
+	}
+	log := p.logger(ctx).WithValues("host", host.Name, "name", name)
+	var inst *provider.Instance
+	err = p.hosts.call(ctx, log, host, func(api libvirtAPI) error {
+		dom, err := api.DomainLookupByName(name)
+		if err != nil {
+			return fmt.Errorf("looking up domain: %w", err)
+		}
+		meta, err := p.metadata(api, dom)
+		if err != nil {
+			return err
+		}
+		inst, err = p.inspect(log, api, host.Name, dom, meta)
+		return err
+	})
+	if err != nil {
+		logAPIError(log, "get", err)
+		return nil, p.wrapErr("get", providerID, err)
+	}
+	log.V(1).Info("KVM domain fetched", "state", inst.State, "ip", inst.IP, "uid", inst.UID)
+	return inst, nil
+}
+
+// inspect assembles an Instance from a domain that is known to exist and
+// whose metadata has already been read (the zero value for a domain
+// without the operator's element, which reports an empty UID as the
+// contract requires).
+func (p *Provider) inspect(log logr.Logger, api libvirtAPI, hostName string, dom libvirt.Domain, meta instanceMetadata) (*provider.Instance, error) {
+	state, _, err := api.DomainGetState(dom, 0)
+	if err != nil {
+		return nil, fmt.Errorf("reading domain state: %w", err)
+	}
+	var ip string
+	if isActive(libvirt.DomainState(state)) {
+		ip = p.address(log, api, dom)
+	}
+	return &provider.Instance{
+		ID:        formatProviderID(hostName, dom.Name),
+		IP:        ip,
+		State:     mapState(libvirt.DomainState(state), ip),
+		UID:       meta.UID,
+		CreatedAt: meta.createdAt(),
+	}, nil
+}
+
+// metadata returns the operator's element, or the zero value when the
+// domain has none (it was not created by this operator).
+func (p *Provider) metadata(api libvirtAPI, dom libvirt.Domain) (instanceMetadata, error) {
+	raw, err := api.DomainGetMetadata(dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, libvirt.DomainAffectCurrent)
+	if hasCode(err, libvirt.ErrNoDomainMetadata) {
+		return instanceMetadata{}, nil
+	}
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("reading domain metadata: %w", err)
+	}
+	meta, err := parseInstanceMetadata(raw)
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
+	}
+	return meta, nil
+}
+
+// address returns the domain's first routable IPv4, asking the host's ARP
+// table first and the guest agent second. Both sources fail routinely while
+// the guest is still booting (no traffic seen yet, agent not installed
+// yet), so their errors mean "no address yet", not "Get failed".
+func (p *Provider) address(log logr.Logger, api libvirtAPI, dom libvirt.Domain) string {
+	for _, source := range []libvirt.DomainInterfaceAddressesSource{
+		libvirt.DomainInterfaceAddressesSrcArp,
+		libvirt.DomainInterfaceAddressesSrcAgent,
+	} {
+		ifaces, err := api.DomainInterfaceAddresses(dom, uint32(source), 0)
+		if err != nil {
+			log.V(2).Info("no addresses from source", "source", source.String(), "error", err.Error())
+			continue
+		}
+		if ip := firstIPv4(ifaces); ip != "" {
+			return ip
+		}
+	}
+	return ""
+}
+
+func firstIPv4(ifaces []libvirt.DomainInterface) string {
+	for _, iface := range ifaces {
+		for _, a := range iface.Addrs {
+			if a.Type != int32(libvirt.IPAddrTypeIpv4) {
+				continue
+			}
+			ip := net.ParseIP(a.Addr)
+			if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsUnspecified() {
+				continue
+			}
+			return a.Addr
+		}
+	}
+	return ""
+}
+
+// isActive reports whether the guest is executing (or blocked on the
+// hypervisor, which is still "up" from the outside).
+func isActive(s libvirt.DomainState) bool {
+	return s == libvirt.DomainRunning || s == libvirt.DomainBlocked
+}
+
+// mapState collapses libvirt's states onto the provider's. An active
+// domain without an address is still Provisioning: an empty IP must never
+// be published as Running. "No state" is also Provisioning — the
+// reconciler answers Stopped with delete-and-recreate, too drastic for a
+// state libvirt could not determine; the next poll decides. Everything
+// else — paused, suspended, shut off, crashed, shutting down — is Stopped.
+func mapState(s libvirt.DomainState, ip string) provider.InstanceState {
+	if s == libvirt.DomainNostate {
+		return provider.StateProvisioning
+	}
+	if !isActive(s) {
+		return provider.StateStopped
+	}
+	if ip == "" {
+		return provider.StateProvisioning
+	}
+	return provider.StateRunning
+}
+
+// Delete tears the VM down and is idempotent at every step: a stopped
+// domain is not destroyed again, a missing domain is not undefined, and
+// the per-VM volumes are removed by name even when the domain is already
+// gone, so a crash between volume creation and define is cleaned up on the
+// next reconcile. A providerID naming a host that is no longer configured
+// is a no-op success rather than an error.
+func (p *Provider) Delete(ctx context.Context, providerID string) error {
+	hostName, name, err := parseProviderID(providerID)
+	if err != nil {
+		return provider.Wrap(provider.ErrPermanent, "delete", p.name, providerID, err)
+	}
+	host, err := p.host(hostName)
+	if err != nil {
+		// The host is no longer configured, so this operator has no way to
+		// reach the domain and nothing left to clean. Same reasoning as
+		// removeVolumes over a missing pool: refusing would pin the Proxy's
+		// finalizer. Whatever is still running there is the admin's to remove,
+		// and orphan GC cannot see it either — see docs/kvm-provider.md.
+		p.logger(ctx).Info("host is not configured; delete has nothing to clean up",
+			"host", hostName, "providerID", providerID)
+		return nil
+	}
+	log := p.logger(ctx).WithValues("host", host.Name, "name", name)
+	err = p.hosts.call(ctx, log, host, func(api libvirtAPI) error {
+		if err := p.removeDomain(log, api, name); err != nil {
+			return err
+		}
+		return p.removeVolumes(log, api, host.Pool, name)
+	})
+	if err != nil {
+		logAPIError(log, "delete", err)
+		return p.wrapErr("delete", providerID, err)
+	}
+	log.V(1).Info("KVM domain deleted")
+	return nil
+}
+
+func (p *Provider) removeDomain(log logr.Logger, api libvirtAPI, name string) error {
+	dom, err := api.DomainLookupByName(name)
+	if hasCode(err, libvirt.ErrNoDomain) {
+		log.V(1).Info("domain already gone")
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("looking up domain: %w", err)
+	}
+	err = api.DomainDestroy(dom)
+	switch {
+	case err == nil:
+		log.V(1).Info("domain destroyed")
+	case hasCode(err, libvirt.ErrOperationInvalid):
+		log.V(1).Info("domain not running, destroy skipped")
+	case hasCode(err, libvirt.ErrNoDomain):
+		return nil
+	default:
+		return fmt.Errorf("destroying domain: %w", err)
+	}
+	if err := api.DomainUndefineFlags(dom, undefineFlags); err != nil && !hasCode(err, libvirt.ErrNoDomain) {
+		return fmt.Errorf("undefining domain: %w", err)
+	}
+	log.V(1).Info("domain undefined")
+	return nil
+}
+
+// removeVolumes deletes the overlay and seed. A missing pool means there
+// is nothing left to clean, not a failure: refusing to finish a Delete
+// over a pool that no longer exists would pin the Proxy's finalizer.
+func (p *Provider) removeVolumes(log logr.Logger, api libvirtAPI, poolName, name string) error {
+	pool, err := api.StoragePoolLookupByName(poolName)
+	if hasCode(err, libvirt.ErrNoStoragePool) {
+		log.Info("pool not found, no volumes to delete", "pool", poolName)
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("looking up pool %s: %w", poolName, err)
+	}
+	for _, vol := range []string{overlayVolumeName(name), seedVolumeName(name)} {
+		if err := deleteIfExists(api, pool, vol); err != nil {
+			return err
+		}
+	}
+	log.V(1).Info("per-VM volumes deleted")
+	return nil
+}
+
+// ListByTag inspects every domain on every host and keeps those carrying
+// the operator's managed metadata. One unreachable host fails the whole
+// sweep, so orphan GC never acts on a partial view of the fleet; a single
+// domain that cannot be read is skipped instead, which is safe in the
+// same direction — a domain missing from the list is never collected.
+func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
+	log := p.logger(ctx)
+	var out []provider.Instance
+	for _, host := range p.cfg.Hosts {
+		hlog := log.WithValues("host", host.Name)
+		err := p.hosts.call(ctx, hlog, host, func(api libvirtAPI) error {
+			doms, _, err := api.ConnectListAllDomains(1, 0)
+			if err != nil {
+				return fmt.Errorf("listing domains: %w", err)
+			}
+			for _, dom := range doms {
+				if inst := p.managedInstance(hlog, api, host.Name, dom); inst != nil {
+					out = append(out, *inst)
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			logAPIError(hlog, "list", err)
+			return nil, p.wrapErr("list", "", err)
+		}
+	}
+	log.V(1).Info("KVM domains listed", "hosts", len(p.cfg.Hosts), "managed", len(out))
+	for _, inst := range out {
+		log.V(2).Info("KVM listed instance", "id", inst.ID, "state", inst.State, "uid", inst.UID, "createdAt", inst.CreatedAt)
+	}
+	return out, nil
+}
+
+// managedInstance returns nil for domains the operator does not own, for
+// domains that vanished between the listing and the inspection, and for
+// domains whose metadata or state cannot be read (corrupt element, mid
+// teardown): one bad domain must not block GC for every other one. A
+// transport failure would also land here, but it fails the next RPC in
+// the same sweep too, so the host still counts as unreachable.
+func (p *Provider) managedInstance(log logr.Logger, api libvirtAPI, hostName string, dom libvirt.Domain) *provider.Instance {
+	log = log.WithValues("name", dom.Name)
+	meta, err := p.metadata(api, dom)
+	if err != nil {
+		log.V(1).Info("skipping domain with unreadable metadata", "error", err.Error())
+		return nil
+	}
+	if !meta.managed() {
+		return nil
+	}
+	inst, err := p.inspect(log, api, hostName, dom, meta)
+	if err != nil {
+		log.V(1).Info("skipping domain that could not be inspected", "error", err.Error())
+		return nil
+	}
+	return inst
+}

internal/provider/kvm/kvm_integration_test.go

@@ -0,0 +1,426 @@
+//go:build integration
+
+package kvm
+
+// Integration test against a real libvirt/KVM hypervisor. It deviates from
+// the repo's testcontainers convention on purpose: libvirt needs /dev/kvm
+// and a bridge, neither of which a container provides, so this runs
+// against whatever host the KVM_TEST_* variables name. It creates one VM
+// from a stock cloud image, waits for it to report an address, deletes it,
+// and checks the pool afterwards. The base image stays cached on the host.
+//
+// Required:
+//
+//	KVM_TEST_HOST       SSH endpoint, host:port
+//	KVM_TEST_USER       SSH user (member of the libvirt group)
+//	KVM_TEST_SSH_KEY    path to the private key
+//	KVM_TEST_BRIDGE     host bridge for the VM's NIC
+//	KVM_TEST_IMAGE_URL  cloud image to fetch (http/https)
+//
+// Optional:
+//
+//	KVM_TEST_KNOWN_HOSTS   known_hosts file; without it host keys are NOT verified
+//	KVM_TEST_IMAGE_SHA256  expected digest of the image
+//	KVM_TEST_POOL          storage pool (default "default")
+//	KVM_TEST_SOCKET        libvirt socket path on the host
+//	KVM_TEST_TIMEOUT       overall budget incl. the image download (default 20m)
+//	KVM_TEST_KEEP          set to anything to leave the VM behind for inspection
+//	KVM_TEST_LOG_LEVEL     provider log verbosity shown in -v output (default 1:
+//	                       one line per libvirt call; 2 adds XML, metadata and
+//	                       addresses; 0 leaves only the t.Log lines)
+//
+// Provider logs are printed through t.Log in a compact form: time, logger
+// name, then the libvirt API function (or the message) and its fields.
+// The provider/host/name values that are the same on every line of a run
+// are dropped; the first t.Log line of the run states them. A run that
+// hits its timeout may log one more line from the RPC the pool abandoned,
+// after the test has ended; that is a symptom of the timeout, not a bug
+// in the tracer.
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+	"k8s.io/apimachinery/pkg/types"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+const (
+	integrationHostName = "kvm-test"
+	integrationImageKey = "test-image"
+	integrationPoll     = 5 * time.Second
+)
+
+type integrationEnv struct {
+	cfg      provider.ProviderConfig
+	timeout  time.Duration
+	keep     bool
+	logLevel int
+}
+
+func loadIntegrationEnv(t *testing.T) integrationEnv {
+	t.Helper()
+	if testing.Short() {
+		t.Skip("integration test skipped under -short")
+	}
+	required := map[string]string{}
+	for _, key := range []string{"KVM_TEST_HOST", "KVM_TEST_USER", "KVM_TEST_SSH_KEY", "KVM_TEST_BRIDGE", "KVM_TEST_IMAGE_URL"} {
+		v := os.Getenv(key)
+		if v == "" {
+			t.Skipf("%s not set; skipping KVM integration test", key)
+		}
+		required[key] = v
+	}
+	cfg := &provider.KVMConfig{
+		SSHKeyFile:     required["KVM_TEST_SSH_KEY"],
+		KnownHostsFile: os.Getenv("KVM_TEST_KNOWN_HOSTS"),
+		Images: map[string]provider.KVMImage{
+			integrationImageKey: {URL: required["KVM_TEST_IMAGE_URL"], SHA256: os.Getenv("KVM_TEST_IMAGE_SHA256")},
+		},
+		MachineTypes: map[string]provider.KVMMachineType{"small": {VCPUs: 1, MemoryMiB: 1024}},
+		Hosts: []provider.KVMHost{{
+			Name:    integrationHostName,
+			Address: required["KVM_TEST_HOST"],
+			User:    required["KVM_TEST_USER"],
+			Socket:  os.Getenv("KVM_TEST_SOCKET"),
+			Pool:    os.Getenv("KVM_TEST_POOL"),
+			Bridge:  required["KVM_TEST_BRIDGE"],
+		}},
+	}
+	if cfg.KnownHostsFile == "" {
+		cfg.InsecureIgnoreHostKey = true
+		t.Log("KVM_TEST_KNOWN_HOSTS not set: host key verification disabled for this run")
+	}
+	pc := provider.ProviderConfig{Name: "kvm-integration", Type: "kvm", KVM: cfg}
+	timeout := 20 * time.Minute
+	if v := os.Getenv("KVM_TEST_TIMEOUT"); v != "" {
+		d, err := time.ParseDuration(v)
+		if err != nil {
+			t.Fatalf("KVM_TEST_TIMEOUT: %v", err)
+		}
+		timeout = d
+	}
+	logLevel := 1
+	if v := os.Getenv("KVM_TEST_LOG_LEVEL"); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil {
+			t.Fatalf("KVM_TEST_LOG_LEVEL: %v", err)
+		}
+		logLevel = n
+	}
+	return integrationEnv{cfg: pc, timeout: timeout, keep: os.Getenv("KVM_TEST_KEEP") != "", logLevel: logLevel}
+}
+
+// integrationLogger routes provider logs into the test output at the
+// requested verbosity, and installs it as controller-runtime's root logger
+// so nothing falls back to the unfulfilled delegating sink (which would
+// print its "SetLogger was never called" warning and swallow the lines).
+func integrationLogger(t *testing.T, verbosity int) logr.Logger {
+	t.Helper()
+	log := logr.New(&compactSink{t: t, verbosity: verbosity})
+	logf.SetLogger(log)
+	return log
+}
+
+// compactSink is a logr sink for test output: one line per record as
+// "HH:MM:SS.mmm name headline k=v ...", where the headline is the rpc
+// field when present (the libvirt API function) and the message
+// otherwise. Keys that are constant for the whole run are omitted.
+type compactSink struct {
+	t         *testing.T
+	verbosity int
+	name      string
+	values    []any
+}
+
+var compactOmit = map[string]bool{"provider": true, "host": true, "name": true}
+
+func (s *compactSink) Init(logr.RuntimeInfo) {}
+
+func (s *compactSink) Enabled(level int) bool { return level <= s.verbosity }
+
+// GetCallStackHelper lets logr mark its frames as helpers, so t.Log
+// attributes each line to the provider code that logged it.
+func (s *compactSink) GetCallStackHelper() func() { return s.t.Helper }
+
+func (s *compactSink) Info(_ int, msg string, kv ...any) {
+	s.t.Helper()
+	s.t.Log(s.format(msg, kv))
+}
+
+func (s *compactSink) Error(err error, msg string, kv ...any) {
+	s.t.Helper()
+	s.t.Log(s.format(msg, append(kv, "error", err)))
+}
+
+func (s *compactSink) WithValues(kv ...any) logr.LogSink {
+	c := *s
+	c.values = append(append([]any(nil), s.values...), kv...)
+	return &c
+}
+
+func (s *compactSink) WithName(name string) logr.LogSink {
+	c := *s
+	if c.name != "" {
+		name = c.name + "/" + name
+	}
+	c.name = name
+	return &c
+}
+
+func (s *compactSink) format(msg string, kv []any) string {
+	all := append(append([]any(nil), s.values...), kv...)
+	headline := msg
+	var b strings.Builder
+	for i := 0; i+1 < len(all); i += 2 {
+		key := fmt.Sprint(all[i])
+		if key == "rpc" {
+			headline = fmt.Sprint(all[i+1])
+			continue
+		}
+		if compactOmit[key] {
+			continue
+		}
+		val := fmt.Sprint(all[i+1])
+		if strings.ContainsAny(val, " \t\"=") {
+			val = strconv.Quote(val)
+		}
+		b.WriteString(" " + key + "=" + val)
+	}
+	name := s.name[strings.LastIndex(s.name, "/")+1:]
+	return fmt.Sprintf("%s %s %s%s", time.Now().Format("15:04:05.000"), name, headline, b.String())
+}
+
+func randomUID(t *testing.T) string {
+	t.Helper()
+	var b [16]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		t.Fatal(err)
+	}
+	h := hex.EncodeToString(b[:])
+	return fmt.Sprintf("%s-%s-%s-%s-%s", h[0:8], h[8:12], h[12:16], h[16:20], h[20:32])
+}
+
+// TestKVM_lifecycle is one sequential story, not parallel subtests: each
+// stage depends on the state the previous one left on the hypervisor.
+func TestKVM_lifecycle(t *testing.T) {
+	env := loadIntegrationEnv(t)
+	log := integrationLogger(t, env.logLevel)
+	ctx, cancel := context.WithTimeout(logr.NewContext(context.Background(), log), env.timeout)
+	defer cancel()
+
+	prov, err := New(ctx, env.cfg)
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	p := prov.(*Provider)
+	host := p.cfg.Hosts[0]
+
+	uid := randomUID(t)
+	req := provider.CreateRequest{
+		Name:      provider.NameFromUID(types.UID(uid)),
+		UID:       uid,
+		Namespace: "integration",
+		ProxyName: "kvm-integration",
+		Placement: provider.Placement{Zone: integrationHostName, MachineType: "small", Image: integrationImageKey},
+		CloudInit: "#cloud-config\n",
+		Port:      3128,
+	}
+	t.Logf("host %s (%s), pool %q, bridge %s, domain %s", host.Name, host.Address, host.Pool, host.Bridge, req.Name)
+
+	// Cleanup runs even when a stage fails, with its own budget: the test
+	// ctx may already be spent by then.
+	t.Cleanup(func() {
+		if env.keep {
+			t.Logf("KVM_TEST_KEEP set: leaving domain %s and its volumes on %s", req.Name, host.Address)
+			return
+		}
+		cctx, ccancel := context.WithTimeout(logr.NewContext(context.Background(), log), 2*time.Minute)
+		defer ccancel()
+		if err := p.Delete(cctx, formatProviderID(host.Name, req.Name)); err != nil {
+			t.Errorf("cleanup Delete: %v", err)
+		}
+	})
+
+	// --- create (includes the image fetch on a cold host) ---
+	start := time.Now()
+	id, err := p.Create(ctx, req)
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	t.Logf("Create returned %s after %s", id, time.Since(start).Round(time.Second))
+	if want := formatProviderID(host.Name, req.Name); id != want {
+		t.Fatalf("providerID = %s, want %s", id, want)
+	}
+
+	// --- create again: must find what it made, not duplicate it ---
+	id2, err := p.Create(ctx, req)
+	if err != nil {
+		t.Fatalf("second Create: %v", err)
+	}
+	if id2 != id {
+		t.Fatalf("second Create returned %s, want %s", id2, id)
+	}
+
+	// --- wait for the guest to come up with an address ---
+	inst := waitForRunning(ctx, t, p, id)
+	t.Logf("domain Running with IP %s (state reported after %s)", inst.IP, time.Since(start).Round(time.Second))
+	if inst.UID != uid {
+		t.Errorf("UID read back = %q, want %q (metadata round-trip through libvirt)", inst.UID, uid)
+	}
+	if age := time.Since(inst.CreatedAt); inst.CreatedAt.IsZero() || age < 0 || age > time.Hour {
+		t.Errorf("CreatedAt = %s (age %s), want a recent timestamp", inst.CreatedAt, age)
+	}
+	if conn, err := net.DialTimeout("tcp", net.JoinHostPort(inst.IP, "22"), 5*time.Second); err != nil {
+		t.Logf("note: tcp %s:22 not reachable from this machine (%v); fine if the bridge network is not routed here", inst.IP, err)
+	} else {
+		conn.Close()
+		t.Logf("tcp %s:22 reachable from this machine", inst.IP)
+	}
+
+	// --- the GC view must include it ---
+	listed, err := p.ListByTag(ctx)
+	if err != nil {
+		t.Fatalf("ListByTag: %v", err)
+	}
+	if !containsID(listed, id) {
+		t.Errorf("ListByTag = %+v, want it to include %s", listed, id)
+	}
+
+	// --- delete, then verify nothing of the VM remains but the base image does ---
+	if err := p.Delete(ctx, id); err != nil {
+		t.Fatalf("Delete: %v", err)
+	}
+	if _, err := p.Get(ctx, id); !errors.Is(err, provider.ErrNotFound) {
+		t.Errorf("Get after Delete = %v, want ErrNotFound", err)
+	}
+	if err := p.Delete(ctx, id); err != nil {
+		t.Errorf("second Delete: %v, want nil (idempotent)", err)
+	}
+	listed, err = p.ListByTag(ctx)
+	if err != nil {
+		t.Fatalf("ListByTag after Delete: %v", err)
+	}
+	if containsID(listed, id) {
+		t.Errorf("ListByTag still includes %s after Delete", id)
+	}
+
+	src, err := resolveImage(p.cfg, integrationImageKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = p.hosts.call(ctx, p.logger(ctx).WithValues("host", host.Name), host, func(api libvirtAPI) error {
+		pool, err := api.StoragePoolLookupByName(host.Pool)
+		if err != nil {
+			return err
+		}
+		for _, gone := range []string{overlayVolumeName(req.Name), seedVolumeName(req.Name)} {
+			if _, err := api.StorageVolLookupByName(pool, gone); !hasCode(err, libvirt.ErrNoStorageVol) {
+				t.Errorf("volume %s after Delete: lookup err = %v, want ErrNoStorageVol", gone, err)
+			}
+		}
+		for _, kept := range []string{src.volumeName(), src.markerName()} {
+			if _, err := api.StorageVolLookupByName(pool, kept); err != nil {
+				t.Errorf("cached image volume %s: %v, want it kept after Delete", kept, err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("inspecting pool after Delete: %v", err)
+	}
+	// KVM_TEST_KEEP is for inspecting a failed run; a passed run has
+	// nothing left to keep, so let cleanup's Delete confirm that.
+	env.keep = false
+}
+
+// TestKVMCheck_realHost runs the provider health check against the host.
+// It creates nothing: it must reach the host and find the pool running.
+func TestKVMCheck_realHost(t *testing.T) {
+	env := loadIntegrationEnv(t)
+	log := integrationLogger(t, env.logLevel)
+	ctx, cancel := context.WithTimeout(logr.NewContext(context.Background(), log), time.Minute)
+	defer cancel()
+
+	prov, err := New(ctx, env.cfg)
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	got := prov.(*Provider).Check(ctx)
+
+	if len(got) != 1 {
+		t.Fatalf("Check returned %d targets, want 1", len(got))
+	}
+	ts := got[0]
+	t.Logf("%s (%s): ready=%v reason=%s latency=%s details=%v",
+		ts.Name, ts.Address, ts.Ready, ts.Reason, ts.Latency.Round(time.Millisecond), ts.Details)
+	if !ts.Ready {
+		t.Fatalf("check failed: %s: %s", ts.Reason, ts.Message)
+	}
+	for _, key := range []string{"hostname", "libvirtVersion", "poolState", "poolAvailable"} {
+		if ts.Details[key] == "" {
+			t.Errorf("details[%s] is empty, want what the host reported", key)
+		}
+	}
+}
+
+// waitForRunning polls Get until the instance is Running with an IP or ctx
+// ends. Transient errors are retried; anything else fails the test.
+func waitForRunning(ctx context.Context, t *testing.T, p *Provider, id string) *provider.Instance {
+	t.Helper()
+	var last provider.InstanceState
+	ticker := time.NewTicker(integrationPoll)
+	defer ticker.Stop()
+	for {
+		inst, err := p.Get(ctx, id)
+		switch {
+		case err == nil:
+			if inst.State != last {
+				t.Logf("state %s ip %q", inst.State, inst.IP)
+				last = inst.State
+			}
+			if inst.State == provider.StateRunning {
+				return inst
+			}
+			if inst.State == provider.StateStopped {
+				t.Fatalf("domain reported %s while waiting for Running; check `virsh console %s` on the host", inst.State, idName(id))
+			}
+		case errors.Is(err, provider.ErrTransient):
+			t.Logf("Get: transient error, retrying: %v", err)
+		default:
+			t.Fatalf("Get: %v", err)
+		}
+		select {
+		case <-ctx.Done():
+			t.Fatalf("timed out waiting for Running (last state %s): %v", last, ctx.Err())
+		case <-ticker.C:
+		}
+	}
+}
+
+func idName(id string) string {
+	_, name, _ := parseProviderID(id)
+	return name
+}
+
+func containsID(list []provider.Instance, id string) bool {
+	for _, inst := range list {
+		if inst.ID == id {
+			return true
+		}
+	}
+	return false
+}

internal/provider/kvm/kvm_test.go

@@ -0,0 +1,999 @@
+package kvm
+
+import (
+	"context"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func TestWithDefaults(t *testing.T) {
+	t.Parallel()
+	cfg := withDefaults(provider.KVMConfig{
+		Images: map[string]provider.KVMImage{
+			"debian": {URL: "https://x/d.qcow2"},
+			"raw":    {URL: "https://x/r.img", Format: "raw", Firmware: "bios"},
+		},
+		Hosts: []provider.KVMHost{
+			{Name: "a", Address: "a:22", User: "virt", Bridge: "br0"},
+			{Name: "b", Address: "b:22", User: "virt", Bridge: "br1", Socket: "/run/s", Pool: "fast"},
+		},
+	})
+	if cfg.DiskSizeGB != defaultDiskSizeGB {
+		t.Errorf("diskSizeGb = %d, want %d", cfg.DiskSizeGB, defaultDiskSizeGB)
+	}
+	if cfg.VendorData != "" {
+		t.Errorf("vendorData = %q, want none: the guest-agent bootstrap is merged into the user-data, not defaulted here", cfg.VendorData)
+	}
+	if cfg.Images["debian"].Format != "qcow2" || cfg.Images["raw"].Format != "raw" {
+		t.Errorf("image formats = %q/%q, want qcow2 default and raw kept", cfg.Images["debian"].Format, cfg.Images["raw"].Format)
+	}
+	if cfg.Images["debian"].Firmware != firmwareUEFI || cfg.Images["raw"].Firmware != firmwareBIOS {
+		t.Errorf("image firmware = %q/%q, want uefi default and bios kept", cfg.Images["debian"].Firmware, cfg.Images["raw"].Firmware)
+	}
+	if a := cfg.Hosts[0]; a.Socket != defaultSocket || a.Pool != defaultPool {
+		t.Errorf("host a = %+v, want default socket and pool", a)
+	}
+	if b := cfg.Hosts[1]; b.Socket != "/run/s" || b.Pool != "fast" {
+		t.Errorf("host b = %+v, want explicit socket and pool kept", b)
+	}
+}
+
+func TestWithDefaults_keepsExplicitValues(t *testing.T) {
+	t.Parallel()
+	cfg := withDefaults(provider.KVMConfig{DiskSizeGB: 42, VendorData: "#cloud-config\n"})
+	if cfg.DiskSizeGB != 42 || cfg.VendorData != "#cloud-config\n" {
+		t.Errorf("cfg = %+v, want explicit values kept", cfg)
+	}
+}
+
+// --- provider over fake hosts ---
+
+const (
+	testName      = "proxy-abc123def456ghij"
+	testUID       = "11111111-2222-3333-4444-555555555555"
+	testImageURL  = "https://images.example/debian-13.qcow2"
+	testCloudInit = "#cloud-config\npackages: [squid]\n"
+)
+
+var (
+	testNow    = time.Date(2026, 9, 8, 10, 0, 0, 0, time.UTC)
+	errHostOff = errors.New("ssh dial: connection refused")
+)
+
+// hostFakes routes each configured host name to its own fakeAPI; a host
+// with no entry is unreachable.
+type hostFakes map[string]*fakeAPI
+
+func (h hostFakes) dial(_ context.Context, host provider.KVMHost) (*hostClient, error) {
+	api, ok := h[host.Name]
+	if !ok {
+		return nil, errHostOff
+	}
+	return &hostClient{api: api, close: func() {}}, nil
+}
+
+func testConfig(hosts ...string) provider.ProviderConfig {
+	cfg := &provider.KVMConfig{
+		SSHKeyFile:     "/k",
+		KnownHostsFile: "/kh",
+		Images:         map[string]provider.KVMImage{"debian-13": {URL: testImageURL}},
+		MachineTypes:   map[string]provider.KVMMachineType{"small": {VCPUs: 2, MemoryMiB: 2048}},
+	}
+	for _, h := range hosts {
+		cfg.Hosts = append(cfg.Hosts, provider.KVMHost{Name: h, Address: h + ":22", User: "virt", Bridge: "br0"})
+	}
+	return provider.ProviderConfig{Name: "kvm-home", Type: "kvm", KVM: cfg}
+}
+
+func newTestProvider(fakes hostFakes, hosts ...string) *Provider {
+	pc := testConfig(hosts...)
+	p := newWithDialer(pc.Name, *pc.KVM, fakes.dial, nil)
+	p.now = func() time.Time { return testNow }
+	return p
+}
+
+// cacheImage pre-populates the pool with the configured image and its
+// marker so Create never touches the network.
+func cacheImage(api *fakeAPI) {
+	src := imageSource{Key: "debian-13", KVMImage: provider.KVMImage{URL: testImageURL, Format: formatQcow2}}
+	api.putVol("default", src.volumeName(), formatQcow2, []byte("qcow2"))
+	api.putVol("default", src.markerName(), formatRaw, []byte(testImageSHA()))
+}
+
+func readyHost() *fakeAPI {
+	api := newFakeAPI("default")
+	cacheImage(api)
+	return api
+}
+
+func testRequest() provider.CreateRequest {
+	return provider.CreateRequest{
+		Name:      testName,
+		UID:       testUID,
+		Namespace: "default",
+		ProxyName: "home-proxy-1",
+		Placement: provider.Placement{Zone: "nas1", MachineType: "small", Image: "debian-13"},
+		CloudInit: testCloudInit,
+		Port:      3128,
+	}
+}
+
+func metadataXML(t *testing.T, uid string, created time.Time) string {
+	t.Helper()
+	out, err := xml.Marshal(newInstanceMetadata(uid, created))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(out)
+}
+
+func ipv4(addr string) libvirt.DomainIPAddr {
+	return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv4), Addr: addr, Prefix: 24}
+}
+
+func ipv6(addr string) libvirt.DomainIPAddr {
+	return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv6), Addr: addr, Prefix: 64}
+}
+
+// managedDomain seeds a domain as Create would have left it.
+func managedDomain(t *testing.T, api *fakeAPI, name, uid string, running bool, addrs map[uint32][]libvirt.DomainInterface) {
+	t.Helper()
+	api.putDomain(&fakeDomain{name: name, metadata: metadataXML(t, uid, testNow), running: running, addrs: addrs})
+	api.putVol("default", overlayVolumeName(name), formatQcow2, nil)
+	api.putVol("default", seedVolumeName(name), formatRaw, []byte("iso"))
+}
+
+func wantClass(t *testing.T, err error, class error) {
+	t.Helper()
+	if err == nil {
+		t.Fatalf("got nil error, want %v", class)
+	}
+	if !errors.Is(err, class) {
+		t.Fatalf("error class = %v, want %v (err: %v)", provider.Class(err), class, err)
+	}
+}
+
+func assertCreated(t *testing.T, api *fakeAPI) {
+	t.Helper()
+	dom := api.domain(testName)
+	if dom == nil {
+		t.Fatal("domain not defined")
+	}
+	if !dom.running {
+		t.Error("domain not running")
+	}
+	overlay := api.vol("default", overlayVolumeName(testName))
+	if overlay == nil {
+		t.Fatal("overlay volume missing")
+	}
+	if overlay.backing != "/var/lib/libvirt/images/base-debian-13.qcow2" || overlay.format != formatQcow2 {
+		t.Errorf("overlay = %+v, want qcow2 backed by the cached base", overlay)
+	}
+	if overlay.capacity != defaultDiskSizeGB*bytesPerGiB {
+		t.Errorf("overlay capacity = %d, want %d", overlay.capacity, defaultDiskSizeGB*bytesPerGiB)
+	}
+	seed := api.vol("default", seedVolumeName(testName))
+	if seed == nil {
+		t.Fatal("seed volume missing")
+	}
+	if seed.format != formatRaw || len(seed.data) == 0 || int64(len(seed.data)) != seed.capacity {
+		t.Errorf("seed = format %s, %d bytes, capacity %d; want raw with capacity == uploaded size", seed.format, len(seed.data), seed.capacity)
+	}
+	_, files := readSeed(t, seed.data)
+	if files["user-data"] != testCloudInit {
+		t.Errorf("seed user-data = %q, want the Proxy's cloud-init verbatim", files["user-data"])
+	}
+	if !strings.Contains(files["vendor-data"], "qemu-guest-agent") {
+		t.Errorf("seed vendor-data = %q, want the guest-agent script", files["vendor-data"])
+	}
+}
+
+func TestCreate_freshHost(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	id, err := p.Create(context.Background(), testRequest())
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if want := "hosts/nas1/domains/" + testName; id != want {
+		t.Errorf("providerID = %s, want %s", id, want)
+	}
+	assertCreated(t, api)
+
+	var d domainXML
+	if err := xml.Unmarshal([]byte(api.domain(testName).xml), &d); err != nil {
+		t.Fatal(err)
+	}
+	if d.VCPU != 2 || d.Memory.Value != 2048 || d.Memory.Unit != "MiB" {
+		t.Errorf("domain size = %d vcpu / %d %s, want 2 / 2048 MiB", d.VCPU, d.Memory.Value, d.Memory.Unit)
+	}
+	if d.Devices.Interfaces[0].Source.Bridge != "br0" {
+		t.Errorf("bridge = %q, want br0", d.Devices.Interfaces[0].Source.Bridge)
+	}
+	wantDisk, wantSeed := api.poolPath+"/"+overlayVolumeName(testName), api.poolPath+"/"+seedVolumeName(testName)
+	if got := d.Devices.Disks[0].Source.File; got != wantDisk {
+		t.Errorf("overlay disk source = %q, want the path libvirt reported, %q", got, wantDisk)
+	}
+	if got := d.Devices.Disks[1].Source.File; got != wantSeed {
+		t.Errorf("seed cdrom source = %q, want the path libvirt reported, %q", got, wantSeed)
+	}
+	if m := d.Metadata.Instance; m.UID != testUID || !m.managed() || !m.createdAt().Equal(testNow) {
+		t.Errorf("metadata = %+v, want managed, uid %s, createdAt %s", m, testUID, testNow)
+	}
+	if api.hasCall("StorageVolDownload") || api.callCount("StorageVolUpload") != 1 {
+		t.Errorf("calls = %v, want no image download and exactly one upload (the seed)", api.calls)
+	}
+}
+
+func TestCreate_fetchesImageOnFirstUse(t *testing.T) {
+	t.Parallel()
+	srv := newImageServer(t)
+	api := newFakeAPI("default")
+	cfg := testConfig("nas1")
+	cfg.KVM.Images["debian-13"] = provider.KVMImage{URL: srv.URL + "/debian.qcow2", SHA256: testImageSHA()}
+	p := newWithDialer(cfg.Name, *cfg.KVM, hostFakes{"nas1": api}.dial, nil)
+
+	if _, err := p.Create(context.Background(), testRequest()); err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("image server hits = %d, want 1", srv.hits.Load())
+	}
+	base := api.vol("default", "base-debian-13.qcow2")
+	if base == nil || string(base.data) != string(testImage) {
+		t.Fatal("base image not cached in the pool")
+	}
+	if api.domain(testName) == nil || !api.domain(testName).running {
+		t.Error("domain not running after image fetch")
+	}
+
+	// A second Proxy with the same image on the same host reuses the cache.
+	req := testRequest()
+	req.Name, req.UID = "proxy-second", "uid-2"
+	if _, err := p.Create(context.Background(), req); err != nil {
+		t.Fatalf("second Create: %v", err)
+	}
+	if srv.hits.Load() != 1 {
+		t.Errorf("image server hits after second Create = %d, want still 1", srv.hits.Load())
+	}
+}
+
+func TestCreate_isIdempotentWhenComplete(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+	if _, err := p.Create(context.Background(), testRequest()); err != nil {
+		t.Fatal(err)
+	}
+	before := len(api.calls)
+
+	id, err := p.Create(context.Background(), testRequest())
+	if err != nil {
+		t.Fatalf("second Create: %v", err)
+	}
+	if id != "hosts/nas1/domains/"+testName {
+		t.Errorf("providerID = %s", id)
+	}
+	for _, m := range []string{"DomainDefineXML", "DomainCreate", "StorageVolUpload"} {
+		if n := api.callCount(m); n != 1 {
+			t.Errorf("%s called %d times across two Creates, want 1", m, n)
+		}
+	}
+	if len(api.calls) == before {
+		t.Error("second Create made no calls at all; expected lookups")
+	}
+}
+
+// Each case breaks Create after a different step, then re-runs it and
+// expects the second attempt to pick up where the first stopped without
+// duplicating what already exists.
+func TestCreate_resumesAfterPartialFailure(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name     string
+		sabotage func(api *fakeAPI)
+		class    error
+		check    func(t *testing.T, api *fakeAPI)
+	}{
+		{
+			name: "seed upload fails after overlay exists",
+			sabotage: func(api *fakeAPI) {
+				api.uploadHook = func(vol string, _ io.Reader) error {
+					api.uploadHook = nil
+					return io.ErrUnexpectedEOF
+				}
+			},
+			class: provider.ErrTransient,
+			check: func(t *testing.T, api *fakeAPI) {
+				if n := api.callCount("StorageVolUpload"); n != 2 {
+					t.Errorf("seed uploads = %d, want 2 (failed, then redone)", n)
+				}
+				if n := api.callCount("StorageVolCreateXML"); n != 4 {
+					t.Errorf("volume creates = %d, want 4 (overlay, seed; overlay rejected as existing, seed again)", n)
+				}
+				if names := api.volNames("default"); len(names) != 4 {
+					t.Errorf("pool holds %v, want base, marker, one overlay, one seed", names)
+				}
+			},
+		},
+		{
+			name:     "define fails after volumes exist",
+			sabotage: func(api *fakeAPI) { api.failOnce["DomainDefineXML"] = lvError(libvirt.ErrInternalError, "boom") },
+			class:    provider.ErrTransient,
+			check: func(t *testing.T, api *fakeAPI) {
+				if n := api.callCount("StorageVolUpload"); n != 2 {
+					t.Errorf("seed uploads = %d, want 2 (seed rebuilt while no domain exists)", n)
+				}
+				if n := api.callCount("DomainDefineXML"); n != 2 {
+					t.Errorf("defines = %d, want 2", n)
+				}
+			},
+		},
+		{
+			name:     "start fails after define",
+			sabotage: func(api *fakeAPI) { api.failOnce["DomainCreate"] = lvError(libvirt.ErrInternalError, "boom") },
+			class:    provider.ErrTransient,
+			check: func(t *testing.T, api *fakeAPI) {
+				if n := api.callCount("DomainDefineXML"); n != 1 {
+					t.Errorf("defines = %d, want 1 (existing domain reused)", n)
+				}
+				if n := api.callCount("DomainCreate"); n != 2 {
+					t.Errorf("starts = %d, want 2", n)
+				}
+			},
+		},
+		{
+			name:     "transport drops mid-create",
+			sabotage: func(api *fakeAPI) { api.failOnce["DomainLookupByName"] = io.EOF },
+			class:    provider.ErrTransient,
+			check: func(t *testing.T, api *fakeAPI) {
+				if n := api.callCount("DomainDefineXML"); n != 1 {
+					t.Errorf("defines = %d, want 1", n)
+				}
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			api := readyHost()
+			tc.sabotage(api)
+			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+			_, err := p.Create(context.Background(), testRequest())
+			wantClass(t, err, tc.class)
+			if _, err := p.Create(context.Background(), testRequest()); err != nil {
+				t.Fatalf("retry Create: %v", err)
+			}
+			assertCreated(t, api)
+			tc.check(t, api)
+		})
+	}
+}
+
+// A reconcile cancelled mid-upload leaves a seed volume whose content is
+// short; the compensating delete cannot run because the pool has already
+// closed the connection. The next Create must not trust that volume.
+func TestCreate_replacesLeftoverSeed(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	api.putVol("default", seedVolumeName(testName), formatRaw, []byte("trunc"))
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	if _, err := p.Create(context.Background(), testRequest()); err != nil {
+		t.Fatal(err)
+	}
+	assertCreated(t, api)
+	if seed := api.vol("default", seedVolumeName(testName)); string(seed.data) == "trunc" {
+		t.Error("leftover seed was kept")
+	}
+	if n := api.callCount("StorageVolUpload"); n != 1 {
+		t.Errorf("uploads = %d, want 1 (the rebuilt seed)", n)
+	}
+}
+
+func TestCreate_startsStoppedDomain(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	managedDomain(t, api, testName, testUID, false, nil)
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	if _, err := p.Create(context.Background(), testRequest()); err != nil {
+		t.Fatal(err)
+	}
+	if !api.domain(testName).running {
+		t.Error("domain still stopped")
+	}
+	if api.hasCall("DomainDefineXML") || api.hasCall("StorageVolUpload") || api.hasCall("StorageVolDelete") {
+		t.Errorf("calls = %v, want no define, upload or volume delete for an existing domain", api.calls)
+	}
+}
+
+func TestCreate_singleHostNeedsNoZone(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+	req := testRequest()
+	req.Placement.Zone = ""
+
+	id, err := p.Create(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if id != "hosts/nas1/domains/"+testName {
+		t.Errorf("providerID = %s, want the single host filled in", id)
+	}
+}
+
+func TestCreate_placementErrors(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		hosts []string
+		edit  func(*provider.CreateRequest)
+		want  string
+	}{
+		{"unknown host", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Zone = "nas9" }, "not in kvm.hosts"},
+		{"empty zone with two hosts", []string{"nas1", "nas2"}, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "placement.zone is required"},
+		{"empty zone with no hosts", nil, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "kvm.hosts is empty"},
+		{"unknown machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "huge" }, "not in kvm.machineTypes"},
+		{"empty machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "" }, "not in kvm.machineTypes"},
+		{"unknown image", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Image = "windows" }, "neither a key"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fakes := hostFakes{}
+			for _, h := range tc.hosts {
+				fakes[h] = readyHost()
+			}
+			p := newTestProvider(fakes, tc.hosts...)
+			req := testRequest()
+			tc.edit(&req)
+
+			_, err := p.Create(context.Background(), req)
+			wantClass(t, err, provider.ErrPermanent)
+			if !strings.Contains(err.Error(), tc.want) {
+				t.Errorf("error = %q, want it to mention %q", err, tc.want)
+			}
+			for _, api := range fakes {
+				if len(api.calls) != 0 {
+					t.Errorf("host was called: %v", api.calls)
+				}
+			}
+		})
+	}
+}
+
+func TestCreate_hostFailures(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		setup func() hostFakes
+		class error
+	}{
+		{
+			name:  "host unreachable",
+			setup: func() hostFakes { return hostFakes{} },
+			class: provider.ErrTransient,
+		},
+		{
+			name:  "pool missing",
+			setup: func() hostFakes { return hostFakes{"nas1": newFakeAPI("other")} },
+			class: provider.ErrPermanent,
+		},
+		{
+			name: "domain xml rejected",
+			setup: func() hostFakes {
+				api := readyHost()
+				api.fail["DomainDefineXML"] = lvError(libvirt.ErrConfigUnsupported, "no q35 here")
+				return hostFakes{"nas1": api}
+			},
+			class: provider.ErrPermanent,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			p := newTestProvider(tc.setup(), "nas1")
+			_, err := p.Create(context.Background(), testRequest())
+			wantClass(t, err, tc.class)
+		})
+	}
+}
+
+func TestCreate_cancelledContext(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	_, err := p.Create(ctx, testRequest())
+	wantClass(t, err, provider.ErrTransient)
+	if !errors.Is(err, context.Canceled) {
+		t.Errorf("error = %v, want it to unwrap to context.Canceled", err)
+	}
+}
+
+func TestGet(t *testing.T) {
+	t.Parallel()
+	arp := map[uint32][]libvirt.DomainInterface{
+		uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Name: "vnet0", Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.50")}}},
+	}
+	agentOnly := map[uint32][]libvirt.DomainInterface{
+		uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
+			{Name: "lo", Addrs: []libvirt.DomainIPAddr{ipv4("127.0.0.1"), ipv6("::1")}},
+			{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv6("fe80::1"), ipv4("192.168.1.51")}},
+		},
+	}
+	linkLocalOnly := map[uint32][]libvirt.DomainInterface{
+		uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
+			{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv4("169.254.10.10")}},
+		},
+	}
+	cases := []struct {
+		name    string
+		running bool
+		addrs   map[uint32][]libvirt.DomainInterface
+		state   provider.InstanceState
+		ip      string
+	}{
+		{"running with arp address", true, arp, provider.StateRunning, "192.168.1.50"},
+		{"running, agent only, skips loopback and v6", true, agentOnly, provider.StateRunning, "192.168.1.51"},
+		{"running, no address yet", true, nil, provider.StateProvisioning, ""},
+		{"running, only link-local", true, linkLocalOnly, provider.StateProvisioning, ""},
+		{"shut off", false, arp, provider.StateStopped, ""},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			api := readyHost()
+			managedDomain(t, api, testName, testUID, tc.running, tc.addrs)
+			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+			inst, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
+			if err != nil {
+				t.Fatalf("Get: %v", err)
+			}
+			want := provider.Instance{ID: "hosts/nas1/domains/" + testName, IP: tc.ip, State: tc.state, UID: testUID, CreatedAt: testNow}
+			if *inst != want {
+				t.Errorf("instance = %+v, want %+v", *inst, want)
+			}
+		})
+	}
+}
+
+func TestGet_unmanagedDomainHasNoUID(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	api.putDomain(&fakeDomain{name: "hand-made", running: true})
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	inst, err := p.Get(context.Background(), "hosts/nas1/domains/hand-made")
+	if err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	if inst.UID != "" || !inst.CreatedAt.IsZero() {
+		t.Errorf("instance = %+v, want empty UID and zero CreatedAt", inst)
+	}
+}
+
+func TestGet_errors(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		id    string
+		setup func(t *testing.T, api *fakeAPI)
+		class error
+	}{
+		{"missing domain", "hosts/nas1/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrNotFound},
+		{"malformed id", "zones/x/instances/y", func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
+		{"host not configured", "hosts/nas9/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
+		{
+			"state read fails", "hosts/nas1/domains/" + testName,
+			func(t *testing.T, api *fakeAPI) {
+				managedDomain(t, api, testName, testUID, true, nil)
+				api.fail["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
+			},
+			provider.ErrTransient,
+		},
+		{
+			"corrupt metadata", "hosts/nas1/domains/" + testName,
+			func(_ *testing.T, api *fakeAPI) {
+				api.putDomain(&fakeDomain{name: testName, metadata: "<instance", running: true})
+			},
+			provider.ErrPermanent,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			api := readyHost()
+			tc.setup(t, api)
+			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+			_, err := p.Get(context.Background(), tc.id)
+			wantClass(t, err, tc.class)
+		})
+	}
+}
+
+func TestGet_hostUnreachable(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(hostFakes{}, "nas1")
+	_, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
+	wantClass(t, err, provider.ErrTransient)
+}
+
+func assertGone(t *testing.T, api *fakeAPI) {
+	t.Helper()
+	if api.domain(testName) != nil {
+		t.Error("domain still defined")
+	}
+	if names := api.volNames("default"); len(names) != 2 {
+		t.Errorf("pool holds %v, want only the cached base image and its marker", names)
+	}
+}
+
+func TestDelete(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		setup func(t *testing.T, api *fakeAPI)
+		check func(t *testing.T, api *fakeAPI)
+	}{
+		{
+			name:  "running domain",
+			setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, true, nil) },
+			check: func(t *testing.T, api *fakeAPI) {
+				if !api.hasCall("DomainDestroy") || !api.hasCall("DomainUndefineFlags") {
+					t.Errorf("calls = %v, want destroy and undefine", api.calls)
+				}
+			},
+		},
+		{
+			name:  "stopped domain",
+			setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, false, nil) },
+			check: func(t *testing.T, api *fakeAPI) {
+				if !api.hasCall("DomainUndefineFlags") {
+					t.Errorf("calls = %v, want undefine despite destroy being invalid", api.calls)
+				}
+			},
+		},
+		{
+			name: "domain gone, volumes left",
+			setup: func(_ *testing.T, api *fakeAPI) {
+				api.putVol("default", overlayVolumeName(testName), formatQcow2, nil)
+				api.putVol("default", seedVolumeName(testName), formatRaw, nil)
+			},
+			check: func(t *testing.T, api *fakeAPI) {
+				if api.hasCall("DomainDestroy") || api.hasCall("DomainUndefineFlags") {
+					t.Errorf("calls = %v, want no domain operations", api.calls)
+				}
+			},
+		},
+		{
+			name:  "nothing exists",
+			setup: func(*testing.T, *fakeAPI) {},
+			check: func(t *testing.T, api *fakeAPI) {
+				if api.hasCall("StorageVolDelete") {
+					t.Errorf("calls = %v, want no deletes", api.calls)
+				}
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			api := readyHost()
+			tc.setup(t, api)
+			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+			if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
+				t.Fatalf("Delete: %v", err)
+			}
+			assertGone(t, api)
+			tc.check(t, api)
+			if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
+				t.Fatalf("second Delete: %v", err)
+			}
+		})
+	}
+}
+
+func TestDelete_missingPoolIsNotAnError(t *testing.T) {
+	t.Parallel()
+	api := newFakeAPI("other")
+	api.putDomain(&fakeDomain{name: testName, metadata: metadataXML(t, testUID, testNow), running: true})
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
+		t.Fatalf("Delete: %v", err)
+	}
+	if api.domain(testName) != nil {
+		t.Error("domain still defined")
+	}
+}
+
+// A host dropped from the config leaves the operator no way to reach the
+// domain, so Delete has nothing left to clean. Failing instead would pin
+// the Proxy's finalizer forever, since the reconciler's delete path cannot
+// make progress without a Delete that can succeed.
+func TestDelete_unconfiguredHostIsNotAnError(t *testing.T) {
+	t.Parallel()
+	api := readyHost()
+	managedDomain(t, api, testName, testUID, true, nil)
+	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
+
+	if err := p.Delete(context.Background(), "hosts/nas9/domains/"+testName); err != nil {
+		t.Fatalf("Delete over an unconfigured host: %v", err)
+	}
+	if api.domain(testName) == nil {
+		t.Error("domain on the configured host was touched; only the named host's state may change")
+	}
+	// Get still refuses, so the live path cannot mistake a config typo for a
+	// missing VM and recreate it somewhere else.
+	_, err := p.Get(context.Background(), "hosts/nas9/domains/"+testName)
+	wantClass(t, err, provider.ErrPermanent)
+}
+
+func TestDelete_errors(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		id    string
+		fakes func(t *testing.T) hostFakes
+		class error
+	}{
+		{"malformed id", "nope", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
+		{"host unreachable", "hosts/nas1/domains/x", func(*testing.T) hostFakes { return hostFakes{} }, provider.ErrTransient},
+		{
+			"undefine fails", "hosts/nas1/domains/" + testName,
+			func(t *testing.T) hostFakes {
+				api := readyHost()
+				managedDomain(t, api, testName, testUID, true, nil)
+				api.fail["DomainUndefineFlags"] = lvError(libvirt.ErrInternalError, "boom")
+				return hostFakes{"nas1": api}
+			},
+			provider.ErrTransient,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			p := newTestProvider(tc.fakes(t), "nas1")
+			wantClass(t, p.Delete(context.Background(), tc.id), tc.class)
+		})
+	}
+}
+
+func TestListByTag_acrossHosts(t *testing.T) {
+	t.Parallel()
+	nas1, nas2 := readyHost(), readyHost()
+	managedDomain(t, nas1, "proxy-one", "uid-1", true, map[uint32][]libvirt.DomainInterface{
+		uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.10")}}},
+	})
+	nas1.putDomain(&fakeDomain{name: "hand-made", running: true})
+	managedDomain(t, nas2, "proxy-two", "uid-2", false, nil)
+	nas2.putDomain(&fakeDomain{name: "foreign", metadata: `<instance xmlns="` + metadataURI + `"><managed>false</managed></instance>`})
+	p := newTestProvider(hostFakes{"nas1": nas1, "nas2": nas2}, "nas1", "nas2")
+
+	got, err := p.ListByTag(context.Background())
+	if err != nil {
+		t.Fatalf("ListByTag: %v", err)
+	}
+	want := []provider.Instance{
+		{ID: "hosts/nas1/domains/proxy-one", IP: "192.168.1.10", State: provider.StateRunning, UID: "uid-1", CreatedAt: testNow},
+		{ID: "hosts/nas2/domains/proxy-two", State: provider.StateStopped, UID: "uid-2", CreatedAt: testNow},
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d instances %+v, want %d", len(got), got, len(want))
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("instance[%d] = %+v, want %+v", i, got[i], want[i])
+		}
+	}
+}
+
+func TestListByTag_oneHostDownFailsSweep(t *testing.T) {
+	t.Parallel()
+	nas1 := readyHost()
+	managedDomain(t, nas1, "proxy-one", "uid-1", true, nil)
+	p := newTestProvider(hostFakes{"nas1": nas1}, "nas1", "nas2")
+
+	got, err := p.ListByTag(context.Background())
+	wantClass(t, err, provider.ErrTransient)
+	if got != nil {
+		t.Errorf("got %+v alongside the error, want nil (no partial view)", got)
+	}
+}
+
+// A domain that cannot be read is left out of the sweep rather than
+// failing it: GC never deletes what it does not see, so skipping is the
+// safe direction, while one bad domain blocking GC for all others is not.
+func TestListByTag_skipsUnreadableDomain(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name  string
+		setup func(t *testing.T, api *fakeAPI)
+	}{
+		{
+			"deleted mid-sweep",
+			func(t *testing.T, api *fakeAPI) {
+				managedDomain(t, api, "proxy-one", "uid-1", true, nil)
+				api.failOnce["DomainGetMetadata"] = lvError(libvirt.ErrNoDomain, "gone")
+			},
+		},
+		{
+			"corrupt metadata",
+			func(_ *testing.T, api *fakeAPI) {
+				api.putDomain(&fakeDomain{name: "proxy-one", metadata: "<instance", running: true})
+			},
+		},
+		{
+			"state unreadable",
+			func(t *testing.T, api *fakeAPI) {
+				managedDomain(t, api, "proxy-one", "uid-1", true, nil)
+				api.failOnce["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			nas1 := readyHost()
+			tc.setup(t, nas1)
+			managedDomain(t, nas1, "proxy-two", "uid-2", false, nil)
+			p := newTestProvider(hostFakes{"nas1": nas1}, "nas1")
+
+			got, err := p.ListByTag(context.Background())
+			if err != nil {
+				t.Fatalf("ListByTag: %v", err)
+			}
+			if len(got) != 1 || got[0].UID != "uid-2" {
+				t.Errorf("got %+v, want only proxy-two", got)
+			}
+		})
+	}
+}
+
+func TestListByTag_noHosts(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(hostFakes{})
+	got, err := p.ListByTag(context.Background())
+	if err != nil || len(got) != 0 {
+		t.Errorf("got %v, %v; want empty, nil", got, err)
+	}
+}
+
+func TestMapState(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		state libvirt.DomainState
+		ip    string
+		want  provider.InstanceState
+	}{
+		{libvirt.DomainRunning, "10.0.0.1", provider.StateRunning},
+		{libvirt.DomainRunning, "", provider.StateProvisioning},
+		{libvirt.DomainBlocked, "10.0.0.1", provider.StateRunning},
+		{libvirt.DomainPaused, "10.0.0.1", provider.StateStopped},
+		{libvirt.DomainShutdown, "", provider.StateStopped},
+		{libvirt.DomainShutoff, "", provider.StateStopped},
+		{libvirt.DomainCrashed, "", provider.StateStopped},
+		{libvirt.DomainPmsuspended, "", provider.StateStopped},
+		{libvirt.DomainNostate, "", provider.StateProvisioning},
+	}
+	for _, tc := range cases {
+		t.Run(fmt.Sprintf("%s/%q", tc.state, tc.ip), func(t *testing.T) {
+			t.Parallel()
+			if got := mapState(tc.state, tc.ip); got != tc.want {
+				t.Errorf("mapState(%s, %q) = %s, want %s", tc.state, tc.ip, got, tc.want)
+			}
+		})
+	}
+}
+
+// runAllOps drives one full lifecycle over a ready fake host so logging
+// tests see every provider method and every libvirt call.
+func runAllOps(t *testing.T, ctx context.Context, req provider.CreateRequest) {
+	t.Helper()
+	p := newTestProvider(hostFakes{"nas1": readyHost()}, "nas1")
+	id, err := p.Create(ctx, req)
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if _, err := p.Get(ctx, id); err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	if _, err := p.ListByTag(ctx); err != nil {
+		t.Fatalf("ListByTag: %v", err)
+	}
+	if err := p.Delete(ctx, id); err != nil {
+		t.Fatalf("Delete: %v", err)
+	}
+}
+
+func TestLogging_verbosityTiers(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name        string
+		verbosity   int
+		wantLines   []string
+		absentLines []string
+	}{
+		{
+			name:        "v0 stays silent",
+			verbosity:   0,
+			absentLines: []string{"libvirt call", "KVM domain ensured running"},
+		},
+		{
+			name:      "v1 logs one line per libvirt call plus the operation outcomes",
+			verbosity: 1,
+			wantLines: []string{
+				`kvm/libvirt "level"=1 "msg"="libvirt host connected"`,
+				`"msg"="libvirt call"`,
+				`"rpc"="virDomainLookupByName"`,
+				`"libvirtError"="ErrNoDomain"`,
+				`"rpc"="virStorageVolUpload"`,
+				`"rpc"="virDomainDefineXML"`,
+				`"rpc"="virDomainCreate"`,
+				`"rpc"="virDomainInterfaceAddresses"`,
+				`"rpc"="virConnectListAllDomains"`,
+				`"rpc"="virDomainUndefineFlags"`,
+				`"rpc"="virStorageVolDelete"`,
+				`"provider"="kvm-home"`,
+				`"host"="nas1"`,
+				`"name"="` + testName + `"`,
+				`"msg"="KVM domain ensured running"`,
+			},
+			absentLines: []string{`"xml"=`, `"metadata"=`},
+		},
+		{
+			name:      "v2 adds xml, metadata and addresses on the same lines",
+			verbosity: 2,
+			wantLines: []string{`"xml"="<domain`, `"xml"="<volume`, `"metadata"="<instance`, `"names"=[`},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			ctx, snapshot := captureContext(tc.verbosity)
+			req := testRequest()
+			const sentinel = "SENTINEL-cloud-init-must-never-be-logged"
+			req.CloudInit = "#cloud-config\nhostname: " + sentinel + "\n"
+
+			runAllOps(t, ctx, req)
+
+			lines := snapshot()
+			joined := strings.Join(lines, "\n")
+			if tc.verbosity == 0 && len(lines) != 0 {
+				t.Errorf("verbosity 0 logged %d lines:\n%s", len(lines), joined)
+			}
+			for _, want := range tc.wantLines {
+				if !strings.Contains(joined, want) {
+					t.Errorf("output missing %s:\n%s", want, joined)
+				}
+			}
+			for _, absent := range tc.absentLines {
+				if strings.Contains(joined, absent) {
+					t.Errorf("output unexpectedly contains %s:\n%s", absent, joined)
+				}
+			}
+			if strings.Contains(joined, sentinel) {
+				t.Errorf("cloud-init content leaked into logs:\n%s", joined)
+			}
+		})
+	}
+}

internal/provider/kvm/metadata.go

@@ -0,0 +1,61 @@
+package kvm
+
+import (
+	"encoding/xml"
+	"fmt"
+	"time"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// metadataURI namespaces the operator's element inside the domain's
+// <metadata>. libvirt has no labels or tags, but it stores any namespaced XML
+// there and hands it back through DomainGetMetadata, which is how the GC
+// contract (managed flag + owning UID) and the creation time survive across
+// operator restarts.
+const metadataURI = "https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm"
+
+type instanceMetadata struct {
+	XMLName   xml.Name `xml:"https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm instance"`
+	Managed   string   `xml:"managed"`
+	UID       string   `xml:"uid"`
+	CreatedAt string   `xml:"createdAt"`
+}
+
+func newInstanceMetadata(uid string, createdAt time.Time) instanceMetadata {
+	return instanceMetadata{
+		Managed:   provider.LabelManagedYes,
+		UID:       uid,
+		CreatedAt: createdAt.UTC().Format(time.RFC3339),
+	}
+}
+
+// parseInstanceMetadata decodes the element as DomainGetMetadata returns it.
+// libvirt selects the element by namespace URI but serialises it without the
+// declaration (observed on libvirt 11.6: a bare <instance>), and other
+// versions may keep a default or prefixed form. The URI is already checked
+// by the lookup, so only the local name is matched here.
+func parseInstanceMetadata(s string) (instanceMetadata, error) {
+	var raw struct {
+		XMLName xml.Name `xml:"instance"`
+		instanceMetadata
+	}
+	if err := xml.Unmarshal([]byte(s), &raw); err != nil {
+		return instanceMetadata{}, fmt.Errorf("parsing domain metadata: %w", err)
+	}
+	return raw.instanceMetadata, nil
+}
+
+func (m instanceMetadata) managed() bool {
+	return m.Managed == provider.LabelManagedYes
+}
+
+// createdAt returns the zero time on a malformed timestamp: orphan GC treats
+// zero as "old", so a corrupt value can never protect a domain forever.
+func (m instanceMetadata) createdAt() time.Time {
+	t, err := time.Parse(time.RFC3339, m.CreatedAt)
+	if err != nil {
+		return time.Time{}
+	}
+	return t
+}

internal/provider/kvm/metadata_test.go

@@ -0,0 +1,84 @@
+package kvm
+
+import (
+	"encoding/xml"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestInstanceMetadata_roundTrip(t *testing.T) {
+	t.Parallel()
+	created := time.Date(2026, 9, 4, 12, 30, 0, 0, time.FixedZone("CEST", 2*3600))
+	raw, err := xml.Marshal(newInstanceMetadata("uid-1", created))
+	if err != nil {
+		t.Fatalf("Marshal() error = %v", err)
+	}
+	if !strings.Contains(string(raw), `xmlns="`+metadataURI+`"`) {
+		t.Errorf("marshalled metadata lacks the namespace declaration: %s", raw)
+	}
+	m, err := parseInstanceMetadata(string(raw))
+	if err != nil {
+		t.Fatalf("parseInstanceMetadata() error = %v", err)
+	}
+	if !m.managed() || m.UID != "uid-1" {
+		t.Errorf("parsed = %+v, want managed with uid-1", m)
+	}
+	if got := m.createdAt(); !got.Equal(created) || got.Location() != time.UTC {
+		t.Errorf("createdAt() = %v, want %v in UTC", got, created)
+	}
+}
+
+func TestParseInstanceMetadata(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name        string
+		in          string
+		wantErr     bool
+		wantManaged bool
+		wantUID     string
+		wantCreated time.Time
+	}{
+		{
+			// libvirt re-serialises the stored element with whatever prefix it
+			// chooses; only the namespace URI is contractual.
+			name: "libvirt prefixed form",
+			in: `<op:instance xmlns:op="` + metadataURI + `">` +
+				`<op:managed>true</op:managed><op:uid>uid-2</op:uid>` +
+				`<op:createdAt>2026-09-04T10:00:00Z</op:createdAt></op:instance>`,
+			wantManaged: true, wantUID: "uid-2", wantCreated: time.Date(2026, 9, 4, 10, 0, 0, 0, time.UTC),
+		},
+		{
+			// What libvirt 11.6 actually returns from DomainGetMetadata: the
+			// element selected by our URI, serialised without any namespace
+			// declaration.
+			name: "libvirt bare form",
+			in: `<instance><managed>true</managed><uid>uid-3</uid>` +
+				`<createdAt>2026-09-10T07:30:00Z</createdAt></instance>`,
+			wantManaged: true, wantUID: "uid-3", wantCreated: time.Date(2026, 9, 10, 7, 30, 0, 0, time.UTC),
+		},
+		{name: "managed false", in: `<instance xmlns="` + metadataURI + `"><managed>false</managed><uid>u</uid></instance>`, wantUID: "u"},
+		{name: "foreign element", in: `<foo xmlns="urn:other"><managed>true</managed></foo>`, wantErr: true},
+		{name: "garbage", in: `not xml`, wantErr: true},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			m, err := parseInstanceMetadata(tc.in)
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("error = %v, wantErr %v", err, tc.wantErr)
+			}
+			if m.managed() != tc.wantManaged || m.UID != tc.wantUID || !m.createdAt().Equal(tc.wantCreated) {
+				t.Errorf("parsed = %+v, want managed=%v uid=%q createdAt=%v", m, tc.wantManaged, tc.wantUID, tc.wantCreated)
+			}
+		})
+	}
+}
+
+func TestInstanceMetadata_createdAtMalformedIsZero(t *testing.T) {
+	t.Parallel()
+	m := instanceMetadata{CreatedAt: "yesterday"}
+	if !m.createdAt().IsZero() {
+		t.Errorf("createdAt() = %v, want zero time", m.createdAt())
+	}
+}

internal/provider/kvm/seed.go

@@ -0,0 +1,133 @@
+package kvm
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/textproto"
+	"strings"
+
+	"github.com/kdomanski/iso9660"
+)
+
+// seedLabel is the volume label cloud-init's NoCloud datasource looks for.
+const seedLabel = "cidata"
+
+// guestAgentScript installs and starts the QEMU guest agent, which is how
+// the provider learns a VM's address before the host's ARP table has seen
+// any traffic from it.
+//
+// It ships as vendor-data, and as a *script* rather than a cloud-config,
+// because that is the only shape cloud-init cannot drop. Vendor-data
+// cloud-config is merged under the Proxy's own user-data with mergers that
+// keep the user-data's value for every key both documents define and never
+// append lists, so a Proxy carrying its own packages: silently lost the
+// agent. The strategy cannot be overridden from either document: merge_how
+// in user-data governs only the first source, and merge_how in vendor-data
+// is consumed and discarded before the document is stored. A vendor-data
+// script bypasses all of it — cloud-init gives vendor-data its own handler
+// chain, writes scripts to a separate directory and runs them from
+// cc_scripts_vendor, after the user's packages are installed and before the
+// user's runcmd.
+const guestAgentScript = `#!/bin/sh
+# Added by egress-proxies-operator so the hypervisor can report this VM's
+# address. Set installGuestAgent: false on the provider to leave it out.
+set -e
+if ! command -v qemu-ga >/dev/null 2>&1 && [ ! -x /usr/sbin/qemu-ga ]; then
+	if command -v apt-get >/dev/null 2>&1; then
+		apt-get update
+		DEBIAN_FRONTEND=noninteractive apt-get install -y qemu-guest-agent
+	elif command -v dnf >/dev/null 2>&1; then
+		dnf install -y qemu-guest-agent
+	elif command -v yum >/dev/null 2>&1; then
+		yum install -y qemu-guest-agent
+	elif command -v zypper >/dev/null 2>&1; then
+		zypper --non-interactive install qemu-guest-agent
+	elif command -v apk >/dev/null 2>&1; then
+		apk add qemu-guest-agent
+	else
+		echo "no known package manager; install qemu-guest-agent in the image" >&2
+		exit 1
+	fi
+fi
+if command -v systemctl >/dev/null 2>&1; then
+	systemctl enable --now qemu-guest-agent
+elif command -v rc-update >/dev/null 2>&1; then
+	rc-update add qemu-guest-agent default
+	rc-service qemu-guest-agent start
+fi
+`
+
+// buildVendorData renders the seed's vendor-data. With nothing configured
+// it is the guest-agent script alone; with both, a MIME multipart carrying
+// the configured document and the script as separate parts, since
+// vendor-data is a single file and the two must not be concatenated.
+func buildVendorData(configured string, installAgent bool) (string, error) {
+	switch {
+	case !installAgent:
+		return configured, nil
+	case strings.TrimSpace(configured) == "":
+		return guestAgentScript, nil
+	default:
+		return multipartVendorData(configured, guestAgentScript)
+	}
+}
+
+// multipartVendorData packs several documents into one vendor-data blob.
+// cloud-init decides a payload is multipart by finding "mime-version:"
+// within its first 4096 bytes, so those headers lead. The configured
+// document goes in as text/plain on purpose: cloud-init re-sniffs that type
+// from the content's own header line, which keeps this code out of the
+// business of classifying what the user wrote.
+func multipartVendorData(configured, script string) (string, error) {
+	var body bytes.Buffer
+	w := multipart.NewWriter(&body)
+	parts := []struct{ contentType, content string }{
+		{"text/plain", configured},
+		{"text/x-shellscript", script},
+	}
+	for _, p := range parts {
+		part, err := w.CreatePart(textproto.MIMEHeader{"Content-Type": {p.contentType}})
+		if err != nil {
+			return "", fmt.Errorf("building vendor-data part: %w", err)
+		}
+		if _, err := io.WriteString(part, p.content); err != nil {
+			return "", fmt.Errorf("writing vendor-data part: %w", err)
+		}
+	}
+	if err := w.Close(); err != nil {
+		return "", fmt.Errorf("closing vendor-data: %w", err)
+	}
+	return fmt.Sprintf("MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=%q\r\n\r\n%s",
+		w.Boundary(), body.String()), nil
+}
+
+// buildSeedISO renders a NoCloud seed: user-data and vendor-data verbatim,
+// and a meta-data whose instance-id is the deterministic VM name, so
+// cloud-init never mistakes a reboot for a new instance. The ISO's volume
+// descriptor carries the build time, so two builds differ in a few
+// timestamp bytes; the files and label are identical.
+func buildSeedISO(name, userData, vendorData string) ([]byte, error) {
+	w, err := iso9660.NewWriter()
+	if err != nil {
+		return nil, fmt.Errorf("creating iso writer: %w", err)
+	}
+	defer w.Cleanup() //nolint:errcheck // temp-dir cleanup; nothing to do on failure
+
+	files := []struct{ name, body string }{
+		{"user-data", userData},
+		{"meta-data", fmt.Sprintf("instance-id: %s\nlocal-hostname: %s\n", name, name)},
+		{"vendor-data", vendorData},
+	}
+	for _, f := range files {
+		if err := w.AddFile(strings.NewReader(f.body), f.name); err != nil {
+			return nil, fmt.Errorf("adding %s to seed iso: %w", f.name, err)
+		}
+	}
+	var buf bytes.Buffer
+	if err := w.WriteTo(&buf, seedLabel); err != nil {
+		return nil, fmt.Errorf("writing seed iso: %w", err)
+	}
+	return buf.Bytes(), nil
+}

internal/provider/kvm/seed_test.go

@@ -0,0 +1,233 @@
+package kvm
+
+import (
+	"bytes"
+	"io"
+	"maps"
+	"mime"
+	"mime/multipart"
+	"strings"
+	"testing"
+
+	"github.com/kdomanski/iso9660"
+)
+
+func readSeed(t *testing.T, iso []byte) (label string, files map[string]string) {
+	t.Helper()
+	img, err := iso9660.OpenImage(bytes.NewReader(iso))
+	if err != nil {
+		t.Fatalf("OpenImage() error = %v", err)
+	}
+	label, err = img.Label()
+	if err != nil {
+		t.Fatalf("Label() error = %v", err)
+	}
+	root, err := img.RootDir()
+	if err != nil {
+		t.Fatalf("RootDir() error = %v", err)
+	}
+	children, err := root.GetChildren()
+	if err != nil {
+		t.Fatalf("GetChildren() error = %v", err)
+	}
+	files = make(map[string]string, len(children))
+	for _, c := range children {
+		body, err := io.ReadAll(c.Reader())
+		if err != nil {
+			t.Fatalf("reading %s: %v", c.Name(), err)
+		}
+		files[c.Name()] = string(body)
+	}
+	return label, files
+}
+
+func TestBuildSeedISO_contents(t *testing.T) {
+	t.Parallel()
+	userData := "#cloud-config\npackages: [squid]\n"
+	iso, err := buildSeedISO("proxy-abc123def456ghij", userData, guestAgentScript)
+	if err != nil {
+		t.Fatalf("buildSeedISO() error = %v", err)
+	}
+	label, files := readSeed(t, iso)
+	if !strings.EqualFold(label, seedLabel) {
+		t.Errorf("label = %q, want %q", label, seedLabel)
+	}
+	if len(files) != 3 {
+		t.Errorf("files = %v, want exactly user-data, meta-data, vendor-data", files)
+	}
+	if files["user-data"] != userData {
+		t.Errorf("user-data = %q, want verbatim cloud-init", files["user-data"])
+	}
+	if files["vendor-data"] != guestAgentScript {
+		t.Errorf("vendor-data = %q", files["vendor-data"])
+	}
+	if got, want := files["meta-data"], "instance-id: proxy-abc123def456ghij\nlocal-hostname: proxy-abc123def456ghij\n"; got != want {
+		t.Errorf("meta-data = %q, want %q", got, want)
+	}
+}
+
+// The writer stamps the build time into the volume descriptor, so two
+// builds are not byte-identical; what must not change between them is
+// what cloud-init reads: the label and the three files.
+func TestBuildSeedISO_deterministicContent(t *testing.T) {
+	t.Parallel()
+	a, err := buildSeedISO("proxy-x", "#cloud-config\n", "")
+	if err != nil {
+		t.Fatalf("first buildSeedISO() error = %v", err)
+	}
+	b, err := buildSeedISO("proxy-x", "#cloud-config\n", "")
+	if err != nil {
+		t.Fatalf("second buildSeedISO() error = %v", err)
+	}
+	labelA, filesA := readSeed(t, a)
+	labelB, filesB := readSeed(t, b)
+	if labelA != labelB || !maps.Equal(filesA, filesB) {
+		t.Errorf("two builds for the same inputs differ: %q %v vs %q %v", labelA, filesA, labelB, filesB)
+	}
+	if len(a) != len(b) {
+		t.Errorf("sizes differ: %d vs %d; the seed volume is created with the ISO's exact size", len(a), len(b))
+	}
+}
+
+func TestBuildSeedISO_emptyUserData(t *testing.T) {
+	t.Parallel()
+	iso, err := buildSeedISO("proxy-x", "", guestAgentScript)
+	if err != nil {
+		t.Fatalf("buildSeedISO() error = %v", err)
+	}
+	_, files := readSeed(t, iso)
+	if body, ok := files["user-data"]; !ok || body != "" {
+		t.Errorf("user-data = %q (present=%v), want an empty file to still exist", body, ok)
+	}
+}
+
+// The whole point of shipping the agent as vendor-data: whatever the Proxy
+// wrote reaches the guest untouched, in any of cloud-init's formats. The
+// previous implementation parsed and re-rendered cloud-config user-data,
+// which dropped comments, reordered keys and re-typed unquoted scalars
+// (version: 1.10 arrived as 1.1).
+func TestBuildSeedISO_userDataIsNeverRewritten(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name     string
+		userData string
+	}{
+		{"cloud-config with its own packages", "#cloud-config\n# keep me\npackages:\n  - squid\nversion: 1.10\nchpasswd: {list: 'u:0123456789'}\n"},
+		{"jinja template", "## template: jinja\n#cloud-config\nhostname: {{ v1.local_hostname }}\n"},
+		{"shell script", "#!/bin/sh\napt-get install -y squid\n"},
+		{"include", "#include\nhttps://example.invalid/x\n"},
+		{"mime multipart", "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"x\"\r\n\r\n--x\r\nContent-Type: text/cloud-config\r\n\r\n#cloud-config\npackages: [squid]\n\r\n--x--\r\n"},
+		{"gzip payload", "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03binary\x00bytes"},
+		{"leading blank line", "\n\n#cloud-config\npackages: [squid]\n"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			vendor, err := buildVendorData("", true)
+			if err != nil {
+				t.Fatalf("buildVendorData() error = %v", err)
+			}
+			iso, err := buildSeedISO("proxy-x", tc.userData, vendor)
+			if err != nil {
+				t.Fatalf("buildSeedISO() error = %v", err)
+			}
+			_, files := readSeed(t, iso)
+			if files["user-data"] != tc.userData {
+				t.Errorf("user-data was rewritten:\n got %q\nwant %q", files["user-data"], tc.userData)
+			}
+			if !strings.Contains(files["vendor-data"], "qemu-guest-agent") {
+				t.Errorf("vendor-data = %q, want the agent script", files["vendor-data"])
+			}
+		})
+	}
+}
+
+func TestBuildVendorData_agentScriptAlone(t *testing.T) {
+	t.Parallel()
+	got, err := buildVendorData("", true)
+	if err != nil {
+		t.Fatalf("buildVendorData() error = %v", err)
+	}
+	if got != guestAgentScript {
+		t.Errorf("vendor-data = %q, want the bare script", got)
+	}
+	if !strings.HasPrefix(got, "#!") {
+		t.Error("vendor-data must start with a shebang or cloud-init treats it as something other than a script")
+	}
+}
+
+func TestBuildVendorData_optOutKeepsConfiguredOnly(t *testing.T) {
+	t.Parallel()
+	configured := "#cloud-config\nntp: {enabled: true}\n"
+	tests := []struct {
+		name       string
+		configured string
+		want       string
+	}{
+		{"nothing configured", "", ""},
+		{"configured document passes through", configured, configured},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := buildVendorData(tc.configured, false)
+			if err != nil {
+				t.Fatalf("buildVendorData() error = %v", err)
+			}
+			if got != tc.want {
+				t.Errorf("vendor-data = %q, want %q", got, tc.want)
+			}
+			if strings.Contains(got, "qemu-guest-agent") {
+				t.Error("agent script present although installGuestAgent is off")
+			}
+		})
+	}
+}
+
+// Both documents have to survive in one file, and cloud-init only looks for
+// the multipart marker in the first 4096 bytes.
+func TestBuildVendorData_multipartCarriesBoth(t *testing.T) {
+	t.Parallel()
+	configured := "#cloud-config\nntp:\n  enabled: true\n"
+	got, err := buildVendorData(configured, true)
+	if err != nil {
+		t.Fatalf("buildVendorData() error = %v", err)
+	}
+	head := got
+	if len(head) > 4096 {
+		head = head[:4096]
+	}
+	if !strings.Contains(strings.ToLower(head), "mime-version:") {
+		t.Fatalf("no mime-version: in the first 4096 bytes; cloud-init would treat this as a single document:\n%s", head)
+	}
+	_, params, err := mime.ParseMediaType(strings.SplitN(got, "\r\n\r\n", 2)[0][strings.Index(got, "Content-Type:")+len("Content-Type: "):])
+	if err != nil {
+		t.Fatalf("parsing the multipart content type: %v", err)
+	}
+	body := strings.SplitN(got, "\r\n\r\n", 2)[1]
+	r := multipart.NewReader(strings.NewReader(body), params["boundary"])
+	var bodies, types []string
+	for {
+		part, err := r.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("reading part: %v", err)
+		}
+		content, err := io.ReadAll(part)
+		if err != nil {
+			t.Fatalf("reading part body: %v", err)
+		}
+		bodies = append(bodies, string(content))
+		types = append(types, part.Header.Get("Content-Type"))
+	}
+	if len(bodies) != 2 || bodies[0] != configured || bodies[1] != guestAgentScript {
+		t.Fatalf("parts = %q, want the configured document then the script", bodies)
+	}
+	// text/plain makes cloud-init re-sniff the user's document from its own
+	// header line, so this code never has to classify it.
+	if types[0] != "text/plain" || types[1] != "text/x-shellscript" {
+		t.Errorf("part types = %v, want [text/plain text/x-shellscript]", types)
+	}
+}

internal/provider/kvm/trace.go

@@ -0,0 +1,232 @@
+package kvm
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+)
+
+// tracingAPI wraps a libvirtAPI and logs one V(1) line per call after it
+// returns: the libvirt API function (rpc=virDomainLookupByName, ...), the
+// identifiers it was given, curated results, duration, and on failure the
+// error with libvirt's code name. At V(2) the same line also carries XML
+// bodies, metadata and address lists. Upload and download bodies are never
+// logged: the seed ISO carries cloud-init.
+type tracingAPI struct {
+	inner libvirtAPI
+	log   logr.Logger
+}
+
+type tracedCall struct {
+	t      *tracingAPI
+	method string
+	start  time.Time
+	kv     []any
+}
+
+// begin names the call by its libvirt API function: go-libvirt's method
+// names are the remote procedure names, which are the public vir* API
+// names without the prefix.
+func (t *tracingAPI) begin(method string, kv ...any) *tracedCall {
+	return &tracedCall{t: t, method: "vir" + method, start: time.Now(), kv: kv}
+}
+
+// detail adds key/values that are only worth the bytes at V(2).
+func (c *tracedCall) detail(kv ...any) *tracedCall {
+	if c.t.log.V(2).Enabled() {
+		c.kv = append(c.kv, kv...)
+	}
+	return c
+}
+
+func (c *tracedCall) done(err error, kv ...any) {
+	fields := make([]any, 0, 2+len(c.kv)+len(kv)+10)
+	fields = append(fields, "rpc", c.method)
+	fields = append(fields, c.kv...)
+	fields = append(fields, kv...)
+	fields = append(fields, "duration", time.Since(c.start).Round(time.Microsecond))
+	if err != nil {
+		fields = append(fields, "error", err.Error())
+		fields = append(fields, libvirtErrorFields(err)...)
+	}
+	c.t.log.V(1).Info("libvirt call", fields...)
+}
+
+// flagKV renders a flags value only when something is set; go-libvirt's
+// generated stringers do not know bitmask combinations.
+func flagKV(v int64) []any {
+	if v == 0 {
+		return nil
+	}
+	return []any{"flags", v}
+}
+
+func optString(s libvirt.OptString) string {
+	if len(s) == 0 {
+		return ""
+	}
+	return s[0]
+}
+
+func (t *tracingAPI) DomainLookupByName(name string) (libvirt.Domain, error) {
+	c := t.begin("DomainLookupByName", "domain", name)
+	d, err := t.inner.DomainLookupByName(name)
+	c.done(err)
+	return d, err
+}
+
+func (t *tracingAPI) DomainDefineXML(xml string) (libvirt.Domain, error) {
+	c := t.begin("DomainDefineXML", "xmlBytes", len(xml)).detail("xml", xml)
+	d, err := t.inner.DomainDefineXML(xml)
+	c.done(err, "domain", d.Name)
+	return d, err
+}
+
+func (t *tracingAPI) DomainCreate(d libvirt.Domain) error {
+	c := t.begin("DomainCreate", "domain", d.Name)
+	err := t.inner.DomainCreate(d)
+	c.done(err)
+	return err
+}
+
+func (t *tracingAPI) DomainDestroy(d libvirt.Domain) error {
+	c := t.begin("DomainDestroy", "domain", d.Name)
+	err := t.inner.DomainDestroy(d)
+	c.done(err)
+	return err
+}
+
+func (t *tracingAPI) DomainUndefineFlags(d libvirt.Domain, flags libvirt.DomainUndefineFlagsValues) error {
+	c := t.begin("DomainUndefineFlags", append([]any{"domain", d.Name}, flagKV(int64(flags))...)...)
+	err := t.inner.DomainUndefineFlags(d, flags)
+	c.done(err)
+	return err
+}
+
+func (t *tracingAPI) DomainGetState(d libvirt.Domain, flags uint32) (int32, int32, error) {
+	c := t.begin("DomainGetState", "domain", d.Name)
+	state, reason, err := t.inner.DomainGetState(d, flags)
+	c.done(err, "state", libvirt.DomainState(state), "reason", reason)
+	return state, reason, err
+}
+
+func (t *tracingAPI) DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, flags libvirt.DomainModificationImpact) (string, error) {
+	c := t.begin("DomainGetMetadata", "domain", d.Name, "type", typ, "uri", optString(uri))
+	s, err := t.inner.DomainGetMetadata(d, typ, uri, flags)
+	c.detail("metadata", s).done(err, "metadataBytes", len(s))
+	return s, err
+}
+
+func (t *tracingAPI) DomainInterfaceAddresses(d libvirt.Domain, source uint32, flags uint32) ([]libvirt.DomainInterface, error) {
+	c := t.begin("DomainInterfaceAddresses", "domain", d.Name, "source", libvirt.DomainInterfaceAddressesSource(source))
+	ifaces, err := t.inner.DomainInterfaceAddresses(d, source, flags)
+	if t.log.V(2).Enabled() {
+		c.detail("addresses", formatAddresses(ifaces))
+	}
+	c.done(err, "interfaces", len(ifaces))
+	return ifaces, err
+}
+
+func formatAddresses(ifaces []libvirt.DomainInterface) []string {
+	var out []string
+	for _, iface := range ifaces {
+		for _, a := range iface.Addrs {
+			out = append(out, fmt.Sprintf("%s=%s/%d", iface.Name, a.Addr, a.Prefix))
+		}
+	}
+	return out
+}
+
+func (t *tracingAPI) ConnectListAllDomains(needResults int32, flags libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error) {
+	c := t.begin("ConnectListAllDomains", flagKV(int64(flags))...)
+	doms, n, err := t.inner.ConnectListAllDomains(needResults, flags)
+	if t.log.V(2).Enabled() {
+		names := make([]string, 0, len(doms))
+		for _, d := range doms {
+			names = append(names, d.Name)
+		}
+		c.detail("names", names)
+	}
+	c.done(err, "count", len(doms))
+	return doms, n, err
+}
+
+func (t *tracingAPI) StoragePoolLookupByName(name string) (libvirt.StoragePool, error) {
+	c := t.begin("StoragePoolLookupByName", "pool", name)
+	pool, err := t.inner.StoragePoolLookupByName(name)
+	c.done(err)
+	return pool, err
+}
+
+func (t *tracingAPI) StoragePoolRefresh(pool libvirt.StoragePool, flags uint32) error {
+	c := t.begin("StoragePoolRefresh", "pool", pool.Name)
+	err := t.inner.StoragePoolRefresh(pool, flags)
+	c.done(err)
+	return err
+}
+
+func (t *tracingAPI) StorageVolLookupByName(pool libvirt.StoragePool, name string) (libvirt.StorageVol, error) {
+	c := t.begin("StorageVolLookupByName", "pool", pool.Name, "volume", name)
+	vol, err := t.inner.StorageVolLookupByName(pool, name)
+	c.done(err)
+	return vol, err
+}
+
+func (t *tracingAPI) StorageVolGetPath(vol libvirt.StorageVol) (string, error) {
+	c := t.begin("StorageVolGetPath", "pool", vol.Pool, "volume", vol.Name)
+	path, err := t.inner.StorageVolGetPath(vol)
+	c.done(err, "path", path)
+	return path, err
+}
+
+func (t *tracingAPI) StorageVolCreateXML(pool libvirt.StoragePool, xml string, flags libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error) {
+	c := t.begin("StorageVolCreateXML", append([]any{"pool", pool.Name, "xmlBytes", len(xml)}, flagKV(int64(flags))...)...).detail("xml", xml)
+	vol, err := t.inner.StorageVolCreateXML(pool, xml, flags)
+	c.done(err, "volume", vol.Name)
+	return vol, err
+}
+
+func (t *tracingAPI) StorageVolUpload(vol libvirt.StorageVol, r io.Reader, offset, length uint64, flags libvirt.StorageVolUploadFlags) error {
+	c := t.begin("StorageVolUpload", append([]any{"pool", vol.Pool, "volume", vol.Name, "offset", offset, "length", length}, flagKV(int64(flags))...)...)
+	err := t.inner.StorageVolUpload(vol, r, offset, length, flags)
+	c.done(err)
+	return err
+}
+
+func (t *tracingAPI) StorageVolDownload(vol libvirt.StorageVol, w io.Writer, offset, length uint64, flags libvirt.StorageVolDownloadFlags) error {
+	c := t.begin("StorageVolDownload", append([]any{"pool", vol.Pool, "volume", vol.Name, "offset", offset, "length", length}, flagKV(int64(flags))...)...)
+	err := t.inner.StorageVolDownload(vol, w, offset, length, flags)
+	c.done(err)
+	return err
+}
+
+func (t *tracingAPI) StorageVolDelete(vol libvirt.StorageVol, flags libvirt.StorageVolDeleteFlags) error {
+	c := t.begin("StorageVolDelete", append([]any{"pool", vol.Pool, "volume", vol.Name}, flagKV(int64(flags))...)...)
+	err := t.inner.StorageVolDelete(vol, flags)
+	c.done(err)
+	return err
+}
+
+func (t *tracingAPI) ConnectGetLibVersion() (uint64, error) {
+	c := t.begin("ConnectGetLibVersion")
+	v, err := t.inner.ConnectGetLibVersion()
+	c.done(err, "version", formatLibVersion(v))
+	return v, err
+}
+
+func (t *tracingAPI) ConnectGetHostname() (string, error) {
+	c := t.begin("ConnectGetHostname")
+	h, err := t.inner.ConnectGetHostname()
+	c.done(err, "hostname", h)
+	return h, err
+}
+
+func (t *tracingAPI) StoragePoolGetInfo(pool libvirt.StoragePool) (uint8, uint64, uint64, uint64, error) {
+	c := t.begin("StoragePoolGetInfo", "pool", pool.Name)
+	state, capacity, allocation, available, err := t.inner.StoragePoolGetInfo(pool)
+	c.done(err, "state", poolStateName(libvirt.StoragePoolState(state)), "capacity", capacity, "available", available)
+	return state, capacity, allocation, available, err
+}

internal/provider/kvm/trace_test.go

@@ -0,0 +1,303 @@
+package kvm
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"reflect"
+	"strings"
+	"sync"
+	"testing"
+
+	libvirt "github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+	"github.com/go-logr/logr/funcr"
+)
+
+// captureLogger returns a funcr logger capped at verbosity (the test
+// stand-in for --zap-log-level=<verbosity>) and a snapshot function for the
+// lines it recorded. The sink is mutex-guarded: a cancelled call logs from
+// the abandoned goroutine.
+func captureLogger(verbosity int) (logr.Logger, func() []string) {
+	var mu sync.Mutex
+	var lines []string
+	log := funcr.New(func(prefix, args string) {
+		mu.Lock()
+		defer mu.Unlock()
+		lines = append(lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: verbosity})
+	return log, func() []string {
+		mu.Lock()
+		defer mu.Unlock()
+		return append([]string(nil), lines...)
+	}
+}
+
+func captureContext(verbosity int) (context.Context, func() []string) {
+	log, snapshot := captureLogger(verbosity)
+	return logr.NewContext(context.Background(), log), snapshot
+}
+
+const stoppedName = "proxy-stopped"
+
+// tracedHost is a fake with one managed running domain and its volumes, so
+// every method of the seam has something to act on.
+func tracedHost(t *testing.T) *fakeAPI {
+	t.Helper()
+	api := readyHost()
+	managedDomain(t, api, testName, testUID, true, map[uint32][]libvirt.DomainInterface{
+		uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Name: "vnet0", Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.50")}}},
+	})
+	api.putDomain(&fakeDomain{name: stoppedName})
+	return api
+}
+
+func TestTracingAPI_logsEveryMethod(t *testing.T) {
+	t.Parallel()
+	pool := libvirt.StoragePool{Name: "default"}
+	dom := libvirt.Domain{Name: testName}
+	overlay := libvirt.StorageVol{Pool: "default", Name: overlayVolumeName(testName)}
+	cases := []struct {
+		method string
+		call   func(api libvirtAPI) error
+	}{
+		{"DomainLookupByName", func(api libvirtAPI) error { _, err := api.DomainLookupByName(testName); return err }},
+		{"DomainDefineXML", func(api libvirtAPI) error {
+			xml, err := buildDomainXML(testDomainParams())
+			if err != nil {
+				return err
+			}
+			_, err = api.DomainDefineXML(xml)
+			return err
+		}},
+		{"DomainCreate", func(api libvirtAPI) error { return api.DomainCreate(libvirt.Domain{Name: stoppedName}) }},
+		{"DomainDestroy", func(api libvirtAPI) error { return api.DomainDestroy(dom) }},
+		{"DomainUndefineFlags", func(api libvirtAPI) error { return api.DomainUndefineFlags(dom, undefineFlags) }},
+		{"DomainGetState", func(api libvirtAPI) error { _, _, err := api.DomainGetState(dom, 0); return err }},
+		{"DomainGetMetadata", func(api libvirtAPI) error {
+			_, err := api.DomainGetMetadata(dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, 0)
+			return err
+		}},
+		{"DomainInterfaceAddresses", func(api libvirtAPI) error {
+			_, err := api.DomainInterfaceAddresses(dom, uint32(libvirt.DomainInterfaceAddressesSrcArp), 0)
+			return err
+		}},
+		{"ConnectListAllDomains", func(api libvirtAPI) error { _, _, err := api.ConnectListAllDomains(1, 0); return err }},
+		{"StoragePoolLookupByName", func(api libvirtAPI) error { _, err := api.StoragePoolLookupByName("default"); return err }},
+		{"StoragePoolRefresh", func(api libvirtAPI) error { return api.StoragePoolRefresh(pool, 0) }},
+		{"StorageVolLookupByName", func(api libvirtAPI) error { _, err := api.StorageVolLookupByName(pool, overlay.Name); return err }},
+		{"StorageVolGetPath", func(api libvirtAPI) error { _, err := api.StorageVolGetPath(overlay); return err }},
+		{"StorageVolCreateXML", func(api libvirtAPI) error {
+			xml, err := buildVolumeXML(rawVolume("new.iso", 3))
+			if err != nil {
+				return err
+			}
+			_, err = api.StorageVolCreateXML(pool, xml, 0)
+			return err
+		}},
+		{"StorageVolUpload", func(api libvirtAPI) error {
+			return api.StorageVolUpload(overlay, strings.NewReader("abc"), 0, 3, 0)
+		}},
+		{"StorageVolDownload", func(api libvirtAPI) error {
+			return api.StorageVolDownload(overlay, io.Discard, 0, 0, 0)
+		}},
+		{"StorageVolDelete", func(api libvirtAPI) error { return api.StorageVolDelete(overlay, 0) }},
+		{"ConnectGetLibVersion", func(api libvirtAPI) error { _, err := api.ConnectGetLibVersion(); return err }},
+		{"ConnectGetHostname", func(api libvirtAPI) error { _, err := api.ConnectGetHostname(); return err }},
+		{"StoragePoolGetInfo", func(api libvirtAPI) error { _, _, _, _, err := api.StoragePoolGetInfo(pool); return err }},
+	}
+	if n := reflect.TypeOf((*libvirtAPI)(nil)).Elem().NumMethod(); n != len(cases) {
+		t.Fatalf("libvirtAPI has %d methods, this test covers %d; trace the new one", n, len(cases))
+	}
+	for _, tc := range cases {
+		t.Run(tc.method, func(t *testing.T) {
+			t.Parallel()
+			log, snapshot := captureLogger(1)
+			api := &tracingAPI{inner: tracedHost(t), log: log}
+			if err := tc.call(api); err != nil {
+				t.Fatalf("%s: %v", tc.method, err)
+			}
+			lines := snapshot()
+			if len(lines) != 1 {
+				t.Fatalf("logged %d lines, want exactly 1:\n%s", len(lines), strings.Join(lines, "\n"))
+			}
+			for _, want := range []string{`"msg"="libvirt call"`, `"rpc"="vir` + tc.method + `"`, `"duration"=`} {
+				if !strings.Contains(lines[0], want) {
+					t.Errorf("line missing %s:\n%s", want, lines[0])
+				}
+			}
+			if strings.Contains(lines[0], `"error"=`) {
+				t.Errorf("successful call logged an error:\n%s", lines[0])
+			}
+		})
+	}
+}
+
+func TestTracingAPI_passesResultsThrough(t *testing.T) {
+	t.Parallel()
+	inner := tracedHost(t)
+	log, _ := captureLogger(1)
+	traced := &tracingAPI{inner: inner, log: log}
+	dom := libvirt.Domain{Name: testName}
+
+	wantState, wantReason, err := inner.DomainGetState(dom, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	state, reason, err := traced.DomainGetState(dom, 0)
+	if err != nil || state != wantState || reason != wantReason {
+		t.Errorf("DomainGetState = %d/%d/%v, want %d/%d/nil", state, reason, err, wantState, wantReason)
+	}
+
+	overlay := libvirt.StorageVol{Pool: "default", Name: overlayVolumeName(testName)}
+	wantPath, _ := inner.StorageVolGetPath(overlay)
+	if path, err := traced.StorageVolGetPath(overlay); err != nil || path != wantPath {
+		t.Errorf("StorageVolGetPath = %q/%v, want %q", path, err, wantPath)
+	}
+
+	wantDoms, _, _ := inner.ConnectListAllDomains(1, 0)
+	doms, _, err := traced.ConnectListAllDomains(1, 0)
+	if err != nil || !reflect.DeepEqual(doms, wantDoms) {
+		t.Errorf("ConnectListAllDomains = %v/%v, want %v", doms, err, wantDoms)
+	}
+
+	inner.fail["DomainCreate"] = lvError(libvirt.ErrOperationInvalid, "already running")
+	if err := traced.DomainCreate(dom); !hasCode(err, libvirt.ErrOperationInvalid) {
+		t.Errorf("DomainCreate error = %v, want the injected libvirt error unchanged", err)
+	}
+}
+
+func TestTracingAPI_errorFields(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name    string
+		inject  error
+		want    []string
+		notWant []string
+	}{
+		{
+			name:   "libvirt error carries code, name and message",
+			inject: lvError(libvirt.ErrNoDomain, "Domain not found: no domain with matching name"),
+			want:   []string{`"error"="Domain not found`, `"libvirtError"="ErrNoDomain"`, `"libvirtCode"=42`},
+		},
+		{
+			name:    "transport error has no libvirt fields",
+			inject:  io.EOF,
+			want:    []string{`"error"="EOF"`},
+			notWant: []string{`"libvirtError"`, `"libvirtCode"`},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			inner := tracedHost(t)
+			inner.fail["DomainLookupByName"] = tc.inject
+			log, snapshot := captureLogger(1)
+			api := &tracingAPI{inner: inner, log: log}
+			if _, err := api.DomainLookupByName(testName); !errors.Is(err, tc.inject) {
+				t.Fatalf("error = %v, want the injected one", err)
+			}
+			joined := strings.Join(snapshot(), "\n")
+			for _, want := range tc.want {
+				if !strings.Contains(joined, want) {
+					t.Errorf("output missing %s:\n%s", want, joined)
+				}
+			}
+			for _, absent := range tc.notWant {
+				if strings.Contains(joined, absent) {
+					t.Errorf("output unexpectedly contains %s:\n%s", absent, joined)
+				}
+			}
+		})
+	}
+}
+
+func TestTracingAPI_detailOnlyAtV2(t *testing.T) {
+	t.Parallel()
+	dom := libvirt.Domain{Name: testName}
+	calls := []struct {
+		method string
+		call   func(api libvirtAPI) error
+		always string
+		detail string
+	}{
+		{"DomainDefineXML", func(api libvirtAPI) error {
+			xml, err := buildDomainXML(testDomainParams())
+			if err != nil {
+				return err
+			}
+			_, err = api.DomainDefineXML(xml)
+			return err
+		}, `"xmlBytes"=`, `"xml"="<domain`},
+		{"StorageVolCreateXML", func(api libvirtAPI) error {
+			xml, err := buildVolumeXML(rawVolume("new.iso", 3))
+			if err != nil {
+				return err
+			}
+			_, err = api.StorageVolCreateXML(libvirt.StoragePool{Name: "default"}, xml, 0)
+			return err
+		}, `"xmlBytes"=`, `"xml"="<volume`},
+		{"DomainGetMetadata", func(api libvirtAPI) error {
+			_, err := api.DomainGetMetadata(dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, 0)
+			return err
+		}, `"metadataBytes"=`, `"metadata"="<instance`},
+		{"DomainInterfaceAddresses", func(api libvirtAPI) error {
+			_, err := api.DomainInterfaceAddresses(dom, uint32(libvirt.DomainInterfaceAddressesSrcArp), 0)
+			return err
+		}, `"interfaces"=1`, `"addresses"=["vnet0=192.168.1.50/24"]`},
+		{"ConnectListAllDomains", func(api libvirtAPI) error {
+			_, _, err := api.ConnectListAllDomains(1, 0)
+			return err
+		}, `"count"=2`, `"names"=[`},
+	}
+	for _, verbosity := range []int{1, 2} {
+		for _, tc := range calls {
+			t.Run(tc.method+"/v"+string(rune('0'+verbosity)), func(t *testing.T) {
+				t.Parallel()
+				log, snapshot := captureLogger(verbosity)
+				api := &tracingAPI{inner: tracedHost(t), log: log}
+				if err := tc.call(api); err != nil {
+					t.Fatal(err)
+				}
+				joined := strings.Join(snapshot(), "\n")
+				if !strings.Contains(joined, tc.always) {
+					t.Errorf("v%d output missing %s:\n%s", verbosity, tc.always, joined)
+				}
+				if has := strings.Contains(joined, tc.detail); has != (verbosity >= 2) {
+					t.Errorf("v%d output contains %s = %v, want %v:\n%s", verbosity, tc.detail, has, verbosity >= 2, joined)
+				}
+			})
+		}
+	}
+}
+
+func TestTracingAPI_uploadLogsSizeNotContent(t *testing.T) {
+	t.Parallel()
+	const sentinel = "SENTINEL-seed-content-must-never-be-logged"
+	log, snapshot := captureLogger(2)
+	api := &tracingAPI{inner: tracedHost(t), log: log}
+	vol := libvirt.StorageVol{Pool: "default", Name: seedVolumeName(testName)}
+	if err := api.StorageVolUpload(vol, bytes.NewReader([]byte(sentinel)), 0, uint64(len(sentinel)), 0); err != nil {
+		t.Fatal(err)
+	}
+	joined := strings.Join(snapshot(), "\n")
+	if !strings.Contains(joined, `"length"=42`) {
+		t.Errorf("upload line missing the length:\n%s", joined)
+	}
+	if strings.Contains(joined, sentinel) {
+		t.Errorf("upload content leaked into logs:\n%s", joined)
+	}
+}
+
+func TestTracingAPI_silentBelowV1(t *testing.T) {
+	t.Parallel()
+	log, snapshot := captureLogger(0)
+	api := &tracingAPI{inner: tracedHost(t), log: log}
+	if _, err := api.DomainLookupByName(testName); err != nil {
+		t.Fatal(err)
+	}
+	if lines := snapshot(); len(lines) != 0 {
+		t.Errorf("verbosity 0 logged %d lines:\n%s", len(lines), strings.Join(lines, "\n"))
+	}
+}

internal/provider/kvm/volume.go

@@ -0,0 +1,78 @@
+package kvm
+
+import (
+	"encoding/xml"
+	"fmt"
+)
+
+const bytesPerGiB = 1 << 30
+
+func overlayVolumeName(name string) string { return name + ".qcow2" }
+
+func seedVolumeName(name string) string { return name + "-seed.iso" }
+
+// volumeSpec describes a storage volume to create. A non-empty BackingPath
+// makes it a copy-on-write overlay of that file.
+type volumeSpec struct {
+	Name          string
+	SizeBytes     int64
+	Format        string
+	BackingPath   string
+	BackingFormat string
+}
+
+func overlayVolume(name string, sizeGB int64, basePath, baseFormat string) volumeSpec {
+	return volumeSpec{
+		Name:          overlayVolumeName(name),
+		SizeBytes:     sizeGB * bytesPerGiB,
+		Format:        formatQcow2,
+		BackingPath:   basePath,
+		BackingFormat: baseFormat,
+	}
+}
+
+func rawVolume(name string, sizeBytes int64) volumeSpec {
+	return volumeSpec{Name: name, SizeBytes: sizeBytes, Format: formatRaw}
+}
+
+// buildVolumeXML renders the volume for StorageVolCreateXML. Capacity is
+// always given in bytes so no libvirt unit-suffix ambiguity (G vs GiB) can
+// creep in.
+func buildVolumeXML(v volumeSpec) (string, error) {
+	x := volumeXML{
+		Type:     "file",
+		Name:     v.Name,
+		Capacity: sizeXML{Unit: "bytes", Value: v.SizeBytes},
+		Target:   volumeTargetXML{Format: formatXML{Type: v.Format}},
+	}
+	if v.BackingPath != "" {
+		x.BackingStore = &backingStoreXML{Path: v.BackingPath, Format: formatXML{Type: v.BackingFormat}}
+	}
+	out, err := xml.Marshal(x)
+	if err != nil {
+		return "", fmt.Errorf("marshalling volume xml: %w", err)
+	}
+	return string(out), nil
+}
+
+type volumeXML struct {
+	XMLName      xml.Name         `xml:"volume"`
+	Type         string           `xml:"type,attr"`
+	Name         string           `xml:"name"`
+	Capacity     sizeXML          `xml:"capacity"`
+	Target       volumeTargetXML  `xml:"target"`
+	BackingStore *backingStoreXML `xml:"backingStore"`
+}
+
+type volumeTargetXML struct {
+	Format formatXML `xml:"format"`
+}
+
+type formatXML struct {
+	Type string `xml:"type,attr"`
+}
+
+type backingStoreXML struct {
+	Path   string    `xml:"path"`
+	Format formatXML `xml:"format"`
+}