kvm2: apply the milestone review

Shorter start wait with immediate verdicts for dead or paused domains,
ListByTag without state or address RPCs, Timeout for an expired check
round, no adoption of unmarked domains, reference-counted keyed locks,
dead options removed, HEAD fallback on any non-2xx.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Makefile

@@ -70,6 +70,10 @@ test: manifests generate fmt vet setup-envtest ## Run tests.
 test-integration-kvm: ## Run the KVM provider test against a real hypervisor. Needs KVM_TEST_HOST/USER/SSH_KEY/BRIDGE/IMAGE_URL; see docs/testing.md.
 	go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/
 
+.PHONY: test-integration-kvm2
+test-integration-kvm2: ## Run the kvm2 provider test against a real hypervisor. Needs KVM2_TEST_HOST/USER/SSH_KEY/BRIDGE/IMAGE_URL; see docs/testing.md.
+	go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM2 ./internal/provider/kvm2/
+
 # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
 # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
 # kubectl kuberc is disabled by default for test isolation; enable with:

README.md

@@ -160,6 +160,10 @@ Managed proxies as VMs on libvirt/KVM hosts you own. Full guide:
    size (`placement.machineType`) and an image (`placement.image`) — see
    [config/samples/proxy_kvm.yaml](config/samples/proxy_kvm.yaml).
 
+The `kvm2` provider uses the same Secret and the same block shape (`type: kvm2`,
+block `kvm2:`), so the steps above apply to it unchanged; its guide is
+[docs/kvm2-provider.md](docs/kvm2-provider.md).
+
 Cloud-init from a Secret: the Secret **must** carry the label
 `crawl.example.com/cloud-init: "true"` — the operator's cache only holds
 labelled Secrets, so an unlabelled one is invisible (the Proxy reports
@@ -187,6 +191,7 @@ can be two differently-configured instances of the same `gcp` type (see
 | `kubernetes` | A real Squid pod (`ubuntu/squid:6.6-24.04_edge` by default) in the same cluster the operator runs in | `image` (optional) | Needs no cloud account — local dev, CI, and the kind quickstart. Pods share the cluster's egress IP, so it exercises the full lifecycle but not distinct egress paths. |
 | `gcp` | A Compute Engine VM with an ephemeral external IP | `project` (required), `network`, `networkTag`, `diskSizeGb` | The real egress fleet. Deliberately uses only four API calls (Insert / Get / Delete / AggregatedList), all fire-and-forget: `Create` returns as soon as the operation is submitted and the reconciler discovers progress by polling `Get`. Auth is Application Default Credentials — workload identity in-cluster, `gcloud` ADC locally; no key-file plumbing. |
 | `kvm` | A libvirt VM on one of your own hypervisors, reached over SSH | `sshKeyFile`, `knownHostsFile` (unless `insecureIgnoreHostKey`), `machineTypes`, `hosts` (required); `images`, `diskSizeGb`, `vendorData`, `installGuestAgent` | Home-lab or on-prem egress. The base image is downloaded into each host's pool on first use; `Create` waits for the VM to start. See [docs/kvm-provider.md](docs/kvm-provider.md). |
+| `kvm2` | The same: a libvirt VM on one of your own hypervisors, reached over SSH | the same block shape as `kvm`, under `kvm2:` | The second-generation libvirt provider, built from the provider contract. Adds a child span per libvirt RPC, a pool-space precheck reported as quota (the reconciler waits instead of failing), SSH credentials re-read on every dial, and image downloads and health checks on connections of their own. It does not adopt `kvm`'s VMs. See [docs/kvm2-provider.md](docs/kvm2-provider.md). |
 
 ### Adding a provider
 

api/v1alpha1/providerhealth_types.go

@@ -68,7 +68,7 @@ type ProviderTarget struct {
 // ProviderHealthStatus is the result of the latest check of one provider.
 type ProviderHealthStatus struct {
 	// type is the provider's type from the providers config: kubernetes,
-	// gcp or kvm.
+	// gcp, kvm or kvm2.
 	// +optional
 	Type string `json:"type,omitempty"`
 

cmd/main.go

@@ -59,6 +59,7 @@ import (
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kvm"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kvm2"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/providerhealth"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
@@ -201,7 +202,8 @@ func main() {
 		"gcp": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
 			return gcp.NewWithWireOptions(ctx, pc, gcp.WireLogOptions{FullPayloads: gcpWireFullPayloads})
 		},
-		"kvm": kvm.New,
+		"kvm":  kvm.New,
+		"kvm2": kvm2.New,
 	})
 	if err != nil {
 		setupLog.Error(err, "Failed to build providers")

config/crd/bases/crawl.example.com_providerhealths.yaml

@@ -183,7 +183,7 @@ spec:
               type:
                 description: |-
                   type is the provider's type from the providers config: kubernetes,
-                  gcp or kvm.
+                  gcp, kvm or kvm2.
                 type: string
             type: object
         type: object

config/manager/manager.yaml

@@ -149,9 +149,10 @@ spec:
         - name: providers-config
           mountPath: /etc/proxy-operator
           readOnly: true
-        # SSH material for kvm providers (sshKeyFile / knownHostsFile in the
-        # providers config point here). Optional: the mount is empty when the
-        # Secret does not exist, and only kvm entries need it. Create it with:
+        # SSH material for the kvm and kvm2 providers (sshKeyFile /
+        # knownHostsFile in the providers config point here). One Secret
+        # serves both types. Optional: the mount is empty when the Secret
+        # does not exist, and only kvm and kvm2 entries need it. Create it with:
         #   kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
         #     --from-file=id_ed25519=<private key> \
         #     --from-file=known_hosts=<known_hosts with every host's key>
@@ -167,6 +168,7 @@ spec:
       - name: providers-config
         configMap:
           name: providers-config
+      # Shared by the kvm and kvm2 providers; see the mount above.
       - name: kvm-ssh
         secret:
           secretName: kvm-ssh

config/manager/providers_config.yaml

@@ -51,4 +51,35 @@ data:
               bridge: br0                 # to a LAN with DHCP; the VM's IP comes from there
               # socket: /var/run/libvirt/libvirt-sock
               # pool: default
+      # kvm2 takes the same block shape as kvm: an entry ports by changing
+      # type to kvm2 and the block name to kvm2. kvm2 does not adopt domains
+      # made by kvm, so delete those Proxies first. See docs/kvm2-provider.md.
+      # - name: kvm2-home
+      #   type: kvm2
+      #   kvm2:
+      #     sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+      #     knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+      #     # insecureIgnoreHostKey: true
+      #     # diskSizeGb: 10
+      #     # installGuestAgent: false
+      #     # vendorData: |
+      #     #   #cloud-config
+      #     #   ...
+      #     images:
+      #       debian-13:
+      #         url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+      #         # sha256: "<64 hex>"
+      #         # firmware: uefi
+      #       ubuntu-24.04:
+      #         url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+      #     machineTypes:
+      #       small:  {vcpus: 1, memoryMiB: 1024}
+      #       medium: {vcpus: 2, memoryMiB: 2048}
+      #     hosts:
+      #       - name: beelink
+      #         address: beelink.tail8c69c.ts.net:22
+      #         user: novakj
+      #         bridge: br0
+      #         # socket: /var/run/libvirt/libvirt-sock
+      #         # pool: default
 

config/samples/kustomization.yaml

@@ -5,5 +5,6 @@ resources:
 - proxy_kubernetes.yaml
 - proxy_gcp.yaml
 - proxy_kvm.yaml
+- proxy_kvm2.yaml
 - proxy_external.yaml
 # +kubebuilder:scaffold:manifestskustomizesamples

config/samples/providers-config.yaml

@@ -55,3 +55,34 @@ providers:
           bridge: br0                 # to a LAN with DHCP; the VM's IP comes from there
           # socket: /var/run/libvirt/libvirt-sock
           # pool: default
+  # kvm2 takes the same block shape as kvm: an entry ports by changing type
+  # to kvm2 and the block name to kvm2. kvm2 does not adopt domains made by
+  # kvm, so delete those Proxies first. See docs/kvm2-provider.md.
+  # - name: kvm2-home
+  #   type: kvm2
+  #   kvm2:
+  #     sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+  #     knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+  #     # insecureIgnoreHostKey: true
+  #     # diskSizeGb: 10
+  #     # installGuestAgent: false
+  #     # vendorData: |
+  #     #   #cloud-config
+  #     #   ...
+  #     images:
+  #       debian-13:
+  #         url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+  #         # sha256: "<64 hex>"
+  #         # firmware: uefi
+  #       ubuntu-24.04:
+  #         url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
+  #     machineTypes:
+  #       small:  {vcpus: 1, memoryMiB: 1024}
+  #       medium: {vcpus: 2, memoryMiB: 2048}
+  #     hosts:
+  #       - name: beelink
+  #         address: beelink:22
+  #         user: novakj
+  #         bridge: br0
+  #         # socket: /var/run/libvirt/libvirt-sock
+  #         # pool: default

config/samples/proxy_kvm2.yaml

@@ -0,0 +1,44 @@
+# A Managed proxy backed by a libvirt/KVM hypervisor through the kvm2
+# provider: the operator boots a stock Debian cloud image as a VM on the
+# named host, with a NIC on the host's bridge (IP from the LAN's DHCP), and
+# installs Squid via cloud-init. Requires a providers-config entry named
+# "kvm2-home" (see providers-config.yaml) and the kvm-ssh Secret, which the
+# kvm and kvm2 providers share (docs/kvm2-provider.md).
+#
+# placement.zone is the host name from kvm2.hosts (optional when only one
+# host is configured); machineType and image are keys in the provider's
+# machineTypes and images maps. The first Proxy for an image on a host
+# waits for the image download; later ones reuse the cached copy.
+apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: proxy-kvm2-sample
+spec:
+  mode: Managed
+  provider: kvm2-home
+  placement:
+    zone: beelink
+    machineType: small
+    image: debian-13
+  port: 3128
+  cloudInit:
+    inline: |
+      #cloud-config
+      packages:
+        - squid
+      write_files:
+        - path: /etc/squid/conf.d/proxy-operator.conf
+          content: |
+            http_port 3128
+            http_access allow all
+            via off
+            forwarded_for off
+      runcmd:
+        - systemctl restart squid
+      # Optional: log in with `ssh debian@<ip>` (the Debian image's default
+      # user) to look around; `virsh console <name>` on the host works too.
+      # ssh_authorized_keys:
+      #   - ssh-ed25519 AAAA... you@laptop
+  attributes:
+    geo: home
+    purpose: crawl

docs/architecture.md

@@ -10,6 +10,7 @@
 | kubernetes provider | `internal/provider/kubernetes` | library | — | real Squid pods in this cluster (local dev/CI) |
 | gcp provider | `internal/provider/gcp` | library | — | Compute Engine VMs, four API calls, fire-and-forget ops |
 | kvm provider | `internal/provider/kvm` | library | — | VMs on libvirt hosts over SSH-forwarded sockets; image cache, overlay disks, NoCloud seeds |
+| kvm2 provider | `internal/provider/kvm2` | library | — | the same backend, built clean-room from the provider contract as a second type: per-RPC child spans, pool-space quota, per-call budgets, its own metadata namespace, credentials re-read on every dial |
 | Health engine | `internal/health` | Runnable | yes | through-the-proxy probes, thresholds, transition events |
 | Lease store | `internal/lease` | Runnable (expiry sweep) | no | in-memory leases + cooldowns, single mutex |
 | Discovery API | `internal/discovery` | Runnable | no | HTTP list/lease/release/report on `:8090` |
@@ -168,6 +169,40 @@ providerID = hosts/<host>/domains/<name>; GC identity lives in the domain's
 <metadata> element (libvirt has no labels). All RPCs run under a context-
 bounded call: go-libvirt never times out on its own, so a cancelled reconcile
 closes the SSH connection to unblock it.
+
+kvm2 provider (internal/provider/kvm2/) - the same transport, a second type built from the contract
+prov.Create ──► placement checked against the catalogue (Permanent before any dial)
+                DomainLookupByName: exists → mark+uid guard (no mark or foreign   ─┐ every step
+                  uid = Permanent; paused = Permanent),                            │
+                  start if needed, wait for running, done                          │ idempotent,
+                else: StoragePoolGetInfo precheck: image (if uncached) + diskSizeGb │ under a
+                        + 16 MiB must fit, else ErrQuotaExceeded (reconciler       │ per-name lock
+                        waits 5m)                                                  │ shared with
+                      ensure base image: marker present = cached; otherwise GET    │ Delete
+                        streamed into StorageVolUpload on a DEDICATED connection,  │
+                        StoragePoolRefresh, marker written last                    │
+                      overlay volume (exists = reuse)                              │
+                      seed ISO (always rebuilt)                                    │
+                      DomainDefineXML (<metadata> in kvm2's own namespace:         │
+                        managed, uid, createdAt) + DomainCreate, poll until        │
+                        running (15s cap; a domain that dies at once = Transient) ─┘
+prov.Get    ──► DomainLookupByName + DomainGetMetadata + DomainGetState
+                  + DomainInterfaceAddresses (guest agent, then ARP; first routable IPv4)
+                  running without IP = Provisioning; nostate = Provisioning;
+                  only ErrNoDomain is NotFound
+prov.Delete ──► destroy (not running = fine) → undefine (nvram, managed save, snapshots)
+                  → delete overlay + seed by name; unconfigured host = logged no-op
+prov.ListByTag ─► ConnectListAllDomains on every host, keep domains whose kvm2
+                    <metadata> says managed=true; one host down fails the sweep,
+                    one unreadable domain is skipped
+Check       ──► per host, on a FRESH connection: version, hostname, pool lookup,
+                  pool info, close; never the pooled one
+providerID = hosts/<host>/domains/<name>. Every RPC has its own budget (dial 10s,
+RPC 30s, DomainCreate 90s, seed upload 60s, image transfer 20m): a call that
+outlives its ctx or budget kills the pooled connection (the only way to unblock
+go-libvirt), so other calls in flight on that host fail once with Transient and
+retry; the next call redials. Each RPC is a child span kvm2.<Method> under
+provider.*, plus kvm2.dial and kvm2.download.
 ```
 
 The reconciler never watches provider-side resources (Pods or GCP VMs).
@@ -343,6 +378,15 @@ incoming trace context):
 - `provider.create|get|delete|list` (client spans, `provider.WithTracing`
   decorator wired outermost around `WithMetrics`); the GCP SDK's own
   otelhttp transport contributes HTTP child spans automatically.
+- kvm2 child spans: under `provider.*`, one `kvm2.<Method>` client span per
+  libvirt RPC (for example `kvm2.DomainDefineXML`, attributes `kvm2.host`,
+  `kvm2.domain`, `kvm2.pool`, `kvm2.volume`, `kvm2.libvirt_code` on
+  failure), `kvm2.dial` around the SSH handshake and libvirt connect, and
+  `kvm2.download` around an image fetch; under `providerhealth.check`,
+  `kvm2.dial` and the four `kvm2.<Method>` spans of a check. They come
+  from a span decorator over the seam, applied once in the dialer, so no
+  RPC is unspanned. The `libvirt call` log line of an RPC is written
+  inside its span. The `kvm` provider has no child spans.
 - Kubernetes API calls become client child spans via a wrapped
   `rest.Config` transport — gated on an existing parent span, so informer
   list/watch long-polls and leader-election renewals never create root
@@ -368,11 +412,12 @@ the slog handler's ctx.
 Answers "can this cluster reach each provider's backend right now?"
 without waiting for a Proxy to fail. A provider opts in by implementing
 `provider.Checker`: `Check(ctx) []TargetStatus` returns one verdict per
-target and never fails as a whole. All three providers do:
+target and never fails as a whole. All four providers do:
 
 | Provider | Target | What one check does |
 |---|---|---|
 | kvm | each `kvm.hosts` entry | over the pooled SSH+libvirt connection: libvirt version and hostname, then the configured pool must exist and be running; reports pool capacity and free space ([details](kvm-provider.md#health-check)) |
+| kvm2 | each `kvm2.hosts` entry | on a fresh SSH+libvirt connection dialled for the check and closed after it: libvirt version and hostname, then the configured pool must exist and be running; `details` carry `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacityGiB` and `poolAvailableGiB` ([details](kvm2-provider.md#health-check)) |
 | kubernetes | `apiserver` | lists at most one managed Pod cluster-wide: the verb and scope `ListByTag` and GC rely on |
 | gcp | the project | one page of an aggregated instance list with `maxResults=1`: credentials, Compute API enabled, `compute.instances.list` |
 
@@ -613,3 +658,48 @@ they read as choices, not accidents. Chronological by build step.
 - **A provider name that is not a valid object name** gets metrics but no
   `ProviderHealth` object, with a warning at start. Tightening config
   validation instead would crash-loop existing deployments on upgrade.
+- **kvm2 is a second provider type built clean-room from the contract,**
+  not a rewrite of `kvm` in place. Its spec
+  (`docs/kvm2-provider-spec.md`) and code were written from
+  [provider-contract.md](provider-contract.md), the kvm user guide and
+  go-libvirt alone, never from `internal/provider/kvm/`, as a test of
+  whether the contract is enough to build a provider. Both types ship,
+  share the `KVMConfig` block shape and the `kvm-ssh` Secret, and may
+  share a host.
+- **kvm2 image downloads run on a dedicated connection.** A cancelled or
+  overlong call kills the connection it ran on (the only way to unblock
+  go-libvirt), so a 435 MB upload on the pooled connection could die
+  because a poll for another Proxy was cancelled. The transfer dials its
+  own connection and closes it afterwards; the cheap checks and the marker
+  stay on the pooled one.
+- **kvm2 health checks dial fresh connections**, the opposite of the kvm
+  decision above. With credentials re-read on every dial a fresh
+  connection does pick up a rotated Secret, and a check that runs out of
+  time then closes only its own connection, so it cannot disrupt a
+  reconcile. The cost is one sshd session per host per round.
+- **A cancelled kvm2 call kills the pooled connection.** Every RPC has a
+  budget; when a call outlives its ctx or budget the adapter closes the
+  socket and the SSH client, returns Transient, and marks the connection
+  dead so the next call redials. Other calls in flight on that host fail
+  once with Transient and their callers retry. Documented in the guide
+  rather than avoided: the alternative, leaving a hung RPC blocked
+  forever, pins a worker.
+- **kvm2 marks domains in its own metadata namespace**, so it never lists
+  or adopts domains the `kvm` provider made. Switching an entry from `kvm`
+  to `kvm2` means deleting its Proxies first; the two types can share a
+  host without one collecting the other's VMs.
+- **kvm2 turns pool free space into the quota signal.** libvirt reports a
+  full pool as an ordinary error, which would retry on the fast backoff.
+  Before creating any volume, `Create` compares `StoragePoolGetInfo`'s
+  available bytes with the image (if uncached) plus `diskSizeGb` plus 16
+  MiB for the seed, and returns `ErrQuotaExceeded` below that, so the
+  reconciler waits 5 minutes instead.
+- **kvm2 classifies SSH authentication and host-key failures as
+  Permanent.** A key missing from `authorized_keys` or a host key missing
+  from or different in known_hosts needs an admin; kvm retried these
+  forever. A missing key or known_hosts file stays Transient, because the
+  optional Secret may land later.
+- **kvm2 re-reads the SSH key and known_hosts on every dial**, so rotating
+  the Secret needs no restart; nothing is cached. The constructor only
+  checks that the files exist, to warn early without failing the provider
+  set.

docs/kvm2-provider-spec.md

@@ -0,0 +1,653 @@
+# Provider spec: kvm2
+
+Filled from [provider-spec-template.md](provider-spec-template.md) against
+[provider-contract.md](provider-contract.md). This is the design the `kvm2`
+package is built from. Where an entry names an axis in backticks, the answer is
+one of that axis's profiles in contract §4. Nothing in this spec was taken from
+the code of the existing `kvm` provider; backend facts come from
+[kvm-provider.md](kvm-provider.md), the execution log of the first build, the
+go-libvirt module in the module cache, and the beelink host.
+
+## Header
+
+- **Type**: `kvm2`
+- **Example configured name**: `kvm2-home`
+- **Backend and API version this spec was written against**: libvirt 11.6.0 and
+  QEMU 10.1.0 on Ubuntu (host `beelink`), reached with go-libvirt
+  `github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858` over an
+  SSH-forwarded unix socket.
+- **Client**: `rpc-over-ssh`. go-libvirt's RPC client (`libvirt.New(conn)`
+  then `Connect()`) over a `direct-streamlocal` channel opened with
+  `golang.org/x/crypto/ssh` to the host's libvirt socket. Both modules are
+  already direct requires in `go.mod`; no new dependency.
+- **Author and date**: Claude (orchestrator session), 2026-09-16.
+
+## 1. Identity and providerID
+
+**Answer:** `hosts/<host>/domains/<name>`, where `<host>` is `hosts[].name`
+from the config block and `<name>` is `req.Name`. Parsing is strict: exactly
+four segments, literal `hosts` and `domains`, non-empty host and name, no
+further slashes; anything else is `ErrPermanent` from `Get` and `Delete`. Host
+names are validated at config load by the shared `kvmIdentifier` regexp in
+`internal/provider/config_kvm.go` (letters, digits, dots, dashes, underscores),
+so a host name can never contain the separator. Instances do not migrate
+between hosts (libvirt domains are per host), so the location belongs in the
+ID.
+
+**Evidence:** contract §6; `kvmIdentifier` in `internal/provider/config_kvm.go`.
+
+## 2. Primary key (`primary-key`)
+
+**Answer:** `name-is-key`. libvirt addresses domains by name
+(`DomainLookupByName`); `req.Name` is used verbatim as the domain name and as
+the base of the per-VM volume names. Create looks the name up first and adopts
+an existing domain only if it carries the kvm2 mark (`managed` true, entry 8)
+and its metadata `uid` is empty or equals `req.UID`. A domain without the
+mark returns `ErrPermanent` saying it was not created by this provider: kvm2
+never starts or takes over a VM someone else defined under that name. A
+different non-empty `uid` returns `ErrPermanent` naming both UIDs. This
+closes the contract's "adoption is not verified" limit for this provider.
+
+**Evidence:** `DomainLookupByName(Name string) (Domain, error)` in the
+go-libvirt module; contract §2.1 idempotency and §13.
+
+## 3. Placement mapping
+
+**Answer:** three fields are catalogue keys checked before any backend call, one is ignored:
+
+| Field | Meaning | Required | Unknown value |
+|---|---|---|---|
+| `Zone` | a `hosts[].name` | only when more than one host is configured; with exactly one host an empty zone selects it | `ErrPermanent`: `placement.zone "x" is not in kvm2.hosts` |
+| `MachineType` | a key of `machineTypes` (`vcpus`, `memoryMiB`) | yes | `ErrPermanent`: `placement.machineType "x" is not in kvm2.machineTypes` |
+| `Image` | a key of `images`, or an absolute http(s) URL used directly | yes | `ErrPermanent`: `placement.image "x" is neither a key in kvm2.images nor an http(s) URL` |
+| `Region` | ignored | | still part of the spec hash: changing it replaces the VM |
+
+All three checks run before any backend call. The guide documents the ignored
+field and that catalogue edits affect only new VMs.
+
+**Evidence:** contract §2.1 (reject early, MAY ignore with documentation);
+`IsHTTPURL` in `internal/provider/config_kvm.go`.
+
+## 4. Image source
+
+**Answer:** a catalogue in the config block (`images.<key>.url`, optional
+`sha256`, `format` qcow2 or raw, `firmware` uefi or bios) or a URL given
+directly in `placement.image`. The provider downloads the image into the
+host's storage pool on first use and caches it there as a base volume named
+`base-<key>.<format>` (for a URL: `base-<first 16 hex of sha256(url)>.<format>`),
+with a sidecar marker volume `base-<key>.<format>.sha256` holding the digest of
+the verified upload, written last. A base volume without a marker is
+incomplete and is deleted and redone. When `sha256` is configured and differs
+from the marker's content, the image is re-downloaded. The download streams
+from the URL through the operator into `StorageVolUpload` with a running
+sha256; nothing is buffered. `Content-Length` is required because the volume's
+capacity must be declared before the upload; a response without it is
+`ErrPermanent` naming the URL. Per `(host, volume)` a keyed mutex makes
+concurrent Creates download once. The download runs on a dedicated
+connection to the host, not the pooled one, so a cancelled poll on another
+Proxy cannot cut it off. After a successful upload `StoragePoolRefresh` makes
+the file usable as a backing image.
+
+Catalogue edit warning for the guide: overlays back onto the base file, so
+replacing or evicting a base volume while Proxies use it corrupts those VMs;
+add a new key instead.
+
+**Evidence:** kvm-provider.md "The image cache" and "Known gaps"; execution log
+of 2026-09-04 (raw qcow2 upload plus pool refresh accepted by libvirt 11.6;
+435 MB trixie image); contract §1.4 (stream, no buffering) and §2.1
+(catalogue changes).
+
+## 5. Cloud-init delivery
+
+**Answer:** a NoCloud seed ISO built in-process with
+`github.com/kdomanski/iso9660`, volume label `cidata`, files `user-data` (the
+Proxy's `CloudInit`, byte for byte, never parsed), `meta-data`
+(`instance-id: <name>` and `local-hostname: <name>`, so a reboot is never taken
+for a new instance) and `vendor-data`. The ISO is written into a temp
+directory created under the default temp dir (`/tmp`, a 64Mi emptyDir in the
+manager), uploaded as a raw volume `<name>-seed.iso` and attached as a CD-ROM.
+The seed is rebuilt on every Create attempt: a previous attempt may have been
+cut off mid-upload, and a truncated seed would boot a VM without cloud-init.
+
+`vendor-data` carries a shell script that installs and starts
+`qemu-guest-agent` with whichever of apt-get, dnf, yum, zypper or apk the image
+has. It is a script and not a cloud-config on purpose: cloud-init merges a
+vendor-data cloud-config under the user-data with mergers that keep the
+user-data's value for every top-level key both define and never append lists,
+so a Proxy with its own `packages:` would silently lose the agent. Scripts
+bypass the merge. When the config's `vendorData` is set, the two travel as
+separate parts of a MIME multipart (`text/plain` for the configured document,
+`text/x-shellscript` for the agent script), headed by `MIME-Version` within
+the first bytes. With `installGuestAgent: false` and no `vendorData` the file
+is empty. Size: cloud-init has no practical limit for NoCloud; the ISO is a
+few hundred KB.
+
+**Evidence:** kvm-provider.md "What lands on the host"; execution log run 7
+(vendor-data as YAML silently dropped) and run 8 (script survives); contract
+§2.1 (byte for byte, never logged) and §11 (sentinel test).
+
+## 6. Machine type mapping
+
+**Answer:** a catalogue: `machineTypes.<key>` is `{vcpus, memoryMiB}`,
+validated at load (both greater than zero, at least one profile). Disk size is
+provider-wide: `diskSizeGb` (default 10) is the capacity of every VM's overlay
+volume, thin-provisioned; cloud-init's growpart expands the root filesystem
+into it.
+
+**Evidence:** `KVMMachineType` and `validateKVM` in
+`internal/provider/config_kvm.go`.
+
+## 7. Network and IP discovery (`readiness`, `ip-source`)
+
+**Answer:** `readiness`: `provisioning-window` (boot plus cloud-init, plus the
+image download on a cold pool). `ip-source`: `discovered`. The VM gets one
+virtio NIC on `hosts[].bridge`, which must lead to a LAN with DHCP that the
+cluster can route to. The address comes from `DomainInterfaceAddresses`, guest
+agent source first, then the ARP table (on beelink the agent answered about
+20 s after start where ARP never did, because the host itself does not talk to
+the guest). The first IPv4 that is not loopback, link-local or unspecified is
+reported; errors from either source mean "no address yet" and `Get` returns
+`StateProvisioning`. The guest needs the agent: it is installed by the
+vendor-data script (entry 5) unless `installGuestAgent: false`, which needs an
+image that ships it or a network where ARP sees guest traffic. `Port` is not
+used by the provider; nothing on the host filters it. Reachability is a
+prerequisite in the guide: the operator's pod network and the clients must
+route to the bridge's LAN.
+
+**Evidence:** kvm-provider.md "Hypervisor prerequisites" and "Known gaps"
+(IP discovery); execution log run 6 (agent 20 s, ARP never); contract §1.3
+and §2.2 (Running only with IP).
+
+## 8. GC marks and CreatedAt (`gc-mark`, `created-at`)
+
+**Answer:** `gc-mark`: `written-metadata`. `created-at`: `written-at-create`.
+The domain XML passed to `DomainDefineXML` carries, under `<metadata>`, an
+element `<instance xmlns="https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm2">`
+with children `<managed>true</managed>`, `<uid>req.UID</uid>` and
+`<createdAt>RFC3339 UTC</createdAt>`. The mark lands in the same call that
+defines the domain, so there is no mark window. Read back with
+`DomainGetMetadata(dom, DomainMetadataElement, uri, DomainAffectCurrent)`;
+libvirt 11.6 returns the element without its namespace declaration, so the
+parser matches on the local name `instance` only (the URI was already used to
+select it). A domain without the element is not managed and reports an empty
+UID. A `createdAt` that does not parse gives the zero time, which GC treats as
+old; that is accepted. The volumes carry no mark (documented GC blind spot,
+entry 24). The namespace is kvm2's own, so kvm2 never lists or adopts domains
+made by the `kvm` provider; the two types may share a host.
+
+**Evidence:** execution log run 3 (namespace asymmetry);
+`DomainGetMetadata(Dom Domain, Type int32, Uri OptString, Flags DomainModificationImpact)`
+in the go-libvirt module; contract §2.1 (mark in the create call) and §4.5.
+
+## 9. State mapping (`provisioning-state`)
+
+**Answer:** `provisioning-state`: `create-blocks`. Create defines and starts the
+domain in one call and returns only once `DomainGetState` reports running, so
+`Get` never sees a domain kvm2 defined but did not start. A domain that is
+shut off later stopped on its own or was stopped by someone, and Stopped is
+the right answer.
+
+| Backend state (`DomainGetState`) | Lock or task | Provider state |
+|---|---|---|
+| nostate (0) | none | Provisioning (libvirt could not determine it; next poll decides) |
+| running (1), blocked (2) | no IP found | Provisioning |
+| running (1), blocked (2) | IP found | Running |
+| paused (3), shutdown (4), shutoff (5), crashed (6), pmsuspended (7) | none | Stopped |
+| any other code | none | Stopped |
+
+libvirt has no locks or tasks; every RPC is synchronous.
+
+**Evidence:** go-libvirt `DomainState` constants; contract §2.2 and §4.11.
+
+## 10. Create: blocking and task tracking (`create-return`, `async-ops`)
+
+**Answer:** `create-return`: `blocking`. `async-ops`: `none`. Create returns
+when the domain is running, holding one reconcile worker for the duration,
+including the first image download on a host (minutes on a cold pool; the
+guide says so). The wait after `DomainCreate` is capped at 15 seconds:
+`DomainCreate` is synchronous in libvirt, so the first `DomainGetState`
+normally answers running and the cap only covers state settling. Two rules
+keep the wait from pinning a worker: a state after the start that is neither
+active nor `nostate` (shutoff, crashed, paused, pmsuspended, shutdown) is
+`ErrTransient` at once, since polling will not bring that guest up; and a
+domain found paused or pmsuspended before the start is `ErrPermanent`
+(`cannot be started by the operator; resume or destroy it on the host`), an
+admin's doing that `DomainCreate` would refuse and the reconciler must not
+retry. Every RPC has its own deadline (entry 18).
+
+Steps, each idempotent, in order, all under the per-name lock (entry 19):
+
+1. Validate placement (entry 3). No backend call yet.
+2. `DomainLookupByName`. Found: adoption guard (entry 2), then start it if it
+   is not running (paused or pmsuspended is `ErrPermanent`; `DomainCreate`;
+   `ErrOperationInvalid` means already running), poll as in step 9, return
+   the ID.
+3. Pool space precheck (entry 13, quota).
+4. Ensure the base image (entry 4): marker present and matching means done.
+5. Overlay volume `<name>.qcow2`, qcow2 backed by the base file with
+   `diskSizeGb` capacity; `ErrStorageVolExist` means reuse (creation is
+   atomic, so an existing overlay is complete).
+6. Seed volume `<name>-seed.iso`: delete if it exists, create, upload
+   (entry 5). Always rebuilt.
+7. `StorageVolGetPath` for the overlay and the seed.
+8. `DomainDefineXML` with file-type disks (entry 15's AppArmor note), the
+   metadata element, firmware per image.
+9. `DomainCreate`, then poll `DomainGetState` until running: `nostate` is
+   polled again, any other inactive state is `ErrTransient` at once, and the
+   15-second cap is `ErrTransient` too.
+10. Return `hosts/<host>/domains/<name>`.
+
+A retry after a crash finds whatever the previous attempt got to: an existing
+overlay is reused, a seed is rebuilt, a defined domain is started, a running
+domain is returned. `ErrNoDomain` and `ErrNoStorageVol` inside Create (a
+volume vanishing between steps) are `ErrTransient`, never `ErrNotFound`. The
+log line for a Create records `cloudInitBytes`, never the content.
+
+**Evidence:** contract §2.1, §4.2, §5 ("do not return NotFound from Create");
+execution log run 1 (AppArmor).
+
+## 11. Delete and secondary resources (`delete-return`, `secondary-resources`)
+
+**Answer:** `delete-return`: `waits`. `secondary-resources`: `explicit`. Under
+the per-name lock:
+
+1. Parse the ID (`ErrPermanent` if malformed). A host no longer in the config
+   is a logged no-op success: the provider cannot reach it and refusing would
+   pin the finalizer; the guide tells admins to delete Proxies before retiring
+   a host.
+2. `DomainLookupByName`; `ErrNoDomain` means the domain is already gone, skip
+   to step 5.
+3. `DomainDestroy`; `ErrOperationInvalid` (not running) and `ErrNoDomain` are
+   success.
+4. `DomainUndefineFlags` with `DomainUndefineNvram`,
+   `DomainUndefineManagedSave` and `DomainUndefineSnapshotsMetadata`
+   (UEFI domains keep NVRAM; undefine refuses without the flag); `ErrNoDomain`
+   is success.
+5. Delete `<name>.qcow2` and `<name>-seed.iso` by name from the host's pool;
+   `ErrNoStorageVol` is success, and `ErrNoStoragePool` means nothing is left
+   to clean.
+6. Return nil. The base image and its marker stay (shared cache).
+
+Everything is gone when Delete returns nil, so `Get` afterwards is NotFound.
+A second Delete during the first waits on the lock and then finds nothing.
+Volumes that exist without a domain (a crash between step 5 and 8 of Create)
+are removed by the same Proxy's next Delete but are invisible to GC (entry 24).
+
+**Evidence:** kvm-provider.md "Known gaps"; execution log run 4 (NVRAM);
+contract §2.3.
+
+## 12. ListByTag scope and partial failure (`list-partial`)
+
+**Answer:** `fail-whole`, per host, with single unreadable domains skipped.
+Scope: every host in `hosts`, in config order, `ConnectListAllDomains` with
+active and inactive domains. For each domain, read the metadata element; keep
+those with `managed` true and build the `Instance` from the domain name and
+the element alone: `ID`, `UID` and `CreatedAt`, which is all GC reads
+(contract §2.4). `State` and `IP` are left empty; no state or address RPC is
+made. A domain whose metadata cannot be read (mid teardown, corrupt element)
+is skipped with a V(1) line, unless ctx ended, which fails the call; a host
+that cannot be dialled or whose listing fails fails the whole call, so GC
+skips this provider for the sweep rather than act on a partial view.
+Filtering is client-side; libvirt has no label index. Cost: exactly one
+`DomainGetMetadata` per domain per sweep, about half a millisecond each over
+the tunnel.
+
+**Evidence:** execution log run 6 (per-domain metadata cost); contract §2.4
+and §4.7.
+
+## 13. Error classification (`not-found`, `quota`)
+
+**Answer:** `not-found`: `error-code`. `quota`: `detected`, from a pool-space
+precheck rather than a backend error: before creating any volume, Create reads
+`StoragePoolGetInfo` and compares `available` with the bytes it is about to
+commit: the image's `Content-Length` when the base is not cached, plus
+`diskSizeGb` in bytes, plus 16 MiB for the seed. Below that it returns
+`ErrQuotaExceeded` naming the pool, available and needed, and the reconciler
+waits `QuotaRetry`.
+
+One `classify(op string, err error)`:
+
+| Backend response | Class | Why |
+|---|---|---|
+| `libvirt.Error` code `ErrNoDomain`, `ErrNoStorageVol` | NotFound (Transient when `op == "create"`) | the object is gone; inside Create it vanished mid-call |
+| `ErrNoStoragePool` | Permanent | config names a pool that does not exist |
+| `ErrXMLError`, `ErrNoSupport`, `ErrInvalidArg` | Permanent | the request itself is wrong |
+| `ErrAuthFailed`, `ErrAccessDenied` | Permanent | the SSH user may not use the socket |
+| `ErrOperationInvalid` | Transient (Delete and Create treat the specific cases as success before classify) | state race; the next poll decides |
+| any other `libvirt.Error` | Transient | unknown means retry |
+| SSH: `ssh: unable to authenticate` (from `x/crypto/ssh`) | Permanent | the key is not in `authorized_keys` |
+| SSH: `knownhosts.KeyError` with a known key that differs, or with no known key | Permanent | host key changed or missing from known_hosts; the admin must act |
+| SSH key or known_hosts file missing (`fs.ErrNotExist`) | Transient | the Secret has not landed yet |
+| SSH key present but does not parse | Permanent | wrong file in the Secret |
+| `context.Canceled`, `context.DeadlineExceeded`, `net.OpError`, `io.EOF`, `io.ErrUnexpectedEOF` | Transient | network or cancellation |
+| HTTP 404, 403, 410 on the image URL; missing `Content-Length`; sha256 mismatch | Permanent | the catalogue entry is wrong |
+| HTTP 5xx, 429, network errors on the image URL | Transient | retry |
+| an error already wrapped with a class (`errors.Is` any sentinel) | that class | validation and builders classify at the source |
+| anything else | Transient | contract default |
+
+The unit test table uses a `libvirt.Error` per code, the real SSH error
+strings produced by `x/crypto/ssh` against the in-process server, and a real
+`knownhosts.KeyError`.
+
+**Evidence:** go-libvirt `ErrorNumber` constants and `Error{Code uint32, Message}`
+(value receiver, so `errors.As` uses a `libvirt.Error` value); contract §5;
+kvm-provider.md "Known gaps" (SSH auth retried forever in v1: closed here).
+
+## 14. Credentials and TLS (`credentials`)
+
+**Answer:** `mounted-files`. The config block names `sshKeyFile` (private key,
+OpenSSH or PEM, parsed with `ssh.ParsePrivateKey`) and `knownHostsFile`
+(OpenSSH `known_hosts`, parsed with `knownhosts.New`), both expected under the
+existing optional `kvm-ssh` Secret mount at `/etc/proxy-operator/kvm`
+(`config/manager/manager.yaml`; the mount is shared with the `kvm` provider and
+its comment says so). `insecureIgnoreHostKey: true` replaces known_hosts with
+`ssh.InsecureIgnoreHostKey()` and is logged as a warning by `New`. Both files
+are read on every dial, so rotating the Secret takes effect on the next
+connection without a restart. `New` tries to read them once to log a warning
+when they are unusable, but never fails on them and never dials.
+
+Minimum privilege: the SSH user must be able to open the libvirt socket
+(member of the `libvirt` group); libvirt itself is used as `qemu:///system`
+through the socket, so no further host permissions.
+
+TLS: none (SSH provides transport security); `system-roots` for the image
+download's HTTPS, which uses the distroless image's CA bundle.
+
+Hygiene: the private key bytes and the known_hosts content never appear in
+logs, spans or error strings; errors from parsing say "parsing ssh key file
+<path>" plus the parser's message. Tested with a sentinel placed in a bad key
+file.
+
+**Evidence:** kvm-provider.md "Cluster-side setup"; execution log run 8
+(lazy load: a missing Secret must not abort the provider set); contract §7.
+
+## 15. Config block
+
+**Answer:** the block is `kvm2:` and reuses the Go type `provider.KVMConfig`
+from `internal/provider/config_kvm.go` unchanged, so an existing `kvm-home`
+entry ports by changing `type: kvm` to `type: kvm2` and `kvm:` to `kvm2:`.
+Validation calls the shared validator with the block name `kvm2` so messages
+read `kvm2.hosts` and not `kvm.hosts`.
+
+| Field | Required | Default | Validation | Ends up in |
+|---|---|---|---|---|
+| `sshKeyFile` | yes | | non-empty | dial |
+| `knownHostsFile` | unless `insecureIgnoreHostKey` | | non-empty | dial |
+| `insecureIgnoreHostKey` | | false | | dial, warning at `New` |
+| `diskSizeGb` | | 10 | not negative | overlay capacity, quota precheck |
+| `vendorData` | | | | seed `vendor-data` |
+| `installGuestAgent` | | true | | seed `vendor-data` |
+| `images.<key>` | | | key matches `kvmIdentifier` | volume name `base-<key>.<format>` |
+| `images.<key>.url` | yes per entry | | http(s) URL | download |
+| `images.<key>.sha256` | | | 64 hex | marker comparison |
+| `images.<key>.format` | | qcow2 | qcow2 or raw | base volume XML, overlay backing format |
+| `images.<key>.firmware` | | uefi | uefi or bios | domain XML `<os>` |
+| `machineTypes.<key>` | at least one | | vcpus and memoryMiB greater than zero | domain XML |
+| `hosts[].name` | yes | | unique, matches `kvmIdentifier` | providerID segment |
+| `hosts[].address` | yes | | non-empty `host:port` | TCP dial |
+| `hosts[].user` | yes | | non-empty | SSH |
+| `hosts[].socket` | | `/var/run/libvirt/libvirt-sock` | | SSH channel target |
+| `hosts[].pool` | | `default` | | every storage RPC |
+| `hosts[].bridge` | yes | | non-empty | domain XML NIC |
+
+Wiring: `ProviderConfig.KVM2 *KVMConfig` with tag `kvm2,omitempty`;
+`setBlocks` reports `kvm2`; the allow-list in `Config.validate` and the
+constructor map in `cmd/main.go` both gain `kvm2` in the same commit.
+
+**Evidence:** `KVMConfig`, `validateKVM`, `kvmIdentifier` in
+`internal/provider/config_kvm.go`; contract §10.
+
+## 16. Checker
+
+**Answer:** one target per `hosts[]` entry, `Name` the host name, `Address`
+the SSH address, checked concurrently in config order on a fresh dedicated
+connection each time (dial, `ConnectGetLibVersion`, `ConnectGetHostname`,
+`StoragePoolLookupByName`, `StoragePoolGetInfo`, close). It never touches the
+pooled connection, so a check that runs out of time closes only its own
+connection and cannot disrupt a reconcile. Cost: one SSH handshake and four
+cheap RPCs per host per round.
+
+Reasons: `Reachable`; `Timeout` (the round's ctx ended before the check
+finished, whichever stage was running, the dial included, or an RPC's own
+budget ran out once the connection was up); `Unreachable` (TCP connect or the
+SSH transport failed, including on the dial's own 10 s budget while the round
+still had time, or the link died mid-check); `Unauthorized` (SSH authentication refused, host key unknown or
+changed); `Forbidden` (`ErrAccessDenied` or `ErrAuthFailed` from libvirt: the
+socket refused the user); `APIError` (any other libvirt error);
+`CredentialsUnavailable` (the key or known_hosts file is missing or
+unparsable, i.e. the Secret is absent or wrong: fixed in the cluster, not on
+the host); `PoolNotFound`; `PoolNotActive`.
+
+Details on success: `user`, `hostname`, `libvirtVersion` (decoded
+major.minor.micro), `pool`, `poolState`, `poolCapacityGiB`, `poolAvailableGiB`
+(the closest thing to a capacity signal, and the same number the quota
+precheck reads). The `status.type` comment in
+`api/v1alpha1/providerhealth_types.go` gains `kvm2` and the CRD is
+regenerated.
+
+**Evidence:** contract §8; `StoragePoolGetInfo` returning state, capacity,
+allocation, available; `ConnectGetLibVersion` returning major*1e6 + minor*1e3
++ micro.
+
+## 17. Client and child spans (`client-kind`, `child-spans`)
+
+**Answer:** `client-kind`: `rpc-over-ssh`. `child-spans`: `span-decorator`.
+
+The seam is an interface over exactly the go-libvirt methods the provider
+uses, with `ctx` as the first parameter of every method (go-libvirt's own
+methods take none and block): `DomainLookupByName`, `DomainDefineXML`,
+`DomainCreate`, `DomainDestroy`, `DomainUndefineFlags`, `DomainGetState`,
+`DomainGetMetadata`, `DomainInterfaceAddresses`, `ConnectListAllDomains`,
+`StoragePoolLookupByName`, `StoragePoolRefresh`, `StoragePoolGetInfo`,
+`StorageVolLookupByName`, `StorageVolGetPath`, `StorageVolCreateXML`,
+`StorageVolUpload`, `StorageVolDownload`, `StorageVolDelete`,
+`ConnectGetLibVersion`, `ConnectGetHostname`, plus `Close`. One production
+adapter wraps `*libvirt.Libvirt` and applies the per-call deadline
+(entry 18); one in-memory fake implements the seam for unit tests. A dialer
+seam (`dial(ctx, host) (backend, error)`) separates the SSH transport so tests
+inject fakes and an in-process SSH server test covers the real dialer.
+
+The decorator is applied once, inside the real dialer, at the point where a
+connection is created, so provider code never sees an undecorated backend. Per
+call it starts a span `kvm2.<Method>` (for example `kvm2.DomainDefineXML`) with
+attributes `kvm2.host` and the identifiers passed (`kvm2.domain`, `kvm2.pool`,
+`kvm2.volume`), records the libvirt error code on failure, and inside the span
+writes the V(1) log line, so the line carries that span's ID. Two more spans:
+`kvm2.dial` around the SSH handshake and libvirt connect (attributes address
+and user) and `kvm2.download` around an image fetch (attributes URL host,
+bytes, whether a sha256 was configured). All spans are children of the
+decorator's `provider.create`, `provider.get`, `provider.delete`,
+`provider.list` or the monitor's `providerhealth.check`. No CloudInit, key
+material or upload bodies in attributes. A reflection test walks the seam
+interface and asserts every method produces exactly one span and one log
+line.
+
+**Evidence:** contract §4.15, §4.18, §9; `tracing.Start` in
+`internal/tracing/logger.go`.
+
+## 18. Timeouts (`timeouts`)
+
+**Answer:** `per-call`. The adapter runs each blocking go-libvirt call in a
+goroutine and waits on the result, `ctx.Done()` and a timer:
+
+| Call | Budget |
+|---|---|
+| TCP connect plus SSH handshake plus libvirt connect | 10 s |
+| ordinary RPC | 30 s |
+| `DomainCreate` | 90 s |
+| `StorageVolUpload` (seed) | 60 s |
+| `StorageVolUpload` (image), HTTP download | ctx, capped at 20 m |
+| wait for running after `DomainCreate` | 15 s (entry 10) |
+| HTTP client for images | no client timeout; the per-request ctx carries the cap |
+
+When a call is abandoned (ctx cancelled or budget exceeded), the adapter
+closes the underlying connection, because that is the only way to unblock
+go-libvirt, and marks it dead so the next call redials; the abandoned call
+returns `ErrTransient` wrapping the ctx or deadline error. Consequence,
+documented in the guide: other RPCs in flight on that host's pooled connection
+fail once with Transient and their callers retry. The image download uses a
+dedicated connection so the expensive case (a cancelled poll killing a
+435 MB upload) cannot happen. The keepalive (`keepalive@openssh.com` request
+every 15 s) marks the connection dead on failure.
+
+**Evidence:** contract §1.5 (no caller deadline; a hung ListByTag stalls GC),
+§4.16; go-libvirt methods take no ctx.
+
+## 19. Concurrency
+
+**Answer:** shared state is the host connection table (one pooled connection
+per host, dialled lazily, under a mutex with a per-host dial mutex so
+concurrent callers dial once and share the result), the per-`(host, volume)`
+download locks, and a lock per instance name. Both lock sets are one instance
+each of `keyedLocks`: a one-slot semaphore per key, so a waiter gives up when
+its ctx ends, reference-counted so an entry is removed from the map once no
+caller holds or waits for it and the map does not grow with every name ever
+seen, while two waiters on one key always share one semaphore. Create and
+Delete for the same name run under that name's lock, so two Deletes (GC and
+the reconciler) never race on destroy and undefine, and a Delete cannot
+interleave with a Create's steps. Get and ListByTag take no name lock. go-libvirt multiplexes
+RPCs on one connection, so calls for different names proceed concurrently
+on the shared connection. Three reconciles, a GC sweep and a check round at
+once are safe; the check uses its own connections.
+
+**Evidence:** contract §1.2 (concurrency), §4.15.
+
+## 20. Logging (`logging`)
+
+**Answer:** `tiered`. Logger from `logf.FromContext(ctx)` on every call,
+named `kvm2`, with `provider` on every line and `host` and `name` on every
+line about an instance.
+
+- V(1): one line per operation outcome (`kvm2 domain ensured running`,
+  `kvm2 domain fetched` with state, ip and uid, `kvm2 domain deleted`,
+  `kvm2 domains listed` with counts), one `libvirt call` line per RPC from the
+  decorator (`rpc`, identifiers, `duration`, on failure `error`, `libvirtCode`
+  and `libvirtError` name), connection events (dialled, dropped with reason),
+  and a line per failed backend call with the raw error.
+- V(2): the same lines also carry domain and volume XML, the metadata element,
+  interface addresses, listed domain names, and the resolved Create parameters
+  (`cloudInitBytes`, never the content).
+- No V(5) wire tier; go-libvirt exposes no hook for it.
+- Never logged at any level: `CloudInit`, key material, known_hosts content,
+  upload or download bodies.
+
+**Evidence:** contract §4.17, §9.
+
+## 21. Tests (`backend-test`)
+
+**Answer:** `backend-test`: `integration-tag`. Unit tests run over the
+in-memory fake behind the seam (and an in-process `x/crypto/ssh` server for
+the dialer); the integration test is `kvm2_integration_test.go` behind
+`//go:build integration`, skipped unless `KVM2_TEST_HOST`, `KVM2_TEST_USER`,
+`KVM2_TEST_SSH_KEY`, `KVM2_TEST_BRIDGE` and `KVM2_TEST_IMAGE_URL` are set,
+optional `KVM2_TEST_KNOWN_HOSTS` (without it host keys are not verified and the
+log says so), `KVM2_TEST_IMAGE_SHA256`, `KVM2_TEST_POOL`, `KVM2_TEST_SOCKET`,
+`KVM2_TEST_TIMEOUT` (default 20m), `KVM2_TEST_KEEP`, `KVM2_TEST_LOG_LEVEL`;
+run with `make test-integration-kvm2`. It asserts, in order: `New` returns
+without dialling; `Check` reports the host Ready with details; `Create`
+returns `hosts/<host>/domains/<name>`; a second `Create` returns the same ID
+and the host's domain count is unchanged; `Get` reaches Running with an IPv4
+within the budget and its `UID` and `CreatedAt` equal what Create wrote;
+`ListByTag` contains it; `Delete` returns nil; `Get` is `ErrNotFound`;
+`Delete` again returns nil; the pool holds the base image and its marker and
+no `<name>.qcow2` or `<name>-seed.iso`; with an in-memory span recorder,
+`kvm2.DomainDefineXML` and `kvm2.dial` spans are children of
+`provider.create`.
+
+Contract §11 list, with the test that covers each item:
+
+- [x] repeat `Create`: `TestCreate_repeatReturnsSameIDAndCreatesNothing`
+- [x] `Get` after `Delete`: `TestGet_afterDeleteReturnsNotFound`
+- [x] `Delete` twice: `TestDelete_twiceSucceeds`
+- [x] malformed ID: `TestGet_malformedIDIsPermanent`, `TestDelete_malformedIDIsPermanent`, `TestParseProviderID_malformed`
+- [x] unknown state maps to `StateStopped`: `TestMapState_table`
+- [x] `StateRunning` requires an IP: `TestGet_runningWithoutIPIsProvisioning`, `TestGet_runningWithIPIsRunning`
+- [x] `UID` and `CreatedAt` read back: `TestGet_readsUIDAndCreatedAtFromBackend`
+- [x] `ListByTag` excludes unmarked instances: `TestListByTag_excludesUnmarkedDomains`, `TestListByTag_emptyUIDStillListed`
+- [x] `Get` during an in-progress delete: `TestGet_duringDeleteStillReturnsInstance`
+- [x] `Get` during an asynchronous operation or lock: `TestGet_backendBusyIsTransientNeverNotFound`
+- [x] never started is never `StateStopped`: `TestCreate_returnsOnlyWhenRunning`
+- [x] cancelled `ctx` returns promptly: `TestProvider_cancelledContextReturnsPromptly`, `TestCall_cancelUnblocksRPC`
+- [x] `classify` table with captured responses: `TestClassify_table`
+- [x] Checker reasons, expired `ctx`, stable order: `TestCheck_reasonsTable`, `TestCheck_expiredContextReturnsPromptly`, `TestCheck_oneEntryPerHostInConfigOrder`
+- [x] sentinel tests for `CloudInit` and the credential: `TestCreate_cloudInitSentinelNeverLoggedOrInErrors`, `TestSSH_keyMaterialNeverInErrorsOrLogs`
+- [x] child spans under the operation span: `TestObserve_childSpansUnderParent`, `TestObserve_everySeamMethodInstrumented`
+- [x] real-backend lifecycle test: `TestKVM2_lifecycle` and `TestKVM2Check_realHost` in `kvm2_integration_test.go`
+
+Further tests per file are listed in the plan
+(`docs/plans/2026-09-16-1333-kvm2-provider.md`).
+
+**Evidence:** contract §11; docs/testing.md conventions.
+
+## 22. Deployment
+
+**Answer:** the existing `kvm-ssh` mount is reused; the rest is samples and a Makefile target:
+
+- `config/manager/manager.yaml`: no new mount; the existing optional `kvm-ssh`
+  Secret at `/etc/proxy-operator/kvm` serves both `kvm` and `kvm2`, and its
+  comment says so. `/tmp` is already an emptyDir.
+- `config/manager/providers_config.yaml` and
+  `config/samples/providers-config.yaml`: a commented `kvm2-home` entry, the
+  same block as `kvm-home` under `kvm2:`.
+- `config/samples/proxy_kvm2.yaml`, listed in `config/samples/kustomization.yaml`:
+  provider `kvm2-home`, zone `beelink` (matching the shipped config, unlike the
+  `kvm` sample's `nas1`), machine type `small`, image `debian-13`, a cloud-init
+  that installs Squid.
+- Makefile: `test-integration-kvm2` (`go test -race -tags=integration -count=1
+  -v -timeout 40m -run TestKVM2 ./internal/provider/kvm2/`). The Secret helper
+  `make kvm-ssh-secret` already exists and is reused.
+- RBAC: none; the provider does not talk to the cluster's API.
+- `hack/providers-dev.yaml`: no entry (needs credentials).
+
+**Evidence:** contract §12; `config/manager/manager.yaml`.
+
+## 23. Docs
+
+**Answer:** `docs/kvm2-provider.md` (hypervisor prerequisites including OVMF
+and the `libvirt` group and a bridge with DHCP; cluster-side setup reusing the
+`kvm-ssh` Secret; config reference; placement mapping; what lands on the host
+including the metadata namespace; the image cache and the quota precheck; the
+health check reason table; logging tiers and span names; migrating an entry
+from `kvm`, which requires deleting its Proxies first because kvm2 does not
+adopt v1 domains; known gaps). A fourth column in the three tables of
+`docs/providers-existing.md` and a kvm2 section. `docs/architecture.md`:
+components row, a `kvm2` block in §5, the span topology in §10, a §11 row, and
+Decisions bullets (dedicated download connection, checks on fresh connections,
+own metadata namespace, quota precheck, SSH auth failures Permanent). A README
+row. `docs/testing.md` rows and a kvm2 integration section. A `CHANGELOG.md`
+entry after the user confirms.
+
+**Evidence:** contract §12.
+
+## 24. Known limits accepted at design time
+
+**Answer:** nine limits, each with its workaround:
+
+- **Create blocks**, including the first image download per host, holding one
+  of three reconcile workers. Workaround: warm the cache with one Proxy per
+  image per host before creating many.
+- **Volumes without a domain are invisible to GC** (crash between volume
+  creation and define). The same Proxy's next reconcile removes them; a Proxy
+  deleted for good leaves them for `virsh vol-list`.
+- **A host removed from the config orphans its VMs**: Delete is a no-op
+  success for it and GC does not sweep it. Delete the Proxies first.
+- **No adoption of `kvm` (v1) domains**: switching an entry's type requires
+  deleting its Proxies first, or the old domains stay behind.
+- **Cancelling one call drops the host's pooled connection**, so other RPCs
+  in flight on that host fail once with Transient and retry. The download is
+  isolated on its own connection.
+- **`Content-Length` is required** for image downloads; a chunked mirror is
+  rejected as Permanent.
+- **A paused or pmsuspended domain is not started.** A repeat `Create` for
+  a domain in that state is `ErrPermanent` (entry 10), so the Proxy latches
+  as Failed until its spec changes, rather than retrying a `DomainCreate`
+  that libvirt refuses. Resume the domain on the host (`virsh resume`) and
+  touch the spec, or delete the Proxy.
+- **No v1 log-format compatibility**: the `libvirt call` lines have the same
+  keys where the same facts exist, but dashboards keyed on `kvm` logger names
+  need `kvm2`.
+- Contract §13 limits that apply unchanged: config read once per process
+  (adding a host needs a restart; credentials do not), no installation identity
+  in GC, an unconfigured provider latches the Proxy.
+
+**Evidence:** contract §13; entries 4, 8, 11, 18 above.

docs/kvm2-provider.md

@@ -0,0 +1,604 @@
+# The kvm2 provider
+
+Runs Managed proxies as VMs on your own libvirt/KVM hypervisors. It is the
+second-generation libvirt provider, built clean-room from
+[provider-contract.md](provider-contract.md) and the design in
+[kvm2-provider-spec.md](kvm2-provider-spec.md), and it ships side by side
+with the first-generation `kvm` provider: both types can be configured at
+once, both take the same config block shape, and both read the same
+`kvm-ssh` Secret. What kvm2 adds over kvm: a child span per libvirt RPC,
+a pool-space precheck that maps to the reconciler's quota wait, SSH
+credentials re-read on every dial, image downloads and health checks on
+connections of their own, an adoption guard on the owning UID, and SSH
+authentication failures that stop retrying.
+
+Like kvm, the operator reaches each host over SSH, forwards the libvirt
+unix socket through that connection and speaks libvirt's own RPC over it,
+so the hosts need nothing beyond `sshd` and `libvirtd` and the operator
+image carries no libvirt libraries. VMs boot a stock distro cloud image
+that the operator downloads into the host's storage pool on first use,
+with the Proxy's cloud-init delivered on a NoCloud seed ISO built
+in-process.
+
+## Hypervisor prerequisites
+
+Per host, once:
+
+1. **libvirt and KVM** (`libvirtd` running, `/dev/kvm` present).
+2. **An SSH user in the `libvirt` group**, so it may open
+   `/var/run/libvirt/libvirt-sock`. The operator authenticates with a
+   private key; nothing else on the host is touched. The socket is used
+   as `qemu:///system`, so no further host permissions are needed.
+3. **A bridge to a network with DHCP** (for example `br0` bridged onto
+   the LAN). Each VM gets one virtio NIC on it and takes its IP from that
+   network's DHCP. The operator's pod network and the crawler clients
+   must route to those IPs: the health engine probes `http://<ip>:<port>`
+   through the proxy and clients get the same address from the discovery
+   API. Nothing on the host filters the port; the provider does not use
+   `spec.port` at all.
+4. **A storage pool** (default `default`, usually `/var/lib/libvirt/images`)
+   with room for one base image per distro (a few hundred MB) plus the
+   per-VM overlays (`diskSizeGb` each, thin-provisioned). The pool must be
+   running: a stopped pool fails the health check with `PoolNotActive` and
+   fails `Create` permanently.
+5. **UEFI firmware for guests**: the `ovmf` package on Debian/Ubuntu,
+   `edk2-ovmf` on Fedora/RHEL. VMs boot UEFI by default with secure boot
+   off (libvirt picks the OVMF binary itself); a domain start fails with
+   "no firmware found" without it. Only `firmware: bios` images skip this.
+6. **Outbound internet from the VMs**, in practice: a vendor-data script
+   installs `qemu-guest-agent` from the distro mirrors on first boot, and
+   the agent is how the operator learns the VM's address. Without it the
+   operator falls back to the host's ARP table, which only sees traffic
+   addressed to the host itself, so on a plain bridge a VM without the
+   agent stays `Provisioning`.
+
+Check from the operator's point of view before wiring it in:
+
+```sh
+ssh novakj@beelink virsh -c qemu:///system list --all      # user can drive libvirt
+ssh novakj@beelink virsh -c qemu:///system pool-info default
+ssh novakj@beelink ip link show br0
+```
+
+## Cluster-side setup
+
+1. Create the SSH Secret from a key pair dedicated to the operator and a
+   `known_hosts` file holding every host's key. It is the same `kvm-ssh`
+   Secret the `kvm` provider uses; one Secret serves both types, and if
+   you already have it there is nothing to do here.
+
+   ```sh
+   # once: a key pair for the operator; the .pub goes on every host
+   ssh-keygen -t ed25519 -f ~/.ssh/proxy-operator_ed25519 -N '' -C proxy-operator
+   ssh-copy-id -i ~/.ssh/proxy-operator_ed25519.pub novakj@beelink
+
+   # once per host: its host key, under the address the *operator* will
+   # dial (hosts[].address), which may differ from what your laptop uses
+   ssh-keyscan -H beelink.tail8c69c.ts.net >> ~/.ssh/kvm_known_hosts
+
+   # the Secret: the PRIVATE key (not the .pub) plus known_hosts
+   kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
+     --from-file=id_ed25519=$HOME/.ssh/proxy-operator_ed25519 \
+     --from-file=known_hosts=$HOME/.ssh/kvm_known_hosts
+   ```
+
+   `make kvm-ssh-secret` runs that last command as an apply, so it also
+   updates an existing Secret; override the paths with
+   `KVM_SSH_KEY=... KVM_KNOWN_HOSTS=...`. Putting the `.pub` in the
+   Secret is the classic mistake: the target's `kvm-ssh-secret` check
+   refuses a file without `PRIVATE KEY` in it, and the provider itself
+   would fail every call permanently with a "parsing ssh key file" error.
+   Without `known_hosts`, set `insecureIgnoreHostKey: true` and accept
+   that a MITM could hand you a fake hypervisor.
+
+   `config/manager/manager.yaml` mounts the Secret (optional) at
+   `/etc/proxy-operator/kvm` as two files, `id_ed25519` and
+   `known_hosts`, which is what `sshKeyFile` and `knownHostsFile` in the
+   provider config point at. The deployment starts fine without the
+   Secret, even with a kvm2 provider configured: `New` only checks that
+   the files exist and logs a `WARNING:` line when one does not; the
+   provider's proxies fail transiently and recover once the Secret lands.
+
+   **Rotating the key or a host key needs no restart.** Both files are
+   read and parsed on every dial (`loadCredentials` in
+   `internal/provider/kvm2/ssh.go`); nothing is cached. Re-run the
+   command or the make target, and the next connection, health check
+   included, uses the new material. Connections that are already open
+   stay open until they drop.
+
+2. Add a provider entry to the providers config
+   (`config/manager/providers_config.yaml` in-cluster; full sample in
+   `config/samples/providers-config.yaml`). The type is `kvm2` and the
+   block is `kvm2:`; its fields are those of the `kvm` block:
+
+   ```yaml
+   - name: kvm2-home
+     type: kvm2
+     kvm2:
+       sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
+       knownHostsFile: /etc/proxy-operator/kvm/known_hosts
+       images:
+         debian-13:
+           url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+       machineTypes:
+         small: {vcpus: 1, memoryMiB: 1024}
+       hosts:
+         - name: beelink
+           address: beelink.tail8c69c.ts.net:22
+           user: novakj
+           bridge: br0
+   ```
+
+   Validation messages name the block, so a mistake reads
+   `kvm2.hosts[0]: bridge is required`, not `kvm.hosts`. The providers
+   config is read once per process: adding a host or an image entry needs
+   a manager restart; credentials do not.
+
+3. Apply a Proxy (`config/samples/proxy_kvm2.yaml`) and watch it:
+
+   ```sh
+   kubectl get px -w
+   # NAME                MODE      PROVIDER    PHASE          IP             HEALTHY
+   # proxy-kvm2-sample   Managed   kvm2-home   Provisioning
+   # proxy-kvm2-sample   Managed   kvm2-home   Running        192.168.1.73
+   # proxy-kvm2-sample   Managed   kvm2-home   Ready          192.168.1.73   True
+   ```
+
+   The first Proxy for a given image on a given host sits in
+   `Provisioning` for the length of the image download; the operator log
+   shows `fetching base image into host pool` with the URL and size and
+   `base image cached` with the size, duration and digest.
+
+## Config reference
+
+The block is `kvm2:` and its Go type is the shared `KVMConfig` in
+`internal/provider/config_kvm.go`, so the fields, defaults and validation
+are the same as for `kvm`. Defaults are applied by `withDefaults` in
+`internal/provider/kvm2/kvm2.go`.
+
+| Field | Required | Default | Meaning |
+|---|---|---|---|
+| `sshKeyFile` | yes | | private key used for every host; OpenSSH or PEM, parsed on every dial |
+| `knownHostsFile` | unless `insecureIgnoreHostKey` | | OpenSSH `known_hosts` to verify host keys against, read on every dial |
+| `insecureIgnoreHostKey` | | `false` | skip host-key verification entirely; `New` logs a `WARNING:` line when it is on |
+| `diskSizeGb` | | `10` | capacity of each VM's overlay disk; cloud-init's growpart expands the root fs into it. Also what the pool-space precheck reserves per VM |
+| `vendorData` | | | extra cloud-init vendor-data placed on every seed next to the Proxy's user-data. Beware that cloud-init drops any top-level key of a vendor-data **cloud-config** that the user-data also defines. Shipped as one part of a MIME multipart alongside the agent script when `installGuestAgent` is on |
+| `installGuestAgent` | | `true` | add the vendor-data script that installs and starts `qemu-guest-agent`. Turn off for images that ship it already or cannot reach a package mirror; IP discovery then depends on the host's ARP table |
+| `images.<key>` | | | the key names the image in `placement.image`. Letters, digits, dots, dashes and underscores only: it becomes the cached volume name `base-<key>.<format>` |
+| `images.<key>.url` | yes per entry | | http(s) URL of a cloud image (qcow2 or raw). The server must send `Content-Length` |
+| `images.<key>.sha256` | | | expected digest, 64 hex characters; verified after download, a change triggers a re-download |
+| `images.<key>.format` | | `qcow2` | `qcow2` or `raw` |
+| `images.<key>.firmware` | | `uefi` | `uefi` (OVMF, secure boot off) or `bios` (SeaBIOS) for images without an EFI system partition |
+| `machineTypes.<key>` | at least one | | `{vcpus, memoryMiB}` size profile, both greater than zero |
+| `hosts[].name` | yes | | the `placement.zone` value that selects this host; unique. Letters, digits, dots, dashes and underscores only: it becomes part of the providerID `hosts/<name>/domains/<vm>` |
+| `hosts[].address` | yes | | SSH endpoint, `host:port` |
+| `hosts[].user` | yes | | SSH user, member of the `libvirt` group |
+| `hosts[].socket` | | `/var/run/libvirt/libvirt-sock` | libvirt socket path on the host |
+| `hosts[].pool` | | `default` | storage pool for base images, overlays and seeds |
+| `hosts[].bridge` | yes | | host bridge for the VM's NIC |
+
+## Placement
+
+All three fields are checked before any host is dialled. An unknown value
+is `ErrPermanent`, which the reconciler latches as `Failed` with the
+message below until the spec changes.
+
+| Proxy field | Meaning for kvm2 | Error when unknown |
+|---|---|---|
+| `placement.zone` | host name from `hosts`. May be omitted when exactly one host is configured; otherwise required | `placement.zone "x" is not in kvm2.hosts`; with several hosts and no zone, `placement.zone is required when kvm2.hosts lists more than one host` |
+| `placement.machineType` | key in `machineTypes` | `placement.machineType "x" is not in kvm2.machineTypes` |
+| `placement.image` | key in `images`, or an `http(s)://` URL used directly with the defaults (`qcow2`, `uefi`) | `placement.image "x" is neither a key in kvm2.images nor an http(s) URL` |
+| `placement.region` | ignored. It is still part of the spec hash, so changing it replaces the VM and does nothing else | |
+
+The messages come from `Create` in `internal/provider/kvm2/create.go`,
+`host` in `kvm2.go` and `resolveImage` in `image.go`. Fix the spec (which
+replaces the VM) or the config. Catalogue edits affect only new VMs: the
+spec hash covers the placement values, not what the config maps them to.
+
+## What lands on the host
+
+For a Proxy whose deterministic name is `proxy-abc…`, on the chosen
+host's pool:
+
+| Object | Name | Lifetime |
+|---|---|---|
+| base image | `base-<key>.<format>` (for a URL image, `<key>` is the first 16 hex characters of the URL's sha256) | cached; survives Proxy deletion |
+| image marker | `base-<key>.<format>.sha256`, a 64-byte raw volume holding the digest | with the base image |
+| overlay disk | `proxy-abc….qcow2`, qcow2 backed by the base file, `diskSizeGb` capacity | with the Proxy |
+| seed ISO | `proxy-abc…-seed.iso`, raw | with the Proxy |
+| domain | `proxy-abc…` | with the Proxy |
+
+The names are built in `internal/provider/kvm2/volume.go`.
+
+The domain (`buildDomainXML` in `xml.go`) is a `q35` machine with a
+host-passthrough CPU, ACPI and APIC, memory in MiB and vcpus from the
+machine type, the overlay on `vda` (virtio) and the seed as a read-only
+SATA cdrom on `sda`, one virtio NIC on the bridge, a serial console on a
+pty, the guest-agent channel `org.qemu.guest_agent.0`, a virtio RNG fed
+from `/dev/urandom`, and no graphics. Both disks are `type="file"` with
+the paths `StorageVolGetPath` returned: AppArmor derives QEMU's profile
+from those paths, and volume-type disks fail with permission denied on
+stock Debian and Ubuntu hosts. UEFI images get `<os firmware="efi">` with
+the `secure-boot` feature disabled; BIOS images get neither.
+
+Its `<metadata>` carries one element in kvm2's own namespace,
+`https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm2`
+(`metadataURI` in `metadata.go`):
+
+```xml
+<instance xmlns="https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm2">
+  <managed>true</managed>
+  <uid>0f3a…</uid>
+  <createdAt>2026-09-16T17:42:11Z</createdAt>
+</instance>
+```
+
+That is how orphan GC recognises its own VMs, since libvirt has no
+labels, and how a repeat `Create` knows whether an existing domain of the
+same name belongs to this Proxy: a domain without the element is a
+permanent error (`carries no kvm2 mark; it was not created by this
+provider`), a different non-empty `uid` is a permanent error naming both
+UIDs, and a marked domain with a matching or empty `uid` is adopted. The
+namespace is kvm2's own, so the `kvm` provider's domains are invisible to
+it (see *Migrating from kvm*).
+
+The seed (`buildSeedISO` in `seed.go`) is an ISO 9660 image labelled
+`cidata` with three files. `user-data` is the Proxy's `cloudInit`, **byte
+for byte**: any format cloud-init accepts works, including a Jinja
+template, a MIME archive, a shell script or a gzipped payload; the
+operator never parses or re-renders it. `meta-data` holds
+`instance-id: <name>` and `local-hostname: <name>`, so a reboot is never
+taken for a new instance. `vendor-data` carries the guest-agent
+bootstrap. The ISO is staged in a temp directory under `/tmp` (the
+manager's 64Mi emptyDir), assembled in memory (tens of KiB) and uploaded
+as a raw volume; the staging directory is removed before the call
+returns. The seed is rebuilt on every `Create` attempt, because a
+previous attempt may have been cut off mid-upload and a truncated seed
+would boot a VM without cloud-init.
+
+The guest agent rides along in `vendor-data` as a **shell script**
+(`guestAgentScript` in `seed.go`), not a cloud-config, on purpose:
+cloud-init merges a vendor-data cloud-config under the user-data with
+mergers that keep the user-data's value for every top-level key both
+define and never append lists, so a Proxy carrying its own `packages:`
+would silently lose the agent. Scripts bypass the merge. The script is
+idempotent, picks whichever of `apt-get`, `dnf`, `yum`, `zypper` or `apk`
+the image has, enables the service with `systemctl` or OpenRC, and logs
+into `/var/log/cloud-init-output.log`. With `vendorData` also configured,
+the two travel as separate parts of a MIME multipart (`text/plain` for
+your document, `text/x-shellscript` for the script), with the
+`MIME-Version` header first so cloud-init detects it. With
+`installGuestAgent: false` the file is your `vendorData` alone, or empty.
+
+## The image cache
+
+The base image is downloaded **by the operator, through libvirt**, into
+the host's pool the first time a Proxy asks for it on that host. Nothing
+is pre-placed on the hypervisor and the operator needs no filesystem
+access there. The bytes stream from the URL through the operator into
+`StorageVolUpload` with a running sha256; nothing is buffered
+(`fetchImage` and `uploadImage` in `internal/provider/kvm2/image.go`).
+
+The upload runs on a **dedicated connection** to the host, dialled for
+the transfer and closed after it, not on the pooled connection the other
+operations share. A cancelled or overlong call kills the connection it
+ran on (see *Timeouts and connections*), and a poll for some other Proxy
+being cancelled must not cut off a 20-minute upload.
+
+The download needs a `Content-Length`: the volume's capacity is declared
+before the upload starts. The request sends `Accept-Encoding: identity`
+so a transparently gzipped body cannot hide the length. A response
+without it, or with length 0, is a permanent error naming the URL, and so
+is any 4xx status; 429 and 5xx are transient and retried.
+
+The marker volume is written **last**, on the pooled connection, after
+`StoragePoolRefresh` has made libvirt read the file's real format. Its
+presence is what marks a base complete: a crash mid-download leaves a
+base volume without a marker, which the next attempt deletes and redoes
+(`cachedImage`). A re-download also happens when the marker's content is
+not a 64-character hex digest, or when `sha256` is configured and differs
+from it. Otherwise the cached copy is used for as long as it exists, even
+if the URL's content changed upstream: a `latest` URL is pinned at the
+moment of first fetch, per host. Concurrent Creates for the same image
+on the same host queue on a per-(host, volume) lock; the first fetches,
+the rest find the marker.
+
+To evict manually:
+
+```sh
+ssh novakj@beelink virsh -c qemu:///system vol-delete --pool default base-debian-13.qcow2.sha256
+ssh novakj@beelink virsh -c qemu:///system vol-delete --pool default base-debian-13.qcow2
+```
+
+**Warning: do not change an image entry's `url` or `sha256`, and do not
+evict its base volume, while Proxies using that image exist on the
+host.** The overlays back onto the base file; deleting or replacing it
+corrupts every VM that uses it. Delete those Proxies first, or add the
+new image under a new key and move Proxies over by changing their
+`placement.image` (which replaces the VM).
+
+## Quota: the pool-space precheck
+
+Before creating any volume, `Create` reads `StoragePoolGetInfo` and
+compares the pool's free space with what it is about to commit
+(`checkPoolSpace` and `neededBytes` in `image.go`): the image's
+`Content-Length` when the base is not cached yet (found with a `HEAD`
+request, or a `GET` whose body is closed at once when `HEAD` answers
+anything but 2xx, since mirrors refuse it with 405, 403 or 501 alike; the
+`GET`'s status is then what counts), plus `diskSizeGb` in bytes for the
+overlay, plus 16 MiB for the
+seed. Below that it returns `ErrQuotaExceeded` with a message naming the
+pool, the bytes available and the bytes needed. The reconciler answers
+quota with a wait, not a failure: the Proxy shows `Provisioned=False`
+with reason `QuotaExceeded` and is retried after `QuotaRetry` (5
+minutes), off the error backoff curve and never `Failed`. Free up the
+pool or lower `diskSizeGb` and the next attempt goes through.
+
+The precheck is the only quota signal; libvirt itself reports a full pool
+as an ordinary error, which would be transient. The health check exposes
+the same number as `poolAvailableGiB`, so a pool that is filling up shows
+there before Creates start waiting.
+
+A pool that is not running fails the precheck permanently (`storage pool
+<pool> is not running`): start it with `virsh pool-start` and
+`virsh pool-autostart`.
+
+## Migrating from `kvm`
+
+The `kvm2` block is the `kvm` block under another name, so an entry ports
+by changing `type: kvm` to `type: kvm2` and `kvm:` to `kvm2:`. The
+`kvm-ssh` Secret is shared; nothing changes on the hosts.
+
+What does not port is the VMs. kvm2 marks its domains in its own metadata
+namespace and neither lists nor adopts domains the `kvm` provider made:
+`ListByTag` skips them, so orphan GC never sees them, and `Create` refuses
+a domain of the same name that lacks the kvm2 mark with a permanent error
+rather than start it (a Proxy recreated under the new entry gets the same
+deterministic name, so this is what happens when step 1 below is skipped).
+A Proxy's `spec.provider` is immutable, so an existing Proxy cannot be
+moved to the new entry anyway. Do it in this order:
+
+1. Delete the Proxies on the `kvm` entry and wait for them to go; the
+   `kvm` provider removes their domains and volumes.
+2. Change the entry (or add a second one under a new name and keep the
+   old one for a while; both types may share a host).
+3. Restart the manager, then recreate the Proxies with the new
+   `spec.provider`.
+
+A `kvm` domain left behind on a host is invisible to kvm2 forever; clean
+it up on the host with `virsh undefine --nvram <name>` and
+`virsh vol-delete` for its `<name>.qcow2` and `<name>-seed.iso`.
+
+## Logging
+
+The provider logs through controller-runtime's logger, named `kvm2`, with
+`provider` (the configured name) on every line, and `host` and `name`
+(the domain) on every line about an instance. Every line inside a traced
+operation also carries `traceID` and `spanID`.
+
+- `--zap-log-level=debug` (logr V(1)):
+  - one line per operation outcome: `kvm2 domain ensured running` with the
+    `id`, `kvm2 domain fetched` with `state`, `ip` and `uid`,
+    `kvm2 domain not found`, `kvm2 domain deleted`, `kvm2 domains listed`
+    with `hosts` and `managed` counts, `kvm2 host checked` with `ready`,
+    `reason` and `latency`; plus the steps inside a Create
+    (`kvm2 domain already defined`, `overlay volume created`,
+    `seed volume uploaded` with `bytes`, `domain defined` with `uid` and
+    `firmware`, `domain started`, `fetching base image into host pool`,
+    `base image cached`);
+  - **one `libvirt call` line per libvirt RPC**, written by the decorator
+    in `internal/provider/kvm2/observe.go` inside that RPC's span: `rpc`
+    (the go-libvirt method name, for example `DomainLookupByName`), the
+    identifiers it was given (`domain`, `pool`, `volume`), `duration`, and
+    on failure `error` plus `libvirtCode` and `libvirtError` (the code's
+    name, for example `ErrNoDomain`). A Get is four or five RPCs; a
+    Create with a cached image is around twenty;
+  - connection events from `internal/provider/kvm2/hosts.go`:
+    `libvirt host connected` with `address`, `user` and `duration`, and
+    `libvirt host connect failed` with the same plus `error` and `stage`
+    (`credentials`, `tcp`, `ssh` or `socket`);
+  - `kvm2 backend call failed` with `op` and the raw `error` once per
+    failed operation, before the error is returned;
+  - a line per domain that `ListByTag` skips, and a line when the image
+    marker forces a refetch.
+- `--zap-log-level=2`: the `libvirt call` lines also carry the domain and
+  volume `xml`, the `metadata` element, the interface `addresses`, the
+  listed `domains`, the pool `state` and sizes and the volume `path`; the
+  `domain defined` line carries the `xml`; `kvm2 create resolved` logs the
+  resolved parameters (pool, bridge, machine type, sizes, image volume,
+  firmware) and `cloudInitBytes`; `kvm2 listed instance` logs each listed
+  domain.
+- Never logged at any level: the cloud-init content, the SSH key, the
+  known_hosts content, and upload or download bodies. The seed and image
+  uploads report `offset` and `length` only. There is no raw wire tier
+  like the GCP provider's V(5); go-libvirt exposes no hook for it.
+
+## Tracing
+
+With an OTLP exporter configured ([architecture.md
+§10](architecture.md#10-tracing-internaltracing)), every backend call is
+a child span of the operation span the decorators open
+(`provider.create`, `provider.get`, `provider.delete`, `provider.list`)
+or of the monitor's `providerhealth.check`:
+
+| Span | Around | Attributes |
+|---|---|---|
+| `kvm2.<Method>`, for example `kvm2.DomainDefineXML` | one libvirt RPC (`observe.go`) | `kvm2.host`, and the identifiers passed: `kvm2.domain`, `kvm2.pool`, `kvm2.volume`; on failure the status is Error and `kvm2.libvirt_code` carries the libvirt error number |
+| `kvm2.dial` | the SSH handshake and the libvirt connect (`hosts.go`) | `kvm2.host`, `kvm2.address`, `kvm2.user` |
+| `kvm2.download` | one image fetch, from the HTTP request to the marker (`image.go`) | `kvm2.host`, `kvm2.volume`, `kvm2.url_host`, `kvm2.sha256_configured`, and `kvm2.bytes` once the response arrived |
+
+All spans are client spans. No attribute carries cloud-init, key material
+or a URL beyond its host. The `libvirt call` log line for an RPC is
+written inside that RPC's span, so its `spanID` points at the span.
+
+## Timeouts and connections
+
+No caller puts a deadline on the context, so every backend call carries
+its own budget (`defaultBudgets` in `internal/provider/kvm2/backend.go`,
+applied by `call`):
+
+| Call | Budget |
+|---|---|
+| TCP connect plus SSH handshake plus libvirt connect | 10 s |
+| ordinary RPC | 30 s |
+| `DomainCreate` | 90 s |
+| `StorageVolUpload` of at most 16 MiB (the seed, the marker) | 60 s |
+| `StorageVolUpload` of an image, `StorageVolDownload`, and the whole image fetch from HTTP request to marker | 20 m; the caller's ctx may end it earlier |
+| wait for running after `DomainCreate` | 15 s, polled every 500 ms (`defaultRunningWait` and `defaultRunningPoll` in `create.go`); `DomainCreate` is synchronous, so the first poll normally answers running, and a domain that is shut off, crashed or paused right after the start fails at once rather than waiting out the cap |
+| the `HEAD` request of the pool-space precheck | 30 s |
+| the courtesy `ConnectClose` when a connection is closed | 5 s |
+
+go-libvirt's calls take no context and block, so the only way to abandon
+one is to close the connection under it. That is what a cancelled or
+overlong call does: the adapter kills the connection (socket and SSH
+client), returns `ErrTransient` wrapping the context or deadline error,
+and marks the connection dead so the next call redials. The consequence:
+**other RPCs in flight on that host's pooled connection fail once with
+`ErrTransient`** and their callers retry on a fresh connection. In the
+reconciler that is one requeue with backoff; in GC it is one skipped
+sweep.
+
+Two callers are kept off the pooled connection for that reason
+(`dialFresh` in `hosts.go`): the image upload, because a 435 MB transfer
+must not die because a poll for another Proxy was cancelled, and the
+health check, because a check that runs out of the monitor's timeout must
+close only its own connection.
+
+Otherwise there is one pooled SSH plus libvirt connection per host,
+dialled lazily on the first call that needs it and shared by every
+operation for that host (go-libvirt multiplexes RPCs on one connection).
+Concurrent callers wait for one dial and share the result; waiting ends
+with the caller's context. A `keepalive@openssh.com` request every 15 s
+marks the connection dead when it fails or gets no reply within another
+15 s, as does go-libvirt noticing the socket is gone; the next call
+redials. Create and Delete for the same domain name are serialised on a
+per-name lock, so two Deletes (GC and the reconciler) never race on
+destroy and undefine, and a Delete cannot interleave with a Create's
+steps; Get and ListByTag take no lock.
+
+## Health check
+
+The operator checks every configured host every `--provider-check-interval`
+(default 1m) and publishes the result in the `ProviderHealth` object named
+after the provider (`kubectl get ph kvm2-home -o yaml`) and in the
+`proxy_operator_provider_target_up{provider,target}` gauge. One target per
+`hosts[]` entry, in config order, all checked concurrently.
+
+Each check dials a **fresh connection** to the host and closes it when
+done (`probeHost` in `internal/provider/kvm2/check.go`); the pooled
+connection is never touched, so a check that runs out of time cannot fail
+a reconcile's call in flight, and a fresh dial picks up a rotated Secret
+at once. The sequence is the SSH and libvirt handshakes, then four RPCs:
+`ConnectGetLibVersion`, `ConnectGetHostname`, `StoragePoolLookupByName`
+and `StoragePoolGetInfo` on the host's `pool`. The cost per host per
+round is one sshd session and those four calls.
+
+Every target's `details` carry `user` and `pool`. As facts arrive they
+gain `libvirtVersion` (decoded `major.minor.micro`), `hostname`,
+`poolState` (virsh's word: `running`, `inactive`, ...),
+`poolCapacityGiB` and `poolAvailableGiB`, so a target that fails late in
+the sequence still says what the host answered before. The free space is
+the number the quota precheck compares against.
+
+A failing target says where the check broke (`checkReason` in `check.go`):
+
+| Reason | Meaning | Look at |
+|---|---|---|
+| `Reachable` | ready | |
+| `CredentialsUnavailable` | the key or known_hosts file is missing or does not parse | the `kvm-ssh` Secret (see *Cluster-side setup*); fixed in the cluster, not on the host |
+| `Unreachable` | the TCP connect failed or ran out of the 10 s dial budget while the round still had time, the SSH transport failed for a reason other than authentication, the forwarded libvirt socket was refused, or the link died mid-check | DNS, Tailscale or a firewall between the cluster and `address`; `libvirtd` running and the `socket` path |
+| `Unauthorized` | the SSH server rejected the key, or the host key is missing from known_hosts or has changed | `user`'s `authorized_keys`; the known_hosts entry (re-run `ssh-keyscan` and update the Secret) |
+| `Forbidden` | SSH worked, but libvirt refused the user (`ErrAccessDenied` or `ErrAuthFailed`) | `user` in the `libvirt` group |
+| `Timeout` | the monitor's `--provider-check-timeout` (default 15s) ran out before the check finished, whichever stage was running, the dial included; or the connection was up and an RPC's own budget ran out | a hung `libvirtd`, a saturated link, or a round timeout shorter than the dial takes |
+| `PoolNotFound` | there is no pool named `pool` | `virsh pool-list --all` |
+| `PoolNotActive` | the pool exists but is not running | `virsh pool-start <pool>` and `virsh pool-autostart <pool>` |
+| `APIError` | any other libvirt error | the message carries libvirt's own text |
+
+## Error classes
+
+Every error is classified once, in `classify` in
+`internal/provider/kvm2/errors.go`, with errors that carry a class from
+their source (validation, the image fetch, the dial) keeping it. The
+reconciler latches `ErrPermanent` as `Failed` until the spec changes,
+waits 5 minutes on `ErrQuotaExceeded`, recreates on `ErrNotFound` from
+`Get`, and retries everything else with backoff.
+
+| Class | What lands there |
+|---|---|
+| `ErrPermanent` | placement values not in the catalogue; a malformed providerID; a host no longer in the config (from `Get`; `Delete` treats it as done); libvirt `ErrXMLError`, `ErrNoSupport`, `ErrInvalidArg`, `ErrAuthFailed`, `ErrAccessDenied` and `ErrNoStoragePool`; a pool that is not running; SSH authentication refused (`unable to authenticate`); a host key missing from known_hosts or changed (`knownhosts.KeyError`); a key or known_hosts file that exists but does not parse; a domain of the same name owned by another UID or carrying no kvm2 mark; a domain of the same name that is paused or pmsuspended; corrupt metadata on a domain; an image URL answering 4xx, a response without `Content-Length` or with length 0, a sha256 that does not match the configured one |
+| `ErrQuotaExceeded` | the pool-space precheck |
+| `ErrNotFound` | libvirt `ErrNoDomain` or `ErrNoStorageVol` from `Get`, `Delete` or `ListByTag`; inside `Create` the same codes are `ErrTransient`, because the contract forbids NotFound from Create |
+| `ErrTransient` | everything else: a missing key or known_hosts file (the Secret has not landed yet), TCP and network errors, context cancellation, an abandoned or budget-exceeded call, a connection killed by another caller, libvirt `ErrOperationInvalid` and every unrecognised libvirt code, an image URL answering 429 or 5xx or ending the body early, the wait for running expiring or the domain going shut off or crashed right after its start |
+
+Error strings name paths, hosts, volumes and libvirt's own text. They
+never contain key bytes, the known_hosts content or cloud-init.
+
+## Known gaps
+
+- **Create blocks**, including the first image download on each host,
+  holding one of the three reconcile workers for its duration (LAN
+  speed, a few hundred MB, once per image per host). Other Proxies keep
+  reconciling on the remaining workers. Warm the cache with one Proxy per
+  image per host before creating many.
+- **Volumes without a domain are invisible to GC.** A crash between
+  creating the overlay or seed and defining the domain leaves volumes
+  that `ListByTag` cannot see (it lists domains). The same Proxy's next
+  Create reuses the overlay and rebuilds the seed, and its Delete removes
+  both by name even with no domain, but a Proxy CR that is gone for good
+  leaves them for `virsh vol-list` to find.
+- **A host removed from the config orphans its VMs.** `Delete` for a
+  providerID naming an unconfigured host is a logged no-op success, so
+  the Proxy finishes deleting instead of pinning its finalizer, but the
+  domain and its volumes stay on the hypervisor and orphan GC cannot see
+  them either (it only sweeps configured hosts). Delete the Proxies before
+  retiring a host, or clean up afterwards with `virsh undefine --nvram`
+  plus `virsh vol-delete`. `Get` fails permanently for such a providerID
+  on purpose: a typo in `hosts` must not look like a missing VM and
+  trigger a rebuild elsewhere.
+- **No adoption of `kvm` domains.** See *Migrating from kvm*.
+- **A cancelled call drops the host's pooled connection once**, so other
+  RPCs in flight on that host fail once with `ErrTransient` and retry.
+  The image upload and the health check are isolated on their own
+  connections.
+- **`Content-Length` is required** for image downloads; a mirror that
+  only chunks is rejected permanently. Point the catalogue at one that
+  sends the length.
+- **A paused or pmsuspended domain is not started.** A repeat `Create`
+  for a domain in that state fails permanently (`is DomainPaused and
+  cannot be started by the operator; resume or destroy it on the host`),
+  so the Proxy shows `Failed` until its spec changes; the operator never
+  resumes a VM an admin paused. Resume it on the host
+  (`virsh resume <name>`) and touch the spec, or delete the Proxy.
+- **Parser text in credential errors.** A key or known_hosts file that
+  does not parse fails with the parser's own message, which can quote a
+  PEM header type or a malformed known_hosts token. Key bytes are never
+  quoted.
+- **The sample cloud-init is an open proxy.**
+  `config/samples/proxy_kvm2.yaml` installs Squid with
+  `http_access allow all`; restrict it before exposing the bridge network
+  to anything but the cluster.
+- **IP discovery needs the agent or guest traffic to the host.** A VM on
+  a bridge with no DHCP never gets an address and stays `Provisioning`
+  forever; check `virsh console <name>` on the host. The same symptom
+  with a booted, addressed VM means the guest agent never came up: check
+  `systemctl status qemu-guest-agent`, `cloud-init status --long` and the
+  vendor script's output in `/var/log/cloud-init-output.log` (a VM without
+  outbound internet cannot install it). A user-data that disables
+  vendor-data (`vendor_data: {enabled: false}`) also removes the agent, by
+  design. `virsh domifaddr <name> --source agent` and `--source arp` show
+  what the operator sees, in that order.
+- **Debian 13 cloud images hang in GRUB under BIOS on a headless VM**
+  ([Debian bug #1111240](https://bugs.debian.org/1111240)), which is why
+  `firmware` defaults to `uefi`; the same image boots under UEFI with the
+  serial console intact. Images that need `firmware: bios` are the
+  exception.
+
+## Testing
+
+Unit tests run over an in-memory fake of the libvirt seam and an
+in-process SSH server. The optional integration test against a real host
+is described in
+[testing.md](testing.md#the-kvm2-integration-test-real-hypervisor); run it
+with `make test-integration-kvm2`. `TestKVM2Check_realHost` runs the
+health check against the same host and creates nothing:
+`go test -tags=integration -count=1 -v -run TestKVM2Check ./internal/provider/kvm2/`.

docs/plans-executions/2026-09-16-1333-kvm2-provider.md

@@ -0,0 +1,382 @@
+# Execution: kvm2, a clean-room libvirt/KVM provider built from the provider contract
+
+Plan: [docs/plans/2026-09-16-1333-kvm2-provider.md](../plans/2026-09-16-1333-kvm2-provider.md)
+
+- [x] Step 0 — Plan file and branch
+- [x] Step 1 — The spec, `docs/kvm2-provider-spec.md`
+- [x] Step 2 — Config wiring, IDs, state mapping, classify
+- [x] Step 3 — Pure builders: domain and volume XML, metadata, seed
+- [x] Step 4 — Transport, seam, decorator, fakes
+- [x] Step 5 — Image cache and pool-space precheck
+- [x] Step 6 — The four contract methods
+- [x] Step 7 — Checker and the CRD comment
+- [x] Step 8 — Deployment files, Makefile, integration test
+- [x] Step 9 — Documentation
+- [x] Step 10 — Milestone review
+- [ ] Step 11 — Real-host validation, changelog, MR
+
+## Step 0 — Plan file and branch
+
+`main` was fast-forwarded to the merge of PR #7 (`0dc25a1`), which carries the
+three provider documents, and `feat/kvm2-provider` was cut from it. The plan
+was committed alone as `854856c`. The pre-existing local modification of
+`.claude/settings.json` stays uncommitted throughout.
+
+## Step 1 — The spec, `docs/kvm2-provider-spec.md`
+
+2026-09-16 13:37. Written by the orchestrator, not a subagent, since the spec
+is the "specifics" a user hands to the implementer. All 24 entries answered;
+every Evidence line cites the v1 user guide, the 2026-09-04 execution log, a
+go-libvirt symbol, or the contract. Nothing was taken from
+`internal/provider/kvm/`.
+
+Design choices that go beyond the user's profile answers, all recorded in the
+spec: an adoption guard on `Create` (foreign UID is Permanent), a pool-space
+precheck mapped to `ErrQuotaExceeded`, a dedicated connection for image
+downloads so a cancelled poll cannot cut one off, the Checker on fresh
+connections rather than the pool, SSH authentication and host-key failures
+classified Permanent (v1 retried them forever), SSH material re-read on every
+dial so rotation needs no restart, kvm2's own metadata namespace (no adoption
+of v1 domains), and the shared `validateKVM` refactored to take the block name
+so messages say `kvm2.`.
+
+Checks run on the spec:
+
+```sh
+grep -c '^\*\*Answer:\*\*$' docs/kvm2-provider-spec.md      # 0 blank answers
+grep -n 'internal/provider/kvm/' docs/kvm2-provider-spec.md   # empty
+```
+
+plus the axis-code check against contract §4 headings and the path-existence
+loop from the docs-split log. All clean.
+
+Worth noting: the spec's entry 21 lists the concrete test name for every
+contract §11 item, so the later steps' briefs can quote it rather than
+re-derive the mapping.
+
+## Step 2 — Config wiring, IDs, state mapping, classify
+
+2026-09-16 16:38. First subagent step. The brief named the allowed files,
+the spec entries, the test names and the prohibition on
+`internal/provider/kvm/`; the agent reported nothing opened there, with one
+honest caveat: a `golangci-lint` run printed seven single source lines from
+v1 files among its findings, which it did not act on. The review greps
+(`grep -rn 'provider/kvm"' internal/provider/kvm2/`, the v1 identifier
+smell list) are clean.
+
+What landed: `ProviderConfig.KVM2` reusing `KVMConfig`, `setBlocks`, the
+allow-list and dispatch in `Config.validate`, `validateKVM` refactored into
+`validateKVMBlock(prefix, block, c)` with `validateKVM` kept as a wrapper so
+v1 messages and tests are untouched, `"kvm2": kvm2.New` in `cmd/main.go`
+(both type lists in one commit), and the package skeleton: `kvm2.go`
+(`New` never dials, warns on unreadable files, `withDefaults`, `host`,
+`wrapErr`, stubbed methods), `id.go`, `state.go`, `errors.go` with 34
+`classify` rows.
+
+Decisions the agent made that the spec left open, all accepted: `New`
+checks file existence only (parsing belongs to the dialer); `withDefaults`
+copies hosts and images; `firstIPv4` also drops an address whose type tag and
+literal disagree; there is no explicit `fs.ErrNotExist` branch in `classify`
+because it falls to the Transient default anyway.
+
+Commands as run by the agent and re-run by the orchestrator:
+
+```sh
+go vet ./... && go build ./... && go test -race -count=1 ./internal/provider/... ./cmd/...
+go test -short ./...
+```
+
+All packages `ok`.
+
+Worth noting: `bin/golangci-lint` panics inside staticcheck's IR builder on
+this machine (`unexpected expr: *ast.KeyValueExpr` on stdlib `internal/poll`)
+because the local toolchain is Go 1.27.0 while `go.mod` says 1.26 and the
+bundled staticcheck predates it; it reproduces on untouched packages, so lint
+is deferred to Step 10 with a toolchain note. The `.claude/settings.json`
+modification is still local and uncommitted.
+
+## Step 3 — Pure builders: domain and volume XML, metadata, seed
+
+2026-09-16 16:55. Second subagent step, new files only: `xml.go`,
+`metadata.go`, `volume.go`, `seed.go` and their tests (26 tests). The
+domain XML is built from `encoding/xml` structs, both disks are
+`type="file"` with resolved paths (the AppArmor lesson), UEFI is the default
+with the secure-boot feature disabled, the metadata element parses with or
+without its namespace declaration, and the seed ISO is staged under
+`os.TempDir()` (the library uses `os.MkdirTemp("", "")`, so `TMPDIR` is
+honoured; the test points it at a `t.TempDir()` and asserts nothing is left
+behind).
+
+Decisions the agent made, all accepted: `buildDomainXML` defaults an empty
+disk format to qcow2 and rejects anything but qcow2 or raw;
+`buildVolumeXML` rejects a non-positive capacity; `buildSeedISO` rejects an
+empty name; the guest-agent script still runs the enable-and-start step when
+the agent is already installed; the multipart boundary is random, so
+vendor-data is not byte-deterministic across calls (the seed is rebuilt every
+Create anyway).
+
+Commands:
+
+```sh
+gofmt -l ./internal/provider/kvm2 ; go vet ./internal/provider/kvm2/ && go test -race -count=1 ./internal/provider/kvm2/
+go build ./... && go test -short ./...
+```
+
+All `ok`. Review greps for a v1 import or v1 identifiers are empty; the only
+`type="volume"` in the package is the comment explaining why it is not used.
+The agent reported "files read outside the allowed set: none".
+
+## Step 4 — Transport, seam, decorator, fakes
+
+2026-09-16 17:42. Third subagent step, the largest: `backend.go` (the
+ctx-first seam `backend`, the production adapter `libvirtBackend`, a
+generic `call[T]` helper with per-call budgets and the abandon policy),
+`ssh.go` (credentials read on every dial, stage-tagged `dialError`, the
+forwarded socket, keepalive), `hosts.go` (`hostTable` with a per-host
+channel semaphore so waiting for a dial ends with the caller's ctx,
+`dialFresh` for dedicated connections, the production `libvirtDialer`
+starting the `kvm2.dial` span and applying the decorator once),
+`observe.go` (one span `kvm2.<Method>` and one `libvirt call` line per
+RPC), plus an in-memory `fakeBackend`, an in-process SSH server and 41
+tests. The agent also drove the real adapter over go-libvirt's own
+`libvirttest` mock through a `net.Pipe`, which the brief did not ask for.
+
+Abandon policy as built: a call that outlives its ctx or budget kills the
+link (closes the socket and the SSH client), returns Transient wrapping the
+ctx or deadline error, and other in-flight calls on that connection fail once
+with go-libvirt's `ErrInterrupted`; the next `get` redials. Results travel
+through a channel so a late-finishing abandoned call never writes into its
+caller's frame; the agent reported that the first `-race` run caught exactly
+that bug in an earlier version.
+
+Decisions the agent made, accepted: liveness is a separate unexported
+interface so the reflection test over `backend` stays at the 20 RPCs plus
+`Close`; `Close` sends a bounded `ConnectClose` before killing the link;
+`observe` and the dialer take `...tracing.Option` so tests can inject a
+recording tracer provider, as `provider.WithTracing` does.
+
+Clean-room note: three type names coincide with names v1 uses
+(`dialError`, `dialStage`, `hostEntry`). The brief described each concept
+("an error type that records which stage failed", "a per-host entry") so the
+names follow from the description; the greps for a v1 import and for the
+listed v1 identifiers are empty and the agent attested to reading nothing
+under `internal/provider/kvm/`.
+
+Commands (agent and orchestrator):
+
+```sh
+gofmt -l ./internal/provider/kvm2 ; go vet ./internal/provider/kvm2/ && go test -race -count=2 ./internal/provider/kvm2/
+go build ./... && go test -short ./...
+grep -n 'context.Background()' internal/provider/kvm2/*.go | grep -v _test.go   # empty
+```
+
+All `ok`; the agent also ran the timing-sensitive tests at `-count=8`.
+
+## Step 5 — Image cache and pool-space precheck
+
+2026-09-16 18:11. Fourth subagent step: `image.go` and `image_test.go`
+(27 tests), plus additive changes to `fake_test.go` (a `twin()` sharing
+state with its own call log, so a test can tell the pooled connection from
+the dedicated one) and `kvm2.go` (`http` client, budgets and tracing
+options behind functional options, so `New` is unchanged). The pipeline:
+marker lookup and bounded read on the pooled connection, then a GET bound to
+ctx plus the 20-minute cap under a `kvm2.download` span, `StorageVolCreateXML`
+and a streamed `StorageVolUpload` through sha256 on a fresh connection,
+`StoragePoolRefresh`, and the marker written last; every failure after the
+volume exists discards it with the cancellation stripped from ctx so the
+cleanup RPC still runs.
+
+Streaming is asserted by counting: a hook on the dedicated fake's upload
+snapshots how many bytes the 64 MiB test server had written when the upload
+began, and asserts under 16 MiB. The agent chose that over a process-wide
+`MemStats` delta because the tests run in parallel.
+
+Decisions the agent made, accepted: the base volume is created on the
+dedicated connection (nothing exists if the dial fails); `Accept-Encoding:
+identity` is sent so transparent gunzip cannot hide the Content-Length; a
+marker whose content is not 64 hex characters counts as debris;
+`Content-Length: 0` is its own Permanent error; the image lock is a
+one-slot channel so a waiter gives up with its ctx instead of sitting out a
+transfer; the fetch logger is derived after the span starts so its lines
+carry the span ID (the agent's first test run caught that).
+
+Commands (agent and orchestrator):
+
+```sh
+gofmt -l ./internal/provider/kvm2 ; go vet ./internal/provider/kvm2/ && go test -race -count=2 ./internal/provider/kvm2/
+go build ./... && go test -short ./...
+grep -n 'io.ReadAll\|bytes.Buffer' internal/provider/kvm2/image.go            # empty
+grep -n 'context.Background()' internal/provider/kvm2/*.go | grep -v _test.go  # empty
+```
+
+All `ok`. Attestation: "files read outside the allowed set: none".
+
+## Step 6 — The four contract methods
+
+2026-09-16 18:36. Fifth subagent step: `create.go`, `get.go`, `delete.go`,
+`list.go` and five test files (the package now has 150 top-level tests).
+Create follows spec entry 10 step for step under a per-name lock, with the
+adoption guard, the quota precheck, the image cache, an overlay that is
+reused and a seed that is always rebuilt, file-type disks from resolved
+paths, the metadata element in the define call, and a wait for running
+capped at two minutes. Get maps only `ErrNoDomain` to NotFound, prefers the
+guest agent over ARP, and reads UID and CreatedAt back every time. Delete is
+idempotent at every step and a no-op success for an unconfigured host.
+ListByTag fails whole per host and skips single unreadable domains.
+
+Decisions the agent made, accepted: the adoption path also waits for
+running, so a repeat Create on a shut-off domain blocks like a first one;
+`address` and `ListByTag` propagate ctx cancellation instead of reading it as
+"no address" or "skip", so a cancelled Get is Transient rather than a false
+Provisioning; NotFound from Get gets its own V(1) line because polling to
+NotFound is expected; the domain XML rides on the `domain defined` line at
+V(2). Flagged, not solved: adopting a paused domain fails Transient on every
+retry (`DomainCreate` refuses and the wait expires), which the spec does not
+cover; recorded as a known gap for the docs step.
+
+The agent found a one-word conflict between the brief and the spec
+(`DomainAffectCurrent` versus `DomainAffectConfig` for `DomainGetMetadata`);
+the code uses `DomainAffectCurrent`, and spec entry 8 was corrected to
+match in this commit.
+
+Commands (agent and orchestrator):
+
+```sh
+gofmt -l ./internal/provider/kvm2 ; go vet ./internal/provider/kvm2/ && go test -race -count=2 ./internal/provider/kvm2/
+go build ./... && go test -short ./...
+grep -n 'CloudInit' internal/provider/kvm2/create.go          # only len(req.CloudInit) and the seed builder call
+grep -n 'not implemented' internal/provider/kvm2/*.go          # empty
+grep -n 'provider.Wrap(' internal/provider/kvm2/*.go | grep -v _test.go | grep -v 'p.name'   # empty
+```
+
+All `ok`. Attestation: "files read outside the allowed set: none".
+
+## Step 7 — Checker and the CRD comment
+
+2026-09-16 18:52. Sixth subagent step: `check.go` and `check_test.go`
+(ten tests, the reasons table has 19 rows). Every host is checked
+concurrently on a fresh dedicated connection that is closed when the probe
+ends; the pooled table is never consulted, which the test proves with a
+counting dialer. Details fill in as facts arrive, so a `PoolNotFound`
+verdict still says which libvirt version answered. The `status.type` comment
+in `api/v1alpha1/providerhealth_types.go` gained `kvm2` and `make manifests`
+changed only the CRD's description line:
+
+```sh
+make manifests && git diff --stat config/ api/
+#  api/v1alpha1/providerhealth_types.go                    | 2 +-
+#  config/crd/bases/crawl.example.com_providerhealths.yaml | 2 +-
+```
+
+Decisions the agent made, accepted: a dial that runs out of time is
+`Unreachable` (the host did not answer), and `Timeout` is reserved for ctx or
+budget expiry on an established connection; `Close` runs synchronously in
+the probe so the close-count assertion is deterministic, at the cost of up
+to five seconds past ctx if a disconnect hangs on a live link.
+
+The agent also reported that the `captureContext` helper in `kvm2_test.go`
+appends to an unlocked slice, which the race detector flagged once `Check`
+logged from several goroutines. The helper was outside its allowed file set,
+so it built its own locked capture; the orchestrator fixed `captureContext`
+with a mutex in this commit and re-ran the package at `-race -count=3`.
+
+Attestation: "files read outside the allowed set: none".
+
+## Step 8 — Deployment files, Makefile, integration test
+
+2026-09-16 19:39. Seventh subagent step. `kvm2_integration_test.go`
+(`//go:build integration`, `KVM2_TEST_*` variables, the eleven ordered
+assertions of spec entry 21 plus `TestKVM2Check_realHost`),
+`config/samples/proxy_kvm2.yaml` (zone `beelink`, so the sample matches the
+shipped config), a commented `kvm2-home` entry in both providers configs
+with the porting note, comment-only edits on the shared `kvm-ssh` mount, the
+`test-integration-kvm2` Makefile target and the kvm2 rows and section in
+`docs/testing.md`. The integration test was not run; that is Step 11 with
+the user's go-ahead.
+
+The agent found that `New` builds the production dialer without tracing
+options, so in production the dialer and decorator use the global tracer
+provider (correct: `cmd/main.go` sets it up), while a test that wants every
+span family in one recorder sets the dialer's option itself. The test does
+so and asserts that `kvm2.DomainDefineXML` and `kvm2.dial` are children of
+`provider.create`.
+
+Decisions the agent made, accepted: the UID is generated from `crypto/rand`
+rather than promoting `google/uuid` to a direct dependency; the sentinel
+check looks for the full cloud-init string because the provider name also
+appears in attributes; `-run TestKVM2` matches both tests, as the kvm
+target's pattern does. Forward links to `docs/kvm2-provider.md` dangle until
+Step 9.
+
+Commands:
+
+```sh
+go vet ./... && go vet -tags=integration ./... && go vet -tags=e2e ./...
+bin/kustomize build config/default > /dev/null && bin/kustomize build config/samples > /dev/null
+make -n test-integration-kvm2
+# go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM2 ./internal/provider/kvm2/
+```
+
+All clean. Attestation: "files read outside the allowed set: none".
+
+## Step 9 — Documentation
+
+2026-09-16 19:54. Eighth subagent step: `docs/kvm2-provider.md` (sixteen
+sections modelled on the kvm guide), a `kvm2` column in the three tables of
+`docs/providers-existing.md` plus a kvm2 section and renumbering, the
+components row, §5 block, §10 span bullet, §11 row and eight Decisions
+bullets in `docs/architecture.md`, a README row and paragraph, and the
+guide in the `docs/providers.md` index. The agent verified each claim
+against the package rather than the spec and dropped a draft sentence
+claiming kvm and kvm2 share the base-image cache, because checking it would
+have meant reading v1's volume names.
+
+Checks re-run by the orchestrator: no em-dash in the guide, no line
+references, each of the 19 axis codes appears once in the appendix table
+with seven cells, every link and anchor across the seven touched documents
+resolves. The one added em-dash is the `Leader-elected` cell of the new
+components-table row, where the table's own convention wins.
+
+Attestation: "files read outside the allowed set: none".
+
+## Step 10 — Milestone review
+
+2026-09-16 21:19. The `go-operator-reviewer` agent reviewed the package
+against the contract and the spec (v1 code off limits to it as well) and
+returned seven findings, all accepted and applied by a fresh fix agent in
+one commit with regression tests:
+
+1. `waitRunning`'s two-minute cap only ever elapsed in the bad cases and
+   pinned a reconcile worker; now 15 s, a domain that dies right after
+   start is Transient at once, and a paused or pmsuspended domain is
+   Permanent before `DomainCreate` (an admin's doing).
+2. `ListByTag` read state and addresses that GC never uses, one agent RPC
+   per active domain per sweep; it now builds instances from the name and
+   the metadata only (`TestListByTag_doesNotReadStateOrAddresses`).
+3. A health round that ran out of time was blamed on the host
+   (`Unreachable`); `checkHost` now reports `Timeout` whenever the round's
+   ctx ended, whichever dial stage was running, and the promptness test
+   uses the production error shape.
+4. `Create` adopted an unmarked domain and never wrote the mark, so GC could
+   not see it; an unmarked domain under the name is now Permanent
+   (`TestCreate_adoptsOnlyMarkedDomains`).
+5. `hostTable.closeAll` had no production caller and `withBudgets` could not
+   work (the dialer captured the defaults first); both removed.
+6. The per-name and per-image lock maps grew without bound; replaced by a
+   reference-counted `keyedLocks` whose entries vanish when uncontended.
+7. `imageSizeIfUncached` falls back to GET on any non-2xx HEAD, and two
+   Delete no-op lines moved from V(0) to V(1).
+
+The spec (entries 2, 10, 12, 16, 18, 19, 24), the guide, the appendix and
+the architecture flow were updated in the same commit. The reviewer also
+listed what it checked and found correct: the error boundary, the state
+rules, Delete's idempotency and the replacement window, ListByTag's
+partial-failure policy, the abandon policy and double-close guards,
+resource handling on every path, and secret hygiene in the format strings.
+
+Lint: `bin/golangci-lint` still panics on this toolchain (Go 1.27 against
+a staticcheck built for 1.26), reproducible on untouched packages, so
+`make lint` could not gate this step; `go vet` with every build tag is
+clean. Tests: `go test -race -count=3 ./internal/provider/kvm2/` (168
+top-level tests) and `go test -short ./...` pass; `make test` with envtest
+runs next.

docs/plans/2026-09-16-1333-kvm2-provider.md

@@ -0,0 +1,301 @@
+# Plan: kvm2, a clean-room libvirt/KVM provider built from the provider contract
+
+**Created:** 2026-09-16 13:33
+
+## Context
+
+The provider documentation was just split into a generic contract
+(`docs/provider-contract.md`), a spec template (`docs/provider-spec-template.md`)
+and an appendix on the existing providers (`docs/providers-existing.md`), so that
+a new provider can be specified and built from those files alone. Before using
+them for Proxmox, the user wants a dry run on a known backend: rebuild the
+libvirt/KVM provider from scratch as a second type, `kvm2`, following the
+documents, without touching or reading the existing `internal/provider/kvm/`
+code. The result is a validated workflow (spec first, then implementation by
+fresh agents against the contract) and a v2 provider that can replace v1 once it
+passes on the real hypervisor.
+
+User decisions:
+
+- **Placement:** new package `internal/provider/kvm2`, type `kvm2`, config block
+  `kvm2:`, side by side with `kvm`. The Go config struct reuses
+  `provider.KVMConfig` (`internal/provider/config_kvm.go` is shared code, not
+  the kvm package), so an entry ports by changing `type` and the block name.
+- **Clean room:** implementers never open, grep or import
+  `internal/provider/kvm/`. Docs (`docs/kvm-provider.md`) and the operational
+  lessons from the first build's execution log are allowed.
+- **Implementer:** the orchestrator (this session) writes the spec and the step
+  briefs; each implementation step runs in a fresh general-purpose subagent
+  given only the three docs, the filled spec, the backend facts and its brief.
+  The orchestrator reviews each result against the contract, runs the tests and
+  commits.
+- **Profiles:** `create-return` blocking, `provisioning-state` create-blocks,
+  `child-spans` span-decorator, `quota` detected (pool free space precheck),
+  `credentials` mounted-files (the existing `kvm-ssh` Secret), `client-kind`
+  rpc-over-ssh, `gc-mark` written-metadata, `created-at` written-at-create,
+  `ip-source` discovered (agent, then ARP), `list-partial` fail-whole per host
+  with unreadable domains skipped, `not-found` error-code, `delete-return`
+  waits, `secondary-resources` explicit, `timeouts` per-call, `logging` tiered,
+  `backend-test` integration-tag.
+
+## Backend facts the spec carries (allowed lessons)
+
+- Real host `beelink`: libvirt 11.6, QEMU 10.1, pool `default` at
+  `/srv/vms/images`, bridge `br0`, `beelink.tail8c69c.ts.net:22`, user `novakj`,
+  key `~/.ssh/proxy-operator_ed25519`, known_hosts `~/.ssh/kvm_known_hosts`.
+  The Debian trixie qcow2 is about 435 MB.
+- AppArmor confines QEMU from the domain XML: only `<disk type="file">` with a
+  `<source file>` path gets a rule, so volume paths are resolved with
+  `StorageVolGetPath` and emitted as file disks.
+- Debian 13 hangs in GRUB under BIOS with a serial console and no video: UEFI
+  by default (`<os firmware="efi">`, secure boot feature disabled), `firmware:
+  bios` per image; OVMF is a host prerequisite; undefine needs the NVRAM flag.
+- libvirt hands the metadata element back without its namespace declaration:
+  parse by local name.
+- Vendor-data must be a shell script, not cloud-config, to survive cloud-init's
+  merge with user-data; it installs `qemu-guest-agent`. The agent answered after
+  about 20 s where ARP never did.
+- Read-only rootfs: `iso9660.NewWriter` needs a temp dir under `/tmp` (64Mi).
+- go-libvirt (`github.com/digitalocean/go-libvirt`, in go.mod): every method
+  blocks and takes no ctx; `libvirt.Error{Code uint32, Message}` has a value
+  receiver; needed `ErrorNumber` constants exist (`ErrNoDomain`,
+  `ErrNoStorageVol`, `ErrNoStoragePool`, `ErrOperationInvalid`,
+  `ErrNoDomainMetadata`, `ErrStorageVolExist`, `ErrXMLError`, `ErrNoSupport`,
+  `ErrInvalidArg`, `ErrAuthFailed`, `ErrAccessDenied`). `libvirt.New(net.Conn)`
+  then `Connect()`. No new dependencies: `x/crypto/ssh`, `kdomanski/iso9660`
+  are already direct requires; use `slices`, never `sort` (depguard).
+
+## Design decisions recorded in the spec (step 1)
+
+- **Package layout** (`internal/provider/kvm2/`): `kvm2.go` (type, `New`,
+  logger, per-name keyed lock), `id.go`, `state.go`, `errors.go`, `xml.go`,
+  `seed.go`, `backend.go` (ctx-first seam over the 20 go-libvirt calls, adapter
+  with a per-RPC deadline), `ssh.go`, `hosts.go` (one cached connection per
+  host, lazy dial, dialer seam), `observe.go` (the single decorator: one span
+  `kvm2.<Call>` per RPC plus the V(1)/V(2) log line), `image.go`, `create.go`,
+  `get.go`, `delete.go`, `list.go`, `check.go`, plus `fake_test.go`,
+  `sshserver_test.go`, `kvm2_integration_test.go`. Names are suggestions for
+  the briefs; the implementer may choose others by responsibility.
+- **Seam:** every seam method takes `ctx`; the adapter runs the blocking call in
+  a goroutine and selects on result, `ctx.Done()` and a timer (dial 10s, RPC
+  30s, `DomainCreate` 90s, upload/download ctx plus a 20m cap). On abandon it
+  closes the connection and marks it dead; the next call redials. The image
+  download runs on a dedicated connection so a cancelled poll cannot cut it off.
+- **Decorator wrapped once:** applied inside the real dialer where a connection
+  is born, so provider code never sees an undecorated backend. Spans `kvm2.dial`
+  (in the dialer) and `kvm2.download` (in `image.go`) are the two non-RPC spans.
+  A reflection test asserts every seam method is instrumented.
+- **Metadata:** own namespace
+  `https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm2`,
+  element `<instance>` with `managed`, `uid`, `createdAt`. kvm2 never lists or
+  adopts v1 domains; a `kvm` and a `kvm2` entry may share a host.
+- **Adoption guard:** an existing domain whose metadata `uid` is non-empty and
+  differs from `req.UID` makes `Create` return `ErrPermanent`.
+- **Quota precheck:** before creating volumes, `StoragePoolGetInfo`; needed =
+  image `Content-Length` (if not cached) + `diskSizeGb` + 16 MiB; below that
+  returns `ErrQuotaExceeded` naming pool, available and needed.
+- **Download** requires `Content-Length` (Permanent otherwise), streams through
+  `sha256` into `StorageVolUpload`, refreshes the pool, writes the marker
+  volume last; a failed attempt deletes the partial volume.
+- **Classify:** `ErrNoDomain`, `ErrNoStorageVol` NotFound, but Transient inside
+  `Create`; `ErrNoStoragePool`, `ErrXMLError`, `ErrNoSupport`, `ErrInvalidArg`,
+  `ErrAuthFailed`, `ErrAccessDenied` Permanent; SSH authentication failures and
+  host-key mismatches or unknown host keys Permanent; missing key or
+  known_hosts file Transient (Secret pending), unparsable key Permanent;
+  cancellation, deadlines, `net.OpError`, `io.EOF` and anything else Transient.
+- **Same-name concurrency:** a keyed mutex per instance name serialises Create
+  and Delete for one name.
+- **State:** nostate Provisioning; running or blocked without IP Provisioning,
+  with IP Running; everything else, including unknown codes, Stopped. IP from
+  the guest agent first, then ARP, first non-loopback non-link-local IPv4.
+- **Checker:** per host concurrently on a fresh dedicated connection (dial,
+  `ConnectGetLibVersion`, `ConnectGetHostname`, pool lookup and info, close), so
+  it never touches the pooled connection. Reasons: shared ones plus
+  `PoolNotFound`, `PoolNotActive`, `CredentialsUnavailable`. Details `user`,
+  `hostname`, `libvirtVersion`, `pool`, `poolState`, `poolCapacityGiB`,
+  `poolAvailableGiB`.
+- **Rotation:** SSH material is read on every dial, so Secret rotation needs no
+  restart.
+- **Config validation messages:** `validateKVM` is refactored into
+  `validateKVMBlock(prefix, block string, c *KVMConfig)` so messages say
+  `kvm2.` for the new block; `validateKVM` stays as a wrapper for `kvm`.
+
+## Steps (one commit each, on `feat/kvm2-provider` off `main`)
+
+PR #7 is merged, so `main` carries the three docs. Step 0 begins with
+`git fetch origin && git switch main && git pull --ff-only`.
+
+Every subagent brief opens and closes with: "Do not open, grep, list or import
+anything under `internal/provider/kvm/`. If a detail seems to need that code,
+stop and report the question instead." Each brief names the files it may create
+or edit and nothing else, lists the inputs (the three docs, the spec,
+`docs/kvm-provider.md` for backend facts, CLAUDE.md conventions), the
+acceptance tests by name and the commands to run, and asks the agent to end
+with "files read outside the allowed set: none".
+
+- **Step 0 (orchestrator):** copy this plan to `docs/plans/<ts>-kvm2-provider.md`
+  with the real timestamp, commit alone, create the branch. Execution summary
+  `docs/plans-executions/<ts>-kvm2-provider.md` with the checklist after step 1.
+- **Step 1 (orchestrator): the spec.** Copy `docs/provider-spec-template.md` to
+  `docs/kvm2-provider-spec.md` and answer all 24 entries with the profiles and
+  decisions above; every Evidence line cites the user guide, a go-libvirt symbol,
+  or "verified on beelink in step 11". Entry 21's checklist lists the concrete
+  test names from steps 4 to 7. Verify: no blank Answer lines, every axis code
+  used matches contract §4.20, no reference to `internal/provider/kvm/`.
+- **Step 2 (subagent): config wiring, IDs, state, classify.** Edits
+  `internal/provider/config.go` (`KVM2 *KVMConfig` field, `setBlocks`,
+  allow-list, dispatch), `config_kvm.go` (`validateKVMBlock`),
+  `config_test.go` and `config_kvm_test.go` rows. Creates `kvm2.go` (type,
+  assertions, `New` that never dials and warns on unreadable SSH files, stub
+  methods), `id.go`, `state.go`, `errors.go` with tests:
+  `TestFormatProviderID_roundTrip`, `TestParseProviderID_malformed`,
+  `TestMapState_table`, `TestFirstIPv4_filters`, `TestClassify_table`. Also
+  registers `"kvm2": kvm2.New` in `cmd/main.go` so the two type lists change in
+  one commit. Verify: three `"kvm2"` sites in `config.go`, no import of
+  `provider/kvm`, `go test -race ./internal/provider/... && go vet ./...`.
+- **Step 3 (subagent): pure builders.** `xml.go`, `seed.go` and tests:
+  `TestDomainXML_fileDisksWithResolvedPaths`, `TestDomainXML_uefiDefaultSecureBootOff`,
+  `TestDomainXML_biosEscape`, `TestDomainXML_metadataElement`,
+  `TestDomainXML_guestAgentChannel`, `TestParseMetadata_withNamespace`,
+  `TestParseMetadata_withoutNamespaceDeclaration`,
+  `TestParseMetadata_absentGivesEmptyUID`, `TestVolumeXML_overlayBackingStore`,
+  `TestSeedISO_filesAndLabel`, `TestSeedISO_userDataByteForByte`,
+  `TestSeedISO_tempDirRemoved`, `TestVendorData_isShellScriptNotCloudConfig`,
+  `TestVendorData_multipartWhenExtraProvided`. Verify: no `type="volume"` in
+  `xml.go`; temp dir under `/tmp` with a deferred removal.
+- **Step 4 (subagent): transport, seam, decorator, fakes.** `backend.go`,
+  `ssh.go`, `hosts.go`, `observe.go`, `fake_test.go`, `sshserver_test.go` and
+  tests: `TestDial_forwardsUnixSocket`, `TestDial_unknownHostKeyRejected`,
+  `TestDial_mismatchedHostKeyRejected`, `TestDial_insecureIgnoreHostKeyAccepts`,
+  `TestDial_missingKeyFileIsTransient`, `TestDial_unparsableKeyIsPermanent`,
+  `TestDial_handshakeTimeout`, `TestCall_deadlineClosesConnectionAndReturnsTransient`,
+  `TestCall_cancelUnblocksRPC`, `TestHosts_oneDialPerHostUnderConcurrency`,
+  `TestHosts_redialsAfterDrop`, `TestHosts_unconfiguredHostIsPermanent`,
+  `TestObserve_everySeamMethodInstrumented`, `TestObserve_childSpansUnderParent`,
+  `TestObserve_logsOneLinePerCallAtV1`, `TestObserve_detailOnlyAtV2`,
+  `TestObserve_uploadBodyNeverLogged`, `TestSSH_keyMaterialNeverInErrorsOrLogs`.
+  Verify: no `context.Background()` outside tests and the keepalive goroutine;
+  `tracing.Start` in the decorator and the dialer; `go test -race -count=2`.
+- **Step 5 (subagent): image cache and pool-space precheck.** `image.go` and
+  tests: `TestEnsureImage_downloadsAndWritesMarkerLast`,
+  `TestEnsureImage_cachedWithMarkerSkipsDownload`,
+  `TestEnsureImage_missingMarkerRedownloads`,
+  `TestEnsureImage_sha256ChangeRedownloads`,
+  `TestEnsureImage_sha256MismatchIsPermanentAndRemovesVolume`,
+  `TestEnsureImage_noContentLengthIsPermanent`, `TestEnsureImage_http404IsPermanent`,
+  `TestEnsureImage_http503IsTransient`, `TestEnsureImage_concurrentCallersFetchOnce`,
+  `TestEnsureImage_cancelledMidStreamReturnsPromptly`,
+  `TestEnsureImage_streamsWithoutBuffering`, `TestCheckPoolSpace_belowNeededIsQuota`.
+  Verify: no `io.ReadAll` or `bytes.Buffer` on the download path; download span.
+- **Step 6 (subagent): the four contract methods.** `create.go`, `get.go`,
+  `delete.go`, `list.go`, `kvm2_test.go`, finishing `kvm2.go`. Tests map 1:1 to
+  contract §11: `TestCreate_repeatReturnsSameIDAndCreatesNothing`,
+  `TestCreate_resumesAfterEachPartialStep`, `TestCreate_returnsOnlyWhenRunning`,
+  `TestCreate_rejectsUnknownPlacement`, `TestCreate_adoptionWithForeignUIDIsPermanent`,
+  `TestCreate_poolOutOfSpaceIsQuotaExceeded`,
+  `TestCreate_libvirtNotFoundIsTransientNotNotFound`,
+  `TestCreate_cloudInitSentinelNeverLoggedOrInErrors`,
+  `TestGet_afterDeleteReturnsNotFound`, `TestGet_duringDeleteStillReturnsInstance`,
+  `TestGet_backendBusyIsTransientNeverNotFound`, `TestGet_runningWithoutIPIsProvisioning`,
+  `TestGet_runningWithIPIsRunning`, `TestGet_agentPreferredOverARP`,
+  `TestGet_addressErrorIsProvisioning`, `TestGet_readsUIDAndCreatedAtFromBackend`,
+  `TestGet_malformedIDIsPermanent`, `TestGet_unconfiguredHostIsPermanent`,
+  `TestDelete_malformedIDIsPermanent`, `TestDelete_twiceSucceeds`,
+  `TestDelete_removesOverlayAndSeed`, `TestDelete_runningDomainDestroyedFirst`,
+  `TestDelete_toleratesMissingVolumesAndPool`, `TestDelete_unconfiguredHostIsNoOp`,
+  `TestDelete_concurrentSameNameSafe`, `TestListByTag_excludesUnmarkedDomains`,
+  `TestListByTag_emptyUIDStillListed`, `TestListByTag_acrossHostsInConfigOrder`,
+  `TestListByTag_unreachableHostFailsWhole`, `TestListByTag_unreadableDomainSkipped`,
+  `TestProvider_cancelledContextReturnsPromptly`,
+  `TestProvider_everyErrorIsWrappedWithConfiguredName`,
+  `TestLogging_v0SilentV1OutcomeV2Detail`. Verify: `CloudInit` appears in
+  `create.go` only as `len(req.CloudInit)`; stub strings gone; `Wrap` always
+  gets the configured name; `go test -race -count=2 ./internal/provider/kvm2/`.
+- **Step 7 (subagent): Checker and the CRD comment.** `check.go`,
+  `check_test.go`: `TestCheck_oneEntryPerHostInConfigOrder`,
+  `TestCheck_hostsCheckedConcurrently`, `TestCheck_expiredContextReturnsPromptly`,
+  `TestCheck_reasonsTable`, `TestCheck_neverUsesPooledConnection`,
+  `TestCheck_detailsKeys`. Edit `api/v1alpha1/providerhealth_types.go`
+  (`status.type` comment) and run `make manifests`. Verify: only the CRD
+  description line changes under `config/`; `go test ./internal/providerhealth/`.
+- **Step 8 (subagent): deployment, Makefile, integration test.** Edit
+  `config/manager/manager.yaml` (comment: the `kvm-ssh` mount serves kvm and
+  kvm2), `config/manager/providers_config.yaml` and
+  `config/samples/providers-config.yaml` (commented `kvm2` entry),
+  `config/samples/kustomization.yaml`, `Makefile` (`test-integration-kvm2`,
+  `-run TestKVM2`, `-timeout 40m`), `docs/testing.md` (suite rows and a kvm2
+  integration section). Create `config/samples/proxy_kvm2.yaml` (zone
+  `beelink`, provider `kvm2-home`) and `kvm2_integration_test.go`
+  (`KVM2_TEST_HOST`, `_USER`, `_SSH_KEY`, `_KNOWN_HOSTS`, `_BRIDGE`,
+  `_IMAGE_URL`; optional `_IMAGE_SHA256`, `_POOL`, `_SOCKET`, `_TIMEOUT`,
+  `_KEEP`, `_LOG_LEVEL`) asserting in order: `New` does not dial; `Check`
+  Ready with details; `Create` returns `hosts/<host>/domains/<name>`; second
+  `Create` same ID, domain count unchanged; `Get` reaches Running with an IPv4
+  and the written UID and CreatedAt; `ListByTag` contains it; `Delete` nil;
+  `Get` NotFound; `Delete` again nil; pool holds base and marker, no per-VM
+  volumes; with an in-memory span recorder, `kvm2.DomainDefineXML` and
+  `kvm2.dial` spans are children of `provider.create`. Verify:
+  `go vet -tags=integration ./... && go vet -tags=e2e ./...`,
+  `bin/kustomize build config/default` and `config/samples` parse,
+  `make -n test-integration-kvm2`.
+- **Step 9 (subagent): documentation.** Create `docs/kvm2-provider.md`
+  (prerequisites incl. OVMF, cluster setup reusing `kvm-ssh`, config reference
+  pointing at the shared `KVMConfig` table, placement mapping and the
+  add-a-key-never-edit warning, what lands on the host incl. the metadata
+  namespace, image cache and quota precheck, health-check reasons, logging and
+  span names, migration from `kvm`, known gaps). Edit
+  `docs/providers-existing.md` (fourth column in the three tables and a kvm2
+  section), `docs/architecture.md` (components row, §5 block, §10 span
+  topology, §11 row, Decisions bullets), `README.md` row, `docs/providers.md`
+  guide link. Verify with the link and symbol sweep from the docs-split
+  execution log; no em-dashes in new prose.
+- **Step 10 (orchestrator): milestone review.** `go-operator-reviewer` on the
+  branch with the spec and contract as inputs, then `/code-review high`,
+  `make lint`, `make test` (full, with envtest). Fix accepted findings with
+  regression tests in one commit; record rejected ones in the summary.
+- **Step 11 (orchestrator with the user): real-host validation, changelog, MR.**
+  Hand the user a host checklist (leftover `proxy-*` domains and volumes from
+  earlier runs, OVMF installed, pool running with free space, `br0` up, the
+  known_hosts entry) and the `KVM2_TEST_*` export lines; do not run remote
+  probes unprompted. With the user's go-ahead run `make test-integration-kvm2`
+  (a permission prompt is expected; the allow-list entry
+  `make test-integration-kvm *` does not match) and, if wanted, the in-cluster
+  pass on kind: build and load the image, deploy, a scratch ConfigMap with the
+  `kvm2-home` entry, apply `proxy_kvm2.yaml`, wait for `Ready`,
+  `kubectl get ph kvm2-home -o yaml`, a `curl -x` through the proxy, delete
+  and confirm the domain and volumes are gone; scratch overrides are not
+  committed. Record cold download duration and time to IP. CHANGELOG entry
+  after the user confirms; push with the token helper; MR via the Gitea API
+  with base `main`; never merge.
+
+## Verification
+
+- Per step: the checks listed above, plus `go build ./... && go test -short ./...`
+  before every commit and `git diff --stat` reviewed for files outside the
+  step's allowed set.
+- Clean-room check on every step's diff: `grep -rn 'provider/kvm"' internal/provider/kvm2/`
+  empty; the agent's closing line "files read outside the allowed set: none";
+  identifiers that coincide with v1's names from the appendix (`libvirtAPI`,
+  `hostPool`, `imageCache`, `tracingAPI`, `lazySSH`) are treated as a smell and
+  questioned.
+- Contract mapping: after step 7, every rule of contract §2 and §8 has a named
+  test in the package; the §11 list maps 1:1 to test names (recorded in the
+  spec's entry 21).
+- End to end: the integration run's ordered assertions (step 8) on beelink, and
+  the optional kind pass (step 11).
+
+## Risks
+
+- Subagent scope creep into `internal/provider/kvm/`: instruction-only guard,
+  mitigated by the closing attestation and the diff greps.
+- `.claude/settings.json` is locally modified and uncommitted; it stays out of
+  feature commits, and the permission prompt for the new make target is
+  accepted rather than pre-approved.
+- `Content-Length` is required for downloads; the Debian and Ubuntu mirrors send
+  it.
+- The cancelled-call policy still drops other in-flight RPCs on that host once;
+  the dedicated download connection removes the expensive case and the spec
+  states the rest.
+- `go-operator-reviewer` hunts over-abstraction; the seam, the decorator and
+  the dialer interface each have one production implementation and one fake,
+  which the contract requires (§4.15, §11).

docs/providers-existing.md

@@ -1,13 +1,17 @@
 # The existing providers
 
 This is the appendix to [provider-contract.md](provider-contract.md): how the
-three providers that exist today (kubernetes, gcp, kvm) meet the contract,
-which profile each one chose on every axis, and what is still missing. It is
-reference material for whoever writes the next provider, not a user guide;
-the user guides are [kvm-provider.md](kvm-provider.md),
+four providers that exist today (kubernetes, gcp, kvm, kvm2) meet the
+contract, which profile each one chose on every axis, and what is still
+missing. It is reference material for whoever writes the next provider, not
+a user guide; the user guides are [kvm-provider.md](kvm-provider.md),
+[kvm2-provider.md](kvm2-provider.md),
 [gcp-vm-validation.md](gcp-vm-validation.md) and
 [gcp-in-specific-project.md](gcp-in-specific-project.md). The README's
-"Implemented providers" table stays the short public summary.
+"Implemented providers" table stays the short public summary. kvm2 is the
+one provider built from the contract itself (clean-room, from
+[kvm2-provider-spec.md](kvm2-provider-spec.md), without reading kvm's code),
+so its column is also a record of what the contract produced.
 
 The per-provider call flows are drawn in
 [architecture.md §5](architecture.md#5-what-provider-calls-do-in-the-outside-world),
@@ -23,59 +27,59 @@ Code references are a file path plus a function or symbol name.
 One row per axis of contract §4, in the same order, with the profile name
 and the symbol that shows it.
 
-| Axis | kubernetes | gcp | kvm |
-|---|---|---|---|
-| `readiness` | immediate: a Pod is scheduled within seconds (`stateFromPod`) | provisioning-window: minutes of boot (`mapState`) | provisioning-window: boot plus, on first use, an image download (`imageCache.ensure`) |
-| `create-return` | submit: returns after `client.Create` | submit: `Insert` is never awaited, `Operation.Wait` is not called | **blocking**: waits through `DomainCreate` (`ensureRunning`), and the first image fetch runs inside (`imageCache.fetch`) |
-| `delete-return` | returns-early: `client.Delete`, Pod terminates on its own | returns-early: `Delete` submits an operation each time | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) |
-| `ip-source` | assigned-at-create: `PodIP` (`instanceFromPod`) | assigned-at-create: the external NAT IP of the first NIC (`toInstance`) | discovered: ARP table first, then the guest agent, first routable IPv4 (`address`, `firstIPv4`) |
-| `gc-mark` | native-labels: Pod labels (`buildPod`) | native-labels: instance labels (`buildInsertRequest`) | written-metadata: a namespaced `<metadata>` element written in `DomainDefineXML` (`metadata.go`) |
-| `created-at` | backend-timestamp: `CreationTimestamp` | backend-timestamp: `CreationTimestamp`, parse error gives zero | written-at-create: `createdAt` in the metadata element (`newInstanceMetadata`) |
-| `list-partial` | n/a: one call, all namespaces | omit-unreachable: `ReturnPartialSuccess` on `AggregatedList` | fail-whole per host, one unreadable domain skipped (`ListByTag`, `managedInstance`) |
-| `quota` | absent: a quota 403 is Permanent (`classify`) | detected: 429, and 403 with reason `quotaExceeded` or `rateLimitExceeded` (`classify`, `hasReason`) | absent (`classify`) |
-| `not-found` | status-404: `apierrors.IsNotFound` | status-404: `isNotFound` | error-code: `ErrNoDomain`, `ErrNoStorageVol` (`hasCode`) |
-| `primary-key` | name-is-key: `namespace/name` | name-is-key: `zones/<zone>/instances/<name>` | name-is-key: `hosts/<host>/domains/<name>` |
-| `provisioning-state` | native: `PodPending` | native: `PROVISIONING`, `STAGING`, `REPAIRING` | create-blocks: a domain is defined and started in one `Create`; `nostate` maps to Provisioning |
-| `async-ops` | none | task-handle, but the handle is discarded: an insert that fails after acceptance is never read | none |
-| `secondary-resources` | none | cascade: `AutoDelete` on the boot disk (`buildInsertRequest`) | explicit: overlay and seed removed by name (`removeVolumes`); base image kept |
-| `credentials` | workload-identity: in-cluster config or kubeconfig (`NewWithTransportWrapper`) | environment: Application Default Credentials | mounted-files: SSH key and known_hosts from the optional `kvm-ssh` Secret, read lazily (`newLazySSH`) |
-| `client-kind` | vendor-sdk: controller-runtime `client.Client`, no seam interface | vendor-sdk: `cloud.google.com/go/compute` behind `instancesAPI` | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind `libvirtAPI` (`conn.go`) |
-| `timeouts` | ctx-only: no `rest.Config.Timeout` is set | ctx-only: SDK defaults, with SDK retries on Get and List | per-call for the dial only (`defaultHandshakeTimeout` in `dialSSH`); RPCs are ctx-only and a cancelled `call` closes the connection |
-| `logging` | none | tiered: V(1) outcomes and `logAPIError`, V(2) detail, V(5) wire log (`wirelog.go`) | tiered: V(1) outcomes and one `libvirt call` line per RPC (`trace.go`), V(2) XML and addresses; no wire tier |
-| `child-spans` | transport-wrapper: `tracing.RestConfigWrapper()` on its own `rest.Config`, parent-gated | sdk-instrumentation: the SDK adds `otelhttp` itself; not parent-gated, but every call site has a parent | **none**: `trace.go` is a log decorator, not spans |
-| `backend-test` | cluster-e2e: kind (`test/e2e`) | manual-runbook: [gcp-vm-validation.md](gcp-vm-validation.md) | integration-tag: `kvm_integration_test.go`, `make test-integration-kvm` |
+| Axis | kubernetes | gcp | kvm | kvm2 |
+|---|---|---|---|---|
+| `readiness` | immediate: a Pod is scheduled within seconds (`stateFromPod`) | provisioning-window: minutes of boot (`mapState`) | provisioning-window: boot plus, on first use, an image download (`imageCache.ensure`) | provisioning-window: boot plus, on first use, an image download (`ensureImage`) |
+| `create-return` | submit: returns after `client.Create` | submit: `Insert` is never awaited, `Operation.Wait` is not called | **blocking**: waits through `DomainCreate` (`ensureRunning`), and the first image fetch runs inside (`imageCache.fetch`) | **blocking**: waits through `DomainCreate` and a 15-second poll for running (`ensureRunning`, `waitRunning`), and the first image fetch runs inside (`fetchImage`) |
+| `delete-return` | returns-early: `client.Delete`, Pod terminates on its own | returns-early: `Delete` submits an operation each time | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) |
+| `ip-source` | assigned-at-create: `PodIP` (`instanceFromPod`) | assigned-at-create: the external NAT IP of the first NIC (`toInstance`) | discovered: ARP table first, then the guest agent, first routable IPv4 (`address`, `firstIPv4`) | discovered: the guest agent first, then the ARP table, first routable IPv4 (`addressSources`, `address`, `firstIPv4`) |
+| `gc-mark` | native-labels: Pod labels (`buildPod`) | native-labels: instance labels (`buildInsertRequest`) | written-metadata: a namespaced `<metadata>` element written in `DomainDefineXML` (`metadata.go`) | written-metadata: an element in kvm2's own namespace written in `DomainDefineXML` (`metadataURI`, `newInstanceMetadata`) |
+| `created-at` | backend-timestamp: `CreationTimestamp` | backend-timestamp: `CreationTimestamp`, parse error gives zero | written-at-create: `createdAt` in the metadata element (`newInstanceMetadata`) | written-at-create: `createdAt` in the metadata element, parse error gives zero (`newInstanceMetadata`, `createdAt`) |
+| `list-partial` | n/a: one call, all namespaces | omit-unreachable: `ReturnPartialSuccess` on `AggregatedList` | fail-whole per host, one unreadable domain skipped (`ListByTag`, `managedInstance`) | fail-whole per host, one unreadable domain skipped unless ctx ended; `ID`, `UID` and `CreatedAt` from the name and the mark only, no state or address RPC (`ListByTag`) |
+| `quota` | absent: a quota 403 is Permanent (`classify`) | detected: 429, and 403 with reason `quotaExceeded` or `rateLimitExceeded` (`classify`, `hasReason`) | absent (`classify`) | detected: a pool-space precheck before any volume is created (`checkPoolSpace`, `neededBytes`) |
+| `not-found` | status-404: `apierrors.IsNotFound` | status-404: `isNotFound` | error-code: `ErrNoDomain`, `ErrNoStorageVol` (`hasCode`) | error-code: `ErrNoDomain`, `ErrNoStorageVol`, Transient inside Create (`classifyLibvirt`) |
+| `primary-key` | name-is-key: `namespace/name` | name-is-key: `zones/<zone>/instances/<name>` | name-is-key: `hosts/<host>/domains/<name>` | name-is-key: `hosts/<host>/domains/<name>`, with a UID guard on adoption (`create`) |
+| `provisioning-state` | native: `PodPending` | native: `PROVISIONING`, `STAGING`, `REPAIRING` | create-blocks: a domain is defined and started in one `Create`; `nostate` maps to Provisioning | create-blocks: defined, started and polled to running in one `Create`; `nostate` maps to Provisioning (`mapState`) |
+| `async-ops` | none | task-handle, but the handle is discarded: an insert that fails after acceptance is never read | none | none |
+| `secondary-resources` | none | cascade: `AutoDelete` on the boot disk (`buildInsertRequest`) | explicit: overlay and seed removed by name (`removeVolumes`); base image kept | explicit: overlay and seed removed by name (`removeVolumes`); base image kept |
+| `credentials` | workload-identity: in-cluster config or kubeconfig (`NewWithTransportWrapper`) | environment: Application Default Credentials | mounted-files: SSH key and known_hosts from the optional `kvm-ssh` Secret, read lazily (`newLazySSH`) | mounted-files: the same `kvm-ssh` Secret, read on every dial and never cached (`loadCredentials`) |
+| `client-kind` | vendor-sdk: controller-runtime `client.Client`, no seam interface | vendor-sdk: `cloud.google.com/go/compute` behind `instancesAPI` | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind `libvirtAPI` (`conn.go`) | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind the ctx-first `backend` seam (`backend.go`) |
+| `timeouts` | ctx-only: no `rest.Config.Timeout` is set | ctx-only: SDK defaults, with SDK retries on Get and List | per-call for the dial only (`defaultHandshakeTimeout` in `dialSSH`); RPCs are ctx-only and a cancelled `call` closes the connection | per-call: a budget around the dial, every RPC, the uploads and the download (`defaultBudgets`, `call`); a cancelled or overlong call kills the pooled connection |
+| `logging` | none | tiered: V(1) outcomes and `logAPIError`, V(2) detail, V(5) wire log (`wirelog.go`) | tiered: V(1) outcomes and one `libvirt call` line per RPC (`trace.go`), V(2) XML and addresses; no wire tier | tiered: V(1) outcomes and one `libvirt call` line per RPC (`observe.go`), V(2) XML and addresses; no wire tier |
+| `child-spans` | transport-wrapper: `tracing.RestConfigWrapper()` on its own `rest.Config`, parent-gated | sdk-instrumentation: the SDK adds `otelhttp` itself; not parent-gated, but every call site has a parent | **none**: `trace.go` is a log decorator, not spans | span-decorator: `kvm2.<Method>` per RPC (`observe`), plus `kvm2.dial` (`libvirtDialer`) and `kvm2.download` (`fetchImage`) |
+| `backend-test` | cluster-e2e: kind (`test/e2e`) | manual-runbook: [gcp-vm-validation.md](gcp-vm-validation.md) | integration-tag: `kvm_integration_test.go`, `make test-integration-kvm` | integration-tag: `kvm2_integration_test.go`, `make test-integration-kvm2` |
 
 ## 2. Other differences
 
 Rows that are not profile axes but that a new provider author compares
 against.
 
-| Feature | kubernetes | gcp | kvm |
-|---|---|---|---|
-| Create verifies the adopted instance's UID | no | no | no |
-| Placement used | none | Zone, MachineType, Image | Zone (host), MachineType, Image |
-| CloudInit delivered | **no**: ignored | yes: `user-data` metadata | yes: NoCloud seed ISO built in-process (`buildSeedISO`) |
-| Port used | yes: Squid config and containerPort (`squidConf`) | no: firewall is manual | no |
-| States produced | Provisioning, Running, Terminated | all four | Provisioning, Running, Stopped |
-| Deleting instance still reports its state | yes: a Pod being deleted reports its phase | n/a | n/a |
-| Error.Provider is the configured name | **no**: hardcoded `"kubernetes"` | yes | yes |
-| Checker target | `apiserver` | the project | each host, concurrently |
-| Checker Details | none | none | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacity`, `poolAvailable` |
-| Credential rotation without restart | n/a | via ADC | no: cached after first successful load |
-| Connection handling | client-go | SDK client, never closed | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostPool`) |
-| Unit-test double | controller-runtime fake client | `fakeAPI` behind `instancesAPI`, plus `httptest` for the real SDK client in `check_test.go` | `fakeAPI` behind `libvirtAPI`, plus an in-process `x/crypto/ssh` server in `conn_test.go` |
-| Proxy software | generated Squid config (`pod.go`) | from `cloudInit`; the image must run cloud-init | from `cloudInit`, plus a vendor-data script that installs the guest agent (`seed.go`) |
-| IP reachable from the operator | Pod network, so in-cluster only (not from `make run-dev` on a laptop) | external IP, once a firewall rule allows the cluster's egress IP | LAN address on the host's bridge; the operator needs a route to that LAN |
-| Distinct egress IP per proxy | no: the cluster's egress | yes: ephemeral external IP | whatever the host's LAN routes through; not per VM by itself |
+| Feature | kubernetes | gcp | kvm | kvm2 |
+|---|---|---|---|---|
+| Create verifies the adopted instance's UID | no | no | no | **yes**: a domain without the kvm2 mark or with a different non-empty UID is Permanent; a marked one with a matching or empty UID is adopted (`create`) |
+| Placement used | none | Zone, MachineType, Image | Zone (host), MachineType, Image | Zone (host), MachineType, Image |
+| CloudInit delivered | **no**: ignored | yes: `user-data` metadata | yes: NoCloud seed ISO built in-process (`buildSeedISO`) | yes: NoCloud seed ISO built in-process (`buildSeedISO`) |
+| Port used | yes: Squid config and containerPort (`squidConf`) | no: firewall is manual | no | no |
+| States produced | Provisioning, Running, Terminated | all four | Provisioning, Running, Stopped | Provisioning, Running, Stopped |
+| Deleting instance still reports its state | yes: a Pod being deleted reports its phase | n/a | n/a | n/a: Delete waits, and a concurrent Get sees the domain until it is undefined |
+| Error.Provider is the configured name | **no**: hardcoded `"kubernetes"` | yes | yes | yes (`wrapErr`) |
+| Checker target | `apiserver` | the project | each host, concurrently | each host, concurrently, on a fresh connection |
+| Checker Details | none | none | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacity`, `poolAvailable` | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacityGiB`, `poolAvailableGiB`, filled in as far as the check got |
+| Credential rotation without restart | n/a | via ADC | no: cached after first successful load | yes: both files read on every dial (`loadCredentials`) |
+| Connection handling | client-go | SDK client, never closed | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostPool`) | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostTable`); dedicated connections for the image upload and the check (`dialFresh`) |
+| Unit-test double | controller-runtime fake client | `fakeAPI` behind `instancesAPI`, plus `httptest` for the real SDK client in `check_test.go` | `fakeAPI` behind `libvirtAPI`, plus an in-process `x/crypto/ssh` server in `conn_test.go` | `fakeBackend` behind `backend` (`fake_test.go`), plus an in-process `x/crypto/ssh` server (`sshserver_test.go`) and go-libvirt's own mock for the adapter |
+| Proxy software | generated Squid config (`pod.go`) | from `cloudInit`; the image must run cloud-init | from `cloudInit`, plus a vendor-data script that installs the guest agent (`seed.go`) | from `cloudInit`, plus a vendor-data script that installs the guest agent (`seed.go`) |
+| IP reachable from the operator | Pod network, so in-cluster only (not from `make run-dev` on a laptop) | external IP, once a firewall rule allows the cluster's egress IP | LAN address on the host's bridge; the operator needs a route to that LAN | LAN address on the host's bridge; the operator needs a route to that LAN |
+| Distinct egress IP per proxy | no: the cluster's egress | yes: ephemeral external IP | whatever the host's LAN routes through; not per VM by itself | whatever the host's LAN routes through; not per VM by itself |
 
 ## 3. Observability status
 
-| Layer | kubernetes | gcp | kvm |
-|---|---|---|---|
-| 1. Operation spans and metrics | yes: decorators | yes: decorators | yes: decorators |
-| 2. Child spans per backend call | yes: `tracing.RestConfigWrapper()`, parent-gated (`cmd/main.go`, `NewWithTransportWrapper`) | yes: the SDK's own `otelhttp`; not parent-gated | **no** |
-| 3. Logs carry `traceID` and `spanID` | n/a: the package does not log | yes, including the V(5) wire log | yes: every `libvirt call` line at V(1) |
-| Tested against a real collector | yes: the kind e2e checks `provider.create` and `provider.delete` in Tempo (`test/e2e/tracing_test.go`), operation spans only | no | no |
+| Layer | kubernetes | gcp | kvm | kvm2 |
+|---|---|---|---|---|
+| 1. Operation spans and metrics | yes: decorators | yes: decorators | yes: decorators | yes: decorators |
+| 2. Child spans per backend call | yes: `tracing.RestConfigWrapper()`, parent-gated (`cmd/main.go`, `NewWithTransportWrapper`) | yes: the SDK's own `otelhttp`; not parent-gated | **no** | yes: `kvm2.<Method>` from the `observe` decorator, `kvm2.dial`, `kvm2.download`; always under the caller's span, since every call carries the caller's ctx |
+| 3. Logs carry `traceID` and `spanID` | n/a: the package does not log | yes, including the V(5) wire log | yes: every `libvirt call` line at V(1) | yes: every `libvirt call` line at V(1), written inside its RPC's span |
+| Tested against a real collector | yes: the kind e2e checks `provider.create` and `provider.delete` in Tempo (`test/e2e/tracing_test.go`), operation spans only | no | no | no; the integration test asserts the parent-child topology in an in-memory recorder (`assertLifecycleSpans`) |
 
 ## 4. kubernetes (`internal/provider/kubernetes`)
 
@@ -329,18 +333,213 @@ the author's view.
     tries ARP first; it says `host` and `name` are on every log line, but
     the check and list paths lack them.
 
-## 7. Gaps shared by the existing providers
+## 7. kvm2 (`internal/provider/kvm2`)
 
-- **Adoption is not verified.** None of the three checks `LabelUID` when
-  `Create` finds an existing instance with the right name.
-- **The Create comment does not match kvm.** The interface comment in
-  `internal/provider/provider.go` says `Create` returns as soon as the
-  request is submitted; kvm blocks until the domain has started.
+The full user guide is [kvm2-provider.md](kvm2-provider.md) and the design
+it was built from is [kvm2-provider-spec.md](kvm2-provider-spec.md); this
+section is the author's view. Nothing in the package was taken from
+`internal/provider/kvm`; where the two agree it is because both follow the
+same libvirt.
+
+- **Package and seam.** `kvm2.go` (`Provider`, `New`, `withDefaults`,
+  `host`, `wrapErr`, the per-name lock), `create.go`, `get.go`,
+  `delete.go`, `list.go` (the four methods), `backend.go` (the seam, the
+  production adapter and the budgets), `ssh.go` (credentials, the forwarded
+  socket, keepalive), `hosts.go` (the pooled table and the production
+  dialer), `observe.go` (the span and log decorator), `image.go` (cache,
+  download, quota precheck), `seed.go`, `xml.go`, `volume.go`,
+  `metadata.go`, `id.go`, `state.go`, `errors.go`, `check.go`. The seam is
+  `backend`: the twenty go-libvirt calls the provider uses, each with `ctx`
+  first (go-libvirt's own methods take none and block), plus `Close`.
+  `libvirtBackend` adapts `*libvirt.Libvirt` and runs every call through
+  `call`, which applies the budget and the abandon policy. The `dialer`
+  seam separates the SSH transport: `newProvider` injects one for tests,
+  `newLibvirtDialer` is production. The `observe` decorator is applied
+  once, in `libvirtDialer.dial`, so provider code never holds an
+  undecorated backend.
+- **Config.** `ProviderConfig.KVM2`, the same `KVMConfig` type as kvm,
+  validated by `validateKVMBlock` with block name `kvm2` so messages read
+  `kvm2.hosts`. Defaults (`diskSizeGb` 10, socket, pool, image format and
+  firmware) are filled by `withDefaults`. `New` never dials; it checks that
+  the credential files exist and warns when one does not, or when
+  `insecureIgnoreHostKey` is on.
+- **ID.** `formatProviderID(host, name)` gives `hosts/<host>/domains/<name>`;
+  `parseProviderID` accepts exactly four segments with the literal
+  `hosts` and `domains` and non-empty host and name, else `ErrPermanent`.
+- **Create.** `Create` validates `Zone` (`host`; optional with one host),
+  `MachineType` (a catalogue key) and `Image` (`resolveImage`: a catalogue
+  key or an http(s) URL with qcow2 and uefi defaults) before any dial, and
+  logs the resolved parameters with `cloudInitBytes` at V(2). Then, under
+  the per-name lock (`lockName`) and on the pooled connection
+  (`hostTable.get`), `create`: `DomainLookupByName`; found means the
+  adoption guard (no kvm2 mark, or a non-empty metadata UID that differs
+  from `req.UID`, is Permanent) and `ensureRunning`. Otherwise
+  `StoragePoolLookupByName`, `imageSizeIfUncached` (marker lookup, then
+  `HEAD`, or a closed `GET` when `HEAD` is not 2xx) and
+  `checkPoolSpace` with `neededBytes` (image if uncached, plus
+  `diskSizeGb`, plus 16 MiB), then `ensureImage` (`cachedImage` on the
+  pooled connection, `fetchImage` streaming into `uploadImage` on a
+  `dialFresh` connection, `StoragePoolRefresh`, `writeMarker` last),
+  `ensureOverlay` (`ErrStorageVolExist` means reuse), `writeSeed` (delete,
+  build, upload; always rebuilt), two `StorageVolGetPath`, `buildDomainXML`
+  with file-type disks and the metadata element, `DomainDefineXML`,
+  `ensureRunning` (`DomainGetState`, where paused or pmsuspended is
+  Permanent; `DomainCreate` with `ErrOperationInvalid` as already running;
+  `waitRunning` polls every 500 ms for up to 15 seconds, Transient on
+  expiry and at once for any inactive state other than `nostate`).
+  `Create` returns only when the domain is running.
+- **Get and state.** `mapState(state, ip)`:
+
+  | libvirt state | IP | Provider state |
+  |---|---|---|
+  | nostate | any | Provisioning |
+  | running, blocked | empty | Provisioning |
+  | running, blocked | set | Running |
+  | paused, shutdown, shutoff, crashed, pmsuspended, anything else | any | Stopped |
+
+  `get`: `DomainLookupByName` (only `ErrNoDomain` reaches NotFound),
+  `readMetadata` (`DomainGetMetadata` with `metadataURI`;
+  `ErrNoDomainMetadata` means unmanaged with an empty UID; an element that
+  does not parse is Permanent), then `DomainGetState` and, for an active
+  domain, `address`. `addressSources` asks the guest agent first,
+  then ARP; a source that fails is skipped at V(2), only ctx ending is an
+  error, so a cancelled Get is Transient and not a false Provisioning.
+  `firstIPv4` drops loopback, link-local and unspecified addresses and an
+  address whose type tag and literal disagree. `UID` and `CreatedAt` are
+  read back on every call; an unparsable `createdAt` gives the zero time.
+- **Delete.** Under the per-name lock: `removeDomain` (`DomainDestroy`,
+  tolerating not running and gone; `DomainUndefineFlags` with NVRAM,
+  managed save and snapshot metadata) and `removeVolumes` (overlay and seed
+  by name through `deleteVolumeIfPresent`; a missing pool is a no-op). A
+  host no longer in the config is a logged no-op success. Nothing is left
+  when nil is returned.
+- **ListByTag.** `ConnectListAllDomains` (active and inactive) on every
+  host in config order; `readMetadata` and `managed` keep kvm2's own
+  domains, and the `Instance` is `ID`, `UID` and `CreatedAt` from the name
+  and the element, with `State` and `IP` empty: GC reads nothing else
+  (contract §2.4), so no state or address RPC is made. A domain whose
+  metadata cannot be read is skipped at V(1), unless ctx ended, which
+  fails the call. A host that cannot be dialled or listed fails the whole
+  call.
+- **Errors.** `classify(op, err)`: an error already carrying a class keeps
+  it; libvirt codes through `classifyLibvirt` (`ErrNoDomain` and
+  `ErrNoStorageVol` NotFound, Transient when `op == "create"`;
+  `ErrNoStoragePool`, `ErrXMLError`, `ErrNoSupport`, `ErrInvalidArg`,
+  `ErrAuthFailed`, `ErrAccessDenied` Permanent; the rest Transient);
+  `knownhosts.KeyError` and the `unable to authenticate` text from
+  `x/crypto/ssh` Permanent; everything else Transient, including a missing
+  credential file. `permanentf` and `transientf` classify at the source:
+  placement, XML builders, a parse failure of the key or known_hosts, an
+  image 4xx, a missing or zero `Content-Length`, a sha256 mismatch, a pool
+  that is not running (Permanent); an image 429 or 5xx, a short body, the
+  running wait expiring (Transient). `checkPoolSpace` is the only source of
+  `ErrQuotaExceeded`. `logAPIError` writes `kvm2 backend call failed` with
+  `libvirtErrorFields`.
+- **Check.** `Check` runs `checkHost` per host concurrently, each through
+  `probeHost` on a `dialFresh` connection that is closed when the probe
+  ends; the pooled table is never consulted. `ConnectGetLibVersion`,
+  `ConnectGetHostname`, `StoragePoolLookupByName`, `StoragePoolGetInfo`;
+  a pool that is not running is `poolNotActiveError`. Details are filled
+  as facts arrive (`user` and `pool` always; `libvirtVersion`, `hostname`,
+  `poolState`, `poolCapacityGiB`, `poolAvailableGiB`). `checkHost` answers
+  `Timeout` whenever the round's ctx ended, whatever stage was running and
+  whatever shape the error has; otherwise `checkReason`: a `dialError` is
+  judged by its stage (`credentials` gives `CredentialsUnavailable`; `ssh`
+  with a `knownhosts.KeyError` or the auth text gives `Unauthorized`; any
+  other stage, including a dial that ran out of its own budget, gives
+  `Unreachable`); then `context.DeadlineExceeded` or `Canceled` `Timeout`,
+  `ErrAccessDenied` or `ErrAuthFailed` `Forbidden`, `ErrNoStoragePool`
+  `PoolNotFound`, any other libvirt code `APIError`, anything else
+  `Unreachable`; `PoolNotActive` from the pool state.
+- **Connections.** `hostTable` keeps one pooled connection per host,
+  dialled lazily under a per-host one-slot semaphore so waiting ends with
+  the caller's ctx; a dead backend (`liveness`) is closed and redialled on
+  the next `get`. `dialSSHSocket` runs credentials, TCP, the SSH handshake
+  and the `direct-streamlocal` channel to the socket, each failure a
+  `dialError` with its stage; `libvirtDialer.connect` bounds the whole
+  sequence plus the libvirt `Connect` with the dial budget and starts the
+  `keepalive` (15 s, `keepalive@openssh.com`) and a watcher on go-libvirt's
+  `Disconnected`. `call` runs each blocking go-libvirt call in a goroutine
+  and waits on the result, ctx and a budget timer; abandoning a call
+  (`link.markDead`) closes the socket and the SSH client, returns Transient
+  wrapping the ctx or deadline error, and every other call waiting on that
+  link fails once with Transient. `loadCredentials` runs on every dial.
+- **Logging.** Logger from the context, named `kvm2`, with `provider`,
+  and `host` and `name` on lines about an instance. V(1): one line per
+  operation outcome (`kvm2 domain ensured running`, `kvm2 domain fetched`,
+  `kvm2 domain deleted`, `kvm2 domains listed`, `kvm2 host checked`), one
+  `libvirt call` line per RPC from `observe` (rpc name, identifiers,
+  duration, error fields), the connection events `libvirt host connected`
+  and `libvirt host connect failed` with `stage`, and `kvm2 backend call
+  failed`. V(2): XML, metadata, addresses, listed names, pool sizes, the
+  resolved Create parameters. Upload and download bodies, cloud-init and
+  key material are never logged.
+- **Tracing.** `observe.call` starts `kvm2.<Method>` per RPC with
+  `kvm2.host` and the identifiers (`kvm2.domain`, `kvm2.pool`,
+  `kvm2.volume`), sets `kvm2.libvirt_code` on failure and writes the
+  `libvirt call` line inside the span; `libvirtDialer.dial` starts
+  `kvm2.dial` (`kvm2.address`, `kvm2.user`); `fetchImage` starts
+  `kvm2.download` (`kvm2.url_host`, `kvm2.sha256_configured`,
+  `kvm2.bytes`). Both the decorator and the dialer take `tracing.Option`
+  so tests inject a recorder; `New` leaves them on the global provider.
+- **Tests.** Pure builders (`xml_test.go`, `volume_test.go`,
+  `seed_test.go`, `id_test.go`, `metadata_test.go`, `state_test.go`);
+  `errors_test.go` (the classify table with real SSH error strings and a
+  real `knownhosts.KeyError`); `backend_test.go` (the adapter over
+  go-libvirt's own mock through a `net.Pipe`: deadline kills the
+  connection, cancel unblocks, killed by another caller); `hosts_test.go`;
+  `ssh_test.go` over `sshserver_test.go` (every dial stage, host-key
+  policies, key material never in errors or logs); `image_test.go`
+  (marker rules, streaming asserted by counting bytes, dedicated upload
+  connection); `create_test.go`, `get_test.go`, `delete_test.go`,
+  `list_test.go`, `kvm2_test.go`, `provider_test.go` (the contract §11
+  list, named test by test in the spec's entry 21); `observe_test.go`
+  (every seam method spanned and logged exactly once, by reflection);
+  `check_test.go` (the reason table, never touches the pool). The double is
+  `fakeBackend` in `fake_test.go`, whose `twin` shares state under its own
+  call log so a test can tell the pooled connection from a dedicated one.
+  Real backend: `kvm2_integration_test.go` behind `//go:build integration`,
+  driven by `KVM2_TEST_*` variables and `make test-integration-kvm2`,
+  described in
+  [testing.md](testing.md#the-kvm2-integration-test-real-hypervisor).
+- **Deployment.** The `kvm-ssh` Secret and its mount are shared with kvm
+  (`config/manager/manager.yaml` says so), `make kvm-ssh-secret` is reused,
+  a commented `kvm2-home` entry in both providers configs,
+  `config/samples/proxy_kvm2.yaml` (zone `beelink`, matching the shipped
+  config), the `kvm2` type in the `status.type` comment of
+  `api/v1alpha1/providerhealth_types.go`. No RBAC.
+- **Known gaps** (see also [kvm2-provider.md § Known gaps](kvm2-provider.md#known-gaps)).
+  - `Create` blocks, including the first image download on each host.
+  - GC blind spots: volumes without a domain are invisible to GC, and so are
+    VMs on a host removed from the config, which `Delete` treats as done.
+  - No adoption of kvm's domains: a switched entry needs its Proxies
+    deleted first.
+  - Cancelling one operation cuts off the others on that host once: a
+    cancelled `call` kills the pooled connection.
+  - `Content-Length` is required for image downloads.
+  - A paused or pmsuspended domain is not started by a repeat `Create`:
+    `ensureRunning` returns Permanent, so the Proxy latches until its spec
+    changes; the admin resumes the VM on the host.
+  - Credential parse errors carry the parser's own text, which can quote a
+    PEM header type or a malformed known_hosts token (never key bytes).
+  - No reused-overlay check: an existing overlay is reused without checking
+    its backing image or size.
+  - The sample cloud-init installs Squid with `http_access allow all`.
+
+## 8. Gaps shared by the existing providers
+
+- **Adoption is not verified in three of the four.** kubernetes, gcp and
+  kvm adopt any existing instance with the right name without checking
+  `LabelUID`; kvm2 returns `ErrPermanent` when the UID differs.
+- **The Create comment does not match kvm and kvm2.** The interface comment
+  in `internal/provider/provider.go` says `Create` returns as soon as the
+  request is submitted; both libvirt providers block until the domain has
+  started.
 - **`Stopped` and `Terminated` are produced inconsistently.** kubernetes
-  never produces Stopped, kvm never produces Terminated, and the reconciler
-  treats them the same.
+  never produces Stopped, kvm and kvm2 never produce Terminated, and the
+  reconciler treats them the same.
 - **Different `ListByTag` partial-failure policies.** gcp leaves failed
-  parts out, kvm fails the whole list. Both are safe.
+  parts out, kvm and kvm2 fail the whole list. Both are safe.
 - **Stale `mock` comments** in `internal/provider/provider.go`,
   `internal/provider/registry/registry.go` and `api/v1alpha1/proxy_types.go`
   (and so the generated CRD's `spec.provider` description).
@@ -351,33 +550,41 @@ the author's view.
   is still open.
 - **Tracing.** kvm has no child spans, so a kvm trace shows `provider.create`
   and nothing under it; correlation works only through `traceID` on its V(1)
-  log lines. Only kubernetes is tested against a real collector, and that
-  test checks only the operation spans; nothing checks that gcp's SDK spans
-  or the kubernetes API spans nest under `provider.*`.
+  log lines. kvm2 has them, but only an in-memory recorder checks their
+  topology. Only kubernetes is tested against a real collector, and that
+  test checks only the operation spans; nothing checks that gcp's SDK spans,
+  the kubernetes API spans or kvm2's spans nest under `provider.*` in a
+  real collector.
 
-## 8. Proposed features
+## 9. Proposed features
 
-Listed only. Nothing below is designed or implemented.
+Listed only. Nothing below is designed or implemented, except where a
+bullet says kvm2 already does it.
 
 - **Child spans in kvm:** extend the `tracingAPI` decorator in
   `kvm/trace.go`, which already wraps every libvirt call, to also start a
   span per RPC, and add spans for the SSH dial and the image download. The
   seam's methods take no `ctx` today (`hostPool.call` hands `fn` only the
-  API), so the span context has to be plumbed in.
-- **Tempo e2e checks for child spans**, and for gcp and kvm through their
-  real backend runbooks.
+  API), so the span context has to be plumbed in. kvm2's `observe.go` over
+  its ctx-first seam is the shape this would take.
+- **Tempo e2e checks for child spans**, and for gcp, kvm and kvm2 through
+  their real backend runbooks.
 - **A shared conformance suite:** for example `internal/provider/providertest`,
   which runs the contract §11 tests against any provider plus a fake
   backend, so a new provider gets the contract tests for free.
 - **Verify the UID when `Create` adopts an instance.** Return `ErrPermanent`
   on a mismatch instead of adopting an instance that belongs to someone else.
+  kvm2 does this; the other three do not.
 - **Report asynchronous create failures:** gcp keeps the operation name, and
   `Get` reports the failure instead of NotFound.
 - **Capacity or quota signals** for kvm, such as pool free space and host
-  memory, mapped to `ErrQuotaExceeded`.
+  memory, mapped to `ErrQuotaExceeded`. kvm2 has the pool-space precheck;
+  host memory is still unchecked in both.
 - **GC for secondary resources:** mark volumes and similar resources, and
   let `ListByTag` or a second listing method cover them.
-- **Reload credentials and config without a restart.**
+- **Reload credentials and config without a restart.** kvm2 re-reads its
+  SSH material on every dial; the providers config is still read once per
+  process for every type.
 - **Checker `Details`** for kubernetes (server version) and gcp (quota
   headroom).
 - **Internal-IP mode** for gcp, and a firewall rule managed per provider.

docs/providers.md

@@ -13,11 +13,13 @@ the existing ones happen to work.
   a new provider's design answers, one entry per decision the contract leaves
   open. Copy it to `docs/<type>-provider-spec.md`, fill it in, and hand it to
   the implementer with the other two files.
-- **[providers-existing.md](providers-existing.md)**: how kubernetes, gcp and
-  kvm meet the contract, a table mapping each of them to every profile, their
-  per-provider reference notes, shared gaps and proposed features.
+- **[providers-existing.md](providers-existing.md)**: how kubernetes, gcp,
+  kvm and kvm2 meet the contract, a table mapping each of them to every
+  profile, their per-provider reference notes, shared gaps and proposed
+  features.
 
 Per-provider user guides: [kvm-provider.md](kvm-provider.md),
+[kvm2-provider.md](kvm2-provider.md),
 [gcp-vm-validation.md](gcp-vm-validation.md) and
 [gcp-in-specific-project.md](gcp-in-specific-project.md).
 

docs/testing.md

@@ -62,6 +62,11 @@ manual runs too.
 | | `trace_test.go`, `conn_test.go`, `errors_test.go` | every `libvirtAPI` method traced exactly once (guarded by reflection on the interface), results and errors pass through, libvirt error fields, detail keys only at V2, upload content never logged; pool connect/drop lines, no wrapping below V1 | `fakeAPI` + `funcr` capture |
 | | `check_test.go`, `conn_test.go` | health check: details, per-host verdicts in config order, hosts checked concurrently, a timeout that keeps the connection and refuses a second check while the first still waits, no waiting behind a reconcile's dial, the reason table; `probe` abandons without closing, but still drops on a transport error; waiting for a host's slot ends with the caller's deadline; an inactive or missing pool and daemon errors keep the connection; every dial stage tagged through the real SSH server, with classification unchanged | `fakeAPI`, fake dialers, in-process SSH server |
 | | `kvm_integration_test.go` (`-tags=integration`) | one real VM: image fetch, Create, idempotent re-Create, Running with IP, ListByTag, Delete, pool state afterwards; `TestKVMCheck_realHost` runs the health check against the host and creates nothing | a **real hypervisor** — see below |
+| `internal/provider/kvm2` | `xml_test.go`, `volume_test.go`, `seed_test.go`, `id_test.go`, `metadata_test.go`, `state_test.go` | pure builders: domain XML (file disks with resolved paths, UEFI with secure boot off, BIOS escape, the metadata element, the guest-agent channel), volume XML with and without a backing store, seed ISO reopened (files, label, user-data byte for byte, temp dir removed) and the vendor-data script vs. multipart rules, providerID round-trip and malformed forms, metadata in both namespace spellings, state mapping and the IPv4 filter | pure |
+| | `errors_test.go`, `backend_test.go`, `hosts_test.go`, `ssh_test.go` | classification table (libvirt codes, SSH auth text, `knownhosts.KeyError`, missing files); the per-call budget adapter (deadline kills the connection and is Transient, cancel unblocks, killed by another caller, close idempotent) against a mocked libvirt socket; the host table (one dial per host under concurrency, redial after drop, `dialFresh` never cached, waiting ends with ctx); the real dialer end to end: socket forwarding, unknown and mismatched host keys, insecure mode, missing vs. unparsable key and known_hosts, wrong key, handshake timeout, cancel, every stage tagged, key material never in errors or logs | fake dialer; a **real in-process `x/crypto/ssh` server** forwarding a unix socket |
+| | `image_test.go`, `create_test.go`, `get_test.go`, `delete_test.go`, `list_test.go`, `kvm2_test.go`, `provider_test.go` | image cache (fetch with the marker written last, cached, marker missing or without a digest, sha256 change or mismatch, 4xx/5xx, no Content-Length, short read, concurrent callers fetch once, cancel mid-stream, streamed without buffering, upload on a dedicated connection) and the pool-space quota precheck; Create resume after every partial step, returns only when running, adoption guard on UID, seed always rebuilt, cloud-init sentinel never logged; Get state and address mapping incl. agent-before-ARP; every Delete already-gone case, NVRAM flag, per-name lock; ListByTag across hosts, one down fails the whole call, unreadable domain skipped; `New` never dials and warns on missing files; every error wrapped with the configured name; logging tiers V0/V1/V2 | `httptest` image server + in-memory `fakeBackend` behind the `backend` seam |
+| | `observe_test.go`, `check_test.go` | every seam method traced and logged exactly once (guarded by reflection), child spans under the parent, detail only at V2, upload bodies never logged, error fields; health check: one entry per host in config order, hosts checked concurrently, expired ctx returns promptly, never touches the pooled connection, the reason table, detail keys | `fakeBackend` + `funcr` capture + span recorder |
+| | `kvm2_integration_test.go` (`-tags=integration`) | one real VM: `New` without a dial, `Check` Ready with details, Create, idempotent re-Create with an unchanged domain count, Running with IP and the metadata round-trip, ListByTag, Delete, NotFound, pool state afterwards, `kvm2.*` spans under `provider.create`; `TestKVM2Check_realHost` runs the health check against the host and creates nothing | a **real hypervisor** — see below |
 | `internal/health` | `probe_test.go` | probes through a **real CONNECT-capable proxy stub** to a real TLS server: tunnel success, refused CONNECT, wrong status, dead proxy, plain-http forward | `httptest` + hijacked bidirectional tunnel |
 | | `engine_test.go` | thresholds, first-verdict, latency suppression matrix, dropped-event retry, stale-probe UID guard, tick scheduling/seeding/pruning, `Start` end-to-end | fake `client.Reader` + injected `probeFn` |
 | `internal/lease` | `store_test.go` | capacity, `MaxLeases=0`, all three selection tie-breaks, target-scoped vs global cooldowns, expiry via fake clock, report-on-retained-lease, idempotent release, **40 concurrent acquires never exceeding capacity** | injectable clock; the concurrency case is why `-race` matters |
@@ -93,12 +98,12 @@ envtest suite the scaffold generated.
   bug (below).
 - **Live GCP**: no test talks to Google. The seam boundary
   (`buildInsertRequest` + classification) is tested exhaustively instead.
-- **Live libvirt, by default**: the KVM unit tests run over an in-memory
-  fake of the 20 libvirt calls the provider uses. Whether libvirt accepts
-  the domain XML, whether a raw qcow2 upload plus pool refresh yields a
-  usable backing image, and whether the guest actually boots and reports
-  an address are provable only on a real host — that is the opt-in
-  integration test below.
+- **Live libvirt, by default**: the KVM and kvm2 unit tests run over an
+  in-memory fake of the 20 libvirt calls each provider uses. Whether
+  libvirt accepts the domain XML, whether a raw qcow2 upload plus pool
+  refresh yields a usable backing image, and whether the guest actually
+  boots and reports an address are provable only on a real host — that is
+  the opt-in integration test below (one per provider).
 
 ## The KVM integration test (real hypervisor)
 
@@ -162,6 +167,95 @@ go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/
 The test is not part of `make test` and not run in CI. Keep it compiling
 with `go vet -tags=integration ./...` when the provider's internals move.
 
+## The kvm2 integration test (real hypervisor)
+
+`internal/provider/kvm2/kvm2_integration_test.go` compiles only under
+`-tags=integration` and skips itself unless the environment names a
+host. Like the kvm one it deviates from the "testcontainers for real
+infra" convention on purpose: libvirt needs `/dev/kvm` and a bridge to a
+DHCP network, and a container has neither. It targets whichever
+hypervisor you point it at and leaves that host's pool with one more
+cached base image (`base-test-image.qcow2` and its `.sha256` marker) than
+it had. The kvm and kvm2 tests can share a host: kvm2 marks its domains
+in its own metadata namespace and never lists or adopts the other's.
+
+```sh
+export KVM2_TEST_HOST=beelink:22               # SSH endpoint
+export KVM2_TEST_USER=novakj                   # must be in the libvirt group
+export KVM2_TEST_SSH_KEY=$HOME/.ssh/id_ed25519 # private key for that user
+export KVM2_TEST_KNOWN_HOSTS=$HOME/.ssh/kvm_known_hosts  # the hypervisors' host keys
+export KVM2_TEST_BRIDGE=br0                    # bridge to a network with DHCP
+export KVM2_TEST_IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
+make test-integration-kvm2
+```
+
+`KVM2_TEST_KNOWN_HOSTS` is technically optional, but leaving it out turns
+host-key verification **off** for the run (the log says so); keep a
+`kvm_known_hosts` holding just the hypervisors rather than pointing the
+test at your everyday `known_hosts`, so a re-keyed hypervisor is a
+one-line fix in a file nothing else uses.
+
+Also optional: `KVM2_TEST_IMAGE_SHA256` (verified after download and
+compared with the cached marker), `KVM2_TEST_POOL` (default `default`),
+`KVM2_TEST_SOCKET` (default `/var/run/libvirt/libvirt-sock`),
+`KVM2_TEST_TIMEOUT` (default `20m`, the budget for the whole lifecycle;
+it must cover the first-time image download), `KVM2_TEST_KEEP` to leave
+the VM behind for `virsh console` when something goes wrong (the log then
+prints the `virsh undefine --nvram` and `virsh vol-delete` lines that
+clean it up), and `KVM2_TEST_LOG_LEVEL` (default `1`) for how much of the
+provider's logging the `-v` output shows: at `1` every libvirt RPC is one
+compact `libvirt call` line (`HH:MM:SS.mmm kvm2 "level"=1 "msg"="libvirt
+call" "rpc"="DomainLookupByName" "domain"=... "duration"=...`, plus the
+per-operation outcome lines and connection events), interleaved with the
+test's own `t.Log` lines; `2` adds the domain and volume XML, metadata
+and addresses; `0` leaves only `t.Log`. The per-run constants (provider,
+host, domain name) are stated once in the first line and left off every
+provider line. See the Logging section of
+[kvm2-provider.md](kvm2-provider.md).
+
+What one run asserts, in order, each step logged with its duration:
+
+1. `New` returns within 100 ms and without dialling (no `libvirt host
+   connected` line before the first call).
+2. `Check` reports the one host `Ready` with reason `Reachable` and the
+   `hostname`, `libvirtVersion`, `poolState` (`running`) and
+   `poolAvailableGiB` details, which are logged.
+3. Every later call goes through `provider.WithTracing` with an in-memory
+   span recorder that the package's own spans also land in.
+4. `Create` returns `hosts/kvm2-test/domains/<name>` (downloading the
+   image on a cold pool — the log prints how long it took).
+5. A second `Create` returns the same ID and the host's domain count,
+   read over a dedicated connection, is unchanged.
+6. `Get`, polled every 5 s, reaches `Running` with an IPv4 within the
+   budget; its `UID` is the one Create wrote and `CreatedAt` is non-zero
+   and within a few minutes of now. The time from Create's return to the
+   first address is logged.
+7. `ListByTag` contains the ID with that UID.
+8. A TCP connect to port 22 on the guest, logged only — the machine
+   running the test may not route to the bridge network.
+9. `Delete` returns nil, `Get` is `ErrNotFound`, a second `Delete` is a
+   no-op.
+10. The pool, inspected over a dedicated connection, holds the base image
+    and its marker but no `<name>.qcow2` or `<name>-seed.iso`.
+11. Among the recorded spans, a `provider.create` span has at least one
+    `kvm2.DomainDefineXML` and one `kvm2.dial` child, and no span
+    attribute carries the cloud-init text.
+
+With `KVM2_TEST_KEEP` steps 9 and 10 are skipped; otherwise a `t.Cleanup`
+attempts the Delete even when an earlier assertion failed.
+
+`TestKVM2Check_realHost` runs the provider health check
+([kvm2-provider.md](kvm2-provider.md)) against the same host. It must
+find the host reachable and the pool running, and it creates nothing. It
+needs the same variables; to run only this test:
+
+```sh
+go test -tags=integration -count=1 -v -run TestKVM2Check ./internal/provider/kvm2/
+```
+
+The test is not part of `make test` and not run in CI. Keep it compiling
+with `go vet -tags=integration ./...` when the provider's internals move.
+
 ## The scaffolded `make test-e2e` suite
 
 `test/e2e/` is the kubebuilder-generated smoke suite (build image → kind

internal/provider/config.go

@@ -22,6 +22,9 @@ type ProviderConfig struct {
 	Kubernetes *KubernetesConfig `json:"kubernetes,omitempty"`
 	GCP        *GCPConfig        `json:"gcp,omitempty"`
 	KVM        *KVMConfig        `json:"kvm,omitempty"`
+	// KVM2 is the same block shape as KVM, for the kvm2 provider type: one
+	// or more libvirt/KVM hypervisors reached over SSH.
+	KVM2 *KVMConfig `json:"kvm2,omitempty"`
 }
 
 // setBlocks names the type-specific blocks that are non-nil, so validate can
@@ -37,6 +40,9 @@ func (p ProviderConfig) setBlocks() []string {
 	if p.KVM != nil {
 		out = append(out, "kvm")
 	}
+	if p.KVM2 != nil {
+		out = append(out, "kvm2")
+	}
 	return out
 }
 
@@ -100,7 +106,7 @@ func (c *Config) validate() error {
 
 		prefix := fmt.Sprintf("providers[%d] %q", i, p.Name)
 		switch p.Type {
-		case "kubernetes", "gcp", "kvm":
+		case "kubernetes", "gcp", "kvm", "kvm2":
 		case "":
 			return fmt.Errorf("%s: type is required", prefix)
 		default:
@@ -120,6 +126,10 @@ func (c *Config) validate() error {
 			if err := validateKVM(prefix, p.KVM); err != nil {
 				return err
 			}
+		case "kvm2":
+			if err := validateKVMBlock(prefix, "kvm2", p.KVM2); err != nil {
+				return err
+			}
 		}
 	}
 	return nil

internal/provider/config_kvm.go

@@ -114,20 +114,27 @@ var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
 var kvmIdentifier = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`)
 
 func validateKVM(prefix string, c *KVMConfig) error {
+	return validateKVMBlock(prefix, "kvm", c)
+}
+
+// validateKVMBlock checks a KVMConfig under the given block name, so the
+// kvm and kvm2 provider types share one validator and each reports its own
+// block in messages.
+func validateKVMBlock(prefix, block string, c *KVMConfig) error {
 	if c == nil {
-		return fmt.Errorf("%s: kvm block is required", prefix)
+		return fmt.Errorf("%s: %s block is required", prefix, block)
 	}
 	if c.SSHKeyFile == "" {
-		return fmt.Errorf("%s: kvm.sshKeyFile is required", prefix)
+		return fmt.Errorf("%s: %s.sshKeyFile is required", prefix, block)
 	}
 	if c.KnownHostsFile == "" && !c.InsecureIgnoreHostKey {
-		return fmt.Errorf("%s: kvm.knownHostsFile is required unless kvm.insecureIgnoreHostKey is true", prefix)
+		return fmt.Errorf("%s: %s.knownHostsFile is required unless %s.insecureIgnoreHostKey is true", prefix, block, block)
 	}
 	if c.DiskSizeGB < 0 {
-		return fmt.Errorf("%s: kvm.diskSizeGb must not be negative", prefix)
+		return fmt.Errorf("%s: %s.diskSizeGb must not be negative", prefix, block)
 	}
 	for name, img := range c.Images {
-		p := fmt.Sprintf("%s: kvm.images[%q]", prefix, name)
+		p := fmt.Sprintf("%s: %s.images[%q]", prefix, block, name)
 		if !kvmIdentifier.MatchString(name) {
 			return fmt.Errorf("%s: key must be letters, digits, dots, dashes or underscores"+
 				" — it becomes the cached volume name base-<key>.<format>", p)
@@ -146,19 +153,19 @@ func validateKVM(prefix string, c *KVMConfig) error {
 		}
 	}
 	if len(c.MachineTypes) == 0 {
-		return fmt.Errorf("%s: kvm.machineTypes must define at least one profile", prefix)
+		return fmt.Errorf("%s: %s.machineTypes must define at least one profile", prefix, block)
 	}
 	for name, mt := range c.MachineTypes {
 		if mt.VCPUs <= 0 || mt.MemoryMiB <= 0 {
-			return fmt.Errorf("%s: kvm.machineTypes[%q] needs vcpus > 0 and memoryMiB > 0", prefix, name)
+			return fmt.Errorf("%s: %s.machineTypes[%q] needs vcpus > 0 and memoryMiB > 0", prefix, block, name)
 		}
 	}
 	if len(c.Hosts) == 0 {
-		return fmt.Errorf("%s: kvm.hosts must list at least one host", prefix)
+		return fmt.Errorf("%s: %s.hosts must list at least one host", prefix, block)
 	}
 	seen := make(map[string]bool, len(c.Hosts))
 	for i, h := range c.Hosts {
-		p := fmt.Sprintf("%s: kvm.hosts[%d]", prefix, i)
+		p := fmt.Sprintf("%s: %s.hosts[%d]", prefix, block, i)
 		if h.Name == "" {
 			return fmt.Errorf("%s: name is required", p)
 		}

internal/provider/config_kvm_test.go

@@ -214,6 +214,100 @@ providers:
 	}
 }
 
+func TestLoadConfig_kvm2Valid(t *testing.T) {
+	t.Parallel()
+	yaml := strings.Replace(validKVMYAML, "type: kvm\n    kvm:\n", "type: kvm2\n    kvm2:\n", 1)
+	cfg, err := LoadConfig([]byte(yaml))
+	if err != nil {
+		t.Fatalf("LoadConfig() error = %v, want nil", err)
+	}
+	if cfg.Providers[0].KVM != nil {
+		t.Errorf("providers[0].kvm = %+v, want nil", cfg.Providers[0].KVM)
+	}
+	k := cfg.Providers[0].KVM2
+	if k == nil {
+		t.Fatal("providers[0].kvm2 = nil")
+	}
+	if k.SSHKeyFile != "/etc/proxy-operator/kvm/id_ed25519" || k.KnownHostsFile != "/etc/proxy-operator/kvm/known_hosts" || k.DiskSizeGB != 20 {
+		t.Errorf("kvm2 = %+v, want sshKeyFile, knownHostsFile and diskSizeGb parsed", k)
+	}
+	if k.Images["ubuntu"].Format != "raw" {
+		t.Errorf("images[ubuntu].format = %q, want raw", k.Images["ubuntu"].Format)
+	}
+	if mt := k.MachineTypes["small"]; mt.VCPUs != 1 || mt.MemoryMiB != 1024 {
+		t.Errorf("machineTypes[small] = %+v, want {1 1024}", mt)
+	}
+	if len(k.Hosts) != 2 || k.Hosts[0].Name != "nas1" || k.Hosts[1].Pool != "fast" {
+		t.Errorf("hosts = %+v, want nas1 and lab2 with lab2's pool parsed", k.Hosts)
+	}
+}
+
+func TestLoadConfig_kvm2Invalid(t *testing.T) {
+	t.Parallel()
+	minimal := func(body string) string {
+		return "providers:\n  - name: kvm2-home\n    type: kvm2\n    kvm2:\n" + indent(body, 6)
+	}
+	hostsAndTypes := `
+machineTypes:
+  small: {vcpus: 1, memoryMiB: 1024}
+hosts:
+  - {name: nas1, address: nas1:22, user: virt, bridge: br0}
+`
+	tests := []struct {
+		name       string
+		yaml       string
+		wantErrSub string
+	}{
+		{
+			name:       "missing kvm2 block",
+			yaml:       "providers:\n  - name: kvm2-home\n    type: kvm2\n",
+			wantErrSub: "kvm2 block is required",
+		},
+		{
+			name:       "missing sshKeyFile",
+			yaml:       minimal("knownHostsFile: /kh\n" + hostsAndTypes),
+			wantErrSub: "kvm2.sshKeyFile is required",
+		},
+		{
+			name:       "missing knownHostsFile without insecure flag",
+			yaml:       minimal("sshKeyFile: /key\n" + hostsAndTypes),
+			wantErrSub: "kvm2.knownHostsFile is required unless kvm2.insecureIgnoreHostKey is true",
+		},
+		{
+			name:       "no machine types",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nhosts:\n  - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
+			wantErrSub: "kvm2.machineTypes must define at least one",
+		},
+		{
+			name:       "no hosts",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\n"),
+			wantErrSub: "kvm2.hosts must list at least one host",
+		},
+		{
+			name:       "duplicate host name",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, user: virt, bridge: br0}\n  - {name: nas1, address: b:22, user: virt, bridge: br0}\n"),
+			wantErrSub: `kvm2.hosts[1]: duplicate host name "nas1"`,
+		},
+		{
+			name:       "image key with a slash",
+			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  ../etc/d: {url: https://x/y.qcow2}\n" + hostsAndTypes),
+			wantErrSub: `kvm2.images["../etc/d"]: key must be letters, digits, dots, dashes or underscores`,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			_, err := LoadConfig([]byte(tc.yaml))
+			if err == nil {
+				t.Fatalf("LoadConfig() error = nil, want error containing %q", tc.wantErrSub)
+			}
+			if !strings.Contains(err.Error(), tc.wantErrSub) {
+				t.Errorf("LoadConfig() error = %q, want substring %q", err.Error(), tc.wantErrSub)
+			}
+		})
+	}
+}
+
 func indent(s string, n int) string {
 	pad := strings.Repeat(" ", n)
 	lines := strings.Split(strings.TrimLeft(s, "\n"), "\n")

internal/provider/config_test.go

@@ -118,6 +118,26 @@ providers:
       image: custom-image`,
 			wantErrSub: "type is gcp but a kubernetes block is set",
 		},
+		{
+			name: "kvm type with kvm2 block",
+			yaml: `
+providers:
+  - name: p1
+    type: kvm
+    kvm2:
+      sshKeyFile: /key`,
+			wantErrSub: "type is kvm but a kvm2 block is set",
+		},
+		{
+			name: "kvm2 type with kvm block",
+			yaml: `
+providers:
+  - name: p1
+    type: kvm2
+    kvm:
+      sshKeyFile: /key`,
+			wantErrSub: "type is kvm2 but a kvm block is set",
+		},
 		{
 			name: "strict mode rejects unknown top-level key",
 			yaml: `

internal/provider/kvm2/backend.go

@@ -0,0 +1,369 @@
+package kvm2
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// backend is the seam over go-libvirt (spec entry 17): exactly the calls the
+// provider uses, each with ctx first, since go-libvirt's own methods take
+// none and block. The production adapter is libvirtBackend; tests use an
+// in-memory fake. Nothing outside this package sees it.
+type backend interface {
+	DomainLookupByName(ctx context.Context, name string) (libvirt.Domain, error)
+	DomainDefineXML(ctx context.Context, xml string) (libvirt.Domain, error)
+	DomainCreate(ctx context.Context, dom libvirt.Domain) error
+	DomainDestroy(ctx context.Context, dom libvirt.Domain) error
+	DomainUndefineFlags(ctx context.Context, dom libvirt.Domain, flags libvirt.DomainUndefineFlagsValues) error
+	DomainGetState(ctx context.Context, dom libvirt.Domain, flags uint32) (state int32, reason int32, err error)
+	DomainGetMetadata(ctx context.Context, dom libvirt.Domain, typ int32, uri libvirt.OptString, flags libvirt.DomainModificationImpact) (string, error)
+	DomainInterfaceAddresses(ctx context.Context, dom libvirt.Domain, source uint32, flags uint32) ([]libvirt.DomainInterface, error)
+	ConnectListAllDomains(ctx context.Context, needResults int32, flags libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error)
+	StoragePoolLookupByName(ctx context.Context, name string) (libvirt.StoragePool, error)
+	StoragePoolRefresh(ctx context.Context, pool libvirt.StoragePool, flags uint32) error
+	StoragePoolGetInfo(ctx context.Context, pool libvirt.StoragePool) (state uint8, capacity, allocation, available uint64, err error)
+	StorageVolLookupByName(ctx context.Context, pool libvirt.StoragePool, name string) (libvirt.StorageVol, error)
+	StorageVolGetPath(ctx context.Context, vol libvirt.StorageVol) (string, error)
+	StorageVolCreateXML(ctx context.Context, pool libvirt.StoragePool, xml string, flags libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error)
+	StorageVolUpload(ctx context.Context, vol libvirt.StorageVol, r io.Reader, offset, length uint64, flags libvirt.StorageVolUploadFlags) error
+	StorageVolDownload(ctx context.Context, vol libvirt.StorageVol, w io.Writer, offset, length uint64, flags libvirt.StorageVolDownloadFlags) error
+	StorageVolDelete(ctx context.Context, vol libvirt.StorageVol, flags libvirt.StorageVolDeleteFlags) error
+	ConnectGetLibVersion(ctx context.Context) (uint64, error)
+	ConnectGetHostname(ctx context.Context) (string, error)
+	Close() error
+}
+
+// liveness is implemented by backends whose connection can die underneath
+// them. The host table drops a backend whose dead channel is closed. It is
+// deliberately not part of backend: the fake needs it, the seam does not.
+type liveness interface {
+	dead() <-chan struct{}
+}
+
+func isDead(b backend) bool {
+	l, ok := b.(liveness)
+	if !ok {
+		return false
+	}
+	select {
+	case <-l.dead():
+		return true
+	default:
+		return false
+	}
+}
+
+// budgets are the per-call deadlines of spec entry 18, independent of the
+// caller's ctx, which by contract carries none.
+type budgets struct {
+	// dial bounds TCP connect, SSH handshake and libvirt connect together.
+	dial time.Duration
+	// rpc bounds every ordinary call.
+	rpc time.Duration
+	// create bounds DomainCreate, which waits for QEMU to start.
+	create time.Duration
+	// seedUpload bounds a StorageVolUpload of at most smallUploadBytes.
+	seedUpload time.Duration
+	// transfer caps a large upload or a download; ctx may end it earlier.
+	transfer time.Duration
+}
+
+var defaultBudgets = budgets{
+	dial:       10 * time.Second,
+	rpc:        30 * time.Second,
+	create:     90 * time.Second,
+	seedUpload: 60 * time.Second,
+	transfer:   20 * time.Minute,
+}
+
+// smallUploadBytes separates a seed ISO (tens of KiB) from a base image
+// (hundreds of MiB) for the upload budget.
+const smallUploadBytes = 16 << 20
+
+// closeBudget bounds the courtesy ConnectClose RPC in Close.
+const closeBudget = 5 * time.Second
+
+func (b budgets) upload(length uint64) time.Duration {
+	if length <= smallUploadBytes {
+		return b.seedUpload
+	}
+	return b.transfer
+}
+
+// link is the liveness of one connection. dead is closed exactly once, by
+// the first of: an abandoned call, a keepalive failure, go-libvirt noticing
+// the socket went away, or Close. kill closes the transport, which is the
+// only way to unblock a go-libvirt call; go-libvirt then fails every other
+// call in flight on the connection with ErrInterrupted.
+type link struct {
+	label string
+	kill  func()
+
+	once   sync.Once
+	closed chan struct{}
+	mu     sync.Mutex
+	reason string
+}
+
+func newLink(label string, kill func()) *link {
+	return &link{label: label, kill: kill, closed: make(chan struct{})}
+}
+
+func (l *link) dead() <-chan struct{} { return l.closed }
+
+func (l *link) isDead() bool {
+	select {
+	case <-l.closed:
+		return true
+	default:
+		return false
+	}
+}
+
+func (l *link) deadReason() string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.reason
+}
+
+// markDead records why the connection is unusable, closes the transport and
+// signals dead. Later calls are no-ops.
+func (l *link) markDead(reason string) {
+	l.once.Do(func() {
+		l.mu.Lock()
+		l.reason = reason
+		l.mu.Unlock()
+		if l.kill != nil {
+			l.kill()
+		}
+		close(l.closed)
+	})
+}
+
+// call executes a blocking go-libvirt call with a budget and returns its
+// result, or Transient when the link is already dead, ctx ends, the budget
+// runs out, or another caller kills the link meanwhile. Abandoning a call
+// kills the link (spec entry 18); the goroutine running fn then returns on
+// its own once go-libvirt sees the closed socket, and its result is dropped.
+// Results travel through the channel so an abandoned call never writes
+// into its caller's frame.
+func call[T any](ctx context.Context, l *link, name string, budget time.Duration, fn func() (T, error)) (T, error) {
+	var zero T
+	if l.isDead() {
+		return zero, transientf("libvirt call %s on %s: connection is closed: %s", name, l.label, l.deadReason())
+	}
+	if err := ctx.Err(); err != nil {
+		return zero, fmt.Errorf("%w: libvirt call %s on %s not started: %w", provider.ErrTransient, name, l.label, err)
+	}
+	type result struct {
+		v   T
+		err error
+	}
+	done := make(chan result, 1)
+	go func() {
+		v, err := fn()
+		done <- result{v, err}
+	}()
+	timer := time.NewTimer(budget)
+	defer timer.Stop()
+	select {
+	case r := <-done:
+		return r.v, r.err
+	case <-ctx.Done():
+		l.markDead(fmt.Sprintf("call %s abandoned: %v", name, ctx.Err()))
+		return zero, fmt.Errorf("%w: libvirt call %s on %s abandoned: %w", provider.ErrTransient, name, l.label, ctx.Err())
+	case <-timer.C:
+		l.markDead(fmt.Sprintf("call %s exceeded its %s budget", name, budget))
+		return zero, fmt.Errorf("%w: libvirt call %s on %s exceeded its %s budget: %w", provider.ErrTransient, name, l.label, budget, context.DeadlineExceeded)
+	case <-l.closed:
+		return zero, transientf("libvirt call %s on %s: connection closed while waiting: %s", name, l.label, l.deadReason())
+	}
+}
+
+// run is call for go-libvirt methods that return only an error.
+func (l *link) run(ctx context.Context, name string, budget time.Duration, fn func() error) error {
+	_, err := call(ctx, l, name, budget, func() (struct{}, error) { return struct{}{}, fn() })
+	return err
+}
+
+// libvirtBackend is the production adapter: a go-libvirt client over one
+// forwarded socket, with the budgets applied around every call. It never
+// logs; the observe decorator does.
+type libvirtBackend struct {
+	lv      *libvirt.Libvirt
+	link    *link
+	budgets budgets
+
+	closeOnce sync.Once
+	closeErr  error
+}
+
+var _ backend = (*libvirtBackend)(nil)
+
+// newLibvirtBackend wraps lv, whose socket runs over transport. Closing the
+// transport is how a hung call is abandoned, so it must close the socket
+// conn given to libvirt.New as well as whatever carries it.
+func newLibvirtBackend(lv *libvirt.Libvirt, label string, transport io.Closer, b budgets) *libvirtBackend {
+	return &libvirtBackend{
+		lv:      lv,
+		link:    newLink(label, func() { _ = transport.Close() }),
+		budgets: b,
+	}
+}
+
+func (b *libvirtBackend) dead() <-chan struct{} { return b.link.dead() }
+
+// connect runs the libvirt handshake under the dial budget and, once it
+// holds, drops the link as soon as go-libvirt reports the socket gone.
+func (b *libvirtBackend) connect(ctx context.Context) error {
+	if err := b.link.run(ctx, "Connect", b.budgets.dial, b.lv.Connect); err != nil {
+		return err
+	}
+	disconnected := b.lv.Disconnected()
+	go func() {
+		select {
+		case <-disconnected:
+			b.link.markDead("connection lost")
+		case <-b.link.closed:
+		}
+	}()
+	return nil
+}
+
+func (b *libvirtBackend) DomainLookupByName(ctx context.Context, name string) (libvirt.Domain, error) {
+	return call(ctx, b.link, "DomainLookupByName", b.budgets.rpc, func() (libvirt.Domain, error) { return b.lv.DomainLookupByName(name) })
+}
+
+func (b *libvirtBackend) DomainDefineXML(ctx context.Context, xml string) (libvirt.Domain, error) {
+	return call(ctx, b.link, "DomainDefineXML", b.budgets.rpc, func() (libvirt.Domain, error) { return b.lv.DomainDefineXML(xml) })
+}
+
+func (b *libvirtBackend) DomainCreate(ctx context.Context, dom libvirt.Domain) error {
+	return b.link.run(ctx, "DomainCreate", b.budgets.create, func() error { return b.lv.DomainCreate(dom) })
+}
+
+func (b *libvirtBackend) DomainDestroy(ctx context.Context, dom libvirt.Domain) error {
+	return b.link.run(ctx, "DomainDestroy", b.budgets.rpc, func() error { return b.lv.DomainDestroy(dom) })
+}
+
+func (b *libvirtBackend) DomainUndefineFlags(ctx context.Context, dom libvirt.Domain, flags libvirt.DomainUndefineFlagsValues) error {
+	return b.link.run(ctx, "DomainUndefineFlags", b.budgets.rpc, func() error { return b.lv.DomainUndefineFlags(dom, flags) })
+}
+
+func (b *libvirtBackend) DomainGetState(ctx context.Context, dom libvirt.Domain, flags uint32) (int32, int32, error) {
+	type stateReason struct{ state, reason int32 }
+	r, err := call(ctx, b.link, "DomainGetState", b.budgets.rpc, func() (stateReason, error) {
+		state, reason, err := b.lv.DomainGetState(dom, flags)
+		return stateReason{state, reason}, err
+	})
+	return r.state, r.reason, err
+}
+
+func (b *libvirtBackend) DomainGetMetadata(ctx context.Context, dom libvirt.Domain, typ int32, uri libvirt.OptString, flags libvirt.DomainModificationImpact) (string, error) {
+	return call(ctx, b.link, "DomainGetMetadata", b.budgets.rpc, func() (string, error) { return b.lv.DomainGetMetadata(dom, typ, uri, flags) })
+}
+
+func (b *libvirtBackend) DomainInterfaceAddresses(ctx context.Context, dom libvirt.Domain, source uint32, flags uint32) ([]libvirt.DomainInterface, error) {
+	return call(ctx, b.link, "DomainInterfaceAddresses", b.budgets.rpc, func() ([]libvirt.DomainInterface, error) {
+		return b.lv.DomainInterfaceAddresses(dom, source, flags)
+	})
+}
+
+func (b *libvirtBackend) ConnectListAllDomains(ctx context.Context, needResults int32, flags libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error) {
+	type listed struct {
+		doms []libvirt.Domain
+		n    uint32
+	}
+	r, err := call(ctx, b.link, "ConnectListAllDomains", b.budgets.rpc, func() (listed, error) {
+		doms, n, err := b.lv.ConnectListAllDomains(needResults, flags)
+		return listed{doms, n}, err
+	})
+	return r.doms, r.n, err
+}
+
+func (b *libvirtBackend) StoragePoolLookupByName(ctx context.Context, name string) (libvirt.StoragePool, error) {
+	return call(ctx, b.link, "StoragePoolLookupByName", b.budgets.rpc, func() (libvirt.StoragePool, error) { return b.lv.StoragePoolLookupByName(name) })
+}
+
+func (b *libvirtBackend) StoragePoolRefresh(ctx context.Context, pool libvirt.StoragePool, flags uint32) error {
+	return b.link.run(ctx, "StoragePoolRefresh", b.budgets.rpc, func() error { return b.lv.StoragePoolRefresh(pool, flags) })
+}
+
+func (b *libvirtBackend) StoragePoolGetInfo(ctx context.Context, pool libvirt.StoragePool) (uint8, uint64, uint64, uint64, error) {
+	type info struct {
+		state                           uint8
+		capacity, allocation, available uint64
+	}
+	r, err := call(ctx, b.link, "StoragePoolGetInfo", b.budgets.rpc, func() (info, error) {
+		state, capacity, allocation, available, err := b.lv.StoragePoolGetInfo(pool)
+		return info{state, capacity, allocation, available}, err
+	})
+	return r.state, r.capacity, r.allocation, r.available, err
+}
+
+func (b *libvirtBackend) StorageVolLookupByName(ctx context.Context, pool libvirt.StoragePool, name string) (libvirt.StorageVol, error) {
+	return call(ctx, b.link, "StorageVolLookupByName", b.budgets.rpc, func() (libvirt.StorageVol, error) { return b.lv.StorageVolLookupByName(pool, name) })
+}
+
+func (b *libvirtBackend) StorageVolGetPath(ctx context.Context, vol libvirt.StorageVol) (string, error) {
+	return call(ctx, b.link, "StorageVolGetPath", b.budgets.rpc, func() (string, error) { return b.lv.StorageVolGetPath(vol) })
+}
+
+func (b *libvirtBackend) StorageVolCreateXML(ctx context.Context, pool libvirt.StoragePool, xml string, flags libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error) {
+	return call(ctx, b.link, "StorageVolCreateXML", b.budgets.rpc, func() (libvirt.StorageVol, error) { return b.lv.StorageVolCreateXML(pool, xml, flags) })
+}
+
+func (b *libvirtBackend) StorageVolUpload(ctx context.Context, vol libvirt.StorageVol, r io.Reader, offset, length uint64, flags libvirt.StorageVolUploadFlags) error {
+	return b.link.run(ctx, "StorageVolUpload", b.budgets.upload(length), func() error {
+		return b.lv.StorageVolUpload(vol, r, offset, length, flags)
+	})
+}
+
+func (b *libvirtBackend) StorageVolDownload(ctx context.Context, vol libvirt.StorageVol, w io.Writer, offset, length uint64, flags libvirt.StorageVolDownloadFlags) error {
+	return b.link.run(ctx, "StorageVolDownload", b.budgets.transfer, func() error {
+		return b.lv.StorageVolDownload(vol, w, offset, length, flags)
+	})
+}
+
+func (b *libvirtBackend) StorageVolDelete(ctx context.Context, vol libvirt.StorageVol, flags libvirt.StorageVolDeleteFlags) error {
+	return b.link.run(ctx, "StorageVolDelete", b.budgets.rpc, func() error { return b.lv.StorageVolDelete(vol, flags) })
+}
+
+func (b *libvirtBackend) ConnectGetLibVersion(ctx context.Context) (uint64, error) {
+	return call(ctx, b.link, "ConnectGetLibVersion", b.budgets.rpc, b.lv.ConnectGetLibVersion)
+}
+
+func (b *libvirtBackend) ConnectGetHostname(ctx context.Context) (string, error) {
+	return call(ctx, b.link, "ConnectGetHostname", b.budgets.rpc, b.lv.ConnectGetHostname)
+}
+
+// Close sends libvirt's ConnectClose when the link is still live, bounded
+// because a hung host would otherwise pin the caller, then kills the link,
+// which closes the socket and the SSH client. Repeat calls return the first
+// result.
+func (b *libvirtBackend) Close() error {
+	b.closeOnce.Do(func() {
+		if b.link.isDead() {
+			return
+		}
+		done := make(chan error, 1)
+		go func() { done <- b.lv.Disconnect() }()
+		timer := time.NewTimer(closeBudget)
+		defer timer.Stop()
+		select {
+		case b.closeErr = <-done:
+		case <-timer.C:
+			b.closeErr = transientf("libvirt disconnect on %s exceeded its %s budget", b.link.label, closeBudget)
+		case <-b.link.closed:
+		}
+		b.link.markDead("closed")
+	})
+	return b.closeErr
+}

internal/provider/kvm2/backend_test.go

@@ -0,0 +1,252 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+	"github.com/digitalocean/go-libvirt/libvirttest"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// blockingFn returns a func that blocks until release is closed, plus a
+// counter of how often it was started.
+func blockingFn(t *testing.T) (fn func() error, started *atomic.Int32) {
+	t.Helper()
+	release := make(chan struct{})
+	t.Cleanup(func() { close(release) })
+	started = &atomic.Int32{}
+	return func() error {
+		started.Add(1)
+		<-release
+		return nil
+	}, started
+}
+
+func TestCall_deadlineClosesConnectionAndReturnsTransient(t *testing.T) {
+	t.Parallel()
+	var killed atomic.Int32
+	l := newLink("beelink", func() { killed.Add(1) })
+	fn, started := blockingFn(t)
+
+	start := time.Now()
+	err := l.run(context.Background(), "DomainCreate", 50*time.Millisecond, fn)
+	elapsed := time.Since(start)
+	if !errors.Is(err, provider.ErrTransient) {
+		t.Fatalf("run() error = %v, want ErrTransient", err)
+	}
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("run() error = %v, want it to wrap context.DeadlineExceeded", err)
+	}
+	if !strings.Contains(err.Error(), "DomainCreate") || !strings.Contains(err.Error(), "beelink") {
+		t.Errorf("run() error = %q, want the call name and host in it", err)
+	}
+	if elapsed > time.Second {
+		t.Errorf("run() took %v, want about the 50ms budget", elapsed)
+	}
+	if killed.Load() != 1 {
+		t.Errorf("kill hook called %d times, want 1", killed.Load())
+	}
+	if !l.isDead() {
+		t.Error("link not dead after an abandoned call")
+	}
+	if !strings.Contains(l.deadReason(), "budget") {
+		t.Errorf("deadReason() = %q, want the budget mentioned", l.deadReason())
+	}
+
+	start = time.Now()
+	err = l.run(context.Background(), "DomainGetState", time.Minute, fn)
+	if !errors.Is(err, provider.ErrTransient) {
+		t.Fatalf("second run() error = %v, want ErrTransient", err)
+	}
+	if time.Since(start) > 100*time.Millisecond {
+		t.Errorf("second run() took %v, want immediate", time.Since(start))
+	}
+	if started.Load() != 1 {
+		t.Errorf("fn started %d times, want 1: a dead link must not touch the socket", started.Load())
+	}
+	if killed.Load() != 1 {
+		t.Errorf("kill hook called %d times, want still 1", killed.Load())
+	}
+}
+
+func TestCall_cancelUnblocksRPC(t *testing.T) {
+	t.Parallel()
+	var killed atomic.Int32
+	l := newLink("beelink", func() { killed.Add(1) })
+	fn, _ := blockingFn(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(10 * time.Millisecond)
+		cancel()
+	}()
+
+	start := time.Now()
+	err := l.run(ctx, "ConnectListAllDomains", time.Minute, fn)
+	if time.Since(start) > 100*time.Millisecond {
+		t.Errorf("run() took %v after cancel, want under 100ms", time.Since(start))
+	}
+	if !errors.Is(err, provider.ErrTransient) || !errors.Is(err, context.Canceled) {
+		t.Errorf("run() error = %v, want ErrTransient wrapping context.Canceled", err)
+	}
+	if killed.Load() != 1 {
+		t.Errorf("kill hook called %d times, want 1", killed.Load())
+	}
+}
+
+func TestCall_alreadyCancelledDoesNotStart(t *testing.T) {
+	t.Parallel()
+	l := newLink("beelink", nil)
+	fn, started := blockingFn(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	err := l.run(ctx, "DomainGetState", time.Minute, fn)
+	if !errors.Is(err, provider.ErrTransient) || !errors.Is(err, context.Canceled) {
+		t.Errorf("run() error = %v, want ErrTransient wrapping context.Canceled", err)
+	}
+	if started.Load() != 0 {
+		t.Error("fn started although ctx was already cancelled")
+	}
+	if l.isDead() {
+		t.Error("link marked dead although nothing was abandoned")
+	}
+}
+
+func TestCall_returnsFnResult(t *testing.T) {
+	t.Parallel()
+	l := newLink("beelink", nil)
+	want := libvirt.Error{Code: uint32(libvirt.ErrNoDomain), Message: "gone"}
+	err := l.run(context.Background(), "DomainLookupByName", time.Second, func() error { return want })
+	if !hasCode(err, libvirt.ErrNoDomain) {
+		t.Errorf("run() error = %v, want the libvirt error through unchanged", err)
+	}
+	if l.isDead() {
+		t.Error("a backend error must not kill the link")
+	}
+}
+
+func TestCall_killedByAnotherCallerFailsTransient(t *testing.T) {
+	t.Parallel()
+	l := newLink("beelink", nil)
+	fn, _ := blockingFn(t)
+	done := make(chan error, 1)
+	go func() { done <- l.run(context.Background(), "DomainGetState", time.Minute, fn) }()
+	time.Sleep(10 * time.Millisecond)
+	l.markDead("keepalive failed")
+	select {
+	case err := <-done:
+		if !errors.Is(err, provider.ErrTransient) || !strings.Contains(err.Error(), "keepalive failed") {
+			t.Errorf("run() error = %v, want ErrTransient with the death reason", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("run() did not return after the link was killed")
+	}
+}
+
+func TestBudgets_upload(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name   string
+		length uint64
+		want   time.Duration
+	}{
+		{name: "seed sized", length: 400 << 10, want: defaultBudgets.seedUpload},
+		{name: "exactly the limit", length: smallUploadBytes, want: defaultBudgets.seedUpload},
+		{name: "one over the limit", length: smallUploadBytes + 1, want: defaultBudgets.transfer},
+		{name: "base image", length: 435 << 20, want: defaultBudgets.transfer},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := defaultBudgets.upload(tc.length); got != tc.want {
+				t.Errorf("upload(%d) = %v, want %v", tc.length, got, tc.want)
+			}
+		})
+	}
+}
+
+// mockAdapter wires a libvirtBackend to go-libvirt's own mock server over a
+// net.Pipe, so the adapter is exercised with the real RPC client.
+func mockAdapter(t *testing.T, b budgets) *libvirtBackend {
+	t.Helper()
+	m := libvirttest.New()
+	conn, err := m.Dial()
+	if err != nil {
+		t.Fatalf("mock dial: %v", err)
+	}
+	ad := newLibvirtBackend(libvirt.New(conn), "mock", conn, b)
+	if err := ad.connect(context.Background()); err != nil {
+		t.Fatalf("connect: %v", err)
+	}
+	t.Cleanup(func() { _ = ad.Close() })
+	return ad
+}
+
+func TestLibvirtBackend_roundTripThroughMock(t *testing.T) {
+	t.Parallel()
+	ad := mockAdapter(t, defaultBudgets)
+	v, err := ad.ConnectGetLibVersion(context.Background())
+	if err != nil {
+		t.Fatalf("ConnectGetLibVersion() error = %v", err)
+	}
+	if v == 0 {
+		t.Error("ConnectGetLibVersion() = 0, want the mock's version")
+	}
+	pool, err := ad.StoragePoolLookupByName(context.Background(), "default")
+	if err != nil {
+		t.Fatalf("StoragePoolLookupByName() error = %v", err)
+	}
+	if pool.Name == "" {
+		t.Error("StoragePoolLookupByName() returned an empty pool")
+	}
+	if isDead(ad) {
+		t.Error("adapter dead after successful calls")
+	}
+}
+
+func TestLibvirtBackend_unansweredCallIsAbandoned(t *testing.T) {
+	t.Parallel()
+	b := defaultBudgets
+	b.rpc = 100 * time.Millisecond
+	ad := mockAdapter(t, b)
+	// The mock does not implement ConnectGetHostname, so it never replies.
+	_, err := ad.ConnectGetHostname(context.Background())
+	if !errors.Is(err, provider.ErrTransient) || !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("ConnectGetHostname() error = %v, want ErrTransient wrapping DeadlineExceeded", err)
+	}
+	if !isDead(ad) {
+		t.Error("adapter not dead after an abandoned call")
+	}
+	select {
+	case <-ad.lv.Disconnected():
+	case <-time.After(time.Second):
+		t.Error("go-libvirt did not notice the closed socket")
+	}
+	if _, err := ad.ConnectGetLibVersion(context.Background()); !errors.Is(err, provider.ErrTransient) {
+		t.Errorf("call on dead adapter error = %v, want ErrTransient", err)
+	}
+	if err := ad.Close(); err != nil {
+		t.Errorf("Close() on dead adapter error = %v, want nil", err)
+	}
+}
+
+func TestLibvirtBackend_closeIsIdempotent(t *testing.T) {
+	t.Parallel()
+	ad := mockAdapter(t, defaultBudgets)
+	first := ad.Close()
+	second := ad.Close()
+	if first != nil || second != nil {
+		t.Errorf("Close() errors = %v, %v, want nil, nil", first, second)
+	}
+	if !isDead(ad) {
+		t.Error("adapter not dead after Close")
+	}
+	if _, err := ad.ConnectGetLibVersion(context.Background()); !errors.Is(err, provider.ErrTransient) {
+		t.Errorf("call after Close error = %v, want ErrTransient", err)
+	}
+}

internal/provider/kvm2/check.go

@@ -0,0 +1,189 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+var _ provider.Checker = (*Provider)(nil)
+
+// Reasons only this provider can tell apart (spec entry 16). Each names a
+// failure an admin fixes somewhere other than where the shared reasons
+// point.
+const (
+	// reasonCredentialsUnavailable: the key or known_hosts file is missing
+	// or unparsable; fix the Secret in the cluster, not the host.
+	reasonCredentialsUnavailable = "CredentialsUnavailable"
+	// reasonPoolNotFound: the configured pool is not defined on the host;
+	// compare with `virsh pool-list --all`.
+	reasonPoolNotFound = "PoolNotFound"
+	// reasonPoolNotActive: the pool exists but is not running; `virsh
+	// pool-start <pool>` (and autostart) brings it back.
+	reasonPoolNotActive = "PoolNotActive"
+)
+
+// Check verifies every configured host concurrently, each on a fresh
+// dedicated connection: dial, libvirt version, hostname, pool lookup, pool
+// info, close. The pooled connection is never touched, so a check that runs
+// out of time closes only its own connection and cannot fail a reconcile's
+// call in flight. Every target gets a verdict, in config order.
+func (p *Provider) Check(ctx context.Context) []provider.TargetStatus {
+	out := make([]provider.TargetStatus, len(p.cfg.Hosts))
+	var wg sync.WaitGroup
+	for i, host := range p.cfg.Hosts {
+		wg.Go(func() { out[i] = p.checkHost(ctx, host) })
+	}
+	wg.Wait()
+	return out
+}
+
+func (p *Provider) checkHost(ctx context.Context, host provider.KVMHost) provider.TargetStatus {
+	start := time.Now()
+	log := p.logger(ctx).WithValues("host", host.Name)
+	ts := provider.TargetStatus{
+		Name:    host.Name,
+		Address: host.Address,
+		Details: map[string]string{"user": host.User, "pool": host.Pool},
+	}
+	err := p.probeHost(ctx, host, ts.Details)
+	ts.Latency = time.Since(start)
+	switch {
+	case err == nil:
+		ts.Ready = true
+		ts.Reason = provider.ReasonReachable
+	default:
+		var inactive *poolNotActiveError
+		switch {
+		case ctx.Err() != nil:
+			// The round ran out of time, whichever stage was running: the
+			// error's shape (a dial that saw the ctx end, an abandoned
+			// call) says nothing the admin should act on.
+			ts.Reason = provider.ReasonTimeout
+		case errors.As(err, &inactive):
+			ts.Reason = reasonPoolNotActive
+		default:
+			ts.Reason = checkReason(err)
+		}
+		ts.Message = err.Error()
+	}
+	log.V(1).Info("kvm2 host checked", "ready", ts.Ready, "reason", ts.Reason, "latency", ts.Latency)
+	return ts
+}
+
+// probeHost runs the check sequence on its own connection and fills details
+// as facts arrive, so a failure late in the sequence still reports what the
+// host said before it.
+func (p *Provider) probeHost(ctx context.Context, host provider.KVMHost, details map[string]string) error {
+	b, err := p.hosts.dialFresh(ctx, host)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = b.Close() }()
+
+	version, err := b.ConnectGetLibVersion(ctx)
+	if err != nil {
+		return err
+	}
+	details["libvirtVersion"] = formatLibVersion(version)
+	hostname, err := b.ConnectGetHostname(ctx)
+	if err != nil {
+		return err
+	}
+	details["hostname"] = hostname
+	pool, err := b.StoragePoolLookupByName(ctx, host.Pool)
+	if err != nil {
+		return err
+	}
+	state, capacity, _, available, err := b.StoragePoolGetInfo(ctx, pool)
+	if err != nil {
+		return err
+	}
+	details["poolState"] = poolStateName(state)
+	details["poolCapacityGiB"] = formatGiB(capacity)
+	details["poolAvailableGiB"] = formatGiB(available)
+	if libvirt.StoragePoolState(state) != libvirt.StoragePoolRunning {
+		return &poolNotActiveError{pool: host.Pool, state: poolStateName(state)}
+	}
+	return nil
+}
+
+// poolNotActiveError is the one verdict that is not an error from the
+// backend: the pool answered, but is not usable.
+type poolNotActiveError struct {
+	pool  string
+	state string
+}
+
+func (e *poolNotActiveError) Error() string {
+	return fmt.Sprintf("storage pool %q is %s", e.pool, e.state)
+}
+
+// checkReason maps a failed probe to a reason once checkHost has ruled out
+// the round's ctx ending. A dial failure is judged by its stage, so a TCP
+// connect or an SSH handshake that ran out of the dial budget on its own is
+// Unreachable, not Timeout: the host did not answer. Timeout is for a
+// budget expiring once the connection was up.
+func checkReason(err error) string {
+	var de *dialError
+	if errors.As(err, &de) {
+		switch de.stage {
+		case stageCredentials:
+			return reasonCredentialsUnavailable
+		case stageSSH:
+			var keyErr *knownhosts.KeyError
+			if errors.As(de.err, &keyErr) || strings.Contains(de.err.Error(), sshAuthFailed) {
+				return provider.ReasonUnauthorized
+			}
+		}
+		return provider.ReasonUnreachable
+	}
+	switch {
+	case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
+		return provider.ReasonTimeout
+	case hasCode(err, libvirt.ErrAccessDenied), hasCode(err, libvirt.ErrAuthFailed):
+		return provider.ReasonForbidden
+	case hasCode(err, libvirt.ErrNoStoragePool):
+		return reasonPoolNotFound
+	}
+	if _, ok := libvirtCode(err); ok {
+		return provider.ReasonAPIError
+	}
+	return provider.ReasonUnreachable
+}
+
+// formatLibVersion decodes ConnectGetLibVersion's major*1_000_000 +
+// minor*1_000 + micro.
+func formatLibVersion(v uint64) string {
+	return fmt.Sprintf("%d.%d.%d", v/1_000_000, v/1_000%1_000, v%1_000)
+}
+
+// poolStateName uses virsh's words for a pool state.
+func poolStateName(state uint8) string {
+	switch libvirt.StoragePoolState(state) {
+	case libvirt.StoragePoolInactive:
+		return "inactive"
+	case libvirt.StoragePoolBuilding:
+		return "building"
+	case libvirt.StoragePoolRunning:
+		return "running"
+	case libvirt.StoragePoolDegraded:
+		return "degraded"
+	case libvirt.StoragePoolInaccessible:
+		return "inaccessible"
+	default:
+		return fmt.Sprintf("state %d", state)
+	}
+}
+
+func formatGiB(bytes uint64) string {
+	return fmt.Sprintf("%.1f", float64(bytes)/(1<<30))
+}

internal/provider/kvm2/check_test.go

@@ -0,0 +1,579 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr/funcr"
+	"golang.org/x/crypto/ssh/knownhosts"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// checkContext is captureContext with a sink that tolerates the concurrent
+// per-host log lines of Check.
+func checkContext() (context.Context, *logCapture) {
+	logs := &logCapture{}
+	log := funcr.New(func(prefix, args string) {
+		logs.mu.Lock()
+		defer logs.mu.Unlock()
+		logs.lines = append(logs.lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: 2})
+	return logf.IntoContext(context.Background(), log), logs
+}
+
+// checkDialer hands out a distinct fake per host name, fails the hosts in
+// errs, counts every dial, and, with block set, waits for ctx to end and
+// then fails the way dialSSHSocket does when the TCP connect sees the ctx
+// end: a dialError at the tcp stage wrapping the ctx's error.
+type checkDialer struct {
+	backends map[string]*fakeBackend
+	errs     map[string]error
+	block    bool
+
+	dials atomic.Int32
+}
+
+func (d *checkDialer) dial(ctx context.Context, host provider.KVMHost) (backend, error) {
+	d.dials.Add(1)
+	if d.block {
+		<-ctx.Done()
+		return nil, &dialError{stage: stageTCP, host: host.Name, err: fmt.Errorf("dial tcp %s: %w", host.Address, ctx.Err())}
+	}
+	if err := d.errs[host.Name]; err != nil {
+		return nil, err
+	}
+	b, ok := d.backends[host.Name]
+	if !ok {
+		return nil, fmt.Errorf("checkDialer: no backend for %s", host.Name)
+	}
+	return b, nil
+}
+
+func threeHosts() provider.KVMConfig {
+	cfg := twoHosts()
+	cfg.Hosts = append(cfg.Hosts, provider.KVMHost{Name: "lab3", Address: "lab3:22", User: "ops", Bridge: "br0", Pool: "ssd"})
+	return cfg
+}
+
+func fakesFor(cfg provider.KVMConfig) map[string]*fakeBackend {
+	out := make(map[string]*fakeBackend, len(cfg.Hosts))
+	for _, h := range cfg.Hosts {
+		f := newFakeBackend()
+		f.hostname = h.Name + ".home.lan"
+		if h.Pool != "" {
+			f.pools[h.Pool] = newFakePool()
+		}
+		out[h.Name] = f
+	}
+	return out
+}
+
+func detailKeys(d map[string]string) []string {
+	keys := make([]string, 0, len(d))
+	for k := range d {
+		keys = append(keys, k)
+	}
+	slices.Sort(keys)
+	return keys
+}
+
+func TestCheck_oneEntryPerHostInConfigOrder(t *testing.T) {
+	t.Parallel()
+	cfg := threeHosts()
+	d := &checkDialer{backends: fakesFor(cfg), errs: map[string]error{"lab2": errors.New("dial tcp 192.0.2.2:22: connect: no route to host")}}
+	p := newProvider(testProvider, cfg, d)
+	ctx, _ := checkContext()
+	got := p.Check(ctx)
+	if len(got) != len(cfg.Hosts) {
+		t.Fatalf("Check() returned %d targets, want %d", len(got), len(cfg.Hosts))
+	}
+	for i, h := range cfg.Hosts {
+		ts := got[i]
+		if ts.Name != h.Name || ts.Address != h.Address {
+			t.Errorf("target %d = %s (%s), want %s (%s)", i, ts.Name, ts.Address, h.Name, h.Address)
+		}
+		if ts.Name == "" || ts.Address == "" {
+			t.Errorf("target %d has an empty name or address: %+v", i, ts)
+		}
+		wantReady := h.Name != "lab2"
+		if ts.Ready != wantReady {
+			t.Errorf("target %s ready = %v, want %v (%s: %s)", ts.Name, ts.Ready, wantReady, ts.Reason, ts.Message)
+		}
+	}
+	if got[1].Reason != provider.ReasonUnreachable || got[0].Reason != provider.ReasonReachable {
+		t.Errorf("reasons = %s, %s, %s", got[0].Reason, got[1].Reason, got[2].Reason)
+	}
+}
+
+func TestCheck_hostsCheckedConcurrently(t *testing.T) {
+	t.Parallel()
+	cfg := twoHosts()
+	fakes := fakesFor(cfg)
+	for _, f := range fakes {
+		release := f.blockOn("ConnectGetLibVersion")
+		time.AfterFunc(200*time.Millisecond, func() { close(release) })
+	}
+	p := newProvider(testProvider, cfg, &checkDialer{backends: fakes})
+	ctx, _ := checkContext()
+	start := time.Now()
+	got := p.Check(ctx)
+	elapsed := time.Since(start)
+	if elapsed > 350*time.Millisecond {
+		t.Errorf("Check() took %v, want under 350ms for two 200ms hosts", elapsed)
+	}
+	for _, ts := range got {
+		if !ts.Ready {
+			t.Errorf("target %s not ready: %s: %s", ts.Name, ts.Reason, ts.Message)
+		}
+		if ts.Latency < 200*time.Millisecond {
+			t.Errorf("target %s latency = %v, want at least the 200ms block", ts.Name, ts.Latency)
+		}
+	}
+}
+
+// TestCheck_expiredContextReturnsPromptly drives the production shape of a
+// round that ran out of time during the dial: the dialer fails at the tcp
+// stage wrapping context.DeadlineExceeded, which checkReason alone would
+// call Unreachable; the verdict must still be Timeout because the round's
+// ctx ended.
+func TestCheck_expiredContextReturnsPromptly(t *testing.T) {
+	t.Parallel()
+	cfg := twoHosts()
+	p := newProvider(testProvider, cfg, &checkDialer{block: true})
+	ctx, _ := checkContext()
+	ctx, cancel := context.WithTimeout(ctx, 0)
+	defer cancel()
+	start := time.Now()
+	got := p.Check(ctx)
+	if elapsed := time.Since(start); elapsed > time.Second {
+		t.Errorf("Check() took %v with an expired ctx, want under 1s", elapsed)
+	}
+	if len(got) != len(cfg.Hosts) {
+		t.Fatalf("Check() returned %d targets, want %d", len(got), len(cfg.Hosts))
+	}
+	for _, ts := range got {
+		if ts.Ready || ts.Reason != provider.ReasonTimeout {
+			t.Errorf("target %s = ready %v reason %s, want not ready, Timeout", ts.Name, ts.Ready, ts.Reason)
+		}
+		if !strings.Contains(ts.Message, context.DeadlineExceeded.Error()) || !strings.Contains(ts.Message, string(stageTCP)) {
+			t.Errorf("target %s message = %q, want the tcp stage and the deadline named", ts.Name, ts.Message)
+		}
+	}
+}
+
+// keyMaterial is the sentinel in the unusable key file of the credentials
+// row; it must never reach a message.
+const keyMaterial = "SENTINEL-PRIVATE-KEY-BYTES-9f3a"
+
+func TestCheck_reasonsTable(t *testing.T) {
+	t.Parallel()
+	sshHandshake := func(cause error) error {
+		return &dialError{stage: stageSSH, host: testHost, err: fmt.Errorf("ssh: handshake failed: %w", cause)}
+	}
+	tests := []struct {
+		name string
+		// dialErr builds the dial failure; nil means the dial succeeds.
+		dialErr func(t *testing.T) error
+		prep    func(f *fakeBackend)
+		timeout time.Duration
+		cancel  bool
+
+		wantReady   bool
+		wantReason  string
+		wantMessage string
+	}{
+		{
+			name: "credentials unusable",
+			dialErr: func(t *testing.T) error {
+				path := filepath.Join(t.TempDir(), "id_ed25519")
+				if err := os.WriteFile(path, []byte(keyMaterial), 0o600); err != nil {
+					t.Fatal(err)
+				}
+				_, err := loadCredentials(provider.KVMConfig{SSHKeyFile: path, KnownHostsFile: "/nonexistent/known_hosts"})
+				if err == nil {
+					t.Fatal("loadCredentials accepted the sentinel file")
+				}
+				return &dialError{stage: stageCredentials, host: testHost, err: err}
+			},
+			wantReason: reasonCredentialsUnavailable,
+		},
+		{
+			name: "credentials missing",
+			dialErr: func(*testing.T) error {
+				_, err := loadCredentials(provider.KVMConfig{SSHKeyFile: "/nonexistent/kvm2/id_ed25519"})
+				return &dialError{stage: stageCredentials, host: testHost, err: err}
+			},
+			wantReason: reasonCredentialsUnavailable,
+		},
+		{
+			name: "tcp refused",
+			dialErr: func(*testing.T) error {
+				return &dialError{stage: stageTCP, host: testHost, err: errors.New("dial tcp 192.0.2.1:22: connect: connection refused")}
+			},
+			wantReason: provider.ReasonUnreachable,
+		},
+		{
+			name: "tcp timed out on the dial budget but ctx is fine",
+			dialErr: func(*testing.T) error {
+				return &dialError{stage: stageTCP, host: testHost, err: fmt.Errorf("dial tcp 192.0.2.1:22: %w", context.DeadlineExceeded)}
+			},
+			wantReason: provider.ReasonUnreachable,
+		},
+		{
+			name:       "ssh host key unknown",
+			dialErr:    func(*testing.T) error { return sshHandshake(&knownhosts.KeyError{}) },
+			wantReason: provider.ReasonUnauthorized,
+		},
+		{
+			name: "ssh host key mismatch",
+			dialErr: func(*testing.T) error {
+				return sshHandshake(&knownhosts.KeyError{Want: []knownhosts.KnownKey{{Filename: "known_hosts", Line: 3}}})
+			},
+			wantReason: provider.ReasonUnauthorized,
+		},
+		{
+			name: "ssh authentication refused",
+			dialErr: func(*testing.T) error {
+				return sshHandshake(errors.New("ssh: unable to authenticate, attempted methods [none publickey], no supported methods remain"))
+			},
+			wantReason: provider.ReasonUnauthorized,
+		},
+		{
+			name:       "ssh handshake reset",
+			dialErr:    func(*testing.T) error { return sshHandshake(errors.New("read tcp: connection reset by peer")) },
+			wantReason: provider.ReasonUnreachable,
+		},
+		{
+			name:       "ssh handshake timed out on the dial budget but ctx is fine",
+			dialErr:    func(*testing.T) error { return sshHandshake(context.DeadlineExceeded) },
+			wantReason: provider.ReasonUnreachable,
+		},
+		{
+			name: "socket refused",
+			dialErr: func(*testing.T) error {
+				return &dialError{stage: stageSocket, host: testHost, err: errors.New("opening /var/run/libvirt/libvirt-sock: ssh: rejected: connect failed (No such file or directory)")}
+			},
+			wantReason:  provider.ReasonUnreachable,
+			wantMessage: "/var/run/libvirt/libvirt-sock",
+		},
+		{
+			name: "libvirt access denied",
+			prep: func(f *fakeBackend) {
+				f.failWith("ConnectGetLibVersion", fakeErr(libvirt.ErrAccessDenied, "access denied"))
+			},
+			wantReason: provider.ReasonForbidden,
+		},
+		{
+			name: "libvirt auth failed",
+			prep: func(f *fakeBackend) {
+				f.failWith("ConnectGetHostname", fakeErr(libvirt.ErrAuthFailed, "authentication failed"))
+			},
+			wantReason: provider.ReasonForbidden,
+		},
+		{
+			name: "pool not found",
+			prep: func(f *fakeBackend) {
+				f.mu.Lock()
+				delete(f.pools, fakePoolName)
+				f.mu.Unlock()
+			},
+			wantReason:  reasonPoolNotFound,
+			wantMessage: fakePoolName,
+		},
+		{
+			name: "other libvirt error",
+			prep: func(f *fakeBackend) {
+				f.failWith("StoragePoolGetInfo", fakeErr(libvirt.ErrInternalError, "internal error: pool info"))
+			},
+			wantReason: provider.ReasonAPIError,
+		},
+		{
+			name:       "rpc outlives the deadline",
+			prep:       func(f *fakeBackend) { f.blockOn("ConnectGetLibVersion") },
+			timeout:    50 * time.Millisecond,
+			wantReason: provider.ReasonTimeout,
+		},
+		{
+			name:       "ctx cancelled mid check",
+			prep:       func(f *fakeBackend) { f.blockOn("StoragePoolLookupByName") },
+			cancel:     true,
+			wantReason: provider.ReasonTimeout,
+		},
+		{
+			name: "link killed mid check",
+			prep: func(f *fakeBackend) {
+				f.failWith("StoragePoolLookupByName", transientf("libvirt call StoragePoolLookupByName on beelink: connection closed while waiting: keepalive: EOF"))
+			},
+			wantReason: provider.ReasonUnreachable,
+		},
+		{
+			name:        "pool inactive",
+			prep:        func(f *fakeBackend) { f.pools[fakePoolName].state = libvirt.StoragePoolInactive },
+			wantReason:  reasonPoolNotActive,
+			wantMessage: `storage pool "default" is inactive`,
+		},
+		{
+			name:       "ok",
+			wantReady:  true,
+			wantReason: provider.ReasonReachable,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			cfg := twoHosts()
+			cfg.Hosts = cfg.Hosts[:1]
+			f := newFakeBackend()
+			d := &checkDialer{backends: map[string]*fakeBackend{testHost: f}}
+			if tc.dialErr != nil {
+				d.errs = map[string]error{testHost: tc.dialErr(t)}
+			}
+			if tc.prep != nil {
+				tc.prep(f)
+			}
+			p := newProvider(testProvider, cfg, d)
+			ctx, _ := checkContext()
+			switch {
+			case tc.timeout > 0:
+				var cancel context.CancelFunc
+				ctx, cancel = context.WithTimeout(ctx, tc.timeout)
+				defer cancel()
+			case tc.cancel:
+				var cancel context.CancelFunc
+				ctx, cancel = context.WithCancel(ctx)
+				time.AfterFunc(50*time.Millisecond, cancel)
+			}
+			got := p.Check(ctx)
+			if len(got) != 1 {
+				t.Fatalf("Check() returned %d targets, want 1", len(got))
+			}
+			ts := got[0]
+			if ts.Ready != tc.wantReady || ts.Reason != tc.wantReason {
+				t.Errorf("verdict = ready %v reason %s (%s), want ready %v reason %s", ts.Ready, ts.Reason, ts.Message, tc.wantReady, tc.wantReason)
+			}
+			if !tc.wantReady && ts.Message == "" {
+				t.Error("failed check has no message")
+			}
+			if tc.wantReady && ts.Message != "" {
+				t.Errorf("successful check has a message: %q", ts.Message)
+			}
+			if strings.Contains(ts.Message, keyMaterial) {
+				t.Errorf("message leaks the key file content: %q", ts.Message)
+			}
+			if tc.wantMessage != "" && !strings.Contains(ts.Message, tc.wantMessage) {
+				t.Errorf("message = %q, want it to contain %q", ts.Message, tc.wantMessage)
+			}
+			if ts.Details["user"] != "virt" || ts.Details["pool"] != "default" {
+				t.Errorf("details = %v, want user and pool", ts.Details)
+			}
+			if tc.wantReady {
+				want := map[string]string{
+					"user": "virt", "pool": "default", "hostname": "fake-host", "libvirtVersion": "11.6.0",
+					"poolState": "running", "poolCapacityGiB": "100.0", "poolAvailableGiB": "50.0",
+				}
+				for k, v := range want {
+					if ts.Details[k] != v {
+						t.Errorf("details[%s] = %q, want %q", k, ts.Details[k], v)
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestCheck_neverUsesPooledConnection(t *testing.T) {
+	t.Parallel()
+	cfg := threeHosts()
+	fakes := fakesFor(cfg)
+	d := &checkDialer{backends: fakes}
+	p := newProvider(testProvider, cfg, d)
+	ctx, _ := checkContext()
+	got := p.Check(ctx)
+	for _, ts := range got {
+		if !ts.Ready {
+			t.Errorf("target %s not ready: %s: %s", ts.Name, ts.Reason, ts.Message)
+		}
+	}
+	if n := d.dials.Load(); int(n) != len(cfg.Hosts) {
+		t.Errorf("dials = %d, want one fresh dial per host (%d)", n, len(cfg.Hosts))
+	}
+	p.hosts.mu.Lock()
+	pooled := len(p.hosts.hosts)
+	p.hosts.mu.Unlock()
+	if pooled != 0 {
+		t.Errorf("pooled table has %d entries after Check, want none", pooled)
+	}
+	for name, f := range fakes {
+		if f.closeCount() != 1 {
+			t.Errorf("fresh backend for %s closed %d times, want 1", name, f.closeCount())
+		}
+	}
+}
+
+func TestCheck_detailsKeys(t *testing.T) {
+	t.Parallel()
+	full := []string{"hostname", "libvirtVersion", "pool", "poolAvailableGiB", "poolCapacityGiB", "poolState", "user"}
+	tests := []struct {
+		name    string
+		dialErr error
+		prep    func(f *fakeBackend)
+		want    []string
+	}{
+		{name: "success", want: full},
+		{name: "pool not active", prep: func(f *fakeBackend) { f.pools[fakePoolName].state = libvirt.StoragePoolDegraded }, want: full},
+		{
+			name:    "dial failure",
+			dialErr: &dialError{stage: stageTCP, host: testHost, err: errors.New("dial tcp: connection refused")},
+			want:    []string{"pool", "user"},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			cfg := twoHosts()
+			cfg.Hosts = cfg.Hosts[:1]
+			f := newFakeBackend()
+			if tc.prep != nil {
+				tc.prep(f)
+			}
+			d := &checkDialer{backends: map[string]*fakeBackend{testHost: f}}
+			if tc.dialErr != nil {
+				d.errs = map[string]error{testHost: tc.dialErr}
+			}
+			p := newProvider(testProvider, cfg, d)
+			ctx, _ := checkContext()
+			got := p.Check(ctx)[0]
+			if keys := detailKeys(got.Details); !slices.Equal(keys, tc.want) {
+				t.Errorf("details keys = %v, want %v", keys, tc.want)
+			}
+			for k, v := range got.Details {
+				if v == "" {
+					t.Errorf("details[%s] is empty", k)
+				}
+			}
+		})
+	}
+}
+
+func TestCheck_logsVerdictPerHost(t *testing.T) {
+	t.Parallel()
+	cfg := twoHosts()
+	d := &checkDialer{backends: fakesFor(cfg), errs: map[string]error{"lab2": &dialError{stage: stageTCP, host: "lab2", err: errors.New("dial tcp: i/o timeout")}}}
+	p := newProvider(testProvider, cfg, d)
+	ctx, logs := checkContext()
+	p.Check(ctx)
+	checked := logs.matching("kvm2 host checked")
+	if len(checked) != 2 {
+		t.Fatalf("got %d verdict lines, want 2: %v", len(checked), checked)
+	}
+	for _, l := range checked {
+		switch {
+		case strings.Contains(l, `"host"="beelink"`):
+			if !strings.Contains(l, `"ready"=true`) || !strings.Contains(l, `"reason"="Reachable"`) || !strings.Contains(l, `"latency"=`) {
+				t.Errorf("beelink verdict line = %s", l)
+			}
+		case strings.Contains(l, `"host"="lab2"`):
+			if !strings.Contains(l, `"ready"=false`) || !strings.Contains(l, `"reason"="Unreachable"`) {
+				t.Errorf("lab2 verdict line = %s", l)
+			}
+		default:
+			t.Errorf("verdict line without a host: %s", l)
+		}
+	}
+}
+
+func TestCheckReason_table(t *testing.T) {
+	t.Parallel()
+	dial := func(stage dialStage, cause error) error {
+		return fmt.Errorf("wrapped: %w", &dialError{stage: stage, host: testHost, err: cause})
+	}
+	tests := []struct {
+		name string
+		err  error
+		want string
+	}{
+		{name: "credentials stage", err: dial(stageCredentials, errors.New("reading ssh key file /k: no such file")), want: reasonCredentialsUnavailable},
+		{name: "credentials stage even when permanent", err: dial(stageCredentials, permanentf("parsing ssh key file /k: ssh: no key found")), want: reasonCredentialsUnavailable},
+		{name: "tcp stage", err: dial(stageTCP, errors.New("connect: connection refused")), want: provider.ReasonUnreachable},
+		{name: "tcp stage with a deadline", err: dial(stageTCP, context.DeadlineExceeded), want: provider.ReasonUnreachable},
+		{name: "ssh stage unknown host key", err: dial(stageSSH, &knownhosts.KeyError{}), want: provider.ReasonUnauthorized},
+		{name: "ssh stage wrapped key error", err: dial(stageSSH, fmt.Errorf("ssh: handshake failed: %w", &knownhosts.KeyError{Want: []knownhosts.KnownKey{{}}})), want: provider.ReasonUnauthorized},
+		{name: "ssh stage unable to authenticate", err: dial(stageSSH, errors.New("ssh: handshake failed: ssh: unable to authenticate, attempted methods [none publickey], no supported methods remain")), want: provider.ReasonUnauthorized},
+		{name: "ssh stage reset", err: dial(stageSSH, errors.New("read tcp: connection reset by peer")), want: provider.ReasonUnreachable},
+		{name: "ssh stage cancelled", err: dial(stageSSH, context.Canceled), want: provider.ReasonUnreachable},
+		{name: "socket stage", err: dial(stageSocket, errors.New("opening /run/libvirt/libvirt-sock: ssh: rejected")), want: provider.ReasonUnreachable},
+		{name: "deadline exceeded", err: fmt.Errorf("%w: libvirt call ConnectGetHostname on beelink exceeded its 30s budget: %w", provider.ErrTransient, context.DeadlineExceeded), want: provider.ReasonTimeout},
+		{name: "cancelled", err: fmt.Errorf("%w: libvirt call ConnectGetHostname on beelink abandoned: %w", provider.ErrTransient, context.Canceled), want: provider.ReasonTimeout},
+		{name: "access denied", err: fakeErr(libvirt.ErrAccessDenied, "access denied"), want: provider.ReasonForbidden},
+		{name: "auth failed wrapped", err: fmt.Errorf("connecting to libvirt on host beelink: %w", fakeErr(libvirt.ErrAuthFailed, "authentication failed")), want: provider.ReasonForbidden},
+		{name: "no storage pool", err: fakeErr(libvirt.ErrNoStoragePool, "Storage pool not found"), want: reasonPoolNotFound},
+		{name: "other libvirt error", err: fakeErr(libvirt.ErrInternalError, "internal error"), want: provider.ReasonAPIError},
+		{name: "link killed", err: transientf("libvirt call X on beelink: connection closed while waiting: keepalive: EOF"), want: provider.ReasonUnreachable},
+		{name: "plain error", err: errors.New("unexpected EOF"), want: provider.ReasonUnreachable},
+		{name: "nil", err: nil, want: provider.ReasonUnreachable},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := checkReason(tc.err); got != tc.want {
+				t.Errorf("checkReason(%v) = %s, want %s", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestFormatLibVersion(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		in   uint64
+		want string
+	}{
+		{0, "0.0.0"},
+		{11006000, "11.6.0"},
+		{10000003, "10.0.3"},
+		{9010012, "9.10.12"},
+		{100999999, "100.999.999"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.want, func(t *testing.T) {
+			t.Parallel()
+			if got := formatLibVersion(tc.in); got != tc.want {
+				t.Errorf("formatLibVersion(%d) = %s, want %s", tc.in, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestPoolStateName(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		in   uint8
+		want string
+	}{
+		{uint8(libvirt.StoragePoolInactive), "inactive"},
+		{uint8(libvirt.StoragePoolBuilding), "building"},
+		{uint8(libvirt.StoragePoolRunning), "running"},
+		{uint8(libvirt.StoragePoolDegraded), "degraded"},
+		{uint8(libvirt.StoragePoolInaccessible), "inaccessible"},
+		{42, "state 42"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.want, func(t *testing.T) {
+			t.Parallel()
+			if got := poolStateName(tc.in); got != tc.want {
+				t.Errorf("poolStateName(%d) = %s, want %s", tc.in, got, tc.want)
+			}
+		})
+	}
+}

internal/provider/kvm2/create.go

@@ -0,0 +1,280 @@
+package kvm2
+
+import (
+	"bytes"
+	"fmt"
+	"time"
+
+	"context"
+
+	"github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// The wait after DomainCreate (spec entry 18): Create returns only once the
+// domain is running, so Get never sees a domain kvm2 defined but did not
+// start (spec entry 9). DomainCreate is synchronous, so the first read
+// normally succeeds; the cap only covers state settling and must not pin a
+// reconcile worker for long.
+const (
+	defaultRunningWait = 15 * time.Second
+	defaultRunningPoll = 500 * time.Millisecond
+)
+
+// Create is spec entry 10: validate placement, then, under the per-name
+// lock, find or build the domain step by step, every step idempotent, and
+// return once it is running.
+func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
+	host, err := p.host(req.Placement.Zone)
+	if err != nil {
+		if req.Placement.Zone != "" {
+			err = permanentf("placement.zone %q is not in kvm2.hosts", req.Placement.Zone)
+		}
+		return "", p.wrapErr("create", "", err)
+	}
+	mt, ok := p.cfg.MachineTypes[req.Placement.MachineType]
+	if !ok {
+		return "", p.wrapErr("create", "", permanentf("placement.machineType %q is not in kvm2.machineTypes", req.Placement.MachineType))
+	}
+	src, err := resolveImage(p.cfg, req.Placement.Image)
+	if err != nil {
+		return "", p.wrapErr("create", "", err)
+	}
+
+	id := formatProviderID(host.Name, req.Name)
+	log := p.logger(ctx).WithValues("host", host.Name, "name", req.Name)
+	log.V(2).Info("kvm2 create resolved",
+		"pool", host.Pool, "bridge", host.Bridge, "machineType", req.Placement.MachineType,
+		"vcpus", mt.VCPUs, "memoryMiB", mt.MemoryMiB, "diskSizeGb", p.cfg.DiskSizeGB,
+		"image", src.volumeName(), "firmware", src.Firmware, "cloudInitBytes", len(req.CloudInit))
+
+	unlock, err := p.lockName(ctx, req.Name)
+	if err != nil {
+		return "", p.wrapErr("create", id, err)
+	}
+	defer unlock()
+
+	be, err := p.hosts.get(ctx, host)
+	if err != nil {
+		logAPIError(log, "create", err)
+		return "", p.wrapErr("create", id, err)
+	}
+	if err := p.create(ctx, log, be, host, req, mt, src); err != nil {
+		logAPIError(log, "create", err)
+		return "", p.wrapErr("create", id, err)
+	}
+	log.V(1).Info("kvm2 domain ensured running", "id", id)
+	return id, nil
+}
+
+func (p *Provider) create(ctx context.Context, log logr.Logger, be backend, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType, src imageSource) error {
+	dom, err := be.DomainLookupByName(ctx, req.Name)
+	switch {
+	case err == nil:
+		meta, err := readMetadata(ctx, be, dom)
+		if err != nil {
+			return err
+		}
+		if !meta.managed() {
+			return permanentf("domain %s exists but carries no kvm2 mark; it was not created by this provider", req.Name)
+		}
+		if meta.UID != "" && meta.UID != req.UID {
+			return permanentf("domain %s belongs to proxy uid %s, not %s", req.Name, meta.UID, req.UID)
+		}
+		log.V(1).Info("kvm2 domain already defined", "uid", meta.UID)
+		return p.ensureRunning(ctx, log, be, dom)
+	case !hasCode(err, libvirt.ErrNoDomain):
+		return fmt.Errorf("looking up domain %s: %w", req.Name, err)
+	}
+
+	pool, err := be.StoragePoolLookupByName(ctx, host.Pool)
+	if err != nil {
+		return fmt.Errorf("looking up storage pool %s: %w", host.Pool, err)
+	}
+	imageBytes, err := p.imageSizeIfUncached(ctx, be, pool, src)
+	if err != nil {
+		return err
+	}
+	if err := checkPoolSpace(ctx, be, pool, host.Pool, neededBytes(imageBytes, p.cfg.DiskSizeGB)); err != nil {
+		return err
+	}
+	basePath, err := p.ensureImage(ctx, host, pool, be, src)
+	if err != nil {
+		return err
+	}
+	overlay, err := ensureOverlay(ctx, log, be, pool, req.Name, basePath, src.Format, p.cfg.DiskSizeGB)
+	if err != nil {
+		return err
+	}
+	seed, err := p.writeSeed(ctx, log, be, pool, req)
+	if err != nil {
+		return err
+	}
+	diskPath, err := be.StorageVolGetPath(ctx, overlay)
+	if err != nil {
+		return fmt.Errorf("reading the path of overlay volume %s: %w", overlay.Name, err)
+	}
+	seedPath, err := be.StorageVolGetPath(ctx, seed)
+	if err != nil {
+		return fmt.Errorf("reading the path of seed volume %s: %w", seed.Name, err)
+	}
+
+	xmlDoc, err := buildDomainXML(domainParams{
+		Name:       req.Name,
+		UID:        req.UID,
+		CreatedAt:  p.now(),
+		VCPUs:      mt.VCPUs,
+		MemoryMiB:  mt.MemoryMiB,
+		Firmware:   src.Firmware,
+		Bridge:     host.Bridge,
+		DiskPath:   diskPath,
+		SeedPath:   seedPath,
+		DiskFormat: provider.KVMFormatQcow2,
+	})
+	if err != nil {
+		return err
+	}
+	dom, err = be.DomainDefineXML(ctx, xmlDoc)
+	if err != nil {
+		return fmt.Errorf("defining domain %s: %w", req.Name, err)
+	}
+	kv := []any{"uid", req.UID, "firmware", src.Firmware}
+	if log.V(2).Enabled() {
+		kv = append(kv, "xml", xmlDoc)
+	}
+	log.V(1).Info("domain defined", kv...)
+	return p.ensureRunning(ctx, log, be, dom)
+}
+
+// ensureOverlay creates the per-VM qcow2 overlay backed by the base image,
+// or reuses the one a previous attempt left: volume creation is atomic, so
+// an existing overlay is complete.
+func ensureOverlay(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, name, basePath, baseFormat string, diskSizeGB int64) (libvirt.StorageVol, error) {
+	volName := overlayVolumeName(name)
+	xmlDoc, err := buildVolumeXML(volumeSpec{
+		Name:          volName,
+		CapacityBytes: diskSizeGB << 30,
+		Format:        provider.KVMFormatQcow2,
+		BackingPath:   basePath,
+		BackingFormat: baseFormat,
+	})
+	if err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	vol, err := be.StorageVolCreateXML(ctx, pool, xmlDoc, 0)
+	switch {
+	case err == nil:
+		log.V(1).Info("overlay volume created", "volume", volName)
+		return vol, nil
+	case hasCode(err, libvirt.ErrStorageVolExist):
+		vol, err = be.StorageVolLookupByName(ctx, pool, volName)
+		if err != nil {
+			return libvirt.StorageVol{}, fmt.Errorf("looking up overlay volume %s: %w", volName, err)
+		}
+		log.V(1).Info("overlay volume already exists", "volume", volName)
+		return vol, nil
+	default:
+		return libvirt.StorageVol{}, fmt.Errorf("creating overlay volume %s: %w", volName, err)
+	}
+}
+
+// writeSeed builds the NoCloud seed and uploads it as a raw volume,
+// replacing whatever a previous attempt left (spec entry 5): a cut-off
+// upload would boot a VM without cloud-init. The ISO is tens of KiB, so it
+// is built in memory. Nothing about its content is logged or put in an
+// error.
+func (p *Provider) writeSeed(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, req provider.CreateRequest) (libvirt.StorageVol, error) {
+	name := seedVolumeName(req.Name)
+	if err := deleteVolumeIfPresent(ctx, be, pool, name); err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	installAgent := p.cfg.InstallGuestAgent == nil || *p.cfg.InstallGuestAgent
+	vendorData, err := buildVendorData(p.cfg.VendorData, installAgent)
+	if err != nil {
+		return libvirt.StorageVol{}, permanentf("building vendor-data for %s: %v", req.Name, err)
+	}
+	var iso bytes.Buffer
+	if err := buildSeedISO(req.Name, req.CloudInit, vendorData, &iso); err != nil {
+		return libvirt.StorageVol{}, permanentf("building seed ISO for %s: %v", req.Name, err)
+	}
+	xmlDoc, err := buildVolumeXML(volumeSpec{Name: name, CapacityBytes: int64(iso.Len()), Format: provider.KVMFormatRaw})
+	if err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	vol, err := be.StorageVolCreateXML(ctx, pool, xmlDoc, 0)
+	if err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("creating seed volume %s: %w", name, err)
+	}
+	if err := be.StorageVolUpload(ctx, vol, bytes.NewReader(iso.Bytes()), 0, uint64(iso.Len()), 0); err != nil {
+		return libvirt.StorageVol{}, fmt.Errorf("uploading seed volume %s: %w", name, err)
+	}
+	log.V(1).Info("seed volume uploaded", "volume", name, "bytes", iso.Len())
+	return vol, nil
+}
+
+// ensureRunning starts the domain unless it is active already, then waits
+// until libvirt reports it running. ErrOperationInvalid from DomainCreate
+// means another caller started it between the state read and the call. A
+// paused or pmsuspended domain is an admin's doing and DomainCreate would
+// refuse it; the verdict is Permanent so the reconciler does not retry it.
+func (p *Provider) ensureRunning(ctx context.Context, log logr.Logger, be backend, dom libvirt.Domain) error {
+	state, _, err := be.DomainGetState(ctx, dom, 0)
+	if err != nil {
+		return fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
+	}
+	switch s := libvirt.DomainState(state); {
+	case isActive(s):
+		return nil
+	case s == libvirt.DomainPaused || s == libvirt.DomainPmsuspended:
+		return permanentf("domain %s is %s and cannot be started by the operator; resume or destroy it on the host", dom.Name, s)
+	}
+	switch err := be.DomainCreate(ctx, dom); {
+	case err == nil:
+		log.V(1).Info("domain started")
+	case hasCode(err, libvirt.ErrOperationInvalid):
+		log.V(1).Info("domain start reported it already running")
+	default:
+		return fmt.Errorf("starting domain %s: %w", dom.Name, err)
+	}
+	return p.waitRunning(ctx, be, dom)
+}
+
+// waitRunning polls DomainGetState until the domain is active, the cap
+// passes (Transient: the next Create attempt finds the domain defined and
+// tries again) or ctx ends. Only nostate is worth another poll; a domain
+// that is shut off, crashed or paused right after its start is not going
+// to come up on its own, so that is Transient at once.
+func (p *Provider) waitRunning(ctx context.Context, be backend, dom libvirt.Domain) error {
+	deadline := time.NewTimer(p.runningWait)
+	defer deadline.Stop()
+	ticker := time.NewTicker(p.runningPoll)
+	defer ticker.Stop()
+	for {
+		state, _, err := be.DomainGetState(ctx, dom, 0)
+		if err != nil {
+			return fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
+		}
+		switch s := libvirt.DomainState(state); {
+		case isActive(s):
+			return nil
+		case s != libvirt.DomainNostate:
+			return transientf("domain %s went %s right after start", dom.Name, s)
+		}
+		select {
+		case <-ticker.C:
+		case <-deadline.C:
+			return transientf("domain %s did not reach running within %s (state %s)", dom.Name, p.runningWait, libvirt.DomainState(state))
+		case <-ctx.Done():
+			return fmt.Errorf("%w: waiting for domain %s to run: %w", provider.ErrTransient, dom.Name, ctx.Err())
+		}
+	}
+}
+
+// logAPIError is the V(1) line per failed backend call of spec entry 20.
+func logAPIError(log logr.Logger, op string, err error) {
+	kv := []any{"op", op, "error", err.Error()}
+	kv = append(kv, libvirtErrorFields(err)...)
+	log.V(1).Info("kvm2 backend call failed", kv...)
+}

internal/provider/kvm2/create_test.go

@@ -0,0 +1,467 @@
+package kvm2
+
+import (
+	"bytes"
+	"errors"
+	"math"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func TestCreate_repeatReturnsSameIDAndCreatesNothing(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	first := fx.create(testName)
+	before := len(fx.fake.callLog())
+	defines, vols, uploads := fx.fake.callCount("DomainDefineXML"), fx.fake.callCount("StorageVolCreateXML"), fx.fake.callCount("StorageVolUpload")
+
+	second := fx.create(testName)
+	if second != first {
+		t.Errorf("second Create = %q, want %q", second, first)
+	}
+	if got := fx.fake.callCount("DomainDefineXML"); got != defines {
+		t.Errorf("DomainDefineXML calls after repeat = %d, want %d", got, defines)
+	}
+	if got := fx.fake.callCount("StorageVolCreateXML"); got != vols {
+		t.Errorf("StorageVolCreateXML calls after repeat = %d, want %d", got, vols)
+	}
+	if got := fx.fake.callCount("StorageVolUpload"); got != uploads {
+		t.Errorf("StorageVolUpload calls after repeat = %d, want %d", got, uploads)
+	}
+	if got := fx.fake.callCount("DomainCreate"); got != 1 {
+		t.Errorf("DomainCreate calls = %d, want 1", got)
+	}
+	repeat := fx.fake.callLog()[before:]
+	for _, rpc := range repeat {
+		switch rpc {
+		case "DomainLookupByName", "DomainGetMetadata", "DomainGetState":
+		default:
+			t.Errorf("repeat Create made %s; want only lookup, metadata and state reads: %v", rpc, repeat)
+		}
+	}
+	if len(fx.logs.matching("kvm2 domain already defined")) != 1 {
+		t.Errorf("no 'already defined' line:\n%s", strings.Join(fx.logs.all(), "\n"))
+	}
+}
+
+func TestCreate_resumesAfterEachPartialStep(t *testing.T) {
+	t.Parallel()
+	overlay, seed := overlayVolumeName(testName), seedVolumeName(testName)
+	tests := []struct {
+		name  string
+		setup func(fx *fixture)
+		check func(t *testing.T, fx *fixture)
+	}{
+		{
+			name: "base without marker is refetched",
+			setup: func(fx *fixture) {
+				fx.putVol("base-debian-13.qcow2", []byte("truncated"))
+			},
+			check: func(t *testing.T, fx *fixture) {
+				if got := fx.fake.vol("base-debian-13.qcow2"); got == nil || !bytes.Equal(got.content, fx.image) {
+					t.Errorf("base volume not refetched")
+				}
+				if fx.fake.vol("base-debian-13.qcow2.sha256") == nil {
+					t.Errorf("marker not written")
+				}
+			},
+		},
+		{
+			name: "overlay exists is reused",
+			setup: func(fx *fixture) {
+				fx.cacheImage()
+				fx.putVol(overlay, []byte("overlay"))
+			},
+			check: func(t *testing.T, fx *fixture) {
+				if got := fx.fake.vol(overlay); got == nil || string(got.content) != "overlay" {
+					t.Errorf("overlay volume was replaced")
+				}
+				if len(fx.logs.matching("overlay volume already exists")) != 1 {
+					t.Errorf("no reuse line:\n%s", strings.Join(fx.logs.all(), "\n"))
+				}
+			},
+		},
+		{
+			name: "truncated seed is replaced",
+			setup: func(fx *fixture) {
+				fx.cacheImage()
+				fx.putVol(seed, []byte("cut"))
+			},
+			check: func(t *testing.T, fx *fixture) {
+				got := fx.fake.vol(seed)
+				if got == nil || len(got.content) <= 3 || !bytes.Contains(got.content, []byte(seedLabel)) {
+					t.Errorf("seed volume not rebuilt")
+				}
+			},
+		},
+		{
+			name: "domain defined but shutoff is started",
+			setup: func(fx *fixture) {
+				fx.cacheImage()
+				fx.putVol(overlay, []byte("overlay"))
+				fx.putVol(seed, []byte("seed"))
+				fx.fake.addDomain(testName, libvirt.DomainShutoff, managedMetadata(testUID, time.Now()))
+			},
+			check: func(t *testing.T, fx *fixture) {
+				if got := fx.fake.callCount("DomainDefineXML"); got != 0 {
+					t.Errorf("DomainDefineXML calls = %d, want 0", got)
+				}
+				if got := fx.fake.callCount("DomainCreate"); got != 1 {
+					t.Errorf("DomainCreate calls = %d, want 1", got)
+				}
+			},
+		},
+		{
+			name: "domain running is returned as is",
+			setup: func(fx *fixture) {
+				fx.fake.addDomain(testName, libvirt.DomainRunning, managedMetadata(testUID, time.Now()))
+			},
+			check: func(t *testing.T, fx *fixture) {
+				for _, rpc := range []string{"DomainDefineXML", "DomainCreate", "StorageVolCreateXML"} {
+					if got := fx.fake.callCount(rpc); got != 0 {
+						t.Errorf("%s calls = %d, want 0", rpc, got)
+					}
+				}
+			},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			tc.setup(fx)
+			fx.create(testName)
+			fx.withDomain(testName, func(d *fakeDomain) {
+				if d.state != libvirt.DomainRunning {
+					t.Errorf("domain state = %s, want running", d.state)
+				}
+			})
+			tc.check(t, fx)
+		})
+	}
+}
+
+func TestCreate_returnsOnlyWhenRunning(t *testing.T) {
+	t.Parallel()
+	t.Run("returns after the domain reports running", func(t *testing.T) {
+		t.Parallel()
+		fx := newFixture(t, 1)
+		fx.fake.startDelayPolls = 3
+		fx.create(testName)
+		fx.withDomain(testName, func(d *fakeDomain) {
+			if d.state != libvirt.DomainRunning {
+				t.Errorf("Create returned with domain state %s", d.state)
+			}
+		})
+		if got := fx.fake.callCount("DomainGetState"); got < 4 {
+			t.Errorf("DomainGetState calls = %d, want at least 4 (one before start, three polls)", got)
+		}
+	})
+	t.Run("never running is transient after the cap", func(t *testing.T) {
+		t.Parallel()
+		fx := newFixture(t, 1, withRunningWait(time.Millisecond, 30*time.Millisecond))
+		fx.fake.startDelayPolls = math.MaxInt32
+		start := time.Now()
+		id, err := fx.p.Create(fx.ctx, fx.req(testName))
+		if id != "" {
+			t.Errorf("Create = %q, want no ID", id)
+		}
+		assertProviderError(t, err, "create", provider.ErrTransient)
+		if !strings.Contains(err.Error(), "did not reach running") {
+			t.Errorf("error = %v, want the wait cap named", err)
+		}
+		if elapsed := time.Since(start); elapsed > 2*time.Second {
+			t.Errorf("Create took %v, want the shrunk cap to end it", elapsed)
+		}
+	})
+}
+
+func TestCreate_domainThatDiesAfterStartIsTransientAtOnce(t *testing.T) {
+	t.Parallel()
+	for _, state := range []libvirt.DomainState{libvirt.DomainCrashed, libvirt.DomainShutoff, libvirt.DomainPaused} {
+		t.Run(state.String(), func(t *testing.T) {
+			t.Parallel()
+			// The cap is well over the bound asserted below, so a poll that
+			// ran to the cap would fail the test.
+			fx := newFixture(t, 1, withRunningWait(time.Millisecond, 10*time.Second))
+			fx.fake.stateAfterStart = state
+			start := time.Now()
+			id, err := fx.p.Create(fx.ctx, fx.req(testName))
+			elapsed := time.Since(start)
+			if id != "" {
+				t.Errorf("Create = %q, want no ID", id)
+			}
+			assertProviderError(t, err, "create", provider.ErrTransient)
+			if !strings.Contains(err.Error(), "right after start") {
+				t.Errorf("error = %v, want the state after start named", err)
+			}
+			if elapsed > time.Second {
+				t.Errorf("Create took %v, want it to give up at once, not poll to the cap", elapsed)
+			}
+			if got := fx.fake.callCount("DomainCreate"); got != 1 {
+				t.Errorf("DomainCreate calls = %d, want 1", got)
+			}
+		})
+	}
+}
+
+func TestCreate_pausedDomainIsPermanent(t *testing.T) {
+	t.Parallel()
+	for _, state := range []libvirt.DomainState{libvirt.DomainPaused, libvirt.DomainPmsuspended} {
+		t.Run(state.String(), func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			fx.fake.addDomain(testName, state, managedMetadata(testUID, time.Now()))
+			id, err := fx.p.Create(fx.ctx, fx.req(testName))
+			if id != "" {
+				t.Errorf("Create = %q, want no ID", id)
+			}
+			assertProviderError(t, err, "create", provider.ErrPermanent)
+			if !strings.Contains(err.Error(), "cannot be started by the operator") {
+				t.Errorf("error = %v, want the admin action named", err)
+			}
+			for _, rpc := range []string{"DomainCreate", "DomainDefineXML", "DomainDestroy"} {
+				if got := fx.fake.callCount(rpc); got != 0 {
+					t.Errorf("%s calls = %d, want 0", rpc, got)
+				}
+			}
+			fx.withDomain(testName, func(d *fakeDomain) {
+				if d.state != state {
+					t.Errorf("domain state = %s, want it left %s", d.state, state)
+				}
+			})
+		})
+	}
+}
+
+func TestCreate_rejectsUnknownPlacement(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name    string
+		mutate  func(req *provider.CreateRequest)
+		wantMsg string
+	}{
+		{
+			name:    "zone",
+			mutate:  func(req *provider.CreateRequest) { req.Placement.Zone = "nas9" },
+			wantMsg: `placement.zone "nas9" is not in kvm2.hosts`,
+		},
+		{
+			name:    "machineType",
+			mutate:  func(req *provider.CreateRequest) { req.Placement.MachineType = "xlarge" },
+			wantMsg: `placement.machineType "xlarge" is not in kvm2.machineTypes`,
+		},
+		{
+			name:    "image",
+			mutate:  func(req *provider.CreateRequest) { req.Placement.Image = "ubuntu-24" },
+			wantMsg: `placement.image "ubuntu-24" is neither a key in kvm2.images nor an http(s) URL`,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			req := fx.req(testName)
+			tc.mutate(&req)
+			id, err := fx.p.Create(fx.ctx, req)
+			if id != "" {
+				t.Errorf("Create = %q, want no ID", id)
+			}
+			assertProviderError(t, err, "create", provider.ErrPermanent)
+			if !strings.Contains(err.Error(), tc.wantMsg) {
+				t.Errorf("error = %q, want substring %q", err, tc.wantMsg)
+			}
+			if got := fx.fake.callLog(); len(got) != 0 {
+				t.Errorf("backend calls = %v, want none before validation", got)
+			}
+			if got := fx.dialer.dials.Load(); got != 0 {
+				t.Errorf("dials = %d, want 0", got)
+			}
+		})
+	}
+}
+
+func TestCreate_adoptionWithForeignUIDIsPermanent(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	fx.fake.addDomain(testName, libvirt.DomainRunning, managedMetadata("other-uid", time.Now()))
+	id, err := fx.p.Create(fx.ctx, fx.req(testName))
+	if id != "" {
+		t.Errorf("Create = %q, want no ID", id)
+	}
+	assertProviderError(t, err, "create", provider.ErrPermanent)
+	if !strings.Contains(err.Error(), "other-uid") || !strings.Contains(err.Error(), testUID) {
+		t.Errorf("error = %q, want both UIDs named", err)
+	}
+	for _, rpc := range []string{"DomainCreate", "DomainDefineXML", "StorageVolCreateXML", "DomainDestroy"} {
+		if got := fx.fake.callCount(rpc); got != 0 {
+			t.Errorf("%s calls = %d, want 0", rpc, got)
+		}
+	}
+}
+
+func TestCreate_adoptsOnlyMarkedDomains(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name      string
+		metadata  string
+		wantAdopt bool
+	}{
+		{name: "matching uid", metadata: managedMetadata(testUID, time.Now()), wantAdopt: true},
+		{name: "managed, empty uid", metadata: managedMetadata("", time.Now()), wantAdopt: true},
+		{name: "no metadata element", metadata: ""},
+		{name: "element without the managed mark", metadata: `<instance><uid>` + testUID + `</uid></instance>`},
+		{name: "managed false", metadata: `<instance><managed>false</managed><uid>` + testUID + `</uid></instance>`},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			fx.fake.addDomain(testName, libvirt.DomainRunning, tc.metadata)
+			id, err := fx.p.Create(fx.ctx, fx.req(testName))
+			if tc.wantAdopt {
+				if err != nil || id != formatProviderID(testHost, testName) {
+					t.Fatalf("Create = %q, %v; want the existing domain adopted", id, err)
+				}
+			} else {
+				if id != "" {
+					t.Errorf("Create = %q, want no ID", id)
+				}
+				assertProviderError(t, err, "create", provider.ErrPermanent)
+				if !strings.Contains(err.Error(), "carries no kvm2 mark") {
+					t.Errorf("error = %v, want the missing mark named", err)
+				}
+			}
+			for _, rpc := range []string{"DomainDefineXML", "DomainCreate", "StorageVolCreateXML", "DomainDestroy"} {
+				if got := fx.fake.callCount(rpc); got != 0 {
+					t.Errorf("%s calls = %d, want 0", rpc, got)
+				}
+			}
+		})
+	}
+}
+
+func TestCreate_poolOutOfSpaceIsQuotaExceeded(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	fx.fake.mu.Lock()
+	fx.fake.pools[fakePoolName].available = 1 << 30
+	fx.fake.mu.Unlock()
+	_, err := fx.p.Create(fx.ctx, fx.req(testName))
+	assertProviderError(t, err, "create", provider.ErrQuotaExceeded)
+	if !strings.Contains(err.Error(), fakePoolName) || !strings.Contains(err.Error(), "needed") {
+		t.Errorf("error = %q, want the pool and the needed bytes named", err)
+	}
+	for _, rpc := range []string{"StorageVolCreateXML", "StorageVolUpload", "DomainDefineXML"} {
+		if got := fx.fake.callCount(rpc); got != 0 {
+			t.Errorf("%s calls = %d, want 0", rpc, got)
+		}
+	}
+}
+
+func TestCreate_libvirtNotFoundIsTransientNotNotFound(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		rpc  string
+		err  error
+	}{
+		{name: "volume vanished", rpc: "StorageVolGetPath", err: fakeErr(libvirt.ErrNoStorageVol, "Storage volume not found")},
+		{name: "domain vanished", rpc: "DomainGetState", err: fakeErr(libvirt.ErrNoDomain, "Domain not found")},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			fx.fake.failWith(tc.rpc, tc.err)
+			_, err := fx.p.Create(fx.ctx, fx.req(testName))
+			assertProviderError(t, err, "create", provider.ErrTransient)
+			if errors.Is(err, provider.ErrNotFound) {
+				t.Errorf("Create error = %v, must never be NotFound", err)
+			}
+		})
+	}
+}
+
+func TestCreate_cloudInitSentinelNeverLoggedOrInErrors(t *testing.T) {
+	t.Parallel()
+	const sentinel = "SENTINEL-s3cr3t-passw0rd-7f3a"
+	fx := newFixture(t, 2)
+	fx.dialer.backend = observe(fx.fake, testProvider, testHost)
+	fx.fake.failWith("StorageVolUpload", fakeErr(libvirt.ErrInternalError, "internal error: upload failed"))
+	req := fx.req(testName)
+	req.CloudInit = "#cloud-config\nwrite_files:\n  - content: " + sentinel + "\n"
+	_, err := fx.p.Create(fx.ctx, req)
+	assertProviderError(t, err, "create", provider.ErrTransient)
+	if strings.Contains(err.Error(), sentinel) {
+		t.Errorf("error carries the cloud-init: %v", err)
+	}
+	lines := fx.logs.all()
+	if len(lines) == 0 {
+		t.Fatal("no log lines captured at V(2)")
+	}
+	for _, l := range lines {
+		if strings.Contains(l, sentinel) {
+			t.Errorf("log line carries the cloud-init: %s", l)
+		}
+	}
+	if got := fx.logs.matching(`"cloudInitBytes"=` + strconv.Itoa(len(req.CloudInit))); len(got) != 1 {
+		t.Errorf("cloudInitBytes line = %d, want 1:\n%s", len(got), strings.Join(lines, "\n"))
+	}
+}
+
+func TestCreate_metadataWrittenInDefineCall(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	fx.p.now = func() time.Time { return time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) }
+	fx.create(testName)
+	fx.withDomain(testName, func(d *fakeDomain) {
+		for _, want := range []string{`xmlns="` + metadataURI + `"`, "<uid>" + testUID + "</uid>", "<managed>true</managed>", "<createdAt>2026-09-16T12:00:00Z</createdAt>"} {
+			if !strings.Contains(d.xml, want) {
+				t.Errorf("defined XML lacks %s:\n%s", want, d.xml)
+			}
+		}
+		meta, err := parseInstanceMetadata(d.metadata)
+		if err != nil {
+			t.Fatalf("metadata the define stored does not parse: %v", err)
+		}
+		if !meta.managed() || meta.UID != testUID || meta.createdAt().IsZero() {
+			t.Errorf("metadata = %+v, want managed, uid %s and a createdAt", meta, testUID)
+		}
+	})
+}
+
+func TestCreate_seedAlwaysRebuilt(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	fx.create(testName)
+	seed := seedVolumeName(testName)
+	first := fx.fake.vol(seed)
+	if first == nil {
+		t.Fatal("no seed volume after Create")
+	}
+	uploads := fx.fake.callCount("StorageVolUpload")
+
+	// A crash between the seed upload and the define leaves the volumes
+	// without a domain; the next attempt must not trust the seed.
+	fx.fake.mu.Lock()
+	delete(fx.fake.domains, testName)
+	fx.fake.mu.Unlock()
+	fx.create(testName)
+	second := fx.fake.vol(seed)
+	if second == nil || second == first {
+		t.Errorf("seed volume reused across attempts")
+	}
+	if got := fx.fake.callCount("StorageVolUpload"); got != uploads+1 {
+		t.Errorf("StorageVolUpload calls = %d, want %d", got, uploads+1)
+	}
+	if len(second.content) != len(first.content) || !bytes.Contains(second.content, []byte(seedLabel)) {
+		t.Errorf("rebuilt seed is %d bytes, first was %d; want a complete seed of the same size", len(second.content), len(first.content))
+	}
+}

internal/provider/kvm2/delete.go

@@ -0,0 +1,98 @@
+package kvm2
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// undefineFlags removes everything libvirt keeps for a domain beside its
+// definition; UEFI domains keep NVRAM and undefine refuses without the flag.
+const undefineFlags = libvirt.DomainUndefineManagedSave | libvirt.DomainUndefineSnapshotsMetadata | libvirt.DomainUndefineNvram
+
+// Delete is spec entry 11: under the per-name lock, destroy and undefine
+// the domain, then remove its overlay and seed. Everything already gone is
+// success, and nothing is left when nil is returned, so Get afterwards is
+// NotFound. The base image stays: it is a shared cache.
+func (p *Provider) Delete(ctx context.Context, providerID string) error {
+	hostName, name, err := parseProviderID(providerID)
+	if err != nil {
+		return provider.Wrap(provider.ErrPermanent, "delete", p.name, providerID, err)
+	}
+	log := p.logger(ctx).WithValues("host", hostName, "name", name)
+	host, err := p.host(hostName)
+	if err != nil {
+		log.V(1).Info("host is not configured; delete has nothing to clean up", "id", providerID)
+		return nil
+	}
+	unlock, err := p.lockName(ctx, name)
+	if err != nil {
+		return p.wrapErr("delete", providerID, err)
+	}
+	defer unlock()
+
+	be, err := p.hosts.get(ctx, host)
+	if err != nil {
+		logAPIError(log, "delete", err)
+		return p.wrapErr("delete", providerID, err)
+	}
+	if err := removeDomain(ctx, log, be, name); err != nil {
+		logAPIError(log, "delete", err)
+		return p.wrapErr("delete", providerID, err)
+	}
+	if err := removeVolumes(ctx, log, be, host.Pool, name); err != nil {
+		logAPIError(log, "delete", err)
+		return p.wrapErr("delete", providerID, err)
+	}
+	log.V(1).Info("kvm2 domain deleted")
+	return nil
+}
+
+func removeDomain(ctx context.Context, log logr.Logger, be backend, name string) error {
+	dom, err := be.DomainLookupByName(ctx, name)
+	if hasCode(err, libvirt.ErrNoDomain) {
+		log.V(1).Info("domain already gone")
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("looking up domain %s: %w", name, err)
+	}
+	switch err := be.DomainDestroy(ctx, dom); {
+	case err == nil:
+		log.V(1).Info("domain destroyed")
+	case hasCode(err, libvirt.ErrOperationInvalid):
+		log.V(1).Info("domain not running, destroy skipped")
+	case hasCode(err, libvirt.ErrNoDomain):
+		log.V(1).Info("domain already gone")
+		return nil
+	default:
+		return fmt.Errorf("destroying domain %s: %w", name, err)
+	}
+	if err := be.DomainUndefineFlags(ctx, dom, undefineFlags); err != nil && !hasCode(err, libvirt.ErrNoDomain) {
+		return fmt.Errorf("undefining domain %s: %w", name, err)
+	}
+	log.V(1).Info("domain undefined")
+	return nil
+}
+
+func removeVolumes(ctx context.Context, log logr.Logger, be backend, poolName, name string) error {
+	pool, err := be.StoragePoolLookupByName(ctx, poolName)
+	if hasCode(err, libvirt.ErrNoStoragePool) {
+		log.V(1).Info("pool not found, no volumes to delete", "pool", poolName)
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("looking up storage pool %s: %w", poolName, err)
+	}
+	for _, vol := range []string{overlayVolumeName(name), seedVolumeName(name)} {
+		if err := deleteVolumeIfPresent(ctx, be, pool, vol); err != nil {
+			return err
+		}
+	}
+	log.V(1).Info("per-VM volumes deleted")
+	return nil
+}

internal/provider/kvm2/delete_test.go

@@ -0,0 +1,236 @@
+package kvm2
+
+import (
+	"errors"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func TestDelete_malformedIDIsPermanent(t *testing.T) {
+	t.Parallel()
+	for _, id := range []string{"", "proxy-x", "hosts/beelink/proxy-x", "hosts//domains/proxy-x", "hosts/beelink/domains/"} {
+		t.Run(id, func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			err := fx.p.Delete(fx.ctx, id)
+			assertProviderError(t, err, "delete", provider.ErrPermanent)
+			var pe *provider.Error
+			if errors.As(err, &pe) && pe.ID != id {
+				t.Errorf("error.ID = %q, want %q", pe.ID, id)
+			}
+			if got := fx.dialer.dials.Load(); got != 0 {
+				t.Errorf("dials = %d, want 0", got)
+			}
+		})
+	}
+}
+
+func TestDelete_twiceSucceeds(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	for i := range 2 {
+		if err := fx.p.Delete(fx.ctx, id); err != nil {
+			t.Fatalf("Delete #%d error = %v", i+1, err)
+		}
+	}
+	if fx.hasDomain(testName) {
+		t.Error("domain still defined after Delete")
+	}
+	if got := fx.logs.matching("domain already gone"); len(got) != 1 {
+		t.Errorf("'already gone' lines = %d, want 1 from the second Delete", len(got))
+	}
+	if got := fx.logs.matching("kvm2 domain deleted"); len(got) != 2 {
+		t.Errorf("'deleted' lines = %d, want 2", len(got))
+	}
+}
+
+func TestDelete_removesOverlayAndSeed(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	if err := fx.p.Delete(fx.ctx, id); err != nil {
+		t.Fatalf("Delete() error = %v", err)
+	}
+	for _, name := range []string{overlayVolumeName(testName), seedVolumeName(testName)} {
+		if fx.fake.vol(name) != nil {
+			t.Errorf("volume %s still present after Delete", name)
+		}
+	}
+	for _, name := range []string{"base-debian-13.qcow2", "base-debian-13.qcow2.sha256"} {
+		if fx.fake.vol(name) == nil {
+			t.Errorf("shared base volume %s removed by Delete", name)
+		}
+	}
+}
+
+func TestDelete_runningDomainDestroyedFirst(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	before := len(fx.fake.callLog())
+	if err := fx.p.Delete(fx.ctx, id); err != nil {
+		t.Fatalf("Delete() error = %v", err)
+	}
+	calls := fx.fake.callLog()[before:]
+	destroy := slices.Index(calls, "DomainDestroy")
+	undefine := slices.Index(calls, "DomainUndefineFlags")
+	volDelete := slices.Index(calls, "StorageVolDelete")
+	if destroy < 0 || undefine < 0 || volDelete < 0 || !(destroy < undefine && undefine < volDelete) {
+		t.Errorf("call order = %v, want destroy before undefine before volume deletes", calls)
+	}
+	if n := strings.Count(strings.Join(calls, " "), "StorageVolDelete"); n != 2 {
+		t.Errorf("StorageVolDelete calls = %d, want 2 (overlay and seed)", n)
+	}
+}
+
+func TestDelete_stoppedDomainSkipsDestroy(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	fx.withDomain(testName, func(d *fakeDomain) { d.state = libvirt.DomainShutoff })
+	if err := fx.p.Delete(fx.ctx, id); err != nil {
+		t.Fatalf("Delete() error = %v", err)
+	}
+	if fx.hasDomain(testName) {
+		t.Error("domain still defined after Delete")
+	}
+	if got := fx.logs.matching("domain not running, destroy skipped"); len(got) != 1 {
+		t.Errorf("'destroy skipped' lines = %d, want 1:\n%s", len(got), strings.Join(fx.logs.all(), "\n"))
+	}
+	if got := fx.fake.callCount("DomainUndefineFlags"); got != 1 {
+		t.Errorf("DomainUndefineFlags calls = %d, want 1", got)
+	}
+}
+
+func TestDelete_toleratesMissingVolumesAndPool(t *testing.T) {
+	t.Parallel()
+	t.Run("domain without volumes", func(t *testing.T) {
+		t.Parallel()
+		fx := newFixture(t, 1)
+		fx.fake.addDomain(testName, libvirt.DomainRunning, managedMetadata(testUID, time.Now()))
+		if err := fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)); err != nil {
+			t.Fatalf("Delete() error = %v", err)
+		}
+		if fx.hasDomain(testName) {
+			t.Error("domain still defined after Delete")
+		}
+		if got := fx.fake.callCount("StorageVolDelete"); got != 0 {
+			t.Errorf("StorageVolDelete calls = %d, want 0 for absent volumes", got)
+		}
+	})
+	t.Run("pool gone", func(t *testing.T) {
+		t.Parallel()
+		fx := newFixture(t, 1)
+		fx.p.cfg.Hosts[0].Pool = "retired"
+		fx.fake.addDomain(testName, libvirt.DomainShutoff, managedMetadata(testUID, time.Now()))
+		if err := fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)); err != nil {
+			t.Fatalf("Delete() error = %v", err)
+		}
+		if fx.hasDomain(testName) {
+			t.Error("domain still defined after Delete")
+		}
+		if got := fx.logs.matching("pool not found, no volumes to delete"); len(got) != 1 || !strings.Contains(got[0], `"pool"="retired"`) {
+			t.Errorf("'pool not found' V(1) line = %v", got)
+		}
+	})
+	t.Run("nothing at all", func(t *testing.T) {
+		t.Parallel()
+		fx := newFixture(t, 1)
+		if err := fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)); err != nil {
+			t.Fatalf("Delete() error = %v", err)
+		}
+	})
+}
+
+func TestDelete_unconfiguredHostIsNoOp(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := formatProviderID("nas9", testName)
+	if err := fx.p.Delete(fx.ctx, id); err != nil {
+		t.Fatalf("Delete() error = %v, want nil", err)
+	}
+	if got := fx.dialer.dials.Load(); got != 0 {
+		t.Errorf("dials = %d, want 0", got)
+	}
+	if got := fx.fake.callLog(); len(got) != 0 {
+		t.Errorf("backend calls = %v, want none", got)
+	}
+	if got := fx.logs.matching("host is not configured; delete has nothing to clean up"); len(got) != 1 || !strings.Contains(got[0], `"host"="nas9"`) {
+		t.Errorf("V(1) line = %v, want one naming the host", got)
+	}
+}
+
+func TestDelete_noOpLinesAreSilentAtV0(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 0)
+	fx.p.cfg.Hosts[0].Pool = "retired"
+	for _, id := range []string{formatProviderID("nas9", testName), formatProviderID(testHost, testName)} {
+		if err := fx.p.Delete(fx.ctx, id); err != nil {
+			t.Fatalf("Delete(%s) error = %v, want nil", id, err)
+		}
+	}
+	if lines := fx.logs.all(); len(lines) != 0 {
+		t.Errorf("Delete logged %d lines at V(0), want none:\n%s", len(lines), strings.Join(lines, "\n"))
+	}
+}
+
+func TestDelete_concurrentSameNameSafe(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	const n = 10
+	errs := make([]error, n)
+	var wg sync.WaitGroup
+	for i := range n {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			errs[i] = fx.p.Delete(fx.ctx, id)
+		}()
+	}
+	wg.Wait()
+	for i, err := range errs {
+		if err != nil {
+			t.Errorf("Delete #%d error = %v, want nil", i, err)
+		}
+	}
+	if fx.hasDomain(testName) {
+		t.Error("domain still defined after concurrent Deletes")
+	}
+	for _, name := range []string{overlayVolumeName(testName), seedVolumeName(testName)} {
+		if fx.fake.vol(name) != nil {
+			t.Errorf("volume %s still present", name)
+		}
+	}
+	if got := fx.fake.callCount("DomainUndefineFlags"); got != 1 {
+		t.Errorf("DomainUndefineFlags calls = %d, want 1 (the lock serialises the rest)", got)
+	}
+}
+
+func TestDelete_undefineUsesNvramFlag(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	if err := fx.p.Delete(fx.ctx, id); err != nil {
+		t.Fatalf("Delete() error = %v", err)
+	}
+	fx.fake.mu.Lock()
+	flags, ok := fx.fake.undefineFlags[testName]
+	fx.fake.mu.Unlock()
+	if !ok {
+		t.Fatal("DomainUndefineFlags was not called")
+	}
+	for _, want := range []libvirt.DomainUndefineFlagsValues{libvirt.DomainUndefineNvram, libvirt.DomainUndefineManagedSave, libvirt.DomainUndefineSnapshotsMetadata} {
+		if flags&want == 0 {
+			t.Errorf("undefine flags = %d, lack %d", flags, want)
+		}
+	}
+}

internal/provider/kvm2/errors.go

@@ -0,0 +1,97 @@
+package kvm2
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/digitalocean/go-libvirt"
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// permanentf and transientf classify an error at the source, so classify
+// keeps that class no matter what wraps it later.
+func permanentf(format string, args ...any) error {
+	return fmt.Errorf("%w: %s", provider.ErrPermanent, fmt.Sprintf(format, args...))
+}
+
+func transientf(format string, args ...any) error {
+	return fmt.Errorf("%w: %s", provider.ErrTransient, fmt.Sprintf(format, args...))
+}
+
+// sshAuthFailed is the substring x/crypto/ssh puts in every authentication
+// failure ("ssh: handshake failed: ssh: unable to authenticate, attempted
+// methods [...]").
+const sshAuthFailed = "unable to authenticate"
+
+// classify maps any error reaching the provider's boundary to one of the
+// four contract sentinels, per spec entry 13. op is the contract operation:
+// a vanished domain or volume inside Create is Transient, because the
+// contract forbids NotFound from Create.
+func classify(op string, err error) error {
+	if err == nil {
+		return provider.ErrTransient
+	}
+	for _, class := range []error{provider.ErrNotFound, provider.ErrQuotaExceeded, provider.ErrPermanent, provider.ErrTransient} {
+		if errors.Is(err, class) {
+			return class
+		}
+	}
+	if code, ok := libvirtCode(err); ok {
+		return classifyLibvirt(op, code)
+	}
+	var keyErr *knownhosts.KeyError
+	if errors.As(err, &keyErr) {
+		return provider.ErrPermanent
+	}
+	if strings.Contains(err.Error(), sshAuthFailed) {
+		return provider.ErrPermanent
+	}
+	// Everything else retries: a missing key or known_hosts file (the
+	// optional Secret has not landed yet), context cancellation and
+	// deadlines, net.OpError, io.EOF, and anything unrecognised.
+	return provider.ErrTransient
+}
+
+func classifyLibvirt(op string, code libvirt.ErrorNumber) error {
+	switch code {
+	case libvirt.ErrNoDomain, libvirt.ErrNoStorageVol:
+		if op == "create" {
+			return provider.ErrTransient
+		}
+		return provider.ErrNotFound
+	case libvirt.ErrNoStoragePool, libvirt.ErrXMLError, libvirt.ErrNoSupport, libvirt.ErrInvalidArg,
+		libvirt.ErrAuthFailed, libvirt.ErrAccessDenied:
+		return provider.ErrPermanent
+	default:
+		return provider.ErrTransient
+	}
+}
+
+// libvirtCode extracts the libvirt error number from err, if it is or wraps
+// a libvirt.Error. That type has a value receiver, so errors.As needs a
+// value target.
+func libvirtCode(err error) (libvirt.ErrorNumber, bool) {
+	var le libvirt.Error
+	if !errors.As(err, &le) {
+		return 0, false
+	}
+	return libvirt.ErrorNumber(le.Code), true
+}
+
+func hasCode(err error, code libvirt.ErrorNumber) bool {
+	got, ok := libvirtCode(err)
+	return ok && got == code
+}
+
+// libvirtErrorFields returns the key/value pairs a log line carries for a
+// libvirt error, or nil when err is not one.
+func libvirtErrorFields(err error) []any {
+	code, ok := libvirtCode(err)
+	if !ok {
+		return nil
+	}
+	return []any{"libvirtCode", int(code), "libvirtError", code.String()}
+}

internal/provider/kvm2/errors_test.go

@@ -0,0 +1,146 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net"
+	"os"
+	"testing"
+
+	"github.com/digitalocean/go-libvirt"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func libvirtErr(code libvirt.ErrorNumber, msg string) error {
+	return libvirt.Error{Code: uint32(code), Message: msg}
+}
+
+// Captured from x/crypto/ssh when the offered key is not in authorized_keys.
+const sshAuthFailedMsg = "ssh: handshake failed: ssh: unable to authenticate, attempted methods [none publickey], no supported methods remain"
+
+func TestClassify_table(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		op   string
+		err  error
+		want error
+	}{
+		{name: "nil is transient", op: "get", err: nil, want: provider.ErrTransient},
+		{name: "ErrNoDomain on get", op: "get", err: libvirtErr(libvirt.ErrNoDomain, "Domain not found: no domain with matching name 'proxy-x'"), want: provider.ErrNotFound},
+		{name: "ErrNoDomain on delete", op: "delete", err: libvirtErr(libvirt.ErrNoDomain, "Domain not found"), want: provider.ErrNotFound},
+		{name: "ErrNoDomain on create is transient", op: "create", err: libvirtErr(libvirt.ErrNoDomain, "Domain not found"), want: provider.ErrTransient},
+		{name: "ErrNoStorageVol on get", op: "get", err: libvirtErr(libvirt.ErrNoStorageVol, "Storage volume not found: no storage vol with matching name 'proxy-x.qcow2'"), want: provider.ErrNotFound},
+		{name: "ErrNoStorageVol on create is transient", op: "create", err: libvirtErr(libvirt.ErrNoStorageVol, "Storage volume not found"), want: provider.ErrTransient},
+		{name: "ErrNoStoragePool", op: "create", err: libvirtErr(libvirt.ErrNoStoragePool, "Storage pool not found: no storage pool with matching name 'default'"), want: provider.ErrPermanent},
+		{name: "ErrXMLError", op: "create", err: libvirtErr(libvirt.ErrXMLError, "XML error: missing domain name"), want: provider.ErrPermanent},
+		{name: "ErrNoSupport", op: "create", err: libvirtErr(libvirt.ErrNoSupport, "this function is not supported by the connection driver"), want: provider.ErrPermanent},
+		{name: "ErrInvalidArg", op: "get", err: libvirtErr(libvirt.ErrInvalidArg, "invalid argument: unsupported flags"), want: provider.ErrPermanent},
+		{name: "ErrAuthFailed", op: "list", err: libvirtErr(libvirt.ErrAuthFailed, "authentication failed: access denied by policy"), want: provider.ErrPermanent},
+		{name: "ErrAccessDenied", op: "list", err: libvirtErr(libvirt.ErrAccessDenied, "access denied"), want: provider.ErrPermanent},
+		{name: "ErrOperationInvalid is transient", op: "create", err: libvirtErr(libvirt.ErrOperationInvalid, "Requested operation is not valid: domain is already running"), want: provider.ErrTransient},
+		{name: "ErrInternalError is transient", op: "get", err: libvirtErr(libvirt.ErrInternalError, "internal error: connection closed"), want: provider.ErrTransient},
+		{name: "unknown libvirt code is transient", op: "get", err: libvirtErr(libvirt.ErrorNumber(9999), "?"), want: provider.ErrTransient},
+		{name: "wrapped libvirt error keeps its code", op: "get", err: fmt.Errorf("looking up domain: %w", libvirtErr(libvirt.ErrNoDomain, "Domain not found")), want: provider.ErrNotFound},
+		{name: "knownhosts unknown host", op: "get", err: &knownhosts.KeyError{}, want: provider.ErrPermanent},
+		{name: "knownhosts mismatch", op: "get", err: &knownhosts.KeyError{Want: []knownhosts.KnownKey{{Filename: "/kh", Line: 1, Key: testPublicKey(t)}}}, want: provider.ErrPermanent},
+		{name: "wrapped knownhosts error", op: "create", err: fmt.Errorf("dialing beelink: %w", &knownhosts.KeyError{}), want: provider.ErrPermanent},
+		{name: "ssh unable to authenticate", op: "create", err: errors.New(sshAuthFailedMsg), want: provider.ErrPermanent},
+		{name: "wrapped ssh unable to authenticate", op: "list", err: fmt.Errorf("dialing beelink: %w", errors.New(sshAuthFailedMsg)), want: provider.ErrPermanent},
+		{name: "missing key file is transient", op: "create", err: fmt.Errorf("reading ssh key file /k: %w", fs.ErrNotExist), want: provider.ErrTransient},
+		{name: "os.PathError not exist is transient", op: "create", err: &os.PathError{Op: "open", Path: "/k", Err: fs.ErrNotExist}, want: provider.ErrTransient},
+		{name: "context canceled", op: "get", err: context.Canceled, want: provider.ErrTransient},
+		{name: "context deadline", op: "get", err: context.DeadlineExceeded, want: provider.ErrTransient},
+		{name: "net.OpError", op: "get", err: &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")}, want: provider.ErrTransient},
+		{name: "io.EOF", op: "list", err: io.EOF, want: provider.ErrTransient},
+		{name: "io.ErrUnexpectedEOF", op: "list", err: io.ErrUnexpectedEOF, want: provider.ErrTransient},
+		{name: "anything else is transient", op: "get", err: errors.New("anything"), want: provider.ErrTransient},
+		{name: "pre-classified NotFound stays", op: "create", err: fmt.Errorf("x: %w", provider.ErrNotFound), want: provider.ErrNotFound},
+		{name: "pre-classified Quota stays", op: "create", err: fmt.Errorf("x: %w", provider.ErrQuotaExceeded), want: provider.ErrQuotaExceeded},
+		{name: "pre-classified Permanent stays", op: "get", err: fmt.Errorf("x: %w", permanentf("bad %s", "thing")), want: provider.ErrPermanent},
+		{name: "pre-classified Transient stays", op: "get", err: fmt.Errorf("x: %w", transientf("blip %d", 1)), want: provider.ErrTransient},
+		{name: "pre-classified beats a libvirt code underneath", op: "get", err: fmt.Errorf("%w: %w", provider.ErrPermanent, libvirtErr(libvirt.ErrNoDomain, "x")), want: provider.ErrPermanent},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := classify(tc.op, tc.err); !errors.Is(got, tc.want) {
+				t.Errorf("classify(%q, %v) = %v, want %v", tc.op, tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestHasCode(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		err  error
+		code libvirt.ErrorNumber
+		want bool
+	}{
+		{name: "matching code", err: libvirtErr(libvirt.ErrNoDomain, "x"), code: libvirt.ErrNoDomain, want: true},
+		{name: "matching code wrapped", err: fmt.Errorf("w: %w", libvirtErr(libvirt.ErrStorageVolExist, "x")), code: libvirt.ErrStorageVolExist, want: true},
+		{name: "other code", err: libvirtErr(libvirt.ErrNoDomain, "x"), code: libvirt.ErrNoStorageVol, want: false},
+		{name: "not a libvirt error", err: errors.New("x"), code: libvirt.ErrNoDomain, want: false},
+		{name: "nil", err: nil, code: libvirt.ErrNoDomain, want: false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := hasCode(tc.err, tc.code); got != tc.want {
+				t.Errorf("hasCode(%v, %v) = %v, want %v", tc.err, tc.code, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestLibvirtErrorFields(t *testing.T) {
+	t.Parallel()
+	got := libvirtErrorFields(fmt.Errorf("w: %w", libvirtErr(libvirt.ErrNoDomain, "x")))
+	want := []any{"libvirtCode", 42, "libvirtError", "ErrNoDomain"}
+	if len(got) != len(want) {
+		t.Fatalf("libvirtErrorFields() = %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("libvirtErrorFields()[%d] = %v (%T), want %v (%T)", i, got[i], got[i], want[i], want[i])
+		}
+	}
+	if fields := libvirtErrorFields(errors.New("x")); fields != nil {
+		t.Errorf("libvirtErrorFields(non-libvirt) = %v, want nil", fields)
+	}
+	if fields := libvirtErrorFields(libvirtErr(libvirt.ErrorNumber(9999), "x")); fields[3] != "ErrorNumber(9999)" {
+		t.Errorf("libvirtErrorFields(unknown code)[3] = %v, want ErrorNumber(9999)", fields[3])
+	}
+}
+
+func TestPermanentfAndTransientf(t *testing.T) {
+	t.Parallel()
+	perm := permanentf("host %q missing", "x")
+	if !errors.Is(perm, provider.ErrPermanent) || perm.Error() != `provider: permanent error: host "x" missing` {
+		t.Errorf("permanentf() = %q, want ErrPermanent with the message", perm)
+	}
+	trans := transientf("retry %d", 3)
+	if !errors.Is(trans, provider.ErrTransient) || trans.Error() != "provider: transient error: retry 3" {
+		t.Errorf("transientf() = %q, want ErrTransient with the message", trans)
+	}
+}
+
+// testPublicKey parses a fixed ed25519 public key so a knownhosts.KeyError
+// with a Want entry can be built without generating keys.
+func testPublicKey(t *testing.T) ssh.PublicKey {
+	t.Helper()
+	const line = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGcZBhnMPTyDwFn9lBhwIutqLb2bSlhcZoqpuWv7VuKI test"
+	key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(line))
+	if err != nil {
+		t.Fatalf("ParseAuthorizedKey: %v", err)
+	}
+	return key
+}

internal/provider/kvm2/fake_test.go

@@ -0,0 +1,623 @@
+package kvm2
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"slices"
+	"sync"
+	"sync/atomic"
+
+	"github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// fakeBackend is the in-memory backend of spec entry 21: enough of a
+// libvirt host for the provider's Create, Get, Delete and ListByTag, with
+// per-method error hooks and blocking, call counters and an ordered log.
+type fakeBackend struct {
+	// mu, domains and pools are the host's state; a twin shares them, so
+	// two connections to the same fake host see the same volumes.
+	mu       *sync.Mutex
+	domains  map[string]*fakeDomain
+	pools    map[string]*fakePool
+	hostname string
+	version  uint64
+
+	// hooks returns the error a method fails with before touching state.
+	hooks map[string]func() error
+	// block makes a method wait on its channel (or ctx) before running.
+	block map[string]chan struct{}
+	// discardUploads makes StorageVolUpload stream the reader through a
+	// sha256 in fixed chunks instead of holding the bytes, for tests that
+	// push more through the fake than they want in memory.
+	discardUploads bool
+	// startDelayPolls, when set, makes DomainCreate leave the domain in
+	// nostate; it turns running on the startDelayPolls-th DomainGetState
+	// after that, as a host whose QEMU is still settling would.
+	startDelayPolls int
+	// stateAfterStart, when not nostate, is what DomainCreate leaves the
+	// domain in instead of running, as a guest that dies at once would.
+	stateAfterStart libvirt.DomainState
+	// undefineFlags records the flags of the last DomainUndefineFlags per
+	// domain name.
+	undefineFlags map[string]libvirt.DomainUndefineFlagsValues
+
+	calls  map[string]int
+	log    []string
+	closed int
+	deadCh chan struct{}
+	deadMu sync.Once
+}
+
+type fakeDomain struct {
+	state    libvirt.DomainState
+	xml      string
+	metadata string
+	addrs    map[uint32][]libvirt.DomainInterface
+	uuid     libvirt.UUID
+	id       int32
+	// pendingStart counts the DomainGetState calls left before a delayed
+	// start (see startDelayPolls) turns the domain running.
+	pendingStart int
+}
+
+type fakePool struct {
+	state     libvirt.StoragePoolState
+	capacity  uint64
+	available uint64
+	vols      map[string]*fakeVol
+}
+
+type fakeVol struct {
+	content  []byte
+	capacity uint64
+	// uploaded and sha256 record a discarded upload (see discardUploads).
+	uploaded int64
+	sha256   string
+}
+
+const fakePoolName = "default"
+
+func newFakeBackend() *fakeBackend {
+	return &fakeBackend{
+		mu:            &sync.Mutex{},
+		domains:       make(map[string]*fakeDomain),
+		pools:         map[string]*fakePool{fakePoolName: newFakePool()},
+		hostname:      "fake-host",
+		version:       11006000,
+		hooks:         make(map[string]func() error),
+		block:         make(map[string]chan struct{}),
+		undefineFlags: make(map[string]libvirt.DomainUndefineFlagsValues),
+		calls:         make(map[string]int),
+		deadCh:        make(chan struct{}),
+	}
+}
+
+// twin returns a second connection to the same fake host: it shares the
+// domains and pools (and their lock) but has its own hooks, counters, log,
+// close count and liveness, so a test can tell which connection did what.
+func (f *fakeBackend) twin() *fakeBackend {
+	return &fakeBackend{
+		mu:             f.mu,
+		domains:        f.domains,
+		pools:          f.pools,
+		hostname:       f.hostname,
+		version:        f.version,
+		hooks:          make(map[string]func() error),
+		block:          make(map[string]chan struct{}),
+		discardUploads: f.discardUploads,
+		undefineFlags:  make(map[string]libvirt.DomainUndefineFlagsValues),
+		calls:          make(map[string]int),
+		deadCh:         make(chan struct{}),
+	}
+}
+
+// vol returns the volume by name in the default pool, or nil.
+func (f *fakeBackend) vol(name string) *fakeVol {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.pools[fakePoolName].vols[name]
+}
+
+// closeCount is closed read under the lock, for twins closed by another
+// goroutine.
+func (f *fakeBackend) closeCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.closed
+}
+
+func newFakePool() *fakePool {
+	return &fakePool{state: libvirt.StoragePoolRunning, capacity: 100 << 30, available: 50 << 30, vols: make(map[string]*fakeVol)}
+}
+
+func (f *fakeBackend) dead() <-chan struct{} { return f.deadCh }
+
+func (f *fakeBackend) markDead() { f.deadMu.Do(func() { close(f.deadCh) }) }
+
+// addDomain defines a domain directly, bypassing DomainDefineXML.
+func (f *fakeBackend) addDomain(name string, state libvirt.DomainState, metadata string) *fakeDomain {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	d := &fakeDomain{state: state, metadata: metadata, addrs: make(map[uint32][]libvirt.DomainInterface), uuid: fakeUUID(name)}
+	f.domains[name] = d
+	return d
+}
+
+func (f *fakeBackend) failWith(method string, err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.hooks[method] = func() error { return err }
+}
+
+func (f *fakeBackend) blockOn(method string) chan struct{} {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	ch := make(chan struct{})
+	f.block[method] = ch
+	return ch
+}
+
+func (f *fakeBackend) callCount(method string) int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.calls[method]
+}
+
+func (f *fakeBackend) callLog() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return slices.Clone(f.log)
+}
+
+func fakeUUID(name string) libvirt.UUID {
+	var u libvirt.UUID
+	copy(u[:], name)
+	return u
+}
+
+func fakeErr(code libvirt.ErrorNumber, format string, args ...any) error {
+	return libvirt.Error{Code: uint32(code), Message: fmt.Sprintf(format, args...)}
+}
+
+// enter records the call, honours a block and runs the hook. It is called
+// without f.mu held; state access happens afterwards under the lock.
+func (f *fakeBackend) enter(ctx context.Context, method string) error {
+	f.mu.Lock()
+	f.calls[method]++
+	f.log = append(f.log, method)
+	block := f.block[method]
+	hook := f.hooks[method]
+	f.mu.Unlock()
+	if block != nil {
+		select {
+		case <-block:
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+	if hook != nil {
+		return hook()
+	}
+	return nil
+}
+
+func (f *fakeBackend) domain(name string) (*fakeDomain, error) {
+	d, ok := f.domains[name]
+	if !ok {
+		return nil, fakeErr(libvirt.ErrNoDomain, "Domain not found: no domain with matching name '%s'", name)
+	}
+	return d, nil
+}
+
+func (f *fakeBackend) pool(name string) (*fakePool, error) {
+	p, ok := f.pools[name]
+	if !ok {
+		return nil, fakeErr(libvirt.ErrNoStoragePool, "Storage pool not found: no storage pool with matching name '%s'", name)
+	}
+	return p, nil
+}
+
+func (f *fakeBackend) volume(vol libvirt.StorageVol) (*fakeVol, error) {
+	p, err := f.pool(vol.Pool)
+	if err != nil {
+		return nil, err
+	}
+	v, ok := p.vols[vol.Name]
+	if !ok {
+		return nil, fakeErr(libvirt.ErrNoStorageVol, "Storage volume not found: no storage vol with matching name '%s'", vol.Name)
+	}
+	return v, nil
+}
+
+func (f *fakeBackend) libvirtDomain(name string, d *fakeDomain) libvirt.Domain {
+	id := int32(-1)
+	if isActive(d.state) {
+		id = d.id
+	}
+	return libvirt.Domain{Name: name, UUID: d.uuid, ID: id}
+}
+
+func (f *fakeBackend) DomainLookupByName(ctx context.Context, name string) (libvirt.Domain, error) {
+	if err := f.enter(ctx, "DomainLookupByName"); err != nil {
+		return libvirt.Domain{}, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	d, err := f.domain(name)
+	if err != nil {
+		return libvirt.Domain{}, err
+	}
+	return f.libvirtDomain(name, d), nil
+}
+
+func (f *fakeBackend) DomainDefineXML(ctx context.Context, doc string) (libvirt.Domain, error) {
+	if err := f.enter(ctx, "DomainDefineXML"); err != nil {
+		return libvirt.Domain{}, err
+	}
+	var parsed struct {
+		Name     string `xml:"name"`
+		Metadata struct {
+			Inner string `xml:",innerxml"`
+		} `xml:"metadata"`
+	}
+	if err := xml.Unmarshal([]byte(doc), &parsed); err != nil || parsed.Name == "" {
+		return libvirt.Domain{}, fakeErr(libvirt.ErrXMLError, "XML error: %v", err)
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	d, ok := f.domains[parsed.Name]
+	if !ok {
+		d = &fakeDomain{state: libvirt.DomainShutoff, addrs: make(map[uint32][]libvirt.DomainInterface), uuid: fakeUUID(parsed.Name)}
+		f.domains[parsed.Name] = d
+	}
+	d.xml = doc
+	d.metadata = parsed.Metadata.Inner
+	return f.libvirtDomain(parsed.Name, d), nil
+}
+
+func (f *fakeBackend) DomainCreate(ctx context.Context, dom libvirt.Domain) error {
+	if err := f.enter(ctx, "DomainCreate"); err != nil {
+		return err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	d, err := f.domain(dom.Name)
+	if err != nil {
+		return err
+	}
+	if isActive(d.state) {
+		return fakeErr(libvirt.ErrOperationInvalid, "Requested operation is not valid: domain is already running")
+	}
+	switch {
+	case f.startDelayPolls > 0:
+		d.state = libvirt.DomainNostate
+		d.pendingStart = f.startDelayPolls
+		return nil
+	case f.stateAfterStart != libvirt.DomainNostate:
+		d.state = f.stateAfterStart
+		return nil
+	}
+	d.state = libvirt.DomainRunning
+	d.id = int32(len(f.domains))
+	return nil
+}
+
+func (f *fakeBackend) DomainDestroy(ctx context.Context, dom libvirt.Domain) error {
+	if err := f.enter(ctx, "DomainDestroy"); err != nil {
+		return err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	d, err := f.domain(dom.Name)
+	if err != nil {
+		return err
+	}
+	if !isActive(d.state) {
+		return fakeErr(libvirt.ErrOperationInvalid, "Requested operation is not valid: domain is not running")
+	}
+	d.state = libvirt.DomainShutoff
+	return nil
+}
+
+func (f *fakeBackend) DomainUndefineFlags(ctx context.Context, dom libvirt.Domain, flags libvirt.DomainUndefineFlagsValues) error {
+	if err := f.enter(ctx, "DomainUndefineFlags"); err != nil {
+		return err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if _, err := f.domain(dom.Name); err != nil {
+		return err
+	}
+	f.undefineFlags[dom.Name] = flags
+	delete(f.domains, dom.Name)
+	return nil
+}
+
+func (f *fakeBackend) DomainGetState(ctx context.Context, dom libvirt.Domain, _ uint32) (int32, int32, error) {
+	if err := f.enter(ctx, "DomainGetState"); err != nil {
+		return 0, 0, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	d, err := f.domain(dom.Name)
+	if err != nil {
+		return 0, 0, err
+	}
+	if d.pendingStart > 0 {
+		d.pendingStart--
+		if d.pendingStart == 0 {
+			d.state = libvirt.DomainRunning
+			d.id = int32(len(f.domains))
+		}
+	}
+	return int32(d.state), 0, nil
+}
+
+func (f *fakeBackend) DomainGetMetadata(ctx context.Context, dom libvirt.Domain, typ int32, uri libvirt.OptString, _ libvirt.DomainModificationImpact) (string, error) {
+	if err := f.enter(ctx, "DomainGetMetadata"); err != nil {
+		return "", err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	d, err := f.domain(dom.Name)
+	if err != nil {
+		return "", err
+	}
+	if typ != int32(libvirt.DomainMetadataElement) || len(uri) != 1 || uri[0] != metadataURI || d.metadata == "" {
+		return "", fakeErr(libvirt.ErrNoDomainMetadata, "Requested metadata element is not present")
+	}
+	return d.metadata, nil
+}
+
+func (f *fakeBackend) DomainInterfaceAddresses(ctx context.Context, dom libvirt.Domain, source uint32, _ uint32) ([]libvirt.DomainInterface, error) {
+	if err := f.enter(ctx, "DomainInterfaceAddresses"); err != nil {
+		return nil, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	d, err := f.domain(dom.Name)
+	if err != nil {
+		return nil, err
+	}
+	if !isActive(d.state) {
+		return nil, fakeErr(libvirt.ErrOperationInvalid, "Requested operation is not valid: domain is not running")
+	}
+	return slices.Clone(d.addrs[source]), nil
+}
+
+func (f *fakeBackend) ConnectListAllDomains(ctx context.Context, _ int32, _ libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error) {
+	if err := f.enter(ctx, "ConnectListAllDomains"); err != nil {
+		return nil, 0, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	names := make([]string, 0, len(f.domains))
+	for name := range f.domains {
+		names = append(names, name)
+	}
+	slices.Sort(names)
+	out := make([]libvirt.Domain, 0, len(names))
+	for _, name := range names {
+		out = append(out, f.libvirtDomain(name, f.domains[name]))
+	}
+	return out, uint32(len(out)), nil
+}
+
+func (f *fakeBackend) StoragePoolLookupByName(ctx context.Context, name string) (libvirt.StoragePool, error) {
+	if err := f.enter(ctx, "StoragePoolLookupByName"); err != nil {
+		return libvirt.StoragePool{}, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if _, err := f.pool(name); err != nil {
+		return libvirt.StoragePool{}, err
+	}
+	return libvirt.StoragePool{Name: name, UUID: fakeUUID(name)}, nil
+}
+
+func (f *fakeBackend) StoragePoolRefresh(ctx context.Context, pool libvirt.StoragePool, _ uint32) error {
+	if err := f.enter(ctx, "StoragePoolRefresh"); err != nil {
+		return err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	_, err := f.pool(pool.Name)
+	return err
+}
+
+func (f *fakeBackend) StoragePoolGetInfo(ctx context.Context, pool libvirt.StoragePool) (uint8, uint64, uint64, uint64, error) {
+	if err := f.enter(ctx, "StoragePoolGetInfo"); err != nil {
+		return 0, 0, 0, 0, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	p, err := f.pool(pool.Name)
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+	return uint8(p.state), p.capacity, p.capacity - p.available, p.available, nil
+}
+
+func (f *fakeBackend) StorageVolLookupByName(ctx context.Context, pool libvirt.StoragePool, name string) (libvirt.StorageVol, error) {
+	if err := f.enter(ctx, "StorageVolLookupByName"); err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	vol := libvirt.StorageVol{Pool: pool.Name, Name: name, Key: fakeVolPath(pool.Name, name)}
+	if _, err := f.volume(vol); err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	return vol, nil
+}
+
+func fakeVolPath(pool, name string) string { return "/pool/" + pool + "/" + name }
+
+func (f *fakeBackend) StorageVolGetPath(ctx context.Context, vol libvirt.StorageVol) (string, error) {
+	if err := f.enter(ctx, "StorageVolGetPath"); err != nil {
+		return "", err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if _, err := f.volume(vol); err != nil {
+		return "", err
+	}
+	return fakeVolPath(vol.Pool, vol.Name), nil
+}
+
+func (f *fakeBackend) StorageVolCreateXML(ctx context.Context, pool libvirt.StoragePool, doc string, _ libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error) {
+	if err := f.enter(ctx, "StorageVolCreateXML"); err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	var parsed struct {
+		Name     string `xml:"name"`
+		Capacity uint64 `xml:"capacity"`
+	}
+	if err := xml.Unmarshal([]byte(doc), &parsed); err != nil || parsed.Name == "" {
+		return libvirt.StorageVol{}, fakeErr(libvirt.ErrXMLError, "XML error: %v", err)
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	p, err := f.pool(pool.Name)
+	if err != nil {
+		return libvirt.StorageVol{}, err
+	}
+	if _, exists := p.vols[parsed.Name]; exists {
+		return libvirt.StorageVol{}, fakeErr(libvirt.ErrStorageVolExist, "storage volume '%s' exists already", parsed.Name)
+	}
+	p.vols[parsed.Name] = &fakeVol{capacity: parsed.Capacity}
+	return libvirt.StorageVol{Pool: pool.Name, Name: parsed.Name, Key: fakeVolPath(pool.Name, parsed.Name)}, nil
+}
+
+func (f *fakeBackend) StorageVolUpload(ctx context.Context, vol libvirt.StorageVol, r io.Reader, _, _ uint64, _ libvirt.StorageVolUploadFlags) error {
+	if err := f.enter(ctx, "StorageVolUpload"); err != nil {
+		return err
+	}
+	var (
+		content  []byte
+		uploaded int64
+		digest   string
+	)
+	if r != nil {
+		var err error
+		if f.discardUploads {
+			h := sha256.New()
+			if uploaded, err = io.CopyBuffer(h, r, make([]byte, 64<<10)); err != nil {
+				return err
+			}
+			digest = hex.EncodeToString(h.Sum(nil))
+		} else if content, err = io.ReadAll(r); err != nil {
+			return err
+		}
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	v, err := f.volume(vol)
+	if err != nil {
+		return err
+	}
+	v.content = content
+	v.uploaded = uploaded
+	v.sha256 = digest
+	return nil
+}
+
+func (f *fakeBackend) StorageVolDownload(ctx context.Context, vol libvirt.StorageVol, w io.Writer, _, _ uint64, _ libvirt.StorageVolDownloadFlags) error {
+	if err := f.enter(ctx, "StorageVolDownload"); err != nil {
+		return err
+	}
+	f.mu.Lock()
+	v, err := f.volume(vol)
+	var content []byte
+	if v != nil {
+		content = slices.Clone(v.content)
+	}
+	f.mu.Unlock()
+	if err != nil {
+		return err
+	}
+	if w == nil {
+		return nil
+	}
+	_, err = w.Write(content)
+	return err
+}
+
+func (f *fakeBackend) StorageVolDelete(ctx context.Context, vol libvirt.StorageVol, _ libvirt.StorageVolDeleteFlags) error {
+	if err := f.enter(ctx, "StorageVolDelete"); err != nil {
+		return err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if _, err := f.volume(vol); err != nil {
+		return err
+	}
+	delete(f.pools[vol.Pool].vols, vol.Name)
+	return nil
+}
+
+func (f *fakeBackend) ConnectGetLibVersion(ctx context.Context) (uint64, error) {
+	if err := f.enter(ctx, "ConnectGetLibVersion"); err != nil {
+		return 0, err
+	}
+	return f.version, nil
+}
+
+func (f *fakeBackend) ConnectGetHostname(ctx context.Context) (string, error) {
+	if err := f.enter(ctx, "ConnectGetHostname"); err != nil {
+		return "", err
+	}
+	return f.hostname, nil
+}
+
+func (f *fakeBackend) Close() error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.closed++
+	return nil
+}
+
+// stubDialer covers every dialer a test needs: it counts dials, fails with
+// err when set, waits on block (or ctx) when set, and hands out backend, or
+// one from fresh per call, or a new fakeBackend when neither is given.
+type stubDialer struct {
+	backend backend
+	fresh   func() backend
+	err     error
+	block   chan struct{}
+
+	dials atomic.Int32
+	mu    sync.Mutex
+	hosts []string
+}
+
+func (d *stubDialer) dial(ctx context.Context, host provider.KVMHost) (backend, error) {
+	d.dials.Add(1)
+	d.mu.Lock()
+	d.hosts = append(d.hosts, host.Name)
+	d.mu.Unlock()
+	if d.block != nil {
+		select {
+		case <-d.block:
+		case <-ctx.Done():
+			return nil, fmt.Errorf("%w: dialing %s: %w", provider.ErrTransient, host.Name, ctx.Err())
+		}
+	}
+	if d.err != nil {
+		return nil, d.err
+	}
+	switch {
+	case d.fresh != nil:
+		return d.fresh(), nil
+	case d.backend != nil:
+		return d.backend, nil
+	default:
+		return newFakeBackend(), nil
+	}
+}

internal/provider/kvm2/get.go

@@ -0,0 +1,119 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// Get is spec entries 7 to 9: look the domain up by name, read the mark
+// back, then state and address. Only ErrNoDomain becomes NotFound; every
+// other failure is classified, never guessed.
+func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instance, error) {
+	hostName, name, err := parseProviderID(providerID)
+	if err != nil {
+		return nil, provider.Wrap(provider.ErrPermanent, "get", p.name, providerID, err)
+	}
+	host, err := p.host(hostName)
+	if err != nil {
+		return nil, p.wrapErr("get", providerID, err)
+	}
+	log := p.logger(ctx).WithValues("host", host.Name, "name", name)
+	be, err := p.hosts.get(ctx, host)
+	if err != nil {
+		logAPIError(log, "get", err)
+		return nil, p.wrapErr("get", providerID, err)
+	}
+	inst, err := get(ctx, log, be, host.Name, name)
+	if err != nil {
+		if hasCode(err, libvirt.ErrNoDomain) {
+			log.V(1).Info("kvm2 domain not found")
+		} else {
+			logAPIError(log, "get", err)
+		}
+		return nil, p.wrapErr("get", providerID, err)
+	}
+	log.V(1).Info("kvm2 domain fetched", "state", inst.State, "ip", inst.IP, "uid", inst.UID)
+	return inst, nil
+}
+
+// get builds the Instance from the domain's metadata, its state and, for an
+// active domain, its discovered address.
+func get(ctx context.Context, log logr.Logger, be backend, hostName, name string) (*provider.Instance, error) {
+	dom, err := be.DomainLookupByName(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("looking up domain %s: %w", name, err)
+	}
+	meta, err := readMetadata(ctx, be, dom)
+	if err != nil {
+		return nil, err
+	}
+	state, _, err := be.DomainGetState(ctx, dom, 0)
+	if err != nil {
+		return nil, fmt.Errorf("reading state of domain %s: %w", dom.Name, err)
+	}
+	s := libvirt.DomainState(state)
+	var ip string
+	if isActive(s) {
+		if ip, err = address(ctx, log, be, dom); err != nil {
+			return nil, err
+		}
+	}
+	return &provider.Instance{
+		ID:        formatProviderID(hostName, dom.Name),
+		IP:        ip,
+		State:     mapState(s, ip),
+		UID:       meta.UID,
+		CreatedAt: meta.createdAt(),
+	}, nil
+}
+
+// readMetadata returns the kvm2 mark of a domain, or the zero value (not
+// managed, empty UID) when the domain carries none. A mark that does not
+// parse is Permanent, from parseInstanceMetadata.
+func readMetadata(ctx context.Context, be backend, dom libvirt.Domain) (instanceMetadata, error) {
+	raw, err := be.DomainGetMetadata(ctx, dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, libvirt.DomainAffectCurrent)
+	if hasCode(err, libvirt.ErrNoDomainMetadata) {
+		return instanceMetadata{}, nil
+	}
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("reading metadata of domain %s: %w", dom.Name, err)
+	}
+	return parseInstanceMetadata(raw)
+}
+
+// addressSources is the discovery order of spec entry 7: the guest agent
+// answers where the host's ARP table never sees guest traffic.
+var addressSources = []struct {
+	name string
+	src  libvirt.DomainInterfaceAddressesSource
+}{
+	{"agent", libvirt.DomainInterfaceAddressesSrcAgent},
+	{"arp", libvirt.DomainInterfaceAddressesSrcArp},
+}
+
+// address returns the first usable IPv4 from the first source that yields
+// one, or "" for "no address yet". A source that fails is logged and
+// skipped; only ctx ending is an error, so a cancelled Get does not read
+// as Provisioning.
+func address(ctx context.Context, log logr.Logger, be backend, dom libvirt.Domain) (string, error) {
+	for _, s := range addressSources {
+		ifaces, err := be.DomainInterfaceAddresses(ctx, dom, uint32(s.src), 0)
+		if err != nil {
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return "", fmt.Errorf("reading addresses of domain %s: %w", dom.Name, err)
+			}
+			log.V(2).Info("no addresses from source", "source", s.name, "error", err.Error())
+			continue
+		}
+		if ip := firstIPv4(ifaces); ip != "" {
+			return ip, nil
+		}
+	}
+	return "", nil
+}

internal/provider/kvm2/get_test.go

@@ -0,0 +1,243 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func TestGet_afterDeleteReturnsNotFound(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	if err := fx.p.Delete(fx.ctx, id); err != nil {
+		t.Fatalf("Delete() error = %v", err)
+	}
+	inst, err := fx.p.Get(fx.ctx, id)
+	if inst != nil {
+		t.Errorf("Get after Delete = %+v, want nil", inst)
+	}
+	assertProviderError(t, err, "get", provider.ErrNotFound)
+}
+
+func TestGet_duringDeleteStillReturnsInstance(t *testing.T) {
+	t.Parallel()
+	t.Run("destroyed but still defined is stopped", func(t *testing.T) {
+		t.Parallel()
+		fx := newFixture(t, 1)
+		id := fx.create(testName)
+		fx.withDomain(testName, func(d *fakeDomain) { d.state = libvirt.DomainShutoff })
+		inst := fx.get(id)
+		if inst.State != provider.StateStopped || inst.UID != testUID {
+			t.Errorf("Get = %+v, want Stopped with uid %s", inst, testUID)
+		}
+	})
+	t.Run("undefine in flight still answers", func(t *testing.T) {
+		t.Parallel()
+		fx := newFixture(t, 1)
+		id := fx.create(testName)
+		release := fx.fake.blockOn("DomainUndefineFlags")
+		done := make(chan error, 1)
+		go func() { done <- fx.p.Delete(fx.ctx, id) }()
+		waitFor(t, "the undefine to block", func() bool { return fx.fake.callCount("DomainUndefineFlags") == 1 })
+
+		inst, err := fx.p.Get(fx.ctx, id)
+		if err != nil {
+			t.Fatalf("Get during Delete error = %v, want the instance", err)
+		}
+		if inst.State != provider.StateStopped || inst.ID != id {
+			t.Errorf("Get during Delete = %+v, want Stopped %s", inst, id)
+		}
+		close(release)
+		if err := <-done; err != nil {
+			t.Fatalf("Delete() error = %v", err)
+		}
+		_, err = fx.p.Get(fx.ctx, id)
+		assertProviderError(t, err, "get", provider.ErrNotFound)
+	})
+}
+
+func TestGet_backendBusyIsTransientNeverNotFound(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		err  error
+	}{
+		{name: "operation invalid", err: fakeErr(libvirt.ErrOperationInvalid, "Requested operation is not valid: domain is being migrated")},
+		{name: "call timed out", err: fmt.Errorf("%w: libvirt call DomainGetState on beelink exceeded its 30s budget: %w", provider.ErrTransient, context.DeadlineExceeded)},
+		{name: "unknown libvirt code", err: fakeErr(libvirt.ErrInternalError, "internal error: qemu monitor unresponsive")},
+		{name: "not a libvirt error", err: errors.New("read tcp: connection reset by peer")},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			id := fx.create(testName)
+			fx.fake.failWith("DomainGetState", tc.err)
+			inst, err := fx.p.Get(fx.ctx, id)
+			if inst != nil {
+				t.Errorf("Get = %+v, want nil", inst)
+			}
+			assertProviderError(t, err, "get", provider.ErrTransient)
+			if errors.Is(err, provider.ErrNotFound) {
+				t.Errorf("Get error = %v, must never be NotFound", err)
+			}
+		})
+	}
+}
+
+func TestGet_runningWithoutIPIsProvisioning(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	inst := fx.get(fx.create(testName))
+	if inst.State != provider.StateProvisioning || inst.IP != "" {
+		t.Errorf("Get = %+v, want Provisioning with no IP", inst)
+	}
+}
+
+func TestGet_runningWithIPIsRunning(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcAgent, "127.0.0.1", "169.254.1.2", "10.0.0.5")
+	inst := fx.get(id)
+	if inst.State != provider.StateRunning || inst.IP != "10.0.0.5" {
+		t.Errorf("Get = %+v, want Running with 10.0.0.5", inst)
+	}
+	if got := fx.logs.matching("kvm2 domain fetched"); len(got) != 1 || !strings.Contains(got[0], `"ip"="10.0.0.5"`) || !strings.Contains(got[0], `"state"="Running"`) {
+		t.Errorf("fetched line = %v, want state and ip", got)
+	}
+}
+
+func TestGet_agentPreferredOverARP(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcAgent, "10.0.0.5")
+	fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcArp, "10.0.0.6")
+	before := fx.fake.callCount("DomainInterfaceAddresses")
+	inst := fx.get(id)
+	if inst.IP != "10.0.0.5" {
+		t.Errorf("IP = %q, want the agent's 10.0.0.5", inst.IP)
+	}
+	if got := fx.fake.callCount("DomainInterfaceAddresses") - before; got != 1 {
+		t.Errorf("DomainInterfaceAddresses calls = %d, want 1 (ARP not consulted)", got)
+	}
+}
+
+func TestGet_arpFallbackWhenAgentFails(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 2)
+	id := fx.create(testName)
+	fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcArp, "10.0.0.6")
+	var calls int
+	fx.fake.mu.Lock()
+	fx.fake.hooks["DomainInterfaceAddresses"] = func() error {
+		calls++
+		if calls%2 == 1 {
+			return fakeErr(libvirt.ErrAgentUnresponsive, "Guest agent is not responding")
+		}
+		return nil
+	}
+	fx.fake.mu.Unlock()
+	inst := fx.get(id)
+	if inst.State != provider.StateRunning || inst.IP != "10.0.0.6" {
+		t.Errorf("Get = %+v, want Running with ARP's 10.0.0.6", inst)
+	}
+	if got := fx.logs.matching("no addresses from source"); len(got) != 1 || !strings.Contains(got[0], `"source"="agent"`) {
+		t.Errorf("agent failure line = %v", got)
+	}
+}
+
+func TestGet_addressErrorIsProvisioning(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	fx.fake.failWith("DomainInterfaceAddresses", fakeErr(libvirt.ErrAgentUnresponsive, "Guest agent is not responding"))
+	inst := fx.get(id)
+	if inst.State != provider.StateProvisioning || inst.IP != "" {
+		t.Errorf("Get = %+v, want Provisioning with no IP", inst)
+	}
+}
+
+func TestGet_readsUIDAndCreatedAtFromBackend(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	created := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)
+	fx.p.now = func() time.Time { return created }
+	id := fx.create(testName)
+	inst := fx.get(id)
+	if inst.UID != testUID || !inst.CreatedAt.Equal(created) {
+		t.Errorf("Get = %+v, want uid %s created %s", inst, testUID, created)
+	}
+
+	later := created.Add(48 * time.Hour)
+	fx.withDomain(testName, func(d *fakeDomain) { d.metadata = managedMetadata("rewritten-uid", later) })
+	inst = fx.get(id)
+	if inst.UID != "rewritten-uid" || !inst.CreatedAt.Equal(later) {
+		t.Errorf("Get after metadata change = %+v, want uid rewritten-uid created %s (read back, not remembered)", inst, later)
+	}
+}
+
+func TestGet_unmanagedDomainHasEmptyUID(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	fx.fake.addDomain("legacy-vm", libvirt.DomainRunning, "")
+	inst := fx.get(formatProviderID(testHost, "legacy-vm"))
+	if inst.UID != "" || !inst.CreatedAt.IsZero() || inst.State != provider.StateProvisioning {
+		t.Errorf("Get = %+v, want empty uid, zero createdAt, Provisioning", inst)
+	}
+}
+
+func TestGet_malformedIDIsPermanent(t *testing.T) {
+	t.Parallel()
+	ids := []string{"", "proxy-x", "beelink/proxy-x", "hosts/beelink/proxy-x", "hosts//domains/proxy-x", "hosts/beelink/domains/", "hosts/beelink/domains/a/b", "nodes/beelink/domains/proxy-x"}
+	for _, id := range ids {
+		t.Run(id, func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			inst, err := fx.p.Get(fx.ctx, id)
+			if inst != nil {
+				t.Errorf("Get(%q) = %+v, want nil", id, inst)
+			}
+			assertProviderError(t, err, "get", provider.ErrPermanent)
+			var pe *provider.Error
+			if errors.As(err, &pe) && pe.ID != id {
+				t.Errorf("error.ID = %q, want %q", pe.ID, id)
+			}
+			if got := fx.dialer.dials.Load(); got != 0 {
+				t.Errorf("dials = %d, want 0", got)
+			}
+		})
+	}
+}
+
+func TestGet_unconfiguredHostIsPermanent(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	_, err := fx.p.Get(fx.ctx, formatProviderID("nas9", testName))
+	assertProviderError(t, err, "get", provider.ErrPermanent)
+	if !strings.Contains(err.Error(), `"nas9"`) {
+		t.Errorf("error = %v, want the host named", err)
+	}
+	if got := fx.dialer.dials.Load(); got != 0 {
+		t.Errorf("dials = %d, want 0", got)
+	}
+}
+
+func TestGet_unknownStateIsStopped(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	fx.fake.addDomain(testName, libvirt.DomainState(42), managedMetadata(testUID, time.Now()))
+	inst := fx.get(formatProviderID(testHost, testName))
+	if inst.State != provider.StateStopped {
+		t.Errorf("State = %s, want Stopped for an unknown backend state", inst.State)
+	}
+}

internal/provider/kvm2/hosts.go

@@ -0,0 +1,162 @@
+package kvm2
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+// dialer opens one connection to a host. The production dialer is
+// libvirtDialer; tests inject one that hands out fakes.
+type dialer interface {
+	dial(ctx context.Context, host provider.KVMHost) (backend, error)
+}
+
+// hostTable holds the pooled connection per host (spec entry 19). One dial
+// per host at a time: concurrent callers wait for it and share the result.
+// A dead backend (abandoned call, keepalive failure, socket lost) is dropped
+// and redialled on the next get.
+//
+// What a cancelled or overlong call does to the shared connection (spec
+// entry 18): the adapter kills it, so every other call in flight on that
+// host fails once with Transient and its caller retries on a fresh
+// connection. Callers that must not be exposed to that, the image transfer
+// and the Checker, use dialFresh and get a connection of their own.
+type hostTable struct {
+	dialer dialer
+
+	mu    sync.Mutex
+	hosts map[string]*hostEntry
+}
+
+// hostEntry serialises dials for one host. The semaphore, not a mutex, so a
+// waiter can give up when its ctx ends.
+type hostEntry struct {
+	sem     chan struct{}
+	backend backend
+}
+
+func newHostTable(d dialer) *hostTable {
+	return &hostTable{dialer: d, hosts: make(map[string]*hostEntry)}
+}
+
+func (t *hostTable) entry(name string) *hostEntry {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	e, ok := t.hosts[name]
+	if !ok {
+		e = &hostEntry{sem: make(chan struct{}, 1)}
+		t.hosts[name] = e
+	}
+	return e
+}
+
+// get returns the live pooled backend for host, dialling it when there is
+// none. Waiting for another caller's dial ends with ctx.
+func (t *hostTable) get(ctx context.Context, host provider.KVMHost) (backend, error) {
+	e := t.entry(host.Name)
+	select {
+	case e.sem <- struct{}{}:
+	case <-ctx.Done():
+		return nil, fmt.Errorf("%w: waiting for the connection to host %s: %w", provider.ErrTransient, host.Name, ctx.Err())
+	}
+	defer func() { <-e.sem }()
+	if e.backend != nil {
+		if !isDead(e.backend) {
+			return e.backend, nil
+		}
+		_ = e.backend.Close()
+		e.backend = nil
+	}
+	b, err := t.dialer.dial(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+	e.backend = b
+	return b, nil
+}
+
+// dialFresh returns a dedicated connection that is never pooled. The
+// caller owns it and must Close it.
+func (t *hostTable) dialFresh(ctx context.Context, host provider.KVMHost) (backend, error) {
+	return t.dialer.dial(ctx, host)
+}
+
+// libvirtDialer is the production dialer: SSH transport, go-libvirt over
+// the forwarded socket, the connect handshake, then the observe decorator,
+// so provider code never holds an undecorated backend.
+type libvirtDialer struct {
+	providerName string
+	cfg          provider.KVMConfig
+	budgets      budgets
+	keepalive    time.Duration
+	tracing      []tracing.Option
+}
+
+func newLibvirtDialer(providerName string, cfg provider.KVMConfig) *libvirtDialer {
+	return &libvirtDialer{
+		providerName: providerName,
+		cfg:          cfg,
+		budgets:      defaultBudgets,
+		keepalive:    defaultKeepaliveInterval,
+	}
+}
+
+func (d *libvirtDialer) dial(ctx context.Context, host provider.KVMHost) (backend, error) {
+	ctx, span := tracing.StartSpan(ctx, tracing.Tracer(d.tracing...), "kvm2.dial",
+		trace.WithSpanKind(trace.SpanKindClient),
+		trace.WithAttributes(
+			attribute.String("kvm2.host", host.Name),
+			attribute.String("kvm2.address", host.Address),
+			attribute.String("kvm2.user", host.User),
+		))
+	defer span.End()
+	log := logf.FromContext(ctx).WithName("kvm2").WithValues("provider", d.providerName, "host", host.Name)
+	start := time.Now()
+	b, err := d.connect(ctx, host)
+	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
+		kv := []any{"address", host.Address, "user", host.User, "duration", time.Since(start), "error", err.Error()}
+		if stage, ok := stageOf(err); ok {
+			kv = append(kv, "stage", string(stage))
+		}
+		kv = append(kv, libvirtErrorFields(err)...)
+		log.V(1).Info("libvirt host connect failed", kv...)
+		return nil, err
+	}
+	log.V(1).Info("libvirt host connected", "address", host.Address, "user", host.User, "duration", time.Since(start))
+	return observe(b, d.providerName, host.Name, d.tracing...), nil
+}
+
+// connect is the dial sequence under one budget: the SSH stages, then the
+// libvirt handshake. On success the keepalive and the disconnect watcher
+// run until the link dies.
+func (d *libvirtDialer) connect(ctx context.Context, host provider.KVMHost) (*libvirtBackend, error) {
+	ctx, cancel := context.WithTimeout(ctx, d.budgets.dial)
+	defer cancel()
+	sock, err := dialSSHSocket(ctx, host, d.cfg, 0)
+	if err != nil {
+		return nil, err
+	}
+	b := newLibvirtBackend(libvirt.New(sock), host.Name, sock, d.budgets)
+	if err := b.connect(ctx); err != nil {
+		b.link.markDead("connect failed")
+		return nil, fmt.Errorf("connecting to libvirt on host %s: %w", host.Name, err)
+	}
+	go func() {
+		if err := sock.keepalive(d.keepalive, b.link.dead()); err != nil {
+			b.link.markDead(err.Error())
+		}
+	}()
+	return b, nil
+}

internal/provider/kvm2/hosts_test.go

@@ -0,0 +1,195 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+var (
+	hostA = provider.KVMHost{Name: "beelink", Address: "beelink:22", User: "virt"}
+	hostB = provider.KVMHost{Name: "lab2", Address: "lab2:22", User: "virt"}
+)
+
+func TestHosts_oneDialPerHostUnderConcurrency(t *testing.T) {
+	t.Parallel()
+	shared := newFakeBackend()
+	d := &stubDialer{backend: shared}
+	table := newHostTable(d)
+
+	const n = 50
+	var wg sync.WaitGroup
+	got := make([]backend, n)
+	errs := make([]error, n)
+	for i := range n {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			got[i], errs[i] = table.get(context.Background(), hostA)
+		}()
+	}
+	wg.Wait()
+	for i := range n {
+		if errs[i] != nil {
+			t.Fatalf("get #%d error = %v", i, errs[i])
+		}
+		if got[i] != backend(shared) {
+			t.Fatalf("get #%d returned a different backend", i)
+		}
+	}
+	if d.dials.Load() != 1 {
+		t.Errorf("dials = %d, want 1", d.dials.Load())
+	}
+}
+
+func TestHosts_redialsAfterDrop(t *testing.T) {
+	t.Parallel()
+	first := newFakeBackend()
+	second := newFakeBackend()
+	backends := []backend{first, second}
+	d := &stubDialer{fresh: func() backend {
+		b := backends[0]
+		backends = backends[1:]
+		return b
+	}}
+	table := newHostTable(d)
+
+	b, err := table.get(context.Background(), hostA)
+	if err != nil || b != backend(first) {
+		t.Fatalf("first get = %v, %v, want the first backend", b, err)
+	}
+	if b, _ := table.get(context.Background(), hostA); b != backend(first) {
+		t.Fatal("second get did not reuse the live backend")
+	}
+	first.markDead()
+	b, err = table.get(context.Background(), hostA)
+	if err != nil || b != backend(second) {
+		t.Fatalf("get after drop = %v, %v, want the second backend", b, err)
+	}
+	if first.closed != 1 {
+		t.Errorf("dropped backend closed %d times, want 1", first.closed)
+	}
+	if d.dials.Load() != 2 {
+		t.Errorf("dials = %d, want 2", d.dials.Load())
+	}
+}
+
+func TestHosts_dialFreshIsNotCached(t *testing.T) {
+	t.Parallel()
+	d := &stubDialer{fresh: func() backend { return newFakeBackend() }}
+	table := newHostTable(d)
+
+	pooled, err := table.get(context.Background(), hostA)
+	if err != nil {
+		t.Fatalf("get error = %v", err)
+	}
+	fresh, err := table.dialFresh(context.Background(), hostA)
+	if err != nil {
+		t.Fatalf("dialFresh error = %v", err)
+	}
+	if fresh == pooled {
+		t.Fatal("dialFresh returned the pooled backend")
+	}
+	again, _ := table.get(context.Background(), hostA)
+	if again != pooled {
+		t.Error("get after dialFresh did not return the pooled backend")
+	}
+	if d.dials.Load() != 2 {
+		t.Errorf("dials = %d, want 2", d.dials.Load())
+	}
+	if fresh.(*fakeBackend).closed != 0 {
+		t.Error("dialFresh's backend was closed by the table; the caller owns it")
+	}
+}
+
+func TestHosts_waitingForDialEndsWithContext(t *testing.T) {
+	t.Parallel()
+	release := make(chan struct{})
+	d := &stubDialer{block: release}
+	table := newHostTable(d)
+
+	firstDone := make(chan error, 1)
+	go func() {
+		_, err := table.get(context.Background(), hostA)
+		firstDone <- err
+	}()
+	waitForDials(t, d, 1)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	_, err := table.get(ctx, hostA)
+	if time.Since(start) > time.Second {
+		t.Errorf("second get took %v, want to give up with its ctx", time.Since(start))
+	}
+	if !errors.Is(err, provider.ErrTransient) || !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("second get error = %v, want ErrTransient wrapping DeadlineExceeded", err)
+	}
+	close(release)
+	if err := <-firstDone; err != nil {
+		t.Errorf("first get error = %v, want nil once the dial completes", err)
+	}
+	if d.dials.Load() != 1 {
+		t.Errorf("dials = %d, want 1: the waiter must not dial itself", d.dials.Load())
+	}
+}
+
+func TestHosts_hostsDialIndependently(t *testing.T) {
+	t.Parallel()
+	release := make(chan struct{})
+	defer close(release)
+	dA := &stubDialer{block: release}
+	dB := &stubDialer{}
+	table := newHostTable(perHostDialer{hostA.Name: dA, hostB.Name: dB})
+
+	go func() { _, _ = table.get(context.Background(), hostA) }()
+	waitForDials(t, dA, 1)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	start := time.Now()
+	if _, err := table.get(ctx, hostB); err != nil {
+		t.Fatalf("get(hostB) error = %v while hostA's dial is blocked", err)
+	}
+	if time.Since(start) > 500*time.Millisecond {
+		t.Errorf("get(hostB) took %v, want not to wait for hostA", time.Since(start))
+	}
+}
+
+func TestHosts_dialErrorIsNotCached(t *testing.T) {
+	t.Parallel()
+	d := &stubDialer{err: transientf("boom")}
+	table := newHostTable(d)
+	if _, err := table.get(context.Background(), hostA); !errors.Is(err, provider.ErrTransient) {
+		t.Fatalf("get error = %v, want the dial error", err)
+	}
+	d.err = nil
+	if _, err := table.get(context.Background(), hostA); err != nil {
+		t.Fatalf("get after the dialer recovered error = %v", err)
+	}
+	if d.dials.Load() != 2 {
+		t.Errorf("dials = %d, want 2", d.dials.Load())
+	}
+}
+
+// perHostDialer routes each host to its own stub.
+type perHostDialer map[string]*stubDialer
+
+func (p perHostDialer) dial(ctx context.Context, host provider.KVMHost) (backend, error) {
+	return p[host.Name].dial(ctx, host)
+}
+
+func waitForDials(t *testing.T, d *stubDialer, n int32) {
+	t.Helper()
+	deadline := time.Now().Add(2 * time.Second)
+	for d.dials.Load() < n {
+		if time.Now().After(deadline) {
+			t.Fatalf("dials = %d, want %d", d.dials.Load(), n)
+		}
+		time.Sleep(time.Millisecond)
+	}
+}

internal/provider/kvm2/id.go

@@ -0,0 +1,28 @@
+package kvm2
+
+import (
+	"strings"
+)
+
+// The providerID is hosts/<host>/domains/<name>. Host names are validated at
+// config load to contain no slash, and domain names come from NameFromUID,
+// so a strict four-segment split is enough to parse it.
+const (
+	idHostsSegment   = "hosts"
+	idDomainsSegment = "domains"
+)
+
+func formatProviderID(host, name string) string {
+	return idHostsSegment + "/" + host + "/" + idDomainsSegment + "/" + name
+}
+
+// parseProviderID splits an ID back into host and domain name. Anything
+// that is not exactly hosts/<host>/domains/<name> with both non-empty is
+// ErrPermanent: no retry will make it resolve.
+func parseProviderID(id string) (host, name string, err error) {
+	parts := strings.Split(id, "/")
+	if len(parts) != 4 || parts[0] != idHostsSegment || parts[2] != idDomainsSegment || parts[1] == "" || parts[3] == "" {
+		return "", "", permanentf("malformed providerID %q: want hosts/<host>/domains/<name>", id)
+	}
+	return parts[1], parts[3], nil
+}

internal/provider/kvm2/id_test.go

@@ -0,0 +1,54 @@
+package kvm2
+
+import (
+	"errors"
+	"strings"
+	"testing"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func TestFormatProviderID_roundTrip(t *testing.T) {
+	t.Parallel()
+	id := formatProviderID("beelink", "proxy-abc123def456ghij")
+	if id != "hosts/beelink/domains/proxy-abc123def456ghij" {
+		t.Fatalf("formatProviderID() = %q", id)
+	}
+	host, name, err := parseProviderID(id)
+	if err != nil || host != "beelink" || name != "proxy-abc123def456ghij" {
+		t.Errorf("parseProviderID(%q) = %q, %q, %v; want beelink, proxy-abc123def456ghij, nil", id, host, name, err)
+	}
+}
+
+func TestParseProviderID_malformed(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		id   string
+	}{
+		{name: "empty", id: ""},
+		{name: "two segments", id: "hosts/h"},
+		{name: "three segments", id: "hosts/h/domains"},
+		{name: "empty host", id: "hosts//domains/n"},
+		{name: "empty name", id: "hosts/h/domains/"},
+		{name: "wrong first literal", id: "nodes/h/domains/n"},
+		{name: "wrong third literal", id: "hosts/h/vms/n"},
+		{name: "extra segment", id: "hosts/h/domains/n/extra"},
+		{name: "slash in name", id: "hosts/h/domains/a/b"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			host, name, err := parseProviderID(tc.id)
+			if err == nil {
+				t.Fatalf("parseProviderID(%q) = %q, %q, nil; want error", tc.id, host, name)
+			}
+			if !errors.Is(err, provider.ErrPermanent) {
+				t.Errorf("parseProviderID(%q) error = %v, want ErrPermanent", tc.id, err)
+			}
+			if !strings.Contains(err.Error(), tc.id) {
+				t.Errorf("parseProviderID(%q) error = %q, want it to name the ID", tc.id, err)
+			}
+		})
+	}
+}

internal/provider/kvm2/image.go

@@ -0,0 +1,467 @@
+package kvm2
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+const (
+	// markerCapacityBytes is the marker volume's size: one lowercase hex
+	// sha256 digest, nothing else.
+	markerCapacityBytes = sha256.Size * 2
+	// markerReadCap bounds what is read back from a marker, so a wrong file
+	// under that name cannot grow memory.
+	markerReadCap = 1 << 10
+	// seedReserveBytes is what the quota precheck sets aside for the seed
+	// ISO next to the image and the overlay (spec entry 13).
+	seedReserveBytes = 16 << 20
+)
+
+var hexDigest = regexp.MustCompile(`^[0-9a-f]{64}$`)
+
+// imageSource is a resolved placement.image (spec entries 3 and 4): the
+// catalogue entry, or a URL given directly with the catalogue defaults, plus
+// the key its base volume is named after.
+type imageSource struct {
+	Key string
+	provider.KVMImage
+}
+
+func (s imageSource) volumeName() string { return baseVolumeName(s.Key, s.Format) }
+func (s imageSource) markerName() string { return markerVolumeName(s.volumeName()) }
+
+// resolveImage maps placement.image to a source. cfg is already defaulted,
+// so a catalogue entry carries its format and firmware; a URL gets qcow2
+// and uefi.
+func resolveImage(cfg provider.KVMConfig, image string) (imageSource, error) {
+	if img, ok := cfg.Images[image]; ok {
+		return imageSource{Key: image, KVMImage: img}, nil
+	}
+	if provider.IsHTTPURL(image) {
+		return imageSource{
+			Key:      urlImageKey(image),
+			KVMImage: provider.KVMImage{URL: image, Format: provider.KVMFormatQcow2, Firmware: provider.KVMFirmwareUEFI},
+		}, nil
+	}
+	return imageSource{}, permanentf("placement.image %q is neither a key in kvm2.images nor an http(s) URL", image)
+}
+
+// lockImage takes the per-(host, volume) lock of spec entry 19, or gives up
+// with ctx: a caller whose ctx ends while another caller's download runs
+// returns promptly (contract §1.5) instead of waiting out a 20-minute
+// transfer.
+func (p *Provider) lockImage(ctx context.Context, hostName, volume string) (func(), error) {
+	release, err := p.imageLocks.acquire(ctx, hostName+"/"+volume)
+	if err != nil {
+		return nil, fmt.Errorf("%w: waiting for image %s on host %s: %w", provider.ErrTransient, volume, hostName, err)
+	}
+	return release, nil
+}
+
+// ensureImage makes the base image present and complete in the host's pool
+// and returns its path (spec entry 4). be is the caller's pooled connection
+// and carries the cheap checks; the upload itself goes over a connection of
+// its own. Concurrent callers for the same (host, volume) queue on the
+// lock: the first fetches, the rest find the marker.
+func (p *Provider) ensureImage(ctx context.Context, host provider.KVMHost, pool libvirt.StoragePool, be backend, src imageSource) (string, error) {
+	unlock, err := p.lockImage(ctx, host.Name, src.volumeName())
+	if err != nil {
+		return "", err
+	}
+	defer unlock()
+	log := p.logger(ctx).WithValues("host", host.Name, "volume", src.volumeName())
+	path, err := cachedImage(ctx, log, pool, be, src)
+	if err != nil || path != "" {
+		return path, err
+	}
+	vol, err := p.fetchImage(ctx, host, pool, be, src)
+	if err != nil {
+		return "", err
+	}
+	path, err = be.StorageVolGetPath(ctx, vol)
+	if err != nil {
+		return "", fmt.Errorf("reading the path of base volume %s: %w", vol.Name, err)
+	}
+	return path, nil
+}
+
+// cachedImage returns the base volume's path when the marker says the base
+// is complete and its digest is the configured one (or none is configured).
+// Anything else is cleared so the fetch starts from nothing, and "" is
+// returned: a base without a marker is an upload that was cut off; a marker
+// without a base, or one whose content is not a digest, is debris; a marker
+// with a different digest means the catalogue entry changed.
+func cachedImage(ctx context.Context, log logr.Logger, pool libvirt.StoragePool, be backend, src imageSource) (string, error) {
+	name, markerName := src.volumeName(), src.markerName()
+	marker, err := be.StorageVolLookupByName(ctx, pool, markerName)
+	switch {
+	case hasCode(err, libvirt.ErrNoStorageVol):
+		return "", deleteVolumeIfPresent(ctx, be, pool, name)
+	case err != nil:
+		return "", fmt.Errorf("looking up marker %s: %w", markerName, err)
+	}
+	digest, err := readMarker(ctx, be, marker)
+	if err != nil {
+		return "", err
+	}
+	switch {
+	case !hexDigest.MatchString(digest):
+		log.V(1).Info("base image marker holds no digest; refetching")
+	case src.SHA256 != "" && !strings.EqualFold(digest, src.SHA256):
+		log.V(1).Info("base image digest differs from the configured sha256; refetching", "cached", digest, "configured", strings.ToLower(src.SHA256))
+	default:
+		base, err := be.StorageVolLookupByName(ctx, pool, name)
+		switch {
+		case hasCode(err, libvirt.ErrNoStorageVol):
+			return "", deleteVolumeIfPresent(ctx, be, pool, markerName)
+		case err != nil:
+			return "", fmt.Errorf("looking up base volume %s: %w", name, err)
+		}
+		path, err := be.StorageVolGetPath(ctx, base)
+		if err != nil {
+			return "", fmt.Errorf("reading the path of base volume %s: %w", name, err)
+		}
+		return path, nil
+	}
+	return "", errors.Join(deleteVolumeIfPresent(ctx, be, pool, markerName), deleteVolumeIfPresent(ctx, be, pool, name))
+}
+
+// readMarker returns the digest a marker volume holds, lowercased and
+// stripped of padding. The volume is 64 bytes; the read is capped anyway.
+func readMarker(ctx context.Context, be backend, marker libvirt.StorageVol) (string, error) {
+	w := &cappedWriter{limit: markerReadCap}
+	if err := be.StorageVolDownload(ctx, marker, w, 0, 0, 0); err != nil {
+		return "", fmt.Errorf("reading marker %s: %w", marker.Name, err)
+	}
+	return strings.ToLower(strings.TrimRight(string(w.buf), "\x00 \r\n\t")), nil
+}
+
+// cappedWriter keeps the first limit bytes and accepts the rest without
+// storing them, so a download into it is bounded whatever its length.
+type cappedWriter struct {
+	limit int
+	buf   []byte
+}
+
+func (w *cappedWriter) Write(b []byte) (int, error) {
+	if room := w.limit - len(w.buf); room > 0 {
+		w.buf = append(w.buf, b[:min(room, len(b))]...)
+	}
+	return len(b), nil
+}
+
+// deleteVolumeIfPresent removes the named volume; one that is already gone
+// is success.
+func deleteVolumeIfPresent(ctx context.Context, be backend, pool libvirt.StoragePool, name string) error {
+	vol, err := be.StorageVolLookupByName(ctx, pool, name)
+	if hasCode(err, libvirt.ErrNoStorageVol) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("looking up volume %s: %w", name, err)
+	}
+	if err := be.StorageVolDelete(ctx, vol, 0); err != nil && !hasCode(err, libvirt.ErrNoStorageVol) {
+		return fmt.Errorf("deleting volume %s: %w", name, err)
+	}
+	return nil
+}
+
+// fetchImage streams the image from its URL into a new base volume and then
+// writes the marker, under the kvm2.download span (spec entry 17). The
+// upload goes over a connection of its own (spec entry 18): abandoning a
+// call kills the connection it ran on, and a 20-minute upload must not die
+// because a poll for another Proxy was cancelled on the pooled one. The
+// marker is written last, on the pooled connection, after
+// StoragePoolRefresh made libvirt read the file's real format; its presence
+// is what marks the base complete.
+//
+// Every exit between the base volume's creation and the marker deletes the
+// base volume (best effort): the next attempt would delete it anyway on
+// finding no marker, and removing it now frees the space at once.
+func (p *Provider) fetchImage(ctx context.Context, host provider.KVMHost, pool libvirt.StoragePool, be backend, src imageSource) (libvirt.StorageVol, error) {
+	name := src.volumeName()
+	ctx, cancel := context.WithTimeout(ctx, defaultBudgets.transfer)
+	defer cancel()
+	ctx, span := tracing.StartSpan(ctx, tracing.Tracer(p.tracing...), "kvm2.download",
+		trace.WithSpanKind(trace.SpanKindClient),
+		trace.WithAttributes(
+			attribute.String("kvm2.host", host.Name),
+			attribute.String("kvm2.volume", name),
+			attribute.String("kvm2.url_host", urlHost(src.URL)),
+			attribute.Bool("kvm2.sha256_configured", src.SHA256 != ""),
+		))
+	defer span.End()
+	// Taken after the span starts, so the lines carry its ID.
+	log := p.logger(ctx).WithValues("host", host.Name, "volume", name)
+	start := time.Now()
+
+	resp, err := p.do(ctx, http.MethodGet, src.URL)
+	if err == nil {
+		err = checkResponse(src.URL, resp)
+	}
+	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
+		return libvirt.StorageVol{}, err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	size := resp.ContentLength
+	span.SetAttributes(attribute.Int64("kvm2.bytes", size))
+	log.V(1).Info("fetching base image into host pool", "url", src.URL, "bytes", size)
+
+	vol, digest, err := p.uploadImage(ctx, host, pool, be, src, resp.Body, size)
+	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
+		return libvirt.StorageVol{}, err
+	}
+	if err := be.StoragePoolRefresh(ctx, pool, 0); err != nil {
+		err = fmt.Errorf("refreshing storage pool %s after the upload: %w", pool.Name, err)
+		p.discardVolume(ctx, log, be, pool, name)
+		span.SetStatus(codes.Error, err.Error())
+		return libvirt.StorageVol{}, err
+	}
+	if err := writeMarker(ctx, be, pool, src.markerName(), digest); err != nil {
+		p.discardVolume(ctx, log, be, pool, name)
+		span.SetStatus(codes.Error, err.Error())
+		return libvirt.StorageVol{}, err
+	}
+	log.V(1).Info("base image cached", "bytes", size, "duration", time.Since(start), "sha256", digest)
+	return vol, nil
+}
+
+// uploadImage creates the base volume and streams body into it over a
+// dedicated connection with a running sha256, then checks length and
+// digest. On any failure the volume is discarded and the error says why:
+// a body that ended early is Transient, a digest that differs from the
+// configured one is Permanent.
+func (p *Provider) uploadImage(ctx context.Context, host provider.KVMHost, pool libvirt.StoragePool, be backend, src imageSource, body io.Reader, size int64) (libvirt.StorageVol, string, error) {
+	name := src.volumeName()
+	xmlDoc, err := buildVolumeXML(volumeSpec{Name: name, CapacityBytes: size, Format: src.Format})
+	if err != nil {
+		return libvirt.StorageVol{}, "", err
+	}
+	fresh, err := p.hosts.dialFresh(ctx, host)
+	if err != nil {
+		return libvirt.StorageVol{}, "", fmt.Errorf("dialling host %s for the image upload: %w", host.Name, err)
+	}
+	defer func() { _ = fresh.Close() }()
+	vol, err := fresh.StorageVolCreateXML(ctx, pool, xmlDoc, 0)
+	if err != nil {
+		return libvirt.StorageVol{}, "", fmt.Errorf("creating base volume %s: %w", name, err)
+	}
+
+	hash := sha256.New()
+	counted := &countingReader{r: io.TeeReader(body, hash)}
+	uploadErr := fresh.StorageVolUpload(ctx, vol, counted, 0, uint64(size), 0)
+	read, readErr := counted.state()
+	var digest string
+	if uploadErr == nil {
+		// Only then has the upload stopped reading; an abandoned call's
+		// goroutine may still be writing into the hash.
+		digest = hex.EncodeToString(hash.Sum(nil))
+	}
+	switch {
+	case readErr != nil:
+		err = fmt.Errorf("%w: image %s: the response ended after %d of %d bytes: %w", provider.ErrTransient, src.URL, read, size, readErr)
+	case uploadErr != nil:
+		err = fmt.Errorf("uploading base volume %s: %w", name, uploadErr)
+	case read != size:
+		err = transientf("image %s: short read, %d of %d bytes", src.URL, read, size)
+	case src.SHA256 != "" && !strings.EqualFold(digest, src.SHA256):
+		err = permanentf("image %s: sha256 %s does not match the configured %s", src.URL, digest, strings.ToLower(src.SHA256))
+	}
+	if err != nil {
+		p.discardVolume(ctx, p.logger(ctx).WithValues("host", host.Name, "volume", name), be, pool, name)
+		return libvirt.StorageVol{}, "", err
+	}
+	return vol, digest, nil
+}
+
+// countingReader counts what the upload consumed and keeps the first
+// non-EOF error the source returned. The adapter can return from an
+// abandoned upload while go-libvirt still reads, so the fields are guarded.
+type countingReader struct {
+	r   io.Reader
+	mu  sync.Mutex
+	n   int64
+	err error
+}
+
+func (c *countingReader) Read(b []byte) (int, error) {
+	n, err := c.r.Read(b)
+	c.mu.Lock()
+	c.n += int64(n)
+	if err != nil && err != io.EOF && c.err == nil {
+		c.err = err
+	}
+	c.mu.Unlock()
+	return n, err
+}
+
+func (c *countingReader) state() (int64, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.n, c.err
+}
+
+// discardVolume removes a base volume a failed fetch left behind. It runs
+// with ctx's values but not its cancellation: the fetch may have failed
+// because ctx ended, and the adapter refuses calls on a finished ctx. The
+// budget bounds it on its own. Failure is logged, not returned: the next
+// attempt finds the volume without a marker and deletes it then.
+func (p *Provider) discardVolume(ctx context.Context, log logr.Logger, be backend, pool libvirt.StoragePool, name string) {
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultBudgets.rpc)
+	defer cancel()
+	if err := deleteVolumeIfPresent(ctx, be, pool, name); err != nil {
+		log.V(1).Info("incomplete base volume left behind; the next attempt removes it", "error", err.Error())
+	}
+}
+
+// writeMarker creates the raw marker volume and uploads the digest into it.
+// A marker that could not be filled is removed again: an empty marker next
+// to a complete base would read as debris and cost a refetch.
+func writeMarker(ctx context.Context, be backend, pool libvirt.StoragePool, name, digest string) error {
+	xmlDoc, err := buildVolumeXML(volumeSpec{Name: name, CapacityBytes: markerCapacityBytes, Format: provider.KVMFormatRaw})
+	if err != nil {
+		return err
+	}
+	vol, err := be.StorageVolCreateXML(ctx, pool, xmlDoc, 0)
+	if err != nil {
+		return fmt.Errorf("creating marker %s: %w", name, err)
+	}
+	if err := be.StorageVolUpload(ctx, vol, strings.NewReader(digest), 0, uint64(len(digest)), 0); err != nil {
+		_ = deleteVolumeIfPresent(ctx, be, pool, name)
+		return fmt.Errorf("writing marker %s: %w", name, err)
+	}
+	return nil
+}
+
+// do issues one request bound to ctx. Accept-Encoding is fixed to identity
+// because a transparently decompressed body reports no Content-Length. A
+// transport error is Transient.
+func (p *Provider) do(ctx context.Context, method, rawURL string) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, method, rawURL, nil)
+	if err != nil {
+		return nil, permanentf("image %s: %v", rawURL, err)
+	}
+	req.Header.Set("Accept-Encoding", "identity")
+	resp, err := p.http.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("%w: fetching image %s: %w", provider.ErrTransient, rawURL, err)
+	}
+	return resp, nil
+}
+
+// checkResponse classifies the response at the source (spec entry 13) and
+// closes its body when it is unusable. 404, 403 and 410, and every other
+// 4xx, mean the catalogue entry is wrong; 5xx and 429 are worth a retry. A
+// missing Content-Length is Permanent: the volume's capacity must be
+// declared before the upload, and a mirror that only chunks is one the
+// catalogue must not point at.
+func checkResponse(rawURL string, resp *http.Response) error {
+	var err error
+	code := resp.StatusCode
+	switch {
+	case code >= 200 && code < 300:
+	case code == http.StatusTooManyRequests || code >= 500:
+		err = transientf("image %s: HTTP %s", rawURL, resp.Status)
+	case code >= 400:
+		err = permanentf("image %s: HTTP %s", rawURL, resp.Status)
+	default:
+		err = transientf("image %s: unexpected HTTP %s", rawURL, resp.Status)
+	}
+	if err == nil && resp.ContentLength < 0 {
+		err = permanentf("image %s: response has no Content-Length; the volume capacity must be known before upload", rawURL)
+	}
+	if err == nil && resp.ContentLength == 0 {
+		err = permanentf("image %s: response is empty (Content-Length 0)", rawURL)
+	}
+	if err != nil {
+		_ = resp.Body.Close()
+	}
+	return err
+}
+
+// urlHost is the only part of the URL a span carries (contract §9).
+func urlHost(rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return ""
+	}
+	return u.Host
+}
+
+// imageSizeIfUncached returns the image's Content-Length when the base is
+// not cached on the host, and 0 when the marker is present, for the quota
+// precheck of spec entry 13. It never downloads: HEAD first, and a GET whose
+// body is closed at once when HEAD answers anything but 2xx, since servers
+// refuse HEAD with 405, 403 or 501 alike; the GET's status is what gets
+// classified. A marker is taken at face value here; ensureImage decides
+// whether it still matches.
+func (p *Provider) imageSizeIfUncached(ctx context.Context, be backend, pool libvirt.StoragePool, src imageSource) (int64, error) {
+	_, err := be.StorageVolLookupByName(ctx, pool, src.markerName())
+	switch {
+	case err == nil:
+		return 0, nil
+	case !hasCode(err, libvirt.ErrNoStorageVol):
+		return 0, fmt.Errorf("looking up marker %s: %w", src.markerName(), err)
+	}
+	ctx, cancel := context.WithTimeout(ctx, defaultBudgets.rpc)
+	defer cancel()
+	resp, err := p.do(ctx, http.MethodHead, src.URL)
+	if err != nil {
+		return 0, err
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		if resp, err = p.do(ctx, http.MethodGet, src.URL); err != nil {
+			return 0, err
+		}
+		_ = resp.Body.Close()
+	}
+	if err := checkResponse(src.URL, resp); err != nil {
+		return 0, err
+	}
+	return resp.ContentLength, nil
+}
+
+// checkPoolSpace is the precheck of spec entry 13: the pool must be running
+// and have needed bytes available, or Create returns ErrQuotaExceeded and
+// the reconciler waits QuotaRetry instead of failing the volume creation.
+func checkPoolSpace(ctx context.Context, be backend, pool libvirt.StoragePool, poolName string, needed uint64) error {
+	state, _, _, available, err := be.StoragePoolGetInfo(ctx, pool)
+	if err != nil {
+		return fmt.Errorf("reading storage pool %s: %w", poolName, err)
+	}
+	if s := libvirt.StoragePoolState(state); s != libvirt.StoragePoolRunning {
+		return permanentf("storage pool %s is not running (state %s)", poolName, s)
+	}
+	if available < needed {
+		return fmt.Errorf("%w: storage pool %s has %d bytes available, %d needed", provider.ErrQuotaExceeded, poolName, available, needed)
+	}
+	return nil
+}
+
+// neededBytes is what one Create commits to the pool: the image when it is
+// not cached yet (0 otherwise), the overlay's capacity and the seed reserve.
+func neededBytes(imageBytes int64, diskSizeGB int64) uint64 {
+	return uint64(max(imageBytes, 0)) + uint64(max(diskSizeGB, 0))<<30 + seedReserveBytes
+}

internal/provider/kvm2/kvm2.go

@@ -0,0 +1,243 @@
+// Package kvm2 implements the provider contract on libvirt/KVM hypervisors
+// reached over SSH: the host's libvirt unix socket is forwarded through the
+// SSH connection and driven with go-libvirt's RPC client. The design is in
+// docs/kvm2-provider-spec.md.
+package kvm2
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/go-logr/logr"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+const (
+	defaultSocket     = "/var/run/libvirt/libvirt-sock"
+	defaultPool       = "default"
+	defaultDiskSizeGB = 10
+)
+
+// Provider is one configured kvm2 entry: a set of hosts sharing one SSH key
+// and one image and machine-type catalogue.
+type Provider struct {
+	name    string
+	cfg     provider.KVMConfig
+	now     func() time.Time
+	hosts   *hostTable
+	http    *http.Client
+	tracing []tracing.Option
+
+	// runningWait caps the wait for a started domain to report running,
+	// polled every runningPoll (spec entry 18).
+	runningWait time.Duration
+	runningPoll time.Duration
+
+	// imageLocks serialises image fetches per (host, volume), spec entry 19.
+	imageLocks *keyedLocks
+
+	// nameLocks serialises Create and Delete per instance name, spec entry
+	// 19, so two Deletes never race on destroy and undefine and a Delete
+	// cannot interleave with a Create's steps.
+	nameLocks *keyedLocks
+}
+
+// providerOption adjusts what newProvider builds; tests inject an HTTP
+// client bound to a test server, a span recorder and a shorter running
+// wait.
+type providerOption func(*Provider)
+
+func withHTTPClient(c *http.Client) providerOption {
+	return func(p *Provider) { p.http = c }
+}
+
+func withTracing(opts ...tracing.Option) providerOption {
+	return func(p *Provider) { p.tracing = opts }
+}
+
+// withRunningWait shrinks the poll interval and the cap of the wait for a
+// domain to run.
+func withRunningWait(poll, cap time.Duration) providerOption {
+	return func(p *Provider) { p.runningPoll, p.runningWait = poll, cap }
+}
+
+var _ provider.Provider = (*Provider)(nil)
+
+// New builds a Provider from pc.KVM2. It never dials a host: the credential
+// files are only checked for existence, and a missing one is a warning, not
+// an error, because the Secret mount is optional and may land later.
+func New(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+	cfg := provider.KVMConfig{}
+	if pc.KVM2 != nil {
+		cfg = *pc.KVM2
+	}
+	cfg = withDefaults(cfg)
+	p := newProvider(pc.Name, cfg, newLibvirtDialer(pc.Name, cfg))
+	log := p.logger(ctx)
+	if _, err := os.Stat(p.cfg.SSHKeyFile); err != nil {
+		log.Info("WARNING: ssh key file is not readable; every call fails until it is", "path", p.cfg.SSHKeyFile, "reason", err.Error())
+	}
+	switch {
+	case p.cfg.InsecureIgnoreHostKey:
+		log.Info("WARNING: insecureIgnoreHostKey is set; host keys are not verified")
+	default:
+		if _, err := os.Stat(p.cfg.KnownHostsFile); err != nil {
+			log.Info("WARNING: known_hosts file is not readable; every call fails until it is", "path", p.cfg.KnownHostsFile, "reason", err.Error())
+		}
+	}
+	return p, nil
+}
+
+// newProvider is the constructor tests use: d hands out the backends, the
+// production one being newLibvirtDialer. Without options the image
+// downloads use http.DefaultClient (system CA roots, no client timeout: the
+// per-request ctx carries the transfer budget).
+func newProvider(name string, cfg provider.KVMConfig, d dialer, opts ...providerOption) *Provider {
+	p := &Provider{
+		name:        name,
+		cfg:         withDefaults(cfg),
+		now:         time.Now,
+		hosts:       newHostTable(d),
+		http:        http.DefaultClient,
+		runningWait: defaultRunningWait,
+		runningPoll: defaultRunningPoll,
+		imageLocks:  newKeyedLocks(),
+		nameLocks:   newKeyedLocks(),
+	}
+	for _, opt := range opts {
+		opt(p)
+	}
+	if p.http == nil {
+		p.http = http.DefaultClient
+	}
+	return p
+}
+
+// withDefaults returns cfg with every optional field filled in, so the rest
+// of the package never re-checks for an empty value. The hosts and images
+// are copied, not shared with the caller.
+func withDefaults(cfg provider.KVMConfig) provider.KVMConfig {
+	if cfg.DiskSizeGB == 0 {
+		cfg.DiskSizeGB = defaultDiskSizeGB
+	}
+	hosts := make([]provider.KVMHost, len(cfg.Hosts))
+	for i, h := range cfg.Hosts {
+		if h.Socket == "" {
+			h.Socket = defaultSocket
+		}
+		if h.Pool == "" {
+			h.Pool = defaultPool
+		}
+		hosts[i] = h
+	}
+	cfg.Hosts = hosts
+	images := make(map[string]provider.KVMImage, len(cfg.Images))
+	for k, img := range cfg.Images {
+		if img.Format == "" {
+			img.Format = provider.KVMFormatQcow2
+		}
+		if img.Firmware == "" {
+			img.Firmware = provider.KVMFirmwareUEFI
+		}
+		images[k] = img
+	}
+	cfg.Images = images
+	return cfg
+}
+
+func (p *Provider) logger(ctx context.Context) logr.Logger {
+	return logf.FromContext(ctx).WithName("kvm2").WithValues("provider", p.name)
+}
+
+// host resolves a placement.zone or a providerID's host segment to a
+// configured host. An empty zone is allowed only when exactly one host is
+// configured.
+func (p *Provider) host(zone string) (provider.KVMHost, error) {
+	if zone == "" {
+		if len(p.cfg.Hosts) == 1 {
+			return p.cfg.Hosts[0], nil
+		}
+		return provider.KVMHost{}, permanentf("placement.zone is required when kvm2.hosts lists more than one host")
+	}
+	for _, h := range p.cfg.Hosts {
+		if h.Name == zone {
+			return h, nil
+		}
+	}
+	return provider.KVMHost{}, permanentf("host %q is not in kvm2.hosts", zone)
+}
+
+// wrapErr is the single exit for errors: it classifies err and wraps it with
+// the configured provider name, op and id.
+func (p *Provider) wrapErr(op, id string, err error) error {
+	return provider.Wrap(classify(op, err), op, p.name, id, err)
+}
+
+// lockName takes the per-name lock of spec entry 19, or gives up with ctx,
+// so a caller whose ctx ends while another caller's Create is downloading
+// an image returns promptly.
+func (p *Provider) lockName(ctx context.Context, name string) (func(), error) {
+	release, err := p.nameLocks.acquire(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("%w: waiting for the lock on %s: %w", provider.ErrTransient, name, err)
+	}
+	return release, nil
+}
+
+// keyedLocks is a set of one-slot semaphores by key (spec entry 19): a
+// semaphore rather than a mutex so a waiter can give up with its ctx. Each
+// entry counts the callers holding or waiting for it and is dropped from
+// the map at zero, so the map does not grow with every name ever seen while
+// two waiters on one key still always share one semaphore.
+type keyedLocks struct {
+	mu    sync.Mutex
+	locks map[string]*keyedLock
+}
+
+type keyedLock struct {
+	sem  chan struct{}
+	refs int
+}
+
+func newKeyedLocks() *keyedLocks {
+	return &keyedLocks{locks: make(map[string]*keyedLock)}
+}
+
+// acquire takes the lock for key, or gives up with ctx's error. The
+// returned func releases it and must be called exactly once.
+func (k *keyedLocks) acquire(ctx context.Context, key string) (func(), error) {
+	k.mu.Lock()
+	l, ok := k.locks[key]
+	if !ok {
+		l = &keyedLock{sem: make(chan struct{}, 1)}
+		k.locks[key] = l
+	}
+	l.refs++
+	k.mu.Unlock()
+	select {
+	case l.sem <- struct{}{}:
+		return func() {
+			<-l.sem
+			k.unref(key, l)
+		}, nil
+	case <-ctx.Done():
+		k.unref(key, l)
+		return nil, ctx.Err()
+	}
+}
+
+func (k *keyedLocks) unref(key string, l *keyedLock) {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	l.refs--
+	if l.refs == 0 {
+		delete(k.locks, key)
+	}
+}

internal/provider/kvm2/kvm2_integration_test.go

@@ -0,0 +1,590 @@
+//go:build integration
+
+package kvm2
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr"
+	"github.com/go-logr/logr/funcr"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	"k8s.io/apimachinery/pkg/types"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+// The real-hypervisor test of spec entry 21. It runs only with
+// -tags=integration and skips itself unless KVM2_TEST_* names a host; see
+// docs/testing.md for the variables and what one run asserts.
+const (
+	integrationProvider = "kvm2-integration"
+	integrationHost     = "kvm2-test"
+	integrationImage    = "test-image"
+	integrationMachine  = "small"
+	integrationPort     = 3128
+
+	// integrationCloudInit is the user-data the VM boots with. Nothing
+	// asserts on the guest running it; it exists so the sentinel check can
+	// prove that no span attribute carries the cloud-init text.
+	integrationCloudInit = "#cloud-config\nruncmd:\n  - echo kvm2-integration\n"
+
+	defaultIntegrationTimeout = 20 * time.Minute
+	defaultIntegrationLogLvl  = 1
+	getPollInterval           = 5 * time.Second
+	cleanupBudget             = 5 * time.Minute
+)
+
+type integrationEnv struct {
+	host, user, sshKey, knownHosts, bridge, imageURL string
+	imageSHA256, pool, socket                        string
+	timeout                                          time.Duration
+	keep                                             bool
+	logLevel                                         int
+}
+
+// loadIntegrationEnv reads the KVM2_TEST_* variables, skipping the test
+// under -short or when a required one is missing.
+func loadIntegrationEnv(t *testing.T) integrationEnv {
+	t.Helper()
+	if testing.Short() {
+		t.Skip("real-hypervisor test skipped under -short")
+	}
+	env := integrationEnv{timeout: defaultIntegrationTimeout, logLevel: defaultIntegrationLogLvl}
+	required := []struct {
+		name string
+		dst  *string
+	}{
+		{"KVM2_TEST_HOST", &env.host},
+		{"KVM2_TEST_USER", &env.user},
+		{"KVM2_TEST_SSH_KEY", &env.sshKey},
+		{"KVM2_TEST_BRIDGE", &env.bridge},
+		{"KVM2_TEST_IMAGE_URL", &env.imageURL},
+	}
+	for _, r := range required {
+		v := os.Getenv(r.name)
+		if v == "" {
+			t.Skipf("%s is not set; the kvm2 integration test needs a real hypervisor (see docs/testing.md)", r.name)
+		}
+		*r.dst = v
+	}
+	env.knownHosts = os.Getenv("KVM2_TEST_KNOWN_HOSTS")
+	env.imageSHA256 = os.Getenv("KVM2_TEST_IMAGE_SHA256")
+	env.pool = os.Getenv("KVM2_TEST_POOL")
+	env.socket = os.Getenv("KVM2_TEST_SOCKET")
+	env.keep = os.Getenv("KVM2_TEST_KEEP") != ""
+	if v := os.Getenv("KVM2_TEST_TIMEOUT"); v != "" {
+		d, err := time.ParseDuration(v)
+		if err != nil {
+			t.Fatalf("KVM2_TEST_TIMEOUT=%q: %v", v, err)
+		}
+		env.timeout = d
+	}
+	if v := os.Getenv("KVM2_TEST_LOG_LEVEL"); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 0 {
+			t.Fatalf("KVM2_TEST_LOG_LEVEL=%q: want a non-negative integer", v)
+		}
+		env.logLevel = n
+	}
+	return env
+}
+
+// providerConfig is what the operator would load for this host: one host,
+// one image, one machine type. Host keys are verified only when
+// KVM2_TEST_KNOWN_HOSTS is set.
+func (e integrationEnv) providerConfig() provider.ProviderConfig {
+	return provider.ProviderConfig{
+		Name: integrationProvider,
+		Type: "kvm2",
+		KVM2: &provider.KVMConfig{
+			SSHKeyFile:            e.sshKey,
+			KnownHostsFile:        e.knownHosts,
+			InsecureIgnoreHostKey: e.knownHosts == "",
+			Images: map[string]provider.KVMImage{
+				integrationImage: {URL: e.imageURL, SHA256: e.imageSHA256},
+			},
+			MachineTypes: map[string]provider.KVMMachineType{
+				integrationMachine: {VCPUs: 1, MemoryMiB: 1024},
+			},
+			Hosts: []provider.KVMHost{{
+				Name:    integrationHost,
+				Address: e.host,
+				User:    e.user,
+				Socket:  e.socket,
+				Pool:    e.pool,
+				Bridge:  e.bridge,
+			}},
+		},
+	}
+}
+
+// integrationLog routes the provider's logr output into t.Log through funcr
+// at the chosen verbosity, keeps every line for assertions, and strips the
+// key/value pairs that are constant for the run (stated once up front) so
+// each line stays short. Lines arriving after the test ended are dropped:
+// t.Log panics then.
+type integrationLog struct {
+	t *testing.T
+
+	mu    sync.Mutex
+	drop  []string
+	lines []string
+	done  bool
+}
+
+func newIntegrationLog(t *testing.T, level int, drop ...string) (*integrationLog, logr.Logger) {
+	t.Helper()
+	sink := &integrationLog{t: t, drop: drop}
+	t.Cleanup(func() {
+		sink.mu.Lock()
+		defer sink.mu.Unlock()
+		sink.done = true
+	})
+	log := funcr.New(func(prefix, args string) { sink.write(prefix, args) }, funcr.Options{Verbosity: level})
+	return sink, log
+}
+
+// dropValue adds a rendered pair (`"key"="value"`) to strip from later lines.
+func (l *integrationLog) dropValue(key, value string) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.drop = append(l.drop, fmt.Sprintf("%q=%q", key, value))
+}
+
+func (l *integrationLog) write(prefix, args string) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.lines = append(l.lines, args)
+	if l.done {
+		return
+	}
+	for _, d := range l.drop {
+		args = strings.ReplaceAll(args, " "+d, "")
+	}
+	l.t.Logf("%s %s %s", time.Now().Format("15:04:05.000"), prefix, args)
+}
+
+func (l *integrationLog) matching(substr string) []string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	var out []string
+	for _, line := range l.lines {
+		if strings.Contains(line, substr) {
+			out = append(out, line)
+		}
+	}
+	return out
+}
+
+// newIntegrationProvider builds the provider the way New does, plus the span
+// recorder's tracer on both places the package starts spans: the Provider
+// (kvm2.download) and the dialer (kvm2.dial and, through observe, every
+// kvm2.<Method>). newLibvirtDialer has no option for it, so the field is set
+// directly; that is what makes the kvm2.* spans land in the same recorder
+// as provider.WithTracing's provider.* spans.
+func newIntegrationProvider(pc provider.ProviderConfig, tp *sdktrace.TracerProvider) *Provider {
+	cfg := withDefaults(*pc.KVM2)
+	opt := tracing.WithTracerProvider(tp)
+	d := newLibvirtDialer(pc.Name, cfg)
+	d.tracing = []tracing.Option{opt}
+	return newProvider(pc.Name, cfg, d, withTracing(opt))
+}
+
+// randomUID is a UUID-shaped string, so the run's domain never collides with
+// a previous run's leftovers or with a real Proxy's.
+func randomUID(t *testing.T) string {
+	t.Helper()
+	var b [16]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		t.Fatalf("reading random bytes: %v", err)
+	}
+	h := hex.EncodeToString(b[:])
+	return h[0:8] + "-" + h[8:12] + "-" + h[12:16] + "-" + h[16:20] + "-" + h[20:32]
+}
+
+// stepDone logs one lifecycle step with its duration; the log is the record
+// of how long a cold image download or a boot took on this host.
+func stepDone(t *testing.T, step string, start time.Time) {
+	t.Helper()
+	t.Logf("step %s: %s", step, time.Since(start).Round(time.Millisecond))
+}
+
+// waitUntil polls cond every interval until it is true or ctx ends.
+func waitUntil(ctx context.Context, t *testing.T, interval time.Duration, what string, cond func() bool) {
+	t.Helper()
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for !cond() {
+		select {
+		case <-ticker.C:
+		case <-ctx.Done():
+			t.Fatalf("gave up waiting for %s: %v", what, ctx.Err())
+		}
+	}
+}
+
+// domainCount lists every domain on the host over a dedicated connection.
+func domainCount(ctx context.Context, t *testing.T, p *Provider, host provider.KVMHost) int {
+	t.Helper()
+	be, err := p.hosts.dialFresh(ctx, host)
+	if err != nil {
+		t.Fatalf("dialling %s to count domains: %v", host.Name, err)
+	}
+	defer func() { _ = be.Close() }()
+	doms, _, err := be.ConnectListAllDomains(ctx, 1, libvirt.ConnectListDomainsActive|libvirt.ConnectListDomainsInactive)
+	if err != nil {
+		t.Fatalf("listing domains on %s: %v", host.Name, err)
+	}
+	return len(doms)
+}
+
+// poolVolumes reports, for each name, whether the host's pool holds a volume
+// of that name, over a dedicated connection. Any error other than
+// ErrNoStorageVol fails the test.
+func poolVolumes(ctx context.Context, t *testing.T, p *Provider, host provider.KVMHost, names ...string) map[string]bool {
+	t.Helper()
+	be, err := p.hosts.dialFresh(ctx, host)
+	if err != nil {
+		t.Fatalf("dialling %s to inspect the pool: %v", host.Name, err)
+	}
+	defer func() { _ = be.Close() }()
+	pool, err := be.StoragePoolLookupByName(ctx, host.Pool)
+	if err != nil {
+		t.Fatalf("looking up pool %s: %v", host.Pool, err)
+	}
+	out := make(map[string]bool, len(names))
+	for _, name := range names {
+		_, err := be.StorageVolLookupByName(ctx, pool, name)
+		switch {
+		case err == nil:
+			out[name] = true
+		case hasCode(err, libvirt.ErrNoStorageVol):
+			out[name] = false
+		default:
+			t.Fatalf("looking up volume %s in pool %s: %v", name, host.Pool, err)
+		}
+	}
+	return out
+}
+
+func TestKVM2_lifecycle(t *testing.T) {
+	env := loadIntegrationEnv(t)
+	pc := env.providerConfig()
+	sink, log := newIntegrationLog(t, env.logLevel,
+		fmt.Sprintf("%q=%q", "provider", integrationProvider),
+		fmt.Sprintf("%q=%q", "host", integrationHost))
+	ctx, cancel := context.WithTimeout(logf.IntoContext(context.Background(), log), env.timeout)
+	defer cancel()
+
+	t.Logf("provider=%s host=%s address=%s user=%s bridge=%s image=%s budget=%s log level %d; provider, host and name are left off every provider log line",
+		integrationProvider, integrationHost, env.host, env.user, env.bridge, env.imageURL, env.timeout, env.logLevel)
+	if env.knownHosts == "" {
+		t.Log("KVM2_TEST_KNOWN_HOSTS is not set: host keys are NOT verified for this run")
+	}
+
+	// 1. New returns without dialling.
+	start := time.Now()
+	built, err := New(ctx, pc)
+	elapsed := time.Since(start)
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+	if elapsed > 100*time.Millisecond {
+		t.Errorf("New() took %s, want under 100ms: it must not dial", elapsed)
+	}
+	if lines := sink.matching("libvirt host connected"); len(lines) > 0 {
+		t.Errorf("New() dialled the host before any call: %q", lines)
+	}
+	stepDone(t, "1 New", start)
+
+	// 2. Check reports the host Ready with details.
+	checker, ok := built.(provider.Checker)
+	if !ok {
+		t.Fatalf("New() returned %T, which is not a provider.Checker", built)
+	}
+	start = time.Now()
+	targets := checker.Check(ctx)
+	if len(targets) != 1 {
+		t.Fatalf("Check() returned %d targets, want 1: %+v", len(targets), targets)
+	}
+	ts := targets[0]
+	if !ts.Ready || ts.Reason != provider.ReasonReachable {
+		t.Fatalf("Check() = ready %v reason %q message %q, want Ready/%s (details %v)", ts.Ready, ts.Reason, ts.Message, provider.ReasonReachable, ts.Details)
+	}
+	for _, key := range []string{"hostname", "libvirtVersion", "poolState", "poolAvailableGiB"} {
+		if ts.Details[key] == "" {
+			t.Errorf("Check() details lack %q: %v", key, ts.Details)
+		}
+	}
+	if ts.Details["poolState"] != "running" {
+		t.Errorf("Check() poolState = %q, want running", ts.Details["poolState"])
+	}
+	t.Logf("host %s: hostname=%s libvirt=%s pool=%s (%s) available=%s GiB capacity=%s GiB latency=%s",
+		ts.Name, ts.Details["hostname"], ts.Details["libvirtVersion"], ts.Details["pool"], ts.Details["poolState"],
+		ts.Details["poolAvailableGiB"], ts.Details["poolCapacityGiB"], ts.Latency.Round(time.Millisecond))
+	stepDone(t, "2 Check", start)
+
+	// 3. Every later call goes through the tracing decorator, with the
+	// package's own spans recorded alongside.
+	sr := tracetest.NewSpanRecorder()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+	t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
+	p := newIntegrationProvider(pc, tp)
+	traced := provider.WithTracing(pc.Name, p, tracing.WithTracerProvider(tp))
+	host := p.cfg.Hosts[0]
+
+	// 4. Create.
+	uid := randomUID(t)
+	name := provider.NameFromUID(types.UID(uid))
+	sink.dropValue("name", name)
+	id := formatProviderID(integrationHost, name)
+	req := provider.CreateRequest{
+		Name:      name,
+		UID:       uid,
+		Namespace: "integration",
+		ProxyName: "kvm2-lifecycle",
+		Placement: provider.Placement{Zone: integrationHost, MachineType: integrationMachine, Image: integrationImage},
+		CloudInit: integrationCloudInit,
+		Port:      integrationPort,
+	}
+	t.Logf("domain name %s, uid %s, id %s", name, uid, id)
+
+	deleted := false
+	if env.keep {
+		t.Logf("KVM2_TEST_KEEP is set: the VM stays behind. Clean up on the host with:\n"+
+			"  virsh undefine --nvram %s\n  virsh vol-delete --pool %s %s\n  virsh vol-delete --pool %s %s",
+			name, host.Pool, overlayVolumeName(name), host.Pool, seedVolumeName(name))
+	} else {
+		t.Cleanup(func() {
+			if deleted {
+				return
+			}
+			cctx, ccancel := context.WithTimeout(logf.IntoContext(context.Background(), log), cleanupBudget)
+			defer ccancel()
+			if err := p.Delete(cctx, id); err != nil {
+				t.Errorf("cleanup Delete(%s) error = %v; remove the domain and its volumes by hand", id, err)
+			}
+		})
+	}
+
+	start = time.Now()
+	gotID, err := traced.Create(ctx, req)
+	if err != nil {
+		t.Fatalf("Create() error = %v", err)
+	}
+	if gotID != id {
+		t.Fatalf("Create() = %q, want %q", gotID, id)
+	}
+	createReturned := time.Now()
+	stepDone(t, "4 Create (includes the image download on a cold pool)", start)
+
+	// 5. Create again: same ID, nothing new on the host.
+	start = time.Now()
+	before := domainCount(ctx, t, p, host)
+	again, err := traced.Create(ctx, req)
+	if err != nil {
+		t.Fatalf("second Create() error = %v", err)
+	}
+	if again != id {
+		t.Errorf("second Create() = %q, want %q", again, id)
+	}
+	if after := domainCount(ctx, t, p, host); after != before {
+		t.Errorf("second Create() changed the host's domain count from %d to %d", before, after)
+	}
+	stepDone(t, "5 Create again", start)
+
+	// 6. Get reaches Running with an IPv4.
+	start = time.Now()
+	var inst *provider.Instance
+	waitUntil(ctx, t, getPollInterval, "the domain to be Running with an IPv4", func() bool {
+		var err error
+		inst, err = traced.Get(ctx, id)
+		if err != nil {
+			t.Fatalf("Get(%s) error = %v", id, err)
+		}
+		return inst.State == provider.StateRunning && inst.IP != ""
+	})
+	if inst.UID != uid {
+		t.Errorf("Get() UID = %q, want %q", inst.UID, uid)
+	}
+	if inst.CreatedAt.IsZero() {
+		t.Error("Get() CreatedAt is zero; the metadata mark did not round-trip")
+	} else if skew := time.Since(inst.CreatedAt).Abs(); skew > 5*time.Minute {
+		t.Errorf("Get() CreatedAt = %s, %s from now; want within a few minutes", inst.CreatedAt, skew)
+	}
+	t.Logf("domain running at %s, %s after Create returned (createdAt %s)", inst.IP, time.Since(createReturned).Round(time.Millisecond), inst.CreatedAt.Format(time.RFC3339))
+	stepDone(t, "6 Get until Running", start)
+
+	// 7. ListByTag contains it.
+	start = time.Now()
+	list, err := traced.ListByTag(ctx)
+	if err != nil {
+		t.Fatalf("ListByTag() error = %v", err)
+	}
+	found := false
+	for _, li := range list {
+		if li.ID == id {
+			found = true
+			if li.UID != uid {
+				t.Errorf("ListByTag() entry %s has UID %q, want %q", id, li.UID, uid)
+			}
+		}
+	}
+	if !found {
+		t.Errorf("ListByTag() = %d instances, none with ID %s", len(list), id)
+	}
+	stepDone(t, "7 ListByTag", start)
+
+	// 8. Reachability, logged only: this machine may not route to the bridge.
+	start = time.Now()
+	addr := net.JoinHostPort(inst.IP, "22")
+	if conn, err := net.DialTimeout("tcp", addr, 3*time.Second); err != nil {
+		t.Logf("TCP connect to %s failed (informational; the test host may not route to the bridge): %v", addr, err)
+	} else {
+		_ = conn.Close()
+		t.Logf("TCP connect to %s succeeded", addr)
+	}
+	stepDone(t, "8 TCP connect", start)
+
+	if env.keep {
+		t.Log("KVM2_TEST_KEEP is set: skipping Delete and the pool inspection")
+	} else {
+		// 9. Delete, Get is NotFound, Delete again.
+		start = time.Now()
+		if err := traced.Delete(ctx, id); err != nil {
+			t.Fatalf("Delete() error = %v", err)
+		}
+		deleted = true
+		if _, err := traced.Get(ctx, id); !errors.Is(err, provider.ErrNotFound) {
+			t.Errorf("Get() after Delete error = %v, want ErrNotFound", err)
+		}
+		if err := traced.Delete(ctx, id); err != nil {
+			t.Errorf("second Delete() error = %v, want nil", err)
+		}
+		stepDone(t, "9 Delete, Get, Delete", start)
+
+		// 10. The pool keeps the shared base image and its marker, and holds
+		// no per-VM volume.
+		start = time.Now()
+		base := baseVolumeName(integrationImage, provider.KVMFormatQcow2)
+		marker := markerVolumeName(base)
+		overlay, seed := overlayVolumeName(name), seedVolumeName(name)
+		present := poolVolumes(ctx, t, p, host, base, marker, overlay, seed)
+		for _, want := range []string{base, marker} {
+			if !present[want] {
+				t.Errorf("pool %s lacks %s after Delete; the base image cache must stay", host.Pool, want)
+			}
+		}
+		for _, gone := range []string{overlay, seed} {
+			if present[gone] {
+				t.Errorf("pool %s still holds %s after Delete", host.Pool, gone)
+			}
+		}
+		stepDone(t, "10 pool inspection", start)
+	}
+
+	// 11. Spans: the package's spans are children of the decorator's.
+	assertLifecycleSpans(t, sr.Ended())
+}
+
+// assertLifecycleSpans checks that at least one provider.create span has a
+// kvm2.DomainDefineXML and a kvm2.dial child (the first Create dials the
+// pooled connection and defines the domain), and that no attribute of any
+// span carries the cloud-init text.
+func assertLifecycleSpans(t *testing.T, spans []sdktrace.ReadOnlySpan) {
+	t.Helper()
+	var creates []sdktrace.ReadOnlySpan
+	for _, s := range spans {
+		if s.Name() == "provider.create" {
+			creates = append(creates, s)
+		}
+	}
+	if len(creates) == 0 {
+		t.Fatalf("no provider.create span among %d spans: %v", len(spans), spanNames(spans))
+	}
+	satisfied := false
+	for _, create := range creates {
+		parent := create.SpanContext().SpanID()
+		var children, define, dial int
+		for _, s := range spans {
+			if s.Parent().SpanID() != parent || !strings.HasPrefix(s.Name(), "kvm2.") {
+				continue
+			}
+			children++
+			switch s.Name() {
+			case "kvm2.DomainDefineXML":
+				define++
+			case "kvm2.dial":
+				dial++
+			}
+		}
+		t.Logf("provider.create span %s: %d kvm2.* children (%d DomainDefineXML, %d dial)", parent, children, define, dial)
+		if define > 0 && dial > 0 {
+			satisfied = true
+		}
+	}
+	if !satisfied {
+		t.Errorf("no provider.create span has both a kvm2.DomainDefineXML and a kvm2.dial child; spans: %v", spanNames(spans))
+	}
+	for _, s := range spans {
+		for _, a := range s.Attributes() {
+			v := a.Value.Emit()
+			if strings.Contains(v, integrationCloudInit) || strings.Contains(v, "#cloud-config") {
+				t.Errorf("span %s attribute %s carries the cloud-init text", s.Name(), a.Key)
+			}
+		}
+	}
+}
+
+// TestKVM2Check_realHost runs the health check against the host and creates
+// nothing.
+func TestKVM2Check_realHost(t *testing.T) {
+	env := loadIntegrationEnv(t)
+	pc := env.providerConfig()
+	_, log := newIntegrationLog(t, env.logLevel,
+		fmt.Sprintf("%q=%q", "provider", integrationProvider),
+		fmt.Sprintf("%q=%q", "host", integrationHost))
+	ctx, cancel := context.WithTimeout(logf.IntoContext(context.Background(), log), env.timeout)
+	defer cancel()
+	if env.knownHosts == "" {
+		t.Log("KVM2_TEST_KNOWN_HOSTS is not set: host keys are NOT verified for this run")
+	}
+
+	built, err := New(ctx, pc)
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+	checker, ok := built.(provider.Checker)
+	if !ok {
+		t.Fatalf("New() returned %T, which is not a provider.Checker", built)
+	}
+	targets := checker.Check(ctx)
+	if len(targets) != 1 {
+		t.Fatalf("Check() returned %d targets, want 1: %+v", len(targets), targets)
+	}
+	ts := targets[0]
+	if !ts.Ready || ts.Reason != provider.ReasonReachable {
+		t.Fatalf("Check() = ready %v reason %q message %q, want Ready/%s (details %v)", ts.Ready, ts.Reason, ts.Message, provider.ReasonReachable, ts.Details)
+	}
+	if ts.Details["poolState"] != "running" {
+		t.Errorf("Check() poolState = %q, want running", ts.Details["poolState"])
+	}
+	t.Logf("host %s (%s): hostname=%s libvirt=%s pool=%s (%s) available=%s GiB latency=%s",
+		ts.Name, ts.Address, ts.Details["hostname"], ts.Details["libvirtVersion"], ts.Details["pool"], ts.Details["poolState"],
+		ts.Details["poolAvailableGiB"], ts.Latency.Round(time.Millisecond))
+}

internal/provider/kvm2/kvm2_test.go

@@ -0,0 +1,349 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/go-logr/logr/funcr"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// captureContext collects funcr output at V(2). The sink is locked because
+// Check logs from one goroutine per host.
+func captureContext() (context.Context, *[]string) {
+	lines := &[]string{}
+	var mu sync.Mutex
+	log := funcr.New(func(prefix, args string) {
+		mu.Lock()
+		defer mu.Unlock()
+		*lines = append(*lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: 2})
+	return logf.IntoContext(context.Background(), log), lines
+}
+
+func twoHosts() provider.KVMConfig {
+	return provider.KVMConfig{
+		SSHKeyFile:     "/nonexistent/kvm2/id_ed25519",
+		KnownHostsFile: "/nonexistent/kvm2/known_hosts",
+		MachineTypes:   map[string]provider.KVMMachineType{"small": {VCPUs: 1, MemoryMiB: 1024}},
+		Hosts: []provider.KVMHost{
+			{Name: "beelink", Address: "beelink:22", User: "virt", Bridge: "br0"},
+			{Name: "lab2", Address: "lab2:22", User: "virt", Bridge: "br1", Socket: "/run/s", Pool: "fast"},
+		},
+	}
+}
+
+func TestNew_doesNotDialAndWarnsOnMissingFiles(t *testing.T) {
+	t.Parallel()
+	ctx, lines := captureContext()
+	cfg := twoHosts()
+	cfg.Hosts[0].Address = "192.0.2.1:22"
+	start := time.Now()
+	p, err := New(ctx, provider.ProviderConfig{Name: "kvm2-home", Type: "kvm2", KVM2: &cfg})
+	elapsed := time.Since(start)
+	if err != nil {
+		t.Fatalf("New() error = %v, want nil", err)
+	}
+	if p == nil {
+		t.Fatal("New() = nil provider")
+	}
+	if elapsed > 100*time.Millisecond {
+		t.Errorf("New() took %v, want under 100ms (it must not dial)", elapsed)
+	}
+	var warnings int
+	for _, l := range *lines {
+		if strings.Contains(l, "WARNING:") {
+			warnings++
+		}
+		if !strings.Contains(l, `"provider"="kvm2-home"`) {
+			t.Errorf("log line lacks the provider value: %s", l)
+		}
+	}
+	if warnings != 2 {
+		t.Errorf("got %d WARNING: lines, want 2 (key and known_hosts):\n%s", warnings, strings.Join(*lines, "\n"))
+	}
+}
+
+func TestNew_insecureIgnoreHostKeyWarns(t *testing.T) {
+	t.Parallel()
+	ctx, lines := captureContext()
+	cfg := twoHosts()
+	cfg.KnownHostsFile = ""
+	cfg.InsecureIgnoreHostKey = true
+	if _, err := New(ctx, provider.ProviderConfig{Name: "kvm2-home", Type: "kvm2", KVM2: &cfg}); err != nil {
+		t.Fatalf("New() error = %v, want nil", err)
+	}
+	found := false
+	for _, l := range *lines {
+		if strings.Contains(l, "WARNING:") && strings.Contains(l, "host keys are not verified") {
+			found = true
+		}
+		if strings.Contains(l, "known_hosts") {
+			t.Errorf("known_hosts warning emitted although host keys are ignored: %s", l)
+		}
+	}
+	if !found {
+		t.Errorf("no insecureIgnoreHostKey warning in:\n%s", strings.Join(*lines, "\n"))
+	}
+}
+
+func TestNew_nilBlockDoesNotPanic(t *testing.T) {
+	t.Parallel()
+	ctx, _ := captureContext()
+	p, err := New(ctx, provider.ProviderConfig{Name: "kvm2-empty", Type: "kvm2"})
+	if err != nil {
+		t.Fatalf("New() error = %v, want nil", err)
+	}
+	kp, ok := p.(*Provider)
+	if !ok {
+		t.Fatalf("New() = %T, want *Provider", p)
+	}
+	if kp.name != "kvm2-empty" || kp.cfg.DiskSizeGB != defaultDiskSizeGB || kp.now == nil {
+		t.Errorf("provider = %+v, want name kvm2-empty, default disk size and a clock", kp)
+	}
+}
+
+func TestHost_table(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name     string
+		hosts    []provider.KVMHost
+		zone     string
+		wantName string
+		wantErr  string
+	}{
+		{
+			name:     "one host and empty zone selects it",
+			hosts:    twoHosts().Hosts[:1],
+			zone:     "",
+			wantName: "beelink",
+		},
+		{
+			name:    "several hosts and empty zone is permanent",
+			hosts:   twoHosts().Hosts,
+			zone:    "",
+			wantErr: "placement.zone is required when kvm2.hosts lists more than one host",
+		},
+		{
+			name:     "known name",
+			hosts:    twoHosts().Hosts,
+			zone:     "lab2",
+			wantName: "lab2",
+		},
+		{
+			name:    "unknown name is permanent",
+			hosts:   twoHosts().Hosts,
+			zone:    "nas9",
+			wantErr: `host "nas9" is not in kvm2.hosts`,
+		},
+		{
+			name:    "no hosts at all is permanent",
+			hosts:   nil,
+			zone:    "",
+			wantErr: "placement.zone is required",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			p := newProvider("kvm2-home", provider.KVMConfig{Hosts: tc.hosts}, &stubDialer{})
+			h, err := p.host(tc.zone)
+			if tc.wantErr != "" {
+				if err == nil {
+					t.Fatalf("host(%q) = %+v, want error containing %q", tc.zone, h, tc.wantErr)
+				}
+				if !errors.Is(err, provider.ErrPermanent) {
+					t.Errorf("host(%q) error = %v, want ErrPermanent", tc.zone, err)
+				}
+				if !strings.Contains(err.Error(), tc.wantErr) {
+					t.Errorf("host(%q) error = %q, want substring %q", tc.zone, err, tc.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("host(%q) error = %v, want nil", tc.zone, err)
+			}
+			if h.Name != tc.wantName {
+				t.Errorf("host(%q).Name = %q, want %q", tc.zone, h.Name, tc.wantName)
+			}
+		})
+	}
+}
+
+func TestWithDefaults_table(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name  string
+		in    provider.KVMConfig
+		check func(t *testing.T, got provider.KVMConfig)
+	}{
+		{
+			name: "empty config gets disk size and empty maps",
+			in:   provider.KVMConfig{},
+			check: func(t *testing.T, got provider.KVMConfig) {
+				if got.DiskSizeGB != 10 {
+					t.Errorf("DiskSizeGB = %d, want 10", got.DiskSizeGB)
+				}
+				if got.Hosts == nil || got.Images == nil {
+					t.Errorf("Hosts = %v, Images = %v, want non-nil", got.Hosts, got.Images)
+				}
+			},
+		},
+		{
+			name: "explicit disk size kept",
+			in:   provider.KVMConfig{DiskSizeGB: 40},
+			check: func(t *testing.T, got provider.KVMConfig) {
+				if got.DiskSizeGB != 40 {
+					t.Errorf("DiskSizeGB = %d, want 40", got.DiskSizeGB)
+				}
+			},
+		},
+		{
+			name: "host socket and pool default",
+			in:   provider.KVMConfig{Hosts: []provider.KVMHost{{Name: "a"}, {Name: "b", Socket: "/run/s", Pool: "fast"}}},
+			check: func(t *testing.T, got provider.KVMConfig) {
+				if got.Hosts[0].Socket != "/var/run/libvirt/libvirt-sock" || got.Hosts[0].Pool != "default" {
+					t.Errorf("hosts[0] = %+v, want default socket and pool", got.Hosts[0])
+				}
+				if got.Hosts[1].Socket != "/run/s" || got.Hosts[1].Pool != "fast" {
+					t.Errorf("hosts[1] = %+v, want explicit socket and pool kept", got.Hosts[1])
+				}
+			},
+		},
+		{
+			name: "image format and firmware default",
+			in: provider.KVMConfig{Images: map[string]provider.KVMImage{
+				"d": {URL: "https://x/d.qcow2"},
+				"u": {URL: "https://x/u.img", Format: provider.KVMFormatRaw, Firmware: provider.KVMFirmwareBIOS},
+			}},
+			check: func(t *testing.T, got provider.KVMConfig) {
+				if got.Images["d"].Format != provider.KVMFormatQcow2 || got.Images["d"].Firmware != provider.KVMFirmwareUEFI {
+					t.Errorf("images[d] = %+v, want qcow2 and uefi", got.Images["d"])
+				}
+				if got.Images["u"].Format != provider.KVMFormatRaw || got.Images["u"].Firmware != provider.KVMFirmwareBIOS {
+					t.Errorf("images[u] = %+v, want raw and bios kept", got.Images["u"])
+				}
+			},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			tc.check(t, withDefaults(tc.in))
+		})
+	}
+}
+
+func TestWithDefaults_doesNotAliasInput(t *testing.T) {
+	t.Parallel()
+	in := provider.KVMConfig{
+		Hosts:  []provider.KVMHost{{Name: "a"}},
+		Images: map[string]provider.KVMImage{"d": {URL: "https://x/d.qcow2"}},
+	}
+	got := withDefaults(in)
+	got.Hosts[0].Name = "changed"
+	got.Images["d"] = provider.KVMImage{URL: "changed"}
+	if in.Hosts[0].Name != "a" || in.Images["d"].URL != "https://x/d.qcow2" {
+		t.Errorf("input mutated through the defaulted copy: %+v", in)
+	}
+}
+
+func lockCount(k *keyedLocks) int {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	return len(k.locks)
+}
+
+func TestKeyedLocks_entryRemovedWhenUncontended(t *testing.T) {
+	t.Parallel()
+	k := newKeyedLocks()
+	release, err := k.acquire(context.Background(), "a")
+	if err != nil {
+		t.Fatalf("acquire error = %v", err)
+	}
+	if lockCount(k) != 1 {
+		t.Errorf("entries while held = %d, want 1", lockCount(k))
+	}
+	release()
+	if lockCount(k) != 0 {
+		t.Errorf("entries after release = %d, want 0", lockCount(k))
+	}
+	if _, err := k.acquire(context.Background(), "a"); err != nil {
+		t.Fatalf("acquire after release error = %v", err)
+	}
+}
+
+func TestKeyedLocks_waitersShareOneLock(t *testing.T) {
+	t.Parallel()
+	k := newKeyedLocks()
+	const n = 20
+	var inside, overlaps atomic.Int32
+	var wg sync.WaitGroup
+	for range n {
+		wg.Go(func() {
+			release, err := k.acquire(context.Background(), "shared")
+			if err != nil {
+				t.Errorf("acquire error = %v", err)
+				return
+			}
+			if inside.Add(1) > 1 {
+				overlaps.Add(1)
+			}
+			time.Sleep(time.Millisecond)
+			inside.Add(-1)
+			release()
+		})
+	}
+	wg.Wait()
+	if overlaps.Load() != 0 {
+		t.Errorf("%d holders overlapped on one key", overlaps.Load())
+	}
+	if lockCount(k) != 0 {
+		t.Errorf("entries after every holder released = %d, want 0", lockCount(k))
+	}
+}
+
+func TestKeyedLocks_waiterGivesUpWithContext(t *testing.T) {
+	t.Parallel()
+	k := newKeyedLocks()
+	release, err := k.acquire(context.Background(), "a")
+	if err != nil {
+		t.Fatalf("acquire error = %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	if _, err := k.acquire(ctx, "a"); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("waiter error = %v, want DeadlineExceeded", err)
+	}
+	if time.Since(start) > time.Second {
+		t.Errorf("waiter took %v, want to give up with its ctx", time.Since(start))
+	}
+	if lockCount(k) != 1 {
+		t.Errorf("entries with the holder still in = %d, want 1: the waiter's exit must not drop the holder's entry", lockCount(k))
+	}
+	release()
+	if lockCount(k) != 0 {
+		t.Errorf("entries after the holder released = %d, want 0", lockCount(k))
+	}
+}
+
+func TestWrapErr_carriesProviderNameOpAndID(t *testing.T) {
+	t.Parallel()
+	p := newProvider("kvm2-home", twoHosts(), &stubDialer{})
+	err := p.wrapErr("get", "hosts/beelink/domains/proxy-x", permanentf("boom"))
+	var pe *provider.Error
+	if !errors.As(err, &pe) {
+		t.Fatalf("wrapErr() = %T, want *provider.Error", err)
+	}
+	if pe.Provider != "kvm2-home" || pe.Op != "get" || pe.ID != "hosts/beelink/domains/proxy-x" || pe.Class != provider.ErrPermanent {
+		t.Errorf("wrapErr() = %+v, want provider kvm2-home, op get, the id and ErrPermanent", pe)
+	}
+}

internal/provider/kvm2/list.go

@@ -0,0 +1,56 @@
+package kvm2
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// ListByTag is spec entry 12: every domain on every configured host, in
+// config order, filtered client-side on the kvm2 mark. GC uses only ID, UID
+// and CreatedAt (contract §2.4), all of which the name and the metadata
+// element give, so no state or address RPC is made and State and IP stay
+// empty. A host that cannot be reached or listed fails the whole call
+// (fail-whole); a single domain whose metadata cannot be read is skipped,
+// unless ctx ended, which fails the call rather than pass off an empty view
+// as the truth.
+func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
+	log := p.logger(ctx)
+	var out []provider.Instance
+	for _, host := range p.cfg.Hosts {
+		hlog := log.WithValues("host", host.Name)
+		be, err := p.hosts.get(ctx, host)
+		if err != nil {
+			logAPIError(hlog, "list", err)
+			return nil, p.wrapErr("list", "", err)
+		}
+		doms, _, err := be.ConnectListAllDomains(ctx, 1, libvirt.ConnectListDomainsActive|libvirt.ConnectListDomainsInactive)
+		if err != nil {
+			err = fmt.Errorf("listing domains on host %s: %w", host.Name, err)
+			logAPIError(hlog, "list", err)
+			return nil, p.wrapErr("list", "", err)
+		}
+		for _, dom := range doms {
+			dlog := hlog.WithValues("name", dom.Name)
+			meta, err := readMetadata(ctx, be, dom)
+			if err != nil {
+				if ctx.Err() != nil {
+					return nil, p.wrapErr("list", "", err)
+				}
+				dlog.V(1).Info("skipping domain with unreadable metadata", "error", err.Error())
+				continue
+			}
+			if !meta.managed() {
+				continue
+			}
+			inst := provider.Instance{ID: formatProviderID(host.Name, dom.Name), UID: meta.UID, CreatedAt: meta.createdAt()}
+			dlog.V(2).Info("kvm2 listed instance", "id", inst.ID, "uid", inst.UID, "createdAt", inst.CreatedAt)
+			out = append(out, inst)
+		}
+	}
+	log.V(1).Info("kvm2 domains listed", "hosts", len(p.cfg.Hosts), "managed", len(out))
+	return out, nil
+}

internal/provider/kvm2/list_test.go

@@ -0,0 +1,213 @@
+package kvm2
+
+import (
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func listIDs(list []provider.Instance) []string {
+	ids := make([]string, len(list))
+	for i, inst := range list {
+		ids[i] = inst.ID
+	}
+	return ids
+}
+
+func TestListByTag_excludesUnmarkedDomains(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	fx.fake.addDomain("legacy-vm", libvirt.DomainRunning, "")
+	fx.fake.addDomain("unmarked", libvirt.DomainRunning, `<instance><managed>false</managed><uid>x</uid></instance>`)
+	fx.fake.addDomain("unrelated", libvirt.DomainShutoff, `<instance><note>hand made</note></instance>`)
+	list, err := fx.p.ListByTag(fx.ctx)
+	if err != nil {
+		t.Fatalf("ListByTag() error = %v", err)
+	}
+	if got := listIDs(list); len(got) != 1 || got[0] != id {
+		t.Errorf("ListByTag = %v, want only %s", got, id)
+	}
+	if list[0].UID != testUID || list[0].CreatedAt.IsZero() {
+		t.Errorf("listed = %+v, want uid and createdAt", list[0])
+	}
+	if got := fx.logs.matching("kvm2 domains listed"); len(got) != 1 || !strings.Contains(got[0], `"hosts"=1`) || !strings.Contains(got[0], `"managed"=1`) {
+		t.Errorf("listed line = %v", got)
+	}
+}
+
+func TestListByTag_emptyUIDStillListed(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	fx.fake.addDomain("proxy-nouid", libvirt.DomainRunning, managedMetadata("", time.Now()))
+	list, err := fx.p.ListByTag(fx.ctx)
+	if err != nil {
+		t.Fatalf("ListByTag() error = %v", err)
+	}
+	if len(list) != 1 || list[0].UID != "" || list[0].ID != formatProviderID(testHost, "proxy-nouid") {
+		t.Errorf("ListByTag = %+v, want the domain listed with an empty UID", list)
+	}
+}
+
+func TestListByTag_acrossHostsInConfigOrder(t *testing.T) {
+	t.Parallel()
+	cfg := twoHosts()
+	cfg.Hosts[1].Pool = ""
+	first, second := newFakeBackend(), newFakeBackend()
+	first.addDomain("proxy-z", libvirt.DomainRunning, managedMetadata("uid-z", time.Now()))
+	first.addDomain("proxy-a", libvirt.DomainShutoff, managedMetadata("uid-a", time.Now()))
+	second.addDomain("proxy-b", libvirt.DomainRunning, managedMetadata("uid-b", time.Now()))
+	p := newProvider(testProvider, cfg, &hostDialer{backends: map[string]backend{"beelink": first, "lab2": second}})
+	ctx, _ := captureContext()
+	list, err := p.ListByTag(ctx)
+	if err != nil {
+		t.Fatalf("ListByTag() error = %v", err)
+	}
+	want := []string{formatProviderID("beelink", "proxy-a"), formatProviderID("beelink", "proxy-z"), formatProviderID("lab2", "proxy-b")}
+	if got := listIDs(list); strings.Join(got, ",") != strings.Join(want, ",") {
+		t.Errorf("ListByTag = %v, want %v (config order, then name order)", got, want)
+	}
+	if list[0].UID != "uid-a" || list[1].UID != "uid-z" || list[2].UID != "uid-b" {
+		t.Errorf("uids = %s, %s, %s, want uid-a, uid-z, uid-b", list[0].UID, list[1].UID, list[2].UID)
+	}
+}
+
+func TestListByTag_doesNotReadStateOrAddresses(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 2)
+	id := fx.create(testName)
+	fx.setAddrs(testName, libvirt.DomainInterfaceAddressesSrcAgent, "10.0.0.5")
+	fx.fake.addDomain("proxy-off", libvirt.DomainShutoff, managedMetadata("uid-off", time.Now()))
+	before := len(fx.fake.callLog())
+
+	list, err := fx.p.ListByTag(fx.ctx)
+	if err != nil {
+		t.Fatalf("ListByTag() error = %v", err)
+	}
+	if len(list) != 2 || list[0].ID != id || list[1].ID != formatProviderID(testHost, "proxy-off") {
+		t.Fatalf("ListByTag = %v, want the two managed domains", listIDs(list))
+	}
+	for _, inst := range list {
+		if inst.State != "" || inst.IP != "" {
+			t.Errorf("listed %s = state %q ip %q, want both empty: GC does not use them", inst.ID, inst.State, inst.IP)
+		}
+		if inst.UID == "" || inst.CreatedAt.IsZero() {
+			t.Errorf("listed %s lacks uid or createdAt: %+v", inst.ID, inst)
+		}
+	}
+	for _, rpc := range fx.fake.callLog()[before:] {
+		switch rpc {
+		case "ConnectListAllDomains", "DomainGetMetadata":
+		default:
+			t.Errorf("ListByTag made %s; want only the listing and one metadata read per domain", rpc)
+		}
+	}
+	if got := fx.logs.matching("kvm2 listed instance"); len(got) != 2 || strings.Contains(got[0], `"state"=`) {
+		t.Errorf("listed instance lines = %v, want two without a state", got)
+	}
+}
+
+func TestListByTag_unreachableHostFailsWhole(t *testing.T) {
+	t.Parallel()
+	first := newFakeBackend()
+	first.addDomain("proxy-a", libvirt.DomainRunning, managedMetadata("uid-a", time.Now()))
+	d := &hostDialer{
+		backends: map[string]backend{"beelink": first},
+		errs:     map[string]error{"lab2": errors.New("dial tcp 192.0.2.2:22: i/o timeout")},
+	}
+	p := newProvider(testProvider, twoHosts(), d)
+	ctx, _ := captureContext()
+	list, err := p.ListByTag(ctx)
+	if list != nil {
+		t.Errorf("ListByTag = %v, want nil with the error", list)
+	}
+	assertProviderError(t, err, "list", provider.ErrTransient)
+}
+
+func TestListByTag_unreadableDomainSkipped(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name  string
+		setup func(fx *fixture)
+		line  string
+	}{
+		{
+			name: "corrupt metadata on one domain",
+			setup: func(fx *fixture) {
+				fx.fake.addDomain("proxy-bad", libvirt.DomainRunning, "<bogus>not an instance</bogus>")
+			},
+			line: "skipping domain with unreadable metadata",
+		},
+		{
+			name: "metadata rpc fails for the first domain only",
+			setup: func(fx *fixture) {
+				fx.fake.addDomain("proxy-bad", libvirt.DomainRunning, managedMetadata("uid-bad", time.Now()))
+				var calls int
+				fx.fake.mu.Lock()
+				fx.fake.hooks["DomainGetMetadata"] = func() error {
+					calls++
+					if calls == 1 {
+						return fakeErr(libvirt.ErrInternalError, "internal error: metadata read failed")
+					}
+					return nil
+				}
+				fx.fake.mu.Unlock()
+			},
+			line: "skipping domain with unreadable metadata",
+		},
+		{
+			name: "domain gone between listing and metadata read",
+			setup: func(fx *fixture) {
+				fx.fake.addDomain("proxy-bad", libvirt.DomainRunning, managedMetadata("uid-bad", time.Now()))
+				var calls int
+				fx.fake.mu.Lock()
+				fx.fake.hooks["DomainGetMetadata"] = func() error {
+					calls++
+					if calls == 1 {
+						return fakeErr(libvirt.ErrNoDomain, "Domain not found")
+					}
+					return nil
+				}
+				fx.fake.mu.Unlock()
+			},
+			line: "skipping domain with unreadable metadata",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			fx.fake.addDomain("proxy-good", libvirt.DomainRunning, managedMetadata("uid-good", time.Now()))
+			tc.setup(fx)
+			list, err := fx.p.ListByTag(fx.ctx)
+			if err != nil {
+				t.Fatalf("ListByTag() error = %v", err)
+			}
+			if got := listIDs(list); len(got) != 1 || got[0] != formatProviderID(testHost, "proxy-good") {
+				t.Errorf("ListByTag = %v, want only proxy-good", got)
+			}
+			if got := fx.logs.matching(tc.line); len(got) != 1 || !strings.Contains(got[0], `"name"="proxy-bad"`) {
+				t.Errorf("skip line = %v, want one naming proxy-bad", got)
+			}
+		})
+	}
+}
+
+func TestListByTag_includesInactiveDomains(t *testing.T) {
+	t.Parallel()
+	fx := newFixture(t, 1)
+	id := fx.create(testName)
+	fx.withDomain(testName, func(d *fakeDomain) { d.state = libvirt.DomainShutoff })
+	list, err := fx.p.ListByTag(fx.ctx)
+	if err != nil {
+		t.Fatalf("ListByTag() error = %v", err)
+	}
+	if len(list) != 1 || list[0].ID != id || list[0].UID != testUID {
+		t.Errorf("ListByTag = %+v, want the shut-off domain listed with its uid", list)
+	}
+}

internal/provider/kvm2/metadata.go

@@ -0,0 +1,78 @@
+package kvm2
+
+import (
+	"encoding/xml"
+	"strings"
+	"time"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// The GC mark lives in the domain's <metadata> as one element in kvm2's own
+// namespace (spec entry 8). The namespace is what selects it in
+// DomainGetMetadata; libvirt 11.6 hands the element back without the xmlns
+// declaration, so parsing matches on the local name only.
+const (
+	metadataURI       = "https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm2"
+	metadataLocalName = "instance"
+)
+
+// instanceMetadata is the mark as it is written and read back. The fields
+// are the element's text as is; managed and createdAt interpret them.
+type instanceMetadata struct {
+	XMLName   xml.Name
+	Managed   string `xml:"managed"`
+	UID       string `xml:"uid"`
+	CreatedAt string `xml:"createdAt"`
+}
+
+func newInstanceMetadata(uid string, createdAt time.Time) instanceMetadata {
+	return instanceMetadata{
+		XMLName:   xml.Name{Space: metadataURI, Local: metadataLocalName},
+		Managed:   provider.LabelManagedYes,
+		UID:       uid,
+		CreatedAt: createdAt.UTC().Format(time.RFC3339),
+	}
+}
+
+func (m instanceMetadata) managed() bool {
+	return m.Managed == provider.LabelManagedYes
+}
+
+// createdAt returns the zero time when the element carries no parsable
+// timestamp; GC treats that as old, which the spec accepts.
+func (m instanceMetadata) createdAt() time.Time {
+	t, err := time.Parse(time.RFC3339, m.CreatedAt)
+	if err != nil {
+		return time.Time{}
+	}
+	return t
+}
+
+// marshalInstanceMetadata renders the namespaced element that goes under
+// the domain's <metadata>.
+func marshalInstanceMetadata(m instanceMetadata) (string, error) {
+	m.XMLName = xml.Name{Space: metadataURI, Local: metadataLocalName}
+	out, err := xml.Marshal(m)
+	if err != nil {
+		return "", permanentf("marshalling instance metadata: %v", err)
+	}
+	return string(out), nil
+}
+
+// parseInstanceMetadata reads the element back, with or without its xmlns
+// declaration. Anything that is not an <instance> element is ErrPermanent:
+// the domain's metadata is corrupt and no retry fixes it.
+func parseInstanceMetadata(s string) (instanceMetadata, error) {
+	var m instanceMetadata
+	if strings.TrimSpace(s) == "" {
+		return instanceMetadata{}, permanentf("parsing instance metadata: empty document")
+	}
+	if err := xml.Unmarshal([]byte(s), &m); err != nil {
+		return instanceMetadata{}, permanentf("parsing instance metadata: %v", err)
+	}
+	if m.XMLName.Local != metadataLocalName {
+		return instanceMetadata{}, permanentf("parsing instance metadata: root element is <%s>, want <%s>", m.XMLName.Local, metadataLocalName)
+	}
+	return m, nil
+}

internal/provider/kvm2/metadata_test.go

@@ -0,0 +1,162 @@
+package kvm2
+
+import (
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func TestMarshalInstanceMetadata_namespacedElement(t *testing.T) {
+	t.Parallel()
+	created := time.Date(2026, 9, 16, 10, 30, 0, 0, time.FixedZone("CEST", 2*3600))
+	got, err := marshalInstanceMetadata(newInstanceMetadata("uid-1", created))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{
+		`<instance xmlns="` + metadataURI + `">`,
+		"<managed>true</managed>",
+		"<uid>uid-1</uid>",
+		"<createdAt>2026-09-16T08:30:00Z</createdAt>",
+		"</instance>",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("marshalled metadata %q lacks %q", got, want)
+		}
+	}
+}
+
+func TestParseMetadata_withNamespace(t *testing.T) {
+	t.Parallel()
+	in := `<instance xmlns="` + metadataURI + `"><managed>true</managed><uid>abc</uid><createdAt>2026-09-16T08:30:00Z</createdAt></instance>`
+	m, err := parseInstanceMetadata(in)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !m.managed() {
+		t.Error("managed() = false, want true")
+	}
+	if m.UID != "abc" {
+		t.Errorf("UID = %q, want abc", m.UID)
+	}
+	if want := time.Date(2026, 9, 16, 8, 30, 0, 0, time.UTC); !m.createdAt().Equal(want) {
+		t.Errorf("createdAt() = %v, want %v", m.createdAt(), want)
+	}
+}
+
+func TestParseMetadata_withoutNamespaceDeclaration(t *testing.T) {
+	t.Parallel()
+	in := "<instance>\n  <managed>true</managed>\n  <uid>abc</uid>\n  <createdAt>2026-09-16T08:30:00Z</createdAt>\n</instance>"
+	m, err := parseInstanceMetadata(in)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !m.managed() || m.UID != "abc" {
+		t.Errorf("got managed=%v uid=%q, want true/abc", m.managed(), m.UID)
+	}
+	if m.createdAt().IsZero() {
+		t.Error("createdAt() is zero, want parsed timestamp")
+	}
+}
+
+func TestParseMetadata_absentGivesEmptyUID(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		in   string
+	}{
+		{name: "empty element", in: "<instance/>"},
+		{name: "empty element with namespace", in: `<instance xmlns="` + metadataURI + `"></instance>`},
+		{name: "empty children", in: "<instance><managed></managed><uid></uid><createdAt></createdAt></instance>"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			m, err := parseInstanceMetadata(tc.in)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if m.managed() {
+				t.Error("managed() = true, want false")
+			}
+			if m.UID != "" {
+				t.Errorf("UID = %q, want empty", m.UID)
+			}
+			if !m.createdAt().IsZero() {
+				t.Errorf("createdAt() = %v, want zero", m.createdAt())
+			}
+		})
+	}
+}
+
+func TestParseMetadata_badCreatedAtIsZeroTime(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		in   string
+	}{
+		{name: "garbage", in: "<instance><managed>true</managed><uid>u</uid><createdAt>yesterday</createdAt></instance>"},
+		{name: "date only", in: "<instance><managed>true</managed><uid>u</uid><createdAt>2026-09-16</createdAt></instance>"},
+		{name: "missing", in: "<instance><managed>true</managed><uid>u</uid></instance>"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			m, err := parseInstanceMetadata(tc.in)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !m.createdAt().IsZero() {
+				t.Errorf("createdAt() = %v, want zero", m.createdAt())
+			}
+			if !m.managed() || m.UID != "u" {
+				t.Errorf("other fields lost: managed=%v uid=%q", m.managed(), m.UID)
+			}
+		})
+	}
+}
+
+func TestParseMetadata_notXMLIsPermanent(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		in   string
+	}{
+		{name: "empty string", in: ""},
+		{name: "whitespace", in: "  \n"},
+		{name: "plain text", in: "managed: true"},
+		{name: "unclosed element", in: "<instance><managed>true"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			_, err := parseInstanceMetadata(tc.in)
+			if !errors.Is(err, provider.ErrPermanent) {
+				t.Errorf("err = %v, want ErrPermanent", err)
+			}
+		})
+	}
+}
+
+func TestParseMetadata_wrongRootIsPermanent(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		in   string
+	}{
+		{name: "other local name", in: "<proxy><managed>true</managed></proxy>"},
+		{name: "other local name with our namespace", in: `<vm xmlns="` + metadataURI + `"><uid>x</uid></vm>`},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			_, err := parseInstanceMetadata(tc.in)
+			if !errors.Is(err, provider.ErrPermanent) {
+				t.Errorf("err = %v, want ErrPermanent", err)
+			}
+		})
+	}
+}

internal/provider/kvm2/observe.go

@@ -0,0 +1,263 @@
+package kvm2
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+// observed is the span-decorator of spec entries 17 and 20: one child span
+// kvm2.<Method> and one V(1) "libvirt call" line per call, with request and
+// response detail at V(2). Applied once, in the dialer, over every backend
+// that leaves it.
+type observed struct {
+	inner    backend
+	provider string
+	host     string
+	tracer   trace.Tracer
+}
+
+var (
+	_ backend  = (*observed)(nil)
+	_ liveness = (*observed)(nil)
+)
+
+func observe(inner backend, providerName, hostName string, opts ...tracing.Option) backend {
+	return &observed{inner: inner, provider: providerName, host: hostName, tracer: tracing.Tracer(opts...)}
+}
+
+// dead forwards the inner backend's liveness; a backend without one is
+// never dropped by the host table.
+func (o *observed) dead() <-chan struct{} {
+	if l, ok := o.inner.(liveness); ok {
+		return l.dead()
+	}
+	return nil
+}
+
+// id is one identifier a call names: a domain, pool or volume. It becomes
+// both the span attribute kvm2.<key> and the log key.
+type id struct{ key, val string }
+
+func domainID(name string) id { return id{"domain", name} }
+func poolID(name string) id   { return id{"pool", name} }
+func volumeID(name string) id { return id{"volume", name} }
+
+// detail returns V(2)-only key/value pairs. It runs only when V(2) is on,
+// so building the list of a hundred domain names costs nothing otherwise.
+type detail func() []any
+
+// call wraps one backend call: span, timing, status and the log line. fn
+// returns the V(2) detail for the line, or nil.
+func (o *observed) call(ctx context.Context, rpc string, ids []id, fn func(ctx context.Context) (detail, error)) error {
+	attrs := make([]attribute.KeyValue, 0, len(ids)+1)
+	attrs = append(attrs, attribute.String("kvm2.host", o.host))
+	for _, i := range ids {
+		attrs = append(attrs, attribute.String("kvm2."+i.key, i.val))
+	}
+	ctx, span := tracing.StartSpan(ctx, o.tracer, "kvm2."+rpc, trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes(attrs...))
+	defer span.End()
+	log := logf.FromContext(ctx).WithName("kvm2").WithValues("provider", o.provider, "host", o.host)
+	start := time.Now()
+	more, err := fn(ctx)
+	kv := make([]any, 0, 8+2*len(ids))
+	kv = append(kv, "rpc", rpc)
+	for _, i := range ids {
+		kv = append(kv, i.key, i.val)
+	}
+	kv = append(kv, "duration", time.Since(start))
+	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
+		if code, ok := libvirtCode(err); ok {
+			span.SetAttributes(attribute.Int("kvm2.libvirt_code", int(code)))
+		}
+		kv = append(kv, "error", err.Error())
+		kv = append(kv, libvirtErrorFields(err)...)
+	}
+	if more != nil && log.V(2).Enabled() {
+		kv = append(kv, more()...)
+	}
+	log.V(1).Info("libvirt call", kv...)
+	return err
+}
+
+func (o *observed) DomainLookupByName(ctx context.Context, name string) (dom libvirt.Domain, err error) {
+	err = o.call(ctx, "DomainLookupByName", []id{domainID(name)}, func(ctx context.Context) (detail, error) {
+		dom, err = o.inner.DomainLookupByName(ctx, name)
+		return nil, err
+	})
+	return dom, err
+}
+
+func (o *observed) DomainDefineXML(ctx context.Context, xml string) (dom libvirt.Domain, err error) {
+	err = o.call(ctx, "DomainDefineXML", nil, func(ctx context.Context) (detail, error) {
+		dom, err = o.inner.DomainDefineXML(ctx, xml)
+		return func() []any { return []any{"domain", dom.Name, "xml", xml} }, err
+	})
+	return dom, err
+}
+
+func (o *observed) DomainCreate(ctx context.Context, dom libvirt.Domain) error {
+	return o.call(ctx, "DomainCreate", []id{domainID(dom.Name)}, func(ctx context.Context) (detail, error) {
+		return nil, o.inner.DomainCreate(ctx, dom)
+	})
+}
+
+func (o *observed) DomainDestroy(ctx context.Context, dom libvirt.Domain) error {
+	return o.call(ctx, "DomainDestroy", []id{domainID(dom.Name)}, func(ctx context.Context) (detail, error) {
+		return nil, o.inner.DomainDestroy(ctx, dom)
+	})
+}
+
+func (o *observed) DomainUndefineFlags(ctx context.Context, dom libvirt.Domain, flags libvirt.DomainUndefineFlagsValues) error {
+	return o.call(ctx, "DomainUndefineFlags", []id{domainID(dom.Name)}, func(ctx context.Context) (detail, error) {
+		return nil, o.inner.DomainUndefineFlags(ctx, dom, flags)
+	})
+}
+
+func (o *observed) DomainGetState(ctx context.Context, dom libvirt.Domain, flags uint32) (state int32, reason int32, err error) {
+	err = o.call(ctx, "DomainGetState", []id{domainID(dom.Name)}, func(ctx context.Context) (detail, error) {
+		state, reason, err = o.inner.DomainGetState(ctx, dom, flags)
+		return func() []any { return []any{"state", libvirt.DomainState(state).String(), "reason", reason} }, err
+	})
+	return state, reason, err
+}
+
+func (o *observed) DomainGetMetadata(ctx context.Context, dom libvirt.Domain, typ int32, uri libvirt.OptString, flags libvirt.DomainModificationImpact) (md string, err error) {
+	err = o.call(ctx, "DomainGetMetadata", []id{domainID(dom.Name)}, func(ctx context.Context) (detail, error) {
+		md, err = o.inner.DomainGetMetadata(ctx, dom, typ, uri, flags)
+		return func() []any { return []any{"metadata", md} }, err
+	})
+	return md, err
+}
+
+func (o *observed) DomainInterfaceAddresses(ctx context.Context, dom libvirt.Domain, source uint32, flags uint32) (ifaces []libvirt.DomainInterface, err error) {
+	err = o.call(ctx, "DomainInterfaceAddresses", []id{domainID(dom.Name)}, func(ctx context.Context) (detail, error) {
+		ifaces, err = o.inner.DomainInterfaceAddresses(ctx, dom, source, flags)
+		return func() []any { return []any{"source", source, "addresses", interfaceAddresses(ifaces)} }, err
+	})
+	return ifaces, err
+}
+
+func (o *observed) ConnectListAllDomains(ctx context.Context, needResults int32, flags libvirt.ConnectListAllDomainsFlags) (doms []libvirt.Domain, n uint32, err error) {
+	err = o.call(ctx, "ConnectListAllDomains", nil, func(ctx context.Context) (detail, error) {
+		doms, n, err = o.inner.ConnectListAllDomains(ctx, needResults, flags)
+		return func() []any { return []any{"count", len(doms), "domains", domainNames(doms)} }, err
+	})
+	return doms, n, err
+}
+
+func (o *observed) StoragePoolLookupByName(ctx context.Context, name string) (pool libvirt.StoragePool, err error) {
+	err = o.call(ctx, "StoragePoolLookupByName", []id{poolID(name)}, func(ctx context.Context) (detail, error) {
+		pool, err = o.inner.StoragePoolLookupByName(ctx, name)
+		return nil, err
+	})
+	return pool, err
+}
+
+func (o *observed) StoragePoolRefresh(ctx context.Context, pool libvirt.StoragePool, flags uint32) error {
+	return o.call(ctx, "StoragePoolRefresh", []id{poolID(pool.Name)}, func(ctx context.Context) (detail, error) {
+		return nil, o.inner.StoragePoolRefresh(ctx, pool, flags)
+	})
+}
+
+func (o *observed) StoragePoolGetInfo(ctx context.Context, pool libvirt.StoragePool) (state uint8, capacity, allocation, available uint64, err error) {
+	err = o.call(ctx, "StoragePoolGetInfo", []id{poolID(pool.Name)}, func(ctx context.Context) (detail, error) {
+		state, capacity, allocation, available, err = o.inner.StoragePoolGetInfo(ctx, pool)
+		return func() []any {
+			return []any{"state", libvirt.StoragePoolState(state).String(), "capacity", capacity, "allocation", allocation, "available", available}
+		}, err
+	})
+	return state, capacity, allocation, available, err
+}
+
+func (o *observed) StorageVolLookupByName(ctx context.Context, pool libvirt.StoragePool, name string) (vol libvirt.StorageVol, err error) {
+	err = o.call(ctx, "StorageVolLookupByName", []id{poolID(pool.Name), volumeID(name)}, func(ctx context.Context) (detail, error) {
+		vol, err = o.inner.StorageVolLookupByName(ctx, pool, name)
+		return nil, err
+	})
+	return vol, err
+}
+
+func (o *observed) StorageVolGetPath(ctx context.Context, vol libvirt.StorageVol) (path string, err error) {
+	err = o.call(ctx, "StorageVolGetPath", []id{poolID(vol.Pool), volumeID(vol.Name)}, func(ctx context.Context) (detail, error) {
+		path, err = o.inner.StorageVolGetPath(ctx, vol)
+		return func() []any { return []any{"path", path} }, err
+	})
+	return path, err
+}
+
+func (o *observed) StorageVolCreateXML(ctx context.Context, pool libvirt.StoragePool, xml string, flags libvirt.StorageVolCreateFlags) (vol libvirt.StorageVol, err error) {
+	err = o.call(ctx, "StorageVolCreateXML", []id{poolID(pool.Name)}, func(ctx context.Context) (detail, error) {
+		vol, err = o.inner.StorageVolCreateXML(ctx, pool, xml, flags)
+		return func() []any { return []any{"volume", vol.Name, "xml", xml} }, err
+	})
+	return vol, err
+}
+
+// StorageVolUpload and StorageVolDownload log the length only; the bodies
+// are a seed ISO carrying CloudInit or a base image.
+func (o *observed) StorageVolUpload(ctx context.Context, vol libvirt.StorageVol, r io.Reader, offset, length uint64, flags libvirt.StorageVolUploadFlags) error {
+	return o.call(ctx, "StorageVolUpload", []id{poolID(vol.Pool), volumeID(vol.Name)}, func(ctx context.Context) (detail, error) {
+		err := o.inner.StorageVolUpload(ctx, vol, r, offset, length, flags)
+		return func() []any { return []any{"offset", offset, "length", length} }, err
+	})
+}
+
+func (o *observed) StorageVolDownload(ctx context.Context, vol libvirt.StorageVol, w io.Writer, offset, length uint64, flags libvirt.StorageVolDownloadFlags) error {
+	return o.call(ctx, "StorageVolDownload", []id{poolID(vol.Pool), volumeID(vol.Name)}, func(ctx context.Context) (detail, error) {
+		err := o.inner.StorageVolDownload(ctx, vol, w, offset, length, flags)
+		return func() []any { return []any{"offset", offset, "length", length} }, err
+	})
+}
+
+func (o *observed) StorageVolDelete(ctx context.Context, vol libvirt.StorageVol, flags libvirt.StorageVolDeleteFlags) error {
+	return o.call(ctx, "StorageVolDelete", []id{poolID(vol.Pool), volumeID(vol.Name)}, func(ctx context.Context) (detail, error) {
+		return nil, o.inner.StorageVolDelete(ctx, vol, flags)
+	})
+}
+
+func (o *observed) ConnectGetLibVersion(ctx context.Context) (v uint64, err error) {
+	err = o.call(ctx, "ConnectGetLibVersion", nil, func(ctx context.Context) (detail, error) {
+		v, err = o.inner.ConnectGetLibVersion(ctx)
+		return func() []any { return []any{"libvirtVersion", v} }, err
+	})
+	return v, err
+}
+
+func (o *observed) ConnectGetHostname(ctx context.Context) (name string, err error) {
+	err = o.call(ctx, "ConnectGetHostname", nil, func(ctx context.Context) (detail, error) {
+		name, err = o.inner.ConnectGetHostname(ctx)
+		return func() []any { return []any{"hostname", name} }, err
+	})
+	return name, err
+}
+
+// Close is passed through: it is not a libvirt call and gets no span.
+func (o *observed) Close() error { return o.inner.Close() }
+
+func domainNames(doms []libvirt.Domain) []string {
+	names := make([]string, len(doms))
+	for i, d := range doms {
+		names[i] = d.Name
+	}
+	return names
+}
+
+func interfaceAddresses(ifaces []libvirt.DomainInterface) []string {
+	var out []string
+	for _, iface := range ifaces {
+		for _, a := range iface.Addrs {
+			out = append(out, iface.Name+"="+a.Addr)
+		}
+	}
+	return out
+}

internal/provider/kvm2/observe_test.go

@@ -0,0 +1,334 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr/funcr"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+// logCapture collects funcr output at a verbosity, safe for concurrent use.
+type logCapture struct {
+	mu    sync.Mutex
+	lines []string
+}
+
+func (c *logCapture) all() []string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return append([]string(nil), c.lines...)
+}
+
+func (c *logCapture) matching(substr string) []string {
+	var out []string
+	for _, l := range c.all() {
+		if strings.Contains(l, substr) {
+			out = append(out, l)
+		}
+	}
+	return out
+}
+
+// observeFixture is a decorated fake with a span recorder and a log capture
+// wired into ctx.
+type observeFixture struct {
+	fake *fakeBackend
+	b    backend
+	sr   *tracetest.SpanRecorder
+	tp   *sdktrace.TracerProvider
+	logs *logCapture
+}
+
+func newObserveFixture(t *testing.T, verbosity int) (*observeFixture, context.Context) {
+	t.Helper()
+	sr := tracetest.NewSpanRecorder()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+	t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
+	logs := &logCapture{}
+	log := funcr.New(func(prefix, args string) {
+		logs.mu.Lock()
+		defer logs.mu.Unlock()
+		logs.lines = append(logs.lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: verbosity})
+	fake := newFakeBackend()
+	fx := &observeFixture{
+		fake: fake,
+		b:    observe(fake, "kvm2-home", "beelink", tracing.WithTracerProvider(tp)),
+		sr:   sr,
+		tp:   tp,
+		logs: logs,
+	}
+	return fx, logf.IntoContext(context.Background(), log)
+}
+
+func TestObserve_everySeamMethodInstrumented(t *testing.T) {
+	t.Parallel()
+	seam := reflect.TypeOf((*backend)(nil)).Elem()
+	for i := range seam.NumMethod() {
+		m := seam.Method(i)
+		if m.Name == "Close" {
+			continue
+		}
+		t.Run(m.Name, func(t *testing.T) {
+			t.Parallel()
+			fx, ctx := newObserveFixture(t, 1)
+			args := make([]reflect.Value, m.Type.NumIn())
+			args[0] = reflect.ValueOf(ctx)
+			for j := 1; j < m.Type.NumIn(); j++ {
+				args[j] = reflect.Zero(m.Type.In(j))
+			}
+			reflect.ValueOf(fx.b).MethodByName(m.Name).Call(args)
+
+			ended := fx.sr.Ended()
+			if len(ended) != 1 {
+				t.Fatalf("got %d spans, want 1", len(ended))
+			}
+			if got, want := ended[0].Name(), "kvm2."+m.Name; got != want {
+				t.Errorf("span name = %q, want %q", got, want)
+			}
+			if !hasAttr(ended[0].Attributes(), attribute.String("kvm2.host", "beelink")) {
+				t.Errorf("span attributes %v lack kvm2.host", ended[0].Attributes())
+			}
+			lines := fx.logs.matching(`"msg"="libvirt call"`)
+			if len(lines) != 1 {
+				t.Fatalf("got %d 'libvirt call' lines, want 1:\n%s", len(lines), strings.Join(fx.logs.all(), "\n"))
+			}
+			for _, want := range []string{`"rpc"="` + m.Name + `"`, `"provider"="kvm2-home"`, `"host"="beelink"`, `"duration"=`, `"traceID"=`} {
+				if !strings.Contains(lines[0], want) {
+					t.Errorf("log line lacks %s: %s", want, lines[0])
+				}
+			}
+			if fx.fake.callCount(m.Name) != 1 {
+				t.Errorf("inner %s called %d times, want 1", m.Name, fx.fake.callCount(m.Name))
+			}
+		})
+	}
+}
+
+func TestObserve_childSpansUnderParent(t *testing.T) {
+	t.Parallel()
+	fx, ctx := newObserveFixture(t, 1)
+	ctx, parent := tracing.StartSpan(ctx, fx.tp.Tracer("test"), "provider.get")
+	_, _ = fx.b.DomainLookupByName(ctx, "proxy-x")
+	parent.End()
+
+	var child sdktrace.ReadOnlySpan
+	for _, s := range fx.sr.Ended() {
+		if s.Name() == "kvm2.DomainLookupByName" {
+			child = s
+		}
+	}
+	if child == nil {
+		t.Fatal("no kvm2.DomainLookupByName span recorded")
+	}
+	if child.Parent().SpanID() != parent.SpanContext().SpanID() {
+		t.Errorf("child parent = %s, want %s", child.Parent().SpanID(), parent.SpanContext().SpanID())
+	}
+	if child.SpanContext().TraceID() != parent.SpanContext().TraceID() {
+		t.Error("child is in a different trace than its parent")
+	}
+	if !hasAttr(child.Attributes(), attribute.String("kvm2.domain", "proxy-x")) {
+		t.Errorf("child attributes %v lack kvm2.domain", child.Attributes())
+	}
+	lines := fx.logs.matching(`"msg"="libvirt call"`)
+	if len(lines) != 1 || !strings.Contains(lines[0], `"spanID"="`+child.SpanContext().SpanID().String()+`"`) {
+		t.Errorf("log line does not carry the child's spanID: %v", lines)
+	}
+}
+
+func TestObserve_logsOneLinePerCallAtV1(t *testing.T) {
+	t.Parallel()
+	fx, ctx := newObserveFixture(t, 1)
+	fx.fake.addDomain("proxy-x", libvirt.DomainRunning, "")
+	pool, _ := fx.b.StoragePoolLookupByName(ctx, fakePoolName)
+	_, _ = fx.b.DomainLookupByName(ctx, "proxy-x")
+	_, _ = fx.b.StorageVolLookupByName(ctx, pool, "proxy-x.qcow2")
+
+	lines := fx.logs.matching(`"msg"="libvirt call"`)
+	if len(lines) != 3 {
+		t.Fatalf("got %d lines, want 3:\n%s", len(lines), strings.Join(fx.logs.all(), "\n"))
+	}
+	if len(fx.logs.all()) != 3 {
+		t.Errorf("got %d lines in total, want only the 3 call lines:\n%s", len(fx.logs.all()), strings.Join(fx.logs.all(), "\n"))
+	}
+	wants := []string{
+		`"rpc"="StoragePoolLookupByName" "pool"="default"`,
+		`"rpc"="DomainLookupByName" "domain"="proxy-x"`,
+		`"rpc"="StorageVolLookupByName" "pool"="default" "volume"="proxy-x.qcow2"`,
+	}
+	for i, want := range wants {
+		if !strings.Contains(lines[i], want) {
+			t.Errorf("line %d = %s, want %s in it", i, lines[i], want)
+		}
+		if !strings.Contains(lines[i], `"level"=1`) {
+			t.Errorf("line %d is not V(1): %s", i, lines[i])
+		}
+	}
+}
+
+func TestObserve_detailOnlyAtV2(t *testing.T) {
+	t.Parallel()
+	const xmlDoc = "<domain><name>proxy-x</name><metadata><instance><uid>u1</uid></instance></metadata></domain>"
+	tests := []struct {
+		name      string
+		verbosity int
+		wantXML   bool
+	}{
+		{name: "V1 has no xml", verbosity: 1, wantXML: false},
+		{name: "V2 has xml", verbosity: 2, wantXML: true},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fx, ctx := newObserveFixture(t, tc.verbosity)
+			dom, err := fx.b.DomainDefineXML(ctx, xmlDoc)
+			if err != nil {
+				t.Fatalf("DomainDefineXML error = %v", err)
+			}
+			_, _ = fx.b.DomainGetMetadata(ctx, dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, 0)
+			_, _, _ = fx.b.ConnectListAllDomains(ctx, 1, 0)
+			all := strings.Join(fx.logs.all(), "\n")
+			if got := strings.Contains(all, "<domain>"); got != tc.wantXML {
+				t.Errorf("xml logged = %v, want %v:\n%s", got, tc.wantXML, all)
+			}
+			if got := strings.Contains(all, `"metadata"="<instance>`); got != tc.wantXML {
+				t.Errorf("metadata logged = %v, want %v:\n%s", got, tc.wantXML, all)
+			}
+			if got := strings.Contains(all, `"domains"=["proxy-x"]`); got != tc.wantXML {
+				t.Errorf("domain names logged = %v, want %v:\n%s", got, tc.wantXML, all)
+			}
+			if got := len(fx.logs.matching(`"msg"="libvirt call"`)); got != 3 {
+				t.Errorf("got %d call lines, want 3", got)
+			}
+		})
+	}
+}
+
+func TestObserve_uploadBodyNeverLogged(t *testing.T) {
+	t.Parallel()
+	const sentinel = "SENTINEL-CLOUDINIT-3c7a"
+	fx, ctx := newObserveFixture(t, 2)
+	pool, _ := fx.b.StoragePoolLookupByName(ctx, fakePoolName)
+	vol, err := fx.b.StorageVolCreateXML(ctx, pool, "<volume><name>proxy-x-seed.iso</name><capacity>1024</capacity></volume>", 0)
+	if err != nil {
+		t.Fatalf("StorageVolCreateXML error = %v", err)
+	}
+	body := "#cloud-config\npassword: " + sentinel + "\n"
+	if err := fx.b.StorageVolUpload(ctx, vol, strings.NewReader(body), 0, uint64(len(body)), 0); err != nil {
+		t.Fatalf("StorageVolUpload error = %v", err)
+	}
+	var sink strings.Builder
+	if err := fx.b.StorageVolDownload(ctx, vol, &sink, 0, 0, 0); err != nil {
+		t.Fatalf("StorageVolDownload error = %v", err)
+	}
+	if sink.String() != body {
+		t.Errorf("downloaded %q, want the uploaded body", sink.String())
+	}
+	all := strings.Join(fx.logs.all(), "\n")
+	if strings.Contains(all, sentinel) {
+		t.Errorf("upload body leaked into logs:\n%s", all)
+	}
+	uploads := fx.logs.matching(`"rpc"="StorageVolUpload"`)
+	if len(uploads) != 1 || !strings.Contains(uploads[0], `"length"=`+strconv.Itoa(len(body))) {
+		t.Errorf("upload line lacks length: %v", uploads)
+	}
+	for _, s := range fx.sr.Ended() {
+		for _, a := range s.Attributes() {
+			if strings.Contains(a.Value.Emit(), sentinel) {
+				t.Errorf("span %s attribute %s carries the body", s.Name(), a.Key)
+			}
+		}
+	}
+}
+
+func TestObserve_errorFieldsOnFailure(t *testing.T) {
+	t.Parallel()
+	fx, ctx := newObserveFixture(t, 1)
+	_, err := fx.b.DomainLookupByName(ctx, "missing")
+	if !hasCode(err, libvirt.ErrNoDomain) {
+		t.Fatalf("error = %v, want ErrNoDomain through the decorator", err)
+	}
+	lines := fx.logs.matching(`"msg"="libvirt call"`)
+	if len(lines) != 1 {
+		t.Fatalf("got %d lines, want 1", len(lines))
+	}
+	for _, want := range []string{`"error"="Domain not found`, `"libvirtCode"=42`, `"libvirtError"="ErrNoDomain"`} {
+		if !strings.Contains(lines[0], want) {
+			t.Errorf("line lacks %s: %s", want, lines[0])
+		}
+	}
+	ended := fx.sr.Ended()
+	if len(ended) != 1 {
+		t.Fatalf("got %d spans, want 1", len(ended))
+	}
+	if ended[0].Status().Code != codes.Error {
+		t.Errorf("span status = %v, want Error", ended[0].Status().Code)
+	}
+	if !hasAttr(ended[0].Attributes(), attribute.Int("kvm2.libvirt_code", 42)) {
+		t.Errorf("span attributes %v lack kvm2.libvirt_code=42", ended[0].Attributes())
+	}
+
+	fx.fake.failWith("ConnectGetHostname", errors.New("plain failure"))
+	_, _ = fx.b.ConnectGetHostname(ctx)
+	lines = fx.logs.matching(`"rpc"="ConnectGetHostname"`)
+	if len(lines) != 1 || strings.Contains(lines[0], "libvirtCode") || !strings.Contains(lines[0], `"error"="plain failure"`) {
+		t.Errorf("non-libvirt failure line = %v, want error without libvirt fields", lines)
+	}
+}
+
+func TestObserve_v0Silent(t *testing.T) {
+	t.Parallel()
+	fx, ctx := newObserveFixture(t, 0)
+	fx.fake.addDomain("proxy-x", libvirt.DomainRunning, "")
+	_, _ = fx.b.DomainLookupByName(ctx, "proxy-x")
+	_, _ = fx.b.DomainLookupByName(ctx, "missing")
+	if lines := fx.logs.all(); len(lines) != 0 {
+		t.Errorf("got %d lines at V(0), want none:\n%s", len(lines), strings.Join(lines, "\n"))
+	}
+	if got := len(fx.sr.Ended()); got != 2 {
+		t.Errorf("got %d spans, want 2: spans do not depend on verbosity", got)
+	}
+}
+
+func TestObserve_closeAndLivenessPassThrough(t *testing.T) {
+	t.Parallel()
+	fx, _ := newObserveFixture(t, 1)
+	if isDead(fx.b) {
+		t.Error("decorated live backend reported dead")
+	}
+	fx.fake.markDead()
+	if !isDead(fx.b) {
+		t.Error("decorated dead backend reported live")
+	}
+	if err := fx.b.Close(); err != nil || fx.fake.closed != 1 {
+		t.Errorf("Close() = %v, closed %d times, want nil and 1", err, fx.fake.closed)
+	}
+	if got := len(fx.sr.Ended()); got != 0 {
+		t.Errorf("Close produced %d spans, want none", got)
+	}
+	if lines := fx.logs.all(); len(lines) != 0 {
+		t.Errorf("Close produced log lines: %v", lines)
+	}
+}
+
+func hasAttr(attrs []attribute.KeyValue, want attribute.KeyValue) bool {
+	for _, a := range attrs {
+		if a.Key == want.Key && a.Value == want.Value {
+			return true
+		}
+	}
+	return false
+}

internal/provider/kvm2/provider_test.go

@@ -0,0 +1,415 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/digitalocean/go-libvirt"
+	"github.com/go-logr/logr/funcr"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+const (
+	testProvider = "kvm2-home"
+	testHost     = "beelink"
+	testUID      = "0f1e2d3c-uid-1"
+	testName     = "proxy-abcdefghijklmnop"
+)
+
+// fixture is one configured host backed by a fake, an image server the
+// catalogue points at, a log capture at a chosen verbosity and a provider
+// with a fast running-wait, so a Create runs end to end in memory.
+type fixture struct {
+	t      *testing.T
+	p      *Provider
+	fake   *fakeBackend
+	dialer *stubDialer
+	logs   *logCapture
+	ctx    context.Context
+	image  []byte
+}
+
+func newFixture(t *testing.T, verbosity int, opts ...providerOption) *fixture {
+	t.Helper()
+	fx := &fixture{t: t, fake: newFakeBackend(), logs: &logCapture{}, image: []byte("QFI\xfb fake base image bytes")}
+	srv := httptest.NewServer(serveBytes(fx.image))
+	t.Cleanup(srv.Close)
+	fx.dialer = &stubDialer{backend: fx.fake}
+	cfg := provider.KVMConfig{
+		Images:       map[string]provider.KVMImage{imageKey: {URL: srv.URL + "/debian-13-generic-amd64.qcow2"}},
+		MachineTypes: map[string]provider.KVMMachineType{"small": {VCPUs: 1, MemoryMiB: 1024}},
+		Hosts:        []provider.KVMHost{{Name: testHost, Address: "beelink:22", User: "virt", Bridge: "br0"}},
+	}
+	opts = append([]providerOption{withHTTPClient(srv.Client()), withRunningWait(time.Millisecond, 500*time.Millisecond)}, opts...)
+	fx.p = newProvider(testProvider, cfg, fx.dialer, opts...)
+	log := funcr.New(func(prefix, args string) {
+		fx.logs.mu.Lock()
+		defer fx.logs.mu.Unlock()
+		fx.logs.lines = append(fx.logs.lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: verbosity})
+	fx.ctx = logf.IntoContext(context.Background(), log)
+	return fx
+}
+
+func (fx *fixture) req(name string) provider.CreateRequest {
+	return provider.CreateRequest{
+		Name:      name,
+		UID:       testUID,
+		Namespace: "default",
+		ProxyName: "egress",
+		Placement: provider.Placement{Zone: testHost, MachineType: "small", Image: imageKey},
+		CloudInit: "#cloud-config\npackages: [squid]\n",
+		Port:      3128,
+	}
+}
+
+func (fx *fixture) create(name string) string {
+	fx.t.Helper()
+	id, err := fx.p.Create(fx.ctx, fx.req(name))
+	if err != nil {
+		fx.t.Fatalf("Create(%s) error = %v", name, err)
+	}
+	if want := formatProviderID(testHost, name); id != want {
+		fx.t.Fatalf("Create(%s) = %q, want %q", name, id, want)
+	}
+	return id
+}
+
+func (fx *fixture) get(id string) *provider.Instance {
+	fx.t.Helper()
+	inst, err := fx.p.Get(fx.ctx, id)
+	if err != nil {
+		fx.t.Fatalf("Get(%s) error = %v", id, err)
+	}
+	return inst
+}
+
+// withDomain runs fn on the fake's domain under its lock.
+func (fx *fixture) withDomain(name string, fn func(d *fakeDomain)) {
+	fx.t.Helper()
+	fx.fake.mu.Lock()
+	defer fx.fake.mu.Unlock()
+	d, ok := fx.fake.domains[name]
+	if !ok {
+		fx.t.Fatalf("fake has no domain %s", name)
+	}
+	fn(d)
+}
+
+func (fx *fixture) hasDomain(name string) bool {
+	fx.fake.mu.Lock()
+	defer fx.fake.mu.Unlock()
+	_, ok := fx.fake.domains[name]
+	return ok
+}
+
+func (fx *fixture) setAddrs(name string, src libvirt.DomainInterfaceAddressesSource, ips ...string) {
+	fx.withDomain(name, func(d *fakeDomain) {
+		var addrs []libvirt.DomainIPAddr
+		for _, ip := range ips {
+			addrs = append(addrs, libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv4), Addr: ip, Prefix: 24})
+		}
+		d.addrs[uint32(src)] = []libvirt.DomainInterface{{Name: "enp1s0", Addrs: addrs}}
+	})
+}
+
+// putVol places a volume in the fake's pool directly.
+func (fx *fixture) putVol(name string, content []byte) *fakeVol {
+	fx.fake.mu.Lock()
+	defer fx.fake.mu.Unlock()
+	v := &fakeVol{content: content, capacity: uint64(len(content))}
+	fx.fake.pools[fakePoolName].vols[name] = v
+	return v
+}
+
+// cacheImage places a complete base image and its marker, as a previous
+// Create on the host would have.
+func (fx *fixture) cacheImage() {
+	src, err := resolveImage(fx.p.cfg, imageKey)
+	if err != nil {
+		fx.t.Fatal(err)
+	}
+	fx.putVol(src.volumeName(), fx.image)
+	fx.putVol(src.markerName(), []byte(digestOf(fx.image)))
+}
+
+func managedMetadata(uid string, createdAt time.Time) string {
+	s, err := marshalInstanceMetadata(newInstanceMetadata(uid, createdAt))
+	if err != nil {
+		panic(err)
+	}
+	return s
+}
+
+// assertProviderError checks err is a *provider.Error from the configured
+// provider with the given op and class.
+func assertProviderError(t *testing.T, err error, op string, class error) {
+	t.Helper()
+	if err == nil {
+		t.Fatalf("error = nil, want op %s class %v", op, class)
+	}
+	var pe *provider.Error
+	if !errors.As(err, &pe) {
+		t.Fatalf("error = %T (%v), want *provider.Error", err, err)
+	}
+	if pe.Provider != testProvider {
+		t.Errorf("error.Provider = %q, want %q", pe.Provider, testProvider)
+	}
+	if pe.Op != op {
+		t.Errorf("error.Op = %q, want %q", pe.Op, op)
+	}
+	if !errors.Is(err, class) {
+		t.Errorf("error = %v, want class %v", err, class)
+	}
+	for _, other := range []error{provider.ErrNotFound, provider.ErrQuotaExceeded, provider.ErrPermanent, provider.ErrTransient} {
+		if other != class && errors.Is(err, other) {
+			t.Errorf("error = %v, also matches %v", err, other)
+		}
+	}
+}
+
+// hostDialer hands out one backend per host name and fails hosts in errs.
+type hostDialer struct {
+	backends map[string]backend
+	errs     map[string]error
+}
+
+func (d *hostDialer) dial(_ context.Context, host provider.KVMHost) (backend, error) {
+	if err := d.errs[host.Name]; err != nil {
+		return nil, err
+	}
+	b, ok := d.backends[host.Name]
+	if !ok {
+		return nil, fmt.Errorf("hostDialer: no backend for %s", host.Name)
+	}
+	return b, nil
+}
+
+// waitFor polls cond until it holds or the deadline passes.
+func waitFor(t *testing.T, what string, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for !cond() {
+		if time.Now().After(deadline) {
+			t.Fatalf("timed out waiting for %s", what)
+		}
+		time.Sleep(time.Millisecond)
+	}
+}
+
+func TestProvider_cancelledContextReturnsPromptly(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name     string
+		firstRPC string
+		op       string
+		call     func(ctx context.Context, fx *fixture) error
+	}{
+		{
+			name: "create", firstRPC: "DomainLookupByName", op: "create",
+			call: func(ctx context.Context, fx *fixture) error { _, err := fx.p.Create(ctx, fx.req(testName)); return err },
+		},
+		{
+			name: "get", firstRPC: "DomainLookupByName", op: "get",
+			call: func(ctx context.Context, fx *fixture) error {
+				_, err := fx.p.Get(ctx, formatProviderID(testHost, testName))
+				return err
+			},
+		},
+		{
+			name: "delete", firstRPC: "DomainLookupByName", op: "delete",
+			call: func(ctx context.Context, fx *fixture) error {
+				return fx.p.Delete(ctx, formatProviderID(testHost, testName))
+			},
+		},
+		{
+			name: "list", firstRPC: "ConnectListAllDomains", op: "list",
+			call: func(ctx context.Context, fx *fixture) error { _, err := fx.p.ListByTag(ctx); return err },
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			release := fx.fake.blockOn(tc.firstRPC)
+			defer close(release)
+			ctx, cancel := context.WithCancel(fx.ctx)
+			done := make(chan error, 1)
+			go func() { done <- tc.call(ctx, fx) }()
+			waitFor(t, "the first RPC to block", func() bool { return fx.fake.callCount(tc.firstRPC) == 1 })
+			cancel()
+			select {
+			case err := <-done:
+				assertProviderError(t, err, tc.op, provider.ErrTransient)
+			case <-time.After(time.Second):
+				t.Fatalf("%s did not return within 1s of cancellation", tc.name)
+			}
+		})
+	}
+}
+
+func TestProvider_everyErrorIsWrappedWithConfiguredName(t *testing.T) {
+	t.Parallel()
+	dialErr := errors.New("dial tcp 192.0.2.1:22: connect: connection refused")
+	tests := []struct {
+		name  string
+		setup func(fx *fixture)
+		call  func(fx *fixture) error
+		op    string
+		class error
+	}{
+		{
+			name: "create unknown zone",
+			call: func(fx *fixture) error {
+				req := fx.req(testName)
+				req.Placement.Zone = "nas9"
+				_, err := fx.p.Create(fx.ctx, req)
+				return err
+			},
+			op: "create", class: provider.ErrPermanent,
+		},
+		{
+			name:  "create dial failure",
+			setup: func(fx *fixture) { fx.dialer.err = dialErr },
+			call:  func(fx *fixture) error { _, err := fx.p.Create(fx.ctx, fx.req(testName)); return err },
+			op:    "create", class: provider.ErrTransient,
+		},
+		{
+			name: "create pool out of space",
+			setup: func(fx *fixture) {
+				fx.fake.mu.Lock()
+				fx.fake.pools[fakePoolName].available = 1
+				fx.fake.mu.Unlock()
+			},
+			call: func(fx *fixture) error { _, err := fx.p.Create(fx.ctx, fx.req(testName)); return err },
+			op:   "create", class: provider.ErrQuotaExceeded,
+		},
+		{
+			name:  "create vanished volume",
+			setup: func(fx *fixture) { fx.fake.failWith("StorageVolGetPath", fakeErr(libvirt.ErrNoStorageVol, "gone")) },
+			call:  func(fx *fixture) error { _, err := fx.p.Create(fx.ctx, fx.req(testName)); return err },
+			op:    "create", class: provider.ErrTransient,
+		},
+		{
+			name: "get malformed id",
+			call: func(fx *fixture) error { _, err := fx.p.Get(fx.ctx, "beelink/"+testName); return err },
+			op:   "get", class: provider.ErrPermanent,
+		},
+		{
+			name: "get missing domain",
+			call: func(fx *fixture) error { _, err := fx.p.Get(fx.ctx, formatProviderID(testHost, testName)); return err },
+			op:   "get", class: provider.ErrNotFound,
+		},
+		{
+			name: "get busy backend",
+			setup: func(fx *fixture) {
+				fx.create(testName)
+				fx.fake.failWith("DomainGetState", fakeErr(libvirt.ErrOperationInvalid, "busy"))
+			},
+			call: func(fx *fixture) error { _, err := fx.p.Get(fx.ctx, formatProviderID(testHost, testName)); return err },
+			op:   "get", class: provider.ErrTransient,
+		},
+		{
+			name: "delete malformed id",
+			call: func(fx *fixture) error { return fx.p.Delete(fx.ctx, "hosts//domains/"+testName) },
+			op:   "delete", class: provider.ErrPermanent,
+		},
+		{
+			name:  "delete dial failure",
+			setup: func(fx *fixture) { fx.dialer.err = dialErr },
+			call:  func(fx *fixture) error { return fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)) },
+			op:    "delete", class: provider.ErrTransient,
+		},
+		{
+			name:  "delete access denied",
+			setup: func(fx *fixture) { fx.fake.failWith("DomainLookupByName", fakeErr(libvirt.ErrAccessDenied, "denied")) },
+			call:  func(fx *fixture) error { return fx.p.Delete(fx.ctx, formatProviderID(testHost, testName)) },
+			op:    "delete", class: provider.ErrPermanent,
+		},
+		{
+			name:  "list dial failure",
+			setup: func(fx *fixture) { fx.dialer.err = dialErr },
+			call:  func(fx *fixture) error { _, err := fx.p.ListByTag(fx.ctx); return err },
+			op:    "list", class: provider.ErrTransient,
+		},
+		{
+			name: "list rpc failure",
+			setup: func(fx *fixture) {
+				fx.fake.failWith("ConnectListAllDomains", fakeErr(libvirt.ErrInternalError, "boom"))
+			},
+			call: func(fx *fixture) error { _, err := fx.p.ListByTag(fx.ctx); return err },
+			op:   "list", class: provider.ErrTransient,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fx := newFixture(t, 1)
+			if tc.setup != nil {
+				tc.setup(fx)
+			}
+			assertProviderError(t, tc.call(fx), tc.op, tc.class)
+		})
+	}
+}
+
+func TestLogging_v0SilentV1OutcomeV2Detail(t *testing.T) {
+	t.Parallel()
+	outcomes := []string{"overlay volume created", "seed volume uploaded", "domain defined", "kvm2 domain ensured running"}
+	t.Run("v0 is silent", func(t *testing.T) {
+		t.Parallel()
+		fx := newFixture(t, 0)
+		fx.create(testName)
+		if lines := fx.logs.all(); len(lines) != 0 {
+			t.Errorf("Create logged %d lines at V(0), want none:\n%s", len(lines), strings.Join(lines, "\n"))
+		}
+	})
+	t.Run("v1 has outcomes without xml", func(t *testing.T) {
+		t.Parallel()
+		fx := newFixture(t, 1)
+		fx.create(testName)
+		for _, want := range outcomes {
+			if len(fx.logs.matching(want)) != 1 {
+				t.Errorf("V(1) lines matching %q = %d, want 1:\n%s", want, len(fx.logs.matching(want)), strings.Join(fx.logs.all(), "\n"))
+			}
+		}
+		if got := fx.logs.matching("<domain"); len(got) != 0 {
+			t.Errorf("V(1) carries domain XML:\n%s", strings.Join(got, "\n"))
+		}
+		if got := fx.logs.matching("kvm2 create resolved"); len(got) != 0 {
+			t.Errorf("V(1) carries the V(2) detail line:\n%s", strings.Join(got, "\n"))
+		}
+		for _, l := range fx.logs.all() {
+			if !strings.Contains(l, `"provider"="kvm2-home"`) || !strings.Contains(l, `"host"="beelink"`) {
+				t.Errorf("line lacks provider or host: %s", l)
+			}
+			if !strings.Contains(l, `"name"="`+testName+`"`) && !strings.Contains(l, `"volume"="base-`) {
+				t.Errorf("instance line lacks the name: %s", l)
+			}
+		}
+	})
+	t.Run("v2 has detail", func(t *testing.T) {
+		t.Parallel()
+		fx := newFixture(t, 2)
+		fx.create(testName)
+		if got := fx.logs.matching("<domain"); len(got) == 0 {
+			t.Errorf("V(2) lacks domain XML:\n%s", strings.Join(fx.logs.all(), "\n"))
+		}
+		resolved := fx.logs.matching("kvm2 create resolved")
+		if len(resolved) != 1 {
+			t.Fatalf("V(2) lines matching resolved = %d, want 1", len(resolved))
+		}
+		for _, key := range []string{`"cloudInitBytes"=`, `"image"="base-debian-13.qcow2"`, `"firmware"="uefi"`, `"vcpus"=1`, `"memoryMiB"=1024`, `"machineType"="small"`, `"bridge"="br0"`, `"pool"="default"`} {
+			if !strings.Contains(resolved[0], key) {
+				t.Errorf("resolved line lacks %s: %s", key, resolved[0])
+			}
+		}
+	})
+}

internal/provider/kvm2/seed.go

@@ -0,0 +1,124 @@
+package kvm2
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/textproto"
+	"strings"
+
+	"github.com/kdomanski/iso9660"
+)
+
+// seedLabel is the volume label cloud-init's NoCloud datasource looks for.
+const seedLabel = "cidata"
+
+// guestAgentScript is the vendor-data payload that gets qemu-guest-agent
+// running so the provider can read the VM's address through libvirt. It is
+// a shell script and not a cloud-config on purpose: cloud-init merges a
+// vendor-data cloud-config under the user-data with mergers that keep the
+// user-data's value for every top-level key and never append lists, so a
+// Proxy with its own packages: would silently lose the agent. Scripts
+// bypass the merge.
+const guestAgentScript = `#!/bin/sh
+# Added by egress-proxies-operator: installs and starts qemu-guest-agent so
+# the operator can learn this VM's IP address through libvirt.
+# To leave this out, set installGuestAgent: false in the provider's config.
+set -e
+if command -v qemu-ga >/dev/null 2>&1 || [ -x /usr/sbin/qemu-ga ]; then
+  echo "egress-proxies-operator: qemu-guest-agent already installed"
+elif command -v apt-get >/dev/null 2>&1; then
+  export DEBIAN_FRONTEND=noninteractive
+  apt-get update
+  apt-get install -y qemu-guest-agent
+elif command -v dnf >/dev/null 2>&1; then
+  dnf install -y qemu-guest-agent
+elif command -v yum >/dev/null 2>&1; then
+  yum install -y qemu-guest-agent
+elif command -v zypper >/dev/null 2>&1; then
+  zypper --non-interactive install qemu-guest-agent
+elif command -v apk >/dev/null 2>&1; then
+  apk add qemu-guest-agent
+else
+  echo "egress-proxies-operator: no supported package manager found; install qemu-guest-agent by hand" >&2
+  exit 1
+fi
+if command -v systemctl >/dev/null 2>&1; then
+  systemctl enable --now qemu-guest-agent
+else
+  rc-update add qemu-guest-agent default
+  rc-service qemu-guest-agent start
+fi
+`
+
+// buildSeedISO writes a NoCloud seed to w: user-data byte for byte,
+// meta-data naming the instance, and vendor-data (an empty file when there
+// is none, so cloud-init does not look elsewhere). iso9660.NewWriter stages
+// the files in os.MkdirTemp("", ""), that is under os.TempDir(): $TMPDIR
+// when set, else /tmp, the manager's 64Mi emptyDir. The staging dir is
+// removed before returning. Nothing about the payload is logged here.
+func buildSeedISO(name, userData, vendorData string, w io.Writer) (err error) {
+	if name == "" {
+		return permanentf("seed ISO: instance name is empty")
+	}
+	iw, err := iso9660.NewWriter()
+	if err != nil {
+		return fmt.Errorf("creating seed ISO staging dir: %w", err)
+	}
+	defer func() {
+		if cerr := iw.Cleanup(); cerr != nil {
+			err = errors.Join(err, fmt.Errorf("removing seed ISO staging dir: %w", cerr))
+		}
+	}()
+	metaData := "instance-id: " + name + "\nlocal-hostname: " + name + "\n"
+	files := []struct{ name, content string }{
+		{"user-data", userData},
+		{"meta-data", metaData},
+		{"vendor-data", vendorData},
+	}
+	for _, f := range files {
+		if err := iw.AddFile(strings.NewReader(f.content), f.name); err != nil {
+			return fmt.Errorf("adding %s to seed ISO: %w", f.name, err)
+		}
+	}
+	if err := iw.WriteTo(w, seedLabel); err != nil {
+		return fmt.Errorf("writing seed ISO: %w", err)
+	}
+	return nil
+}
+
+// buildVendorData composes the seed's vendor-data from the configured
+// document and the guest agent script. With both, they travel as separate
+// parts of a MIME multipart, so cloud-init handles each by its own type; the
+// MIME-Version header comes first because cloud-init detects multipart by
+// finding "mime-version:" within the first 4096 bytes.
+func buildVendorData(configured string, installAgent bool) (string, error) {
+	switch {
+	case !installAgent:
+		return configured, nil
+	case strings.TrimSpace(configured) == "":
+		return guestAgentScript, nil
+	}
+	var buf bytes.Buffer
+	mw := multipart.NewWriter(&buf)
+	fmt.Fprintf(&buf, "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=%q\r\n\r\n", mw.Boundary())
+	parts := []struct{ contentType, body string }{
+		{"text/plain", configured},
+		{"text/x-shellscript", guestAgentScript},
+	}
+	for _, p := range parts {
+		pw, err := mw.CreatePart(textproto.MIMEHeader{"Content-Type": {p.contentType}})
+		if err != nil {
+			return "", fmt.Errorf("creating vendor-data part %s: %w", p.contentType, err)
+		}
+		if _, err := io.WriteString(pw, p.body); err != nil {
+			return "", fmt.Errorf("writing vendor-data part %s: %w", p.contentType, err)
+		}
+	}
+	if err := mw.Close(); err != nil {
+		return "", fmt.Errorf("closing vendor-data multipart: %w", err)
+	}
+	return buf.String(), nil
+}

internal/provider/kvm2/seed_test.go

@@ -0,0 +1,264 @@
+package kvm2
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"mime"
+	"mime/multipart"
+	"net/textproto"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/kdomanski/iso9660"
+)
+
+// readSeed builds a seed and returns its files by name plus the label.
+func readSeed(t *testing.T, name, userData, vendorData string) (map[string]string, string) {
+	t.Helper()
+	var buf bytes.Buffer
+	if err := buildSeedISO(name, userData, vendorData, &buf); err != nil {
+		t.Fatalf("buildSeedISO: %v", err)
+	}
+	img, err := iso9660.OpenImage(bytes.NewReader(buf.Bytes()))
+	if err != nil {
+		t.Fatalf("OpenImage: %v", err)
+	}
+	root, err := img.RootDir()
+	if err != nil {
+		t.Fatalf("RootDir: %v", err)
+	}
+	children, err := root.GetChildren()
+	if err != nil {
+		t.Fatalf("GetChildren: %v", err)
+	}
+	files := make(map[string]string, len(children))
+	for _, c := range children {
+		if c.IsDir() {
+			t.Errorf("unexpected directory %q in seed", c.Name())
+			continue
+		}
+		data, err := io.ReadAll(c.Reader())
+		if err != nil {
+			t.Fatalf("reading %s: %v", c.Name(), err)
+		}
+		files[c.Name()] = string(data)
+	}
+	label, err := img.Label()
+	if err != nil {
+		t.Fatalf("Label: %v", err)
+	}
+	return files, label
+}
+
+func TestSeedISO_filesAndLabel(t *testing.T) {
+	t.Parallel()
+	userData := "#cloud-config\npackages: [squid]\n"
+	vendorData := "#!/bin/sh\necho hi\n"
+	files, label := readSeed(t, "proxy-abc", userData, vendorData)
+	if label != seedLabel {
+		t.Errorf("label = %q, want %q", label, seedLabel)
+	}
+	want := map[string]string{
+		"user-data":   userData,
+		"meta-data":   "instance-id: proxy-abc\nlocal-hostname: proxy-abc\n",
+		"vendor-data": vendorData,
+	}
+	if len(files) != len(want) {
+		t.Errorf("seed has %d files %v, want exactly %d", len(files), keys(files), len(want))
+	}
+	for name, content := range want {
+		if got, ok := files[name]; !ok {
+			t.Errorf("seed lacks %s", name)
+		} else if got != content {
+			t.Errorf("%s = %q, want %q", name, got, content)
+		}
+	}
+}
+
+func keys(m map[string]string) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	return out
+}
+
+func TestSeedISO_userDataByteForByte(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name     string
+		userData string
+	}{
+		{name: "mime multipart with CRLF", userData: "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"b\"\r\n\r\n--b\r\nContent-Type: text/cloud-config\r\n\r\n#cloud-config\r\npackages: [squid]\r\n--b--\r\n"},
+		{name: "jinja header", userData: "## template: jinja\n#cloud-config\nhostname: {{ v1.local_hostname }}\n"},
+		{name: "no trailing newline", userData: "#cloud-config\nruncmd:\n  - echo done"},
+		{name: "bytes above 0x7f", userData: "#cloud-config\nwrite_files:\n  - content: \"caf\xc3\xa9 \xff\xfe\x80\"\n"},
+		{name: "empty", userData: ""},
+		{name: "trailing whitespace preserved", userData: "#!/bin/sh\necho x \n\n\n"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			files, _ := readSeed(t, "proxy-abc", tc.userData, "")
+			if got := files["user-data"]; got != tc.userData {
+				t.Errorf("user-data changed:\n got %q\nwant %q", got, tc.userData)
+			}
+		})
+	}
+}
+
+// Not parallel: it changes TMPDIR for the process.
+func TestSeedISO_tempDirRemoved(t *testing.T) {
+	tmp := t.TempDir()
+	t.Setenv("TMPDIR", tmp)
+	if got := os.TempDir(); got != tmp {
+		t.Fatalf("os.TempDir() = %q, want %q", got, tmp)
+	}
+	var buf bytes.Buffer
+	if err := buildSeedISO("proxy-abc", "#cloud-config\n", guestAgentScript, &buf); err != nil {
+		t.Fatalf("buildSeedISO: %v", err)
+	}
+	if buf.Len() == 0 {
+		t.Fatal("seed ISO is empty")
+	}
+	entries, err := os.ReadDir(tmp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 0 {
+		names := make([]string, 0, len(entries))
+		for _, e := range entries {
+			names = append(names, e.Name())
+		}
+		t.Errorf("staging left behind in TMPDIR: %v", names)
+	}
+}
+
+func TestSeedISO_emptyVendorDataStillHasFile(t *testing.T) {
+	t.Parallel()
+	files, _ := readSeed(t, "proxy-abc", "#cloud-config\n", "")
+	got, ok := files["vendor-data"]
+	if !ok {
+		t.Fatalf("seed lacks vendor-data; files: %v", keys(files))
+	}
+	if got != "" {
+		t.Errorf("vendor-data = %q, want empty", got)
+	}
+}
+
+func TestSeedISO_rejectsEmptyName(t *testing.T) {
+	t.Parallel()
+	var buf bytes.Buffer
+	if err := buildSeedISO("", "#cloud-config\n", "", &buf); err == nil {
+		t.Fatal("buildSeedISO with empty name succeeded")
+	}
+	if buf.Len() != 0 {
+		t.Errorf("wrote %d bytes alongside the error", buf.Len())
+	}
+}
+
+func TestVendorData_isShellScriptNotCloudConfig(t *testing.T) {
+	t.Parallel()
+	got, err := buildVendorData("", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(got, "#!/bin/sh\n") {
+		t.Errorf("vendor-data starts with %q, want #!/bin/sh", firstLine(got))
+	}
+	if strings.HasPrefix(got, "#cloud-config") {
+		t.Error("vendor-data is a cloud-config; cloud-init would merge it away")
+	}
+	for _, want := range []string{"qemu-guest-agent", "set -e", "apt-get", "dnf", "yum", "zypper", "apk", "systemctl enable --now qemu-guest-agent", "rc-update", "installGuestAgent: false", "DEBIAN_FRONTEND=noninteractive"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("script lacks %q", want)
+		}
+	}
+	if strings.Contains(got, "MIME-Version") {
+		t.Error("script alone should not be wrapped in a multipart")
+	}
+}
+
+func firstLine(s string) string {
+	line, _, _ := strings.Cut(s, "\n")
+	return line
+}
+
+func bufioReader(s string) *bufio.Reader {
+	return bufio.NewReader(strings.NewReader(s))
+}
+
+func TestVendorData_multipartWhenExtraProvided(t *testing.T) {
+	t.Parallel()
+	configured := "#cloud-config\nntp:\n  enabled: true\n"
+	got, err := buildVendorData(configured, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(got, "MIME-Version: 1.0\r\n") {
+		t.Errorf("vendor-data starts with %q, want MIME-Version: 1.0", firstLine(got))
+	}
+	r := textproto.NewReader(bufioReader(got))
+	hdr, err := r.ReadMIMEHeader()
+	if err != nil {
+		t.Fatalf("reading top-level headers: %v", err)
+	}
+	mediaType, params, err := mime.ParseMediaType(hdr.Get("Content-Type"))
+	if err != nil {
+		t.Fatalf("parsing Content-Type %q: %v", hdr.Get("Content-Type"), err)
+	}
+	if mediaType != "multipart/mixed" {
+		t.Errorf("media type = %q, want multipart/mixed", mediaType)
+	}
+	mr := multipart.NewReader(r.R, params["boundary"])
+	type part struct{ contentType, body string }
+	var parts []part
+	for {
+		p, err := mr.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("NextPart: %v", err)
+		}
+		body, err := io.ReadAll(p)
+		if err != nil {
+			t.Fatalf("reading part: %v", err)
+		}
+		parts = append(parts, part{p.Header.Get("Content-Type"), string(body)})
+	}
+	want := []part{{"text/plain", configured}, {"text/x-shellscript", guestAgentScript}}
+	if len(parts) != len(want) {
+		t.Fatalf("got %d parts, want %d", len(parts), len(want))
+	}
+	for i := range want {
+		if parts[i] != want[i] {
+			t.Errorf("part %d = {%q, %q}, want {%q, %q}", i, parts[i].contentType, firstLine(parts[i].body), want[i].contentType, firstLine(want[i].body))
+		}
+	}
+}
+
+func TestVendorData_offWithoutConfiguredIsEmpty(t *testing.T) {
+	t.Parallel()
+	got, err := buildVendorData("", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != "" {
+		t.Errorf("vendor-data = %q, want empty", got)
+	}
+}
+
+func TestVendorData_offWithConfiguredPassesThrough(t *testing.T) {
+	t.Parallel()
+	configured := "#cloud-config\nntp:\n  enabled: true"
+	got, err := buildVendorData(configured, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != configured {
+		t.Errorf("vendor-data = %q, want the configured document unchanged", got)
+	}
+}

internal/provider/kvm2/ssh.go

@@ -0,0 +1,216 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"net"
+	"os"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// dialStage names the step of a dial that failed, so the Checker (spec
+// entry 16) can pick a reason without parsing messages.
+type dialStage string
+
+const (
+	// stageCredentials: the key or known_hosts file is missing or unparsable.
+	stageCredentials dialStage = "credentials"
+	// stageTCP: the TCP connection to host.Address failed.
+	stageTCP dialStage = "tcp"
+	// stageSSH: the SSH handshake failed, including host key and auth.
+	stageSSH dialStage = "ssh"
+	// stageSocket: the forwarded channel to the libvirt socket was refused.
+	stageSocket dialStage = "socket"
+)
+
+// dialError tags a dial failure with its stage and host. Unwrap keeps the
+// cause visible, so classify still sees *knownhosts.KeyError, fs.ErrNotExist,
+// the "unable to authenticate" text and *net.OpError.
+type dialError struct {
+	stage dialStage
+	host  string
+	err   error
+}
+
+func (e *dialError) Error() string {
+	return fmt.Sprintf("dialing host %s (%s): %v", e.host, e.stage, e.err)
+}
+
+func (e *dialError) Unwrap() error { return e.err }
+
+// stageOf returns the failed stage of a dial error, if err is one.
+func stageOf(err error) (dialStage, bool) {
+	var de *dialError
+	if !errors.As(err, &de) {
+		return "", false
+	}
+	return de.stage, true
+}
+
+const (
+	defaultKeepaliveInterval = 15 * time.Second
+	keepaliveRequest         = "keepalive@openssh.com"
+)
+
+// credentials is what one dial needs from the mounted files. They are read
+// on every dial and never cached (spec entry 14), so rotating the Secret
+// takes effect on the next connection.
+type credentials struct {
+	signer  ssh.Signer
+	hostKey ssh.HostKeyCallback
+}
+
+// loadCredentials reads and parses the key and known_hosts files. A missing
+// file keeps fs.ErrNotExist (Transient: the Secret has not landed); a file
+// that does not parse is Permanent. Messages name the path and carry the
+// parser's own error, never the file's content.
+func loadCredentials(cfg provider.KVMConfig) (credentials, error) {
+	keyBytes, err := os.ReadFile(cfg.SSHKeyFile)
+	if err != nil {
+		return credentials{}, fmt.Errorf("reading ssh key file %s: %w", cfg.SSHKeyFile, err)
+	}
+	signer, err := ssh.ParsePrivateKey(keyBytes)
+	if err != nil {
+		return credentials{}, permanentf("parsing ssh key file %s: %v", cfg.SSHKeyFile, err)
+	}
+	if cfg.InsecureIgnoreHostKey {
+		return credentials{signer: signer, hostKey: ssh.InsecureIgnoreHostKey()}, nil
+	}
+	hostKey, err := knownhosts.New(cfg.KnownHostsFile)
+	if err != nil {
+		var pathErr *fs.PathError
+		if errors.As(err, &pathErr) {
+			return credentials{}, fmt.Errorf("reading known_hosts file %s: %w", cfg.KnownHostsFile, err)
+		}
+		return credentials{}, permanentf("parsing known_hosts file %s: %v", cfg.KnownHostsFile, err)
+	}
+	return credentials{signer: signer, hostKey: hostKey}, nil
+}
+
+// sshSocket is the libvirt socket forwarded through one SSH connection. The
+// embedded Conn is the channel; Close also closes the SSH client, which is
+// what unblocks anything reading the channel.
+type sshSocket struct {
+	net.Conn
+	client *ssh.Client
+
+	closeOnce sync.Once
+	closeErr  error
+}
+
+func (s *sshSocket) Close() error {
+	s.closeOnce.Do(func() {
+		s.closeErr = errors.Join(s.Conn.Close(), s.client.Close())
+	})
+	return s.closeErr
+}
+
+// keepalive sends keepalive@openssh.com every interval until stop is closed
+// or a request fails or takes longer than the interval, and returns why it
+// stopped (nil for stop). The caller marks its link dead on an error.
+func (s *sshSocket) keepalive(interval time.Duration, stop <-chan struct{}) error {
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-stop:
+			return nil
+		case <-ticker.C:
+		}
+		done := make(chan error, 1)
+		go func() {
+			_, _, err := s.client.SendRequest(keepaliveRequest, true, nil)
+			done <- err
+		}()
+		timer := time.NewTimer(interval)
+		select {
+		case err := <-done:
+			timer.Stop()
+			if err != nil {
+				return fmt.Errorf("keepalive: %w", err)
+			}
+		case <-timer.C:
+			return fmt.Errorf("keepalive: no reply within %s", interval)
+		case <-stop:
+			timer.Stop()
+			return nil
+		}
+	}
+}
+
+// dialSSHSocket opens host's libvirt socket through SSH: credentials, TCP,
+// handshake, channel. timeout bounds the whole sequence on top of ctx. Each
+// failure is a *dialError naming its stage. Nothing here logs.
+func dialSSHSocket(ctx context.Context, host provider.KVMHost, cfg provider.KVMConfig, timeout time.Duration) (*sshSocket, error) {
+	if timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+		defer cancel()
+	}
+	creds, err := loadCredentials(cfg)
+	if err != nil {
+		return nil, &dialError{stage: stageCredentials, host: host.Name, err: err}
+	}
+	var nd net.Dialer
+	tcp, err := nd.DialContext(ctx, "tcp", host.Address)
+	if err != nil {
+		return nil, &dialError{stage: stageTCP, host: host.Name, err: err}
+	}
+	clientCfg := &ssh.ClientConfig{
+		User:            host.User,
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(creds.signer)},
+		HostKeyCallback: creds.hostKey,
+		Timeout:         timeout,
+	}
+	client, err := await(ctx, tcp.Close, func() (*ssh.Client, error) {
+		c, chans, reqs, err := ssh.NewClientConn(tcp, host.Address, clientCfg)
+		if err != nil {
+			return nil, err
+		}
+		return ssh.NewClient(c, chans, reqs), nil
+	})
+	if err != nil {
+		_ = tcp.Close()
+		return nil, &dialError{stage: stageSSH, host: host.Name, err: err}
+	}
+	conn, err := await(ctx, func() error { return client.Close() }, func() (net.Conn, error) {
+		return client.Dial("unix", host.Socket)
+	})
+	if err != nil {
+		_ = client.Close()
+		return nil, &dialError{stage: stageSocket, host: host.Name, err: fmt.Errorf("opening %s: %w", host.Socket, err)}
+	}
+	return &sshSocket{Conn: conn, client: client}, nil
+}
+
+// await runs a blocking step in a goroutine. When ctx ends first it calls
+// abort, which closes the transport the step is blocked on, waits for the
+// step to give up, and returns ctx's error instead.
+func await[T any](ctx context.Context, abort func() error, step func() (T, error)) (T, error) {
+	type result struct {
+		v   T
+		err error
+	}
+	done := make(chan result, 1)
+	go func() {
+		v, err := step()
+		done <- result{v, err}
+	}()
+	select {
+	case r := <-done:
+		return r.v, r.err
+	case <-ctx.Done():
+		_ = abort()
+		<-done
+		var zero T
+		return zero, ctx.Err()
+	}
+}

internal/provider/kvm2/ssh_test.go

@@ -0,0 +1,343 @@
+package kvm2
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"net"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// sshFixture is one in-process server with a client key it accepts and a
+// known_hosts file naming its host key.
+type sshFixture struct {
+	server *testSSHServer
+	socket string
+	cfg    provider.KVMConfig
+	host   provider.KVMHost
+}
+
+func newSSHFixture(t *testing.T) *sshFixture {
+	t.Helper()
+	dir := t.TempDir()
+	socket := startEchoSocket(t)
+	keyPath, pub := writeKeyFile(t, dir, "id_ed25519")
+	server := startSSHServer(t, pub, socket)
+	return &sshFixture{
+		server: server,
+		socket: socket,
+		cfg: provider.KVMConfig{
+			SSHKeyFile:     keyPath,
+			KnownHostsFile: writeKnownHosts(t, dir, "known_hosts", server.addr, server.hostKey),
+		},
+		host: provider.KVMHost{Name: "beelink", Address: server.addr, User: "virt", Socket: socket},
+	}
+}
+
+const dialTestTimeout = 5 * time.Second
+
+func TestDial_forwardsUnixSocket(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	for round := range 2 {
+		sock, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, dialTestTimeout)
+		if err != nil {
+			t.Fatalf("round %d: dial error = %v", round, err)
+		}
+		msg := []byte("ping over libvirt-sock")
+		if _, err := sock.Write(msg); err != nil {
+			t.Fatalf("round %d: write: %v", round, err)
+		}
+		buf := make([]byte, len(msg))
+		_ = sock.SetReadDeadline(time.Now().Add(dialTestTimeout))
+		if _, err := readFull(sock, buf); err != nil {
+			t.Fatalf("round %d: read: %v", round, err)
+		}
+		if string(buf) != string(msg) {
+			t.Errorf("round %d: echoed %q, want %q", round, buf, msg)
+		}
+		if err := sock.Close(); err != nil {
+			t.Errorf("round %d: close error = %v", round, err)
+		}
+		if err := sock.Close(); err != nil {
+			t.Errorf("round %d: second close error = %v, want nil", round, err)
+		}
+	}
+	for _, p := range fx.server.requestedPaths() {
+		if p != fx.socket {
+			t.Errorf("server saw channel for %q, want %q", p, fx.socket)
+		}
+	}
+}
+
+func readFull(sock *sshSocket, buf []byte) (int, error) {
+	n := 0
+	for n < len(buf) {
+		m, err := sock.Read(buf[n:])
+		n += m
+		if err != nil {
+			return n, err
+		}
+	}
+	return n, nil
+}
+
+func TestDial_unknownHostKeyRejected(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.cfg.KnownHostsFile = writeKnownHosts(t, t.TempDir(), "known_hosts", "other.example:22", fx.server.hostKey)
+	_, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, dialTestTimeout)
+	assertDialFailure(t, err, stageSSH, provider.ErrPermanent)
+	var keyErr *knownhosts.KeyError
+	if !errors.As(err, &keyErr) {
+		t.Fatalf("error = %v, want *knownhosts.KeyError", err)
+	}
+	if len(keyErr.Want) != 0 {
+		t.Errorf("KeyError.Want = %v, want empty (host unknown)", keyErr.Want)
+	}
+}
+
+func TestDial_mismatchedHostKeyRejected(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.cfg.KnownHostsFile = writeKnownHosts(t, t.TempDir(), "known_hosts", fx.server.addr, otherPublicKey(t))
+	_, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, dialTestTimeout)
+	assertDialFailure(t, err, stageSSH, provider.ErrPermanent)
+	var keyErr *knownhosts.KeyError
+	if !errors.As(err, &keyErr) {
+		t.Fatalf("error = %v, want *knownhosts.KeyError", err)
+	}
+	if len(keyErr.Want) == 0 {
+		t.Error("KeyError.Want empty, want the mismatching known key")
+	}
+}
+
+func TestDial_insecureIgnoreHostKeyAccepts(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.cfg.KnownHostsFile = ""
+	fx.cfg.InsecureIgnoreHostKey = true
+	sock, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, dialTestTimeout)
+	if err != nil {
+		t.Fatalf("dial error = %v, want nil with host keys ignored", err)
+	}
+	_ = sock.Close()
+}
+
+func TestDial_missingKeyFileIsTransient(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.cfg.SSHKeyFile = fx.cfg.SSHKeyFile + ".missing"
+	_, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, dialTestTimeout)
+	assertDialFailure(t, err, stageCredentials, provider.ErrTransient)
+	if !errors.Is(err, fs.ErrNotExist) {
+		t.Errorf("error = %v, want fs.ErrNotExist", err)
+	}
+	if !strings.Contains(err.Error(), fx.cfg.SSHKeyFile) {
+		t.Errorf("error = %q, want the path in it", err)
+	}
+}
+
+func TestDial_missingKnownHostsIsTransient(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.cfg.KnownHostsFile = fx.cfg.KnownHostsFile + ".missing"
+	_, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, dialTestTimeout)
+	assertDialFailure(t, err, stageCredentials, provider.ErrTransient)
+	if !errors.Is(err, fs.ErrNotExist) {
+		t.Errorf("error = %v, want fs.ErrNotExist", err)
+	}
+}
+
+func TestDial_unparsableKeyIsPermanent(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.cfg.SSHKeyFile = writeFile(t, t.TempDir(), "id_bad", "this is not a key\n")
+	_, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, dialTestTimeout)
+	assertDialFailure(t, err, stageCredentials, provider.ErrPermanent)
+	if !strings.Contains(err.Error(), "parsing ssh key file "+fx.cfg.SSHKeyFile) {
+		t.Errorf("error = %q, want 'parsing ssh key file <path>'", err)
+	}
+}
+
+func TestDial_unparsableKnownHostsIsPermanent(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.cfg.KnownHostsFile = writeFile(t, t.TempDir(), "known_hosts", "host.example ssh-ed25519 not-base64!\n")
+	_, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, dialTestTimeout)
+	assertDialFailure(t, err, stageCredentials, provider.ErrPermanent)
+	if !strings.Contains(err.Error(), "parsing known_hosts file "+fx.cfg.KnownHostsFile) {
+		t.Errorf("error = %q, want 'parsing known_hosts file <path>'", err)
+	}
+}
+
+func TestDial_wrongKeyIsUnauthorized(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.cfg.SSHKeyFile, _ = writeKeyFile(t, t.TempDir(), "id_other")
+	_, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, dialTestTimeout)
+	assertDialFailure(t, err, stageSSH, provider.ErrPermanent)
+	if !strings.Contains(err.Error(), sshAuthFailed) {
+		t.Errorf("error = %q, want %q in it", err, sshAuthFailed)
+	}
+}
+
+func TestDial_handshakeTimeout(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.host.Address = silentListener(t)
+	const budget = 200 * time.Millisecond
+	start := time.Now()
+	_, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, budget)
+	elapsed := time.Since(start)
+	assertDialFailure(t, err, stageSSH, provider.ErrTransient)
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("error = %v, want context.DeadlineExceeded", err)
+	}
+	if elapsed > budget+time.Second {
+		t.Errorf("dial took %v, want about the %v budget", elapsed, budget)
+	}
+}
+
+func TestDial_cancelledContext(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.host.Address = silentListener(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(20 * time.Millisecond)
+		cancel()
+	}()
+	start := time.Now()
+	_, err := dialSSHSocket(ctx, fx.host, fx.cfg, dialTestTimeout)
+	if time.Since(start) > time.Second {
+		t.Errorf("dial took %v after cancel", time.Since(start))
+	}
+	assertDialFailure(t, err, stageSSH, provider.ErrTransient)
+	if !errors.Is(err, context.Canceled) {
+		t.Errorf("error = %v, want context.Canceled", err)
+	}
+}
+
+func TestDial_refusedIsTransient(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.host.Address = closedPort(t)
+	_, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, dialTestTimeout)
+	assertDialFailure(t, err, stageTCP, provider.ErrTransient)
+	var opErr *net.OpError
+	if !errors.As(err, &opErr) {
+		t.Errorf("error = %v, want *net.OpError", err)
+	}
+}
+
+func TestDial_socketRefusedIsSocketStage(t *testing.T) {
+	t.Parallel()
+	fx := newSSHFixture(t)
+	fx.host.Socket = "/run/nonexistent/libvirt-sock"
+	_, err := dialSSHSocket(context.Background(), fx.host, fx.cfg, dialTestTimeout)
+	assertDialFailure(t, err, stageSocket, provider.ErrTransient)
+	if !strings.Contains(err.Error(), fx.host.Socket) {
+		t.Errorf("error = %q, want the socket path in it", err)
+	}
+}
+
+func TestSSH_keyMaterialNeverInErrorsOrLogs(t *testing.T) {
+	t.Parallel()
+	const keySentinel = "SENTINEL-KEY-4f9c2b"
+	const hostsSentinel = "SENTINEL-HOSTS-8a1d7e"
+	fx := newSSHFixture(t)
+	dir := t.TempDir()
+	tests := []struct {
+		name string
+		cfg  func(c *provider.KVMConfig)
+	}{
+		{
+			name: "bad key file",
+			cfg: func(c *provider.KVMConfig) {
+				c.SSHKeyFile = writeFile(t, dir, "id_bad", "-----BEGIN NOT A KEY-----\n"+keySentinel+"\n-----END NOT A KEY-----\n")
+			},
+		},
+		{
+			name: "bad known_hosts file",
+			cfg: func(c *provider.KVMConfig) {
+				c.KnownHostsFile = writeFile(t, dir, "known_hosts_bad", "beelink ssh-ed25519 "+hostsSentinel+"\n")
+			},
+		},
+		{
+			name: "good key, host unknown",
+			cfg: func(c *provider.KVMConfig) {
+				c.KnownHostsFile = writeFile(t, dir, "known_hosts_other", knownhosts.Line([]string{"other.example:22"}, fx.server.hostKey)+"\n")
+			},
+		},
+	}
+	keyBytes := readFileT(t, fx.cfg.SSHKeyFile)
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			cfg := fx.cfg
+			tc.cfg(&cfg)
+			_, err := dialSSHSocket(context.Background(), fx.host, cfg, dialTestTimeout)
+			if err == nil {
+				t.Fatal("dial succeeded, want an error")
+			}
+			msg := err.Error()
+			for _, s := range []string{keySentinel, hostsSentinel} {
+				if strings.Contains(msg, s) {
+					t.Errorf("error %q contains sentinel %q", msg, s)
+				}
+			}
+			if strings.Contains(msg, keyBytes) || strings.Contains(msg, "PRIVATE KEY") {
+				t.Errorf("error %q contains key material", msg)
+			}
+		})
+	}
+}
+
+func readFileT(t *testing.T, path string) string {
+	t.Helper()
+	b, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("reading %s: %v", path, err)
+	}
+	return strings.TrimSpace(string(b))
+}
+
+func assertDialFailure(t *testing.T, err error, wantStage dialStage, wantClass error) {
+	t.Helper()
+	if err == nil {
+		t.Fatal("dial succeeded, want an error")
+	}
+	stage, ok := stageOf(err)
+	if !ok {
+		t.Fatalf("error %v is not a *dialError", err)
+	}
+	if stage != wantStage {
+		t.Errorf("stage = %q, want %q (error: %v)", stage, wantStage, err)
+	}
+	if got := classify("get", err); !errors.Is(got, wantClass) {
+		t.Errorf("classify() = %v, want %v (error: %v)", got, wantClass, err)
+	}
+}
+
+func TestDialError_unwrapsToCause(t *testing.T) {
+	t.Parallel()
+	cause := &knownhosts.KeyError{}
+	err := &dialError{stage: stageSSH, host: "beelink", err: cause}
+	if !errors.Is(err, cause) {
+		t.Error("errors.Is(dialError, cause) = false")
+	}
+	if !strings.Contains(err.Error(), "beelink") || !strings.Contains(err.Error(), "(ssh)") {
+		t.Errorf("Error() = %q, want the host and the stage", err)
+	}
+	if _, ok := stageOf(errors.New("plain")); ok {
+		t.Error("stageOf(plain error) reported a stage")
+	}
+}

internal/provider/kvm2/sshserver_test.go

@@ -0,0 +1,289 @@
+package kvm2
+
+import (
+	"bytes"
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/pem"
+	"errors"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+)
+
+// testSSHServer is an in-process x/crypto/ssh server that accepts one
+// authorised public key and forwards direct-streamlocal@openssh.com channels
+// for socketPath to that unix socket, rejecting every other channel.
+type testSSHServer struct {
+	addr       string
+	hostKey    ssh.PublicKey
+	socketPath string
+
+	listener net.Listener
+	mu       sync.Mutex
+	conns    []net.Conn
+	wg       sync.WaitGroup
+	stopOnce sync.Once
+
+	reqMu     sync.Mutex
+	requested []string
+}
+
+func startSSHServer(t *testing.T, authorized ssh.PublicKey, socketPath string) *testSSHServer {
+	t.Helper()
+	_, hostPriv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatalf("generating host key: %v", err)
+	}
+	hostSigner, err := ssh.NewSignerFromKey(hostPriv)
+	if err != nil {
+		t.Fatalf("host signer: %v", err)
+	}
+	cfg := &ssh.ServerConfig{
+		PublicKeyCallback: func(_ ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
+			if bytes.Equal(key.Marshal(), authorized.Marshal()) {
+				return &ssh.Permissions{}, nil
+			}
+			return nil, errors.New("key not authorised")
+		},
+	}
+	cfg.AddHostKey(hostSigner)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	s := &testSSHServer{addr: ln.Addr().String(), hostKey: hostSigner.PublicKey(), socketPath: socketPath, listener: ln}
+	s.wg.Add(1)
+	go s.serve(cfg)
+	t.Cleanup(s.stop)
+	return s
+}
+
+func (s *testSSHServer) serve(cfg *ssh.ServerConfig) {
+	defer s.wg.Done()
+	for {
+		conn, err := s.listener.Accept()
+		if err != nil {
+			return
+		}
+		s.mu.Lock()
+		s.conns = append(s.conns, conn)
+		s.mu.Unlock()
+		s.wg.Add(1)
+		go s.handle(conn, cfg)
+	}
+}
+
+func (s *testSSHServer) handle(conn net.Conn, cfg *ssh.ServerConfig) {
+	defer s.wg.Done()
+	sc, chans, reqs, err := ssh.NewServerConn(conn, cfg)
+	if err != nil {
+		_ = conn.Close()
+		return
+	}
+	defer sc.Close()
+	go ssh.DiscardRequests(reqs)
+	for nc := range chans {
+		if nc.ChannelType() != "direct-streamlocal@openssh.com" {
+			_ = nc.Reject(ssh.UnknownChannelType, "only direct-streamlocal@openssh.com is served")
+			continue
+		}
+		var open struct {
+			SocketPath string
+			Reserved0  string
+			Reserved1  uint32
+		}
+		if err := ssh.Unmarshal(nc.ExtraData(), &open); err != nil {
+			_ = nc.Reject(ssh.ConnectionFailed, "bad direct-streamlocal payload")
+			continue
+		}
+		s.reqMu.Lock()
+		s.requested = append(s.requested, open.SocketPath)
+		s.reqMu.Unlock()
+		if open.SocketPath != s.socketPath {
+			_ = nc.Reject(ssh.ConnectionFailed, "no such socket")
+			continue
+		}
+		target, err := net.Dial("unix", s.socketPath)
+		if err != nil {
+			_ = nc.Reject(ssh.ConnectionFailed, "socket unavailable")
+			continue
+		}
+		ch, chReqs, err := nc.Accept()
+		if err != nil {
+			_ = target.Close()
+			continue
+		}
+		go ssh.DiscardRequests(chReqs)
+		s.wg.Add(1)
+		go func() {
+			defer s.wg.Done()
+			proxy(ch, target)
+		}()
+	}
+}
+
+func proxy(ch ssh.Channel, target net.Conn) {
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		_, _ = io.Copy(target, ch)
+		_ = target.Close()
+	}()
+	go func() {
+		defer wg.Done()
+		_, _ = io.Copy(ch, target)
+		_ = ch.Close()
+	}()
+	wg.Wait()
+}
+
+func (s *testSSHServer) stop() {
+	s.stopOnce.Do(func() {
+		_ = s.listener.Close()
+		s.mu.Lock()
+		for _, c := range s.conns {
+			_ = c.Close()
+		}
+		s.mu.Unlock()
+		s.wg.Wait()
+	})
+}
+
+func (s *testSSHServer) requestedPaths() []string {
+	s.reqMu.Lock()
+	defer s.reqMu.Unlock()
+	return append([]string(nil), s.requested...)
+}
+
+// startEchoSocket listens on a unix socket that echoes what it reads. The
+// path is kept short: unix socket paths are limited to about 100 bytes and
+// t.TempDir() names are long.
+func startEchoSocket(t *testing.T) string {
+	t.Helper()
+	dir, err := os.MkdirTemp("", "kvm2")
+	if err != nil {
+		t.Fatalf("temp dir: %v", err)
+	}
+	t.Cleanup(func() { _ = os.RemoveAll(dir) })
+	path := filepath.Join(dir, "libvirt-sock")
+	ln, err := net.Listen("unix", path)
+	if err != nil {
+		t.Fatalf("listen unix: %v", err)
+	}
+	t.Cleanup(func() { _ = ln.Close() })
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			go func() {
+				_, _ = io.Copy(c, c)
+				_ = c.Close()
+			}()
+		}
+	}()
+	return path
+}
+
+// silentListener accepts TCP connections and never speaks, for handshake
+// timeouts. It returns the address.
+func silentListener(t *testing.T) string {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	var conns []net.Conn
+	var mu sync.Mutex
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			mu.Lock()
+			conns = append(conns, c)
+			mu.Unlock()
+		}
+	}()
+	t.Cleanup(func() {
+		_ = ln.Close()
+		mu.Lock()
+		defer mu.Unlock()
+		for _, c := range conns {
+			_ = c.Close()
+		}
+	})
+	return ln.Addr().String()
+}
+
+// closedPort returns an address nothing listens on.
+func closedPort(t *testing.T) string {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	addr := ln.Addr().String()
+	_ = ln.Close()
+	return addr
+}
+
+// writeKeyFile generates an ed25519 key, writes it in OpenSSH format and
+// returns the path and the public half.
+func writeKeyFile(t *testing.T, dir, name string) (string, ssh.PublicKey) {
+	t.Helper()
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatalf("generating key: %v", err)
+	}
+	block, err := ssh.MarshalPrivateKey(priv, "kvm2 test")
+	if err != nil {
+		t.Fatalf("marshalling key: %v", err)
+	}
+	path := writeFile(t, dir, name, string(pem.EncodeToMemory(block)))
+	sshPub, err := ssh.NewPublicKey(pub)
+	if err != nil {
+		t.Fatalf("public key: %v", err)
+	}
+	return path, sshPub
+}
+
+// writeKnownHosts writes one known_hosts line for addr and key.
+func writeKnownHosts(t *testing.T, dir, name, addr string, key ssh.PublicKey) string {
+	t.Helper()
+	return writeFile(t, dir, name, knownhosts.Line([]string{addr}, key)+"\n")
+}
+
+func writeFile(t *testing.T, dir, name, content string) string {
+	t.Helper()
+	path := filepath.Join(dir, name)
+	if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
+		t.Fatalf("writing %s: %v", path, err)
+	}
+	return path
+}
+
+// otherPublicKey returns a public key unrelated to any file, for mismatch
+// cases.
+func otherPublicKey(t *testing.T) ssh.PublicKey {
+	t.Helper()
+	pub, _, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatalf("generating key: %v", err)
+	}
+	sshPub, err := ssh.NewPublicKey(pub)
+	if err != nil {
+		t.Fatalf("public key: %v", err)
+	}
+	return sshPub
+}

internal/provider/kvm2/state.go

@@ -0,0 +1,49 @@
+package kvm2
+
+import (
+	"net/netip"
+
+	"github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// mapState turns a libvirt domain state plus the discovered address into the
+// contract's state. nostate means libvirt could not tell, so the next poll
+// decides; an active domain is Running only once it has an address; every
+// inactive state, and anything unknown, is Stopped, which the reconciler
+// answers by replacing the instance.
+func mapState(s libvirt.DomainState, ip string) provider.InstanceState {
+	switch {
+	case s == libvirt.DomainNostate:
+		return provider.StateProvisioning
+	case isActive(s) && ip == "":
+		return provider.StateProvisioning
+	case isActive(s):
+		return provider.StateRunning
+	default:
+		return provider.StateStopped
+	}
+}
+
+func isActive(s libvirt.DomainState) bool {
+	return s == libvirt.DomainRunning || s == libvirt.DomainBlocked
+}
+
+// firstIPv4 picks the first usable IPv4 address from the guest's interfaces:
+// not loopback, not link-local, not unspecified. Empty means no address yet.
+func firstIPv4(ifaces []libvirt.DomainInterface) string {
+	for _, iface := range ifaces {
+		for _, a := range iface.Addrs {
+			if libvirt.IPAddrType(a.Type) != libvirt.IPAddrTypeIpv4 {
+				continue
+			}
+			addr, err := netip.ParseAddr(a.Addr)
+			if err != nil || !addr.Is4() || addr.IsLoopback() || addr.IsLinkLocalUnicast() || addr.IsUnspecified() {
+				continue
+			}
+			return addr.String()
+		}
+	}
+	return ""
+}

internal/provider/kvm2/state_test.go

@@ -0,0 +1,92 @@
+package kvm2
+
+import (
+	"testing"
+
+	"github.com/digitalocean/go-libvirt"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+func TestMapState_table(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		state   libvirt.DomainState
+		noIP    provider.InstanceState
+		withIP  provider.InstanceState
+		display string
+	}{
+		{libvirt.DomainNostate, provider.StateProvisioning, provider.StateProvisioning, "nostate"},
+		{libvirt.DomainRunning, provider.StateProvisioning, provider.StateRunning, "running"},
+		{libvirt.DomainBlocked, provider.StateProvisioning, provider.StateRunning, "blocked"},
+		{libvirt.DomainPaused, provider.StateStopped, provider.StateStopped, "paused"},
+		{libvirt.DomainShutdown, provider.StateStopped, provider.StateStopped, "shutdown"},
+		{libvirt.DomainShutoff, provider.StateStopped, provider.StateStopped, "shutoff"},
+		{libvirt.DomainCrashed, provider.StateStopped, provider.StateStopped, "crashed"},
+		{libvirt.DomainPmsuspended, provider.StateStopped, provider.StateStopped, "pmsuspended"},
+		{libvirt.DomainState(99), provider.StateStopped, provider.StateStopped, "unknown code 99"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.display, func(t *testing.T) {
+			t.Parallel()
+			if got := mapState(tc.state, ""); got != tc.noIP {
+				t.Errorf("mapState(%s, \"\") = %s, want %s", tc.display, got, tc.noIP)
+			}
+			if got := mapState(tc.state, "10.0.0.5"); got != tc.withIP {
+				t.Errorf("mapState(%s, ip) = %s, want %s", tc.display, got, tc.withIP)
+			}
+		})
+	}
+}
+
+func TestIsActive(t *testing.T) {
+	t.Parallel()
+	for s, want := range map[libvirt.DomainState]bool{
+		libvirt.DomainRunning: true, libvirt.DomainBlocked: true,
+		libvirt.DomainNostate: false, libvirt.DomainPaused: false, libvirt.DomainShutoff: false, libvirt.DomainState(99): false,
+	} {
+		if got := isActive(s); got != want {
+			t.Errorf("isActive(%d) = %v, want %v", s, got, want)
+		}
+	}
+}
+
+func addr(typ libvirt.IPAddrType, a string) libvirt.DomainIPAddr {
+	return libvirt.DomainIPAddr{Type: int32(typ), Addr: a}
+}
+
+func TestFirstIPv4_filters(t *testing.T) {
+	t.Parallel()
+	v4 := libvirt.IPAddrTypeIpv4
+	v6 := libvirt.IPAddrTypeIpv6
+	tests := []struct {
+		name   string
+		ifaces []libvirt.DomainInterface
+		want   string
+	}{
+		{name: "empty list", ifaces: nil, want: ""},
+		{name: "loopback only", ifaces: []libvirt.DomainInterface{{Name: "lo", Addrs: []libvirt.DomainIPAddr{addr(v4, "127.0.0.1")}}}, want: ""},
+		{name: "link-local only", ifaces: []libvirt.DomainInterface{{Name: "eth0", Addrs: []libvirt.DomainIPAddr{addr(v4, "169.254.10.1")}}}, want: ""},
+		{name: "ipv6 only", ifaces: []libvirt.DomainInterface{{Name: "eth0", Addrs: []libvirt.DomainIPAddr{addr(v6, "2001:db8::1"), addr(v6, "fe80::1")}}}, want: ""},
+		{name: "unspecified", ifaces: []libvirt.DomainInterface{{Name: "eth0", Addrs: []libvirt.DomainIPAddr{addr(v4, "0.0.0.0")}}}, want: ""},
+		{name: "unparsable", ifaces: []libvirt.DomainInterface{{Name: "eth0", Addrs: []libvirt.DomainIPAddr{addr(v4, "not-an-ip")}}}, want: ""},
+		{name: "ipv4 literal tagged as ipv6 is skipped", ifaces: []libvirt.DomainInterface{{Name: "eth0", Addrs: []libvirt.DomainIPAddr{addr(v6, "10.0.0.7")}}}, want: ""},
+		{name: "ipv6 literal tagged as ipv4 is skipped", ifaces: []libvirt.DomainInterface{{Name: "eth0", Addrs: []libvirt.DomainIPAddr{addr(v4, "2001:db8::1")}}}, want: ""},
+		{
+			name: "good one after bad ones across interfaces",
+			ifaces: []libvirt.DomainInterface{
+				{Name: "lo", Addrs: []libvirt.DomainIPAddr{addr(v4, "127.0.0.1")}},
+				{Name: "eth0", Addrs: []libvirt.DomainIPAddr{addr(v6, "fe80::5054:ff:fe12:3456"), addr(v4, "169.254.1.1"), addr(v4, "bogus"), addr(v4, "192.168.1.42"), addr(v4, "192.168.1.43")}},
+			},
+			want: "192.168.1.42",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := firstIPv4(tc.ifaces); got != tc.want {
+				t.Errorf("firstIPv4() = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}

internal/provider/kvm2/volume.go

@@ -0,0 +1,95 @@
+package kvm2
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/xml"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// Volume names in the host's pool (spec entries 4 and 10). The per-VM
+// volumes derive from the domain name; the base image from its catalogue
+// key or, for a URL image, from a hash of the URL.
+func overlayVolumeName(name string) string { return name + ".qcow2" }
+func seedVolumeName(name string) string    { return name + "-seed.iso" }
+
+func baseVolumeName(key, format string) string { return "base-" + key + "." + format }
+func markerVolumeName(baseName string) string  { return baseName + ".sha256" }
+
+// urlImageKey stands in for the catalogue key of an image given as a URL:
+// the first 16 hex characters of sha256(url).
+func urlImageKey(url string) string {
+	sum := sha256.Sum256([]byte(url))
+	return hex.EncodeToString(sum[:])[:16]
+}
+
+// volumeSpec is what StorageVolCreateXML needs. BackingPath and
+// BackingFormat are set for an overlay only.
+type volumeSpec struct {
+	Name          string
+	CapacityBytes int64
+	Format        string
+	BackingPath   string
+	BackingFormat string
+}
+
+type volumeXML struct {
+	XMLName      xml.Name          `xml:"volume"`
+	Type         string            `xml:"type,attr"`
+	Name         string            `xml:"name"`
+	Capacity     volumeCapacityXML `xml:"capacity"`
+	Target       volumeTargetXML   `xml:"target"`
+	BackingStore *backingStoreXML  `xml:"backingStore"`
+}
+
+type volumeCapacityXML struct {
+	Unit  string `xml:"unit,attr"`
+	Value int64  `xml:",chardata"`
+}
+
+type volumeTargetXML struct {
+	Format volumeFormatXML `xml:"format"`
+}
+
+type volumeFormatXML struct {
+	Type string `xml:"type,attr"`
+}
+
+type backingStoreXML struct {
+	Path   string          `xml:"path"`
+	Format volumeFormatXML `xml:"format"`
+}
+
+func validVolumeFormat(f string) bool {
+	return f == provider.KVMFormatQcow2 || f == provider.KVMFormatRaw
+}
+
+func buildVolumeXML(v volumeSpec) (string, error) {
+	if v.Name == "" {
+		return "", permanentf("volume XML: name is empty")
+	}
+	if v.CapacityBytes <= 0 {
+		return "", permanentf("volume XML for %q: capacity must be positive, got %d", v.Name, v.CapacityBytes)
+	}
+	if !validVolumeFormat(v.Format) {
+		return "", permanentf("volume XML for %q: unknown format %q (want %s or %s)", v.Name, v.Format, provider.KVMFormatQcow2, provider.KVMFormatRaw)
+	}
+	x := volumeXML{
+		Type:     "file",
+		Name:     v.Name,
+		Capacity: volumeCapacityXML{Unit: "bytes", Value: v.CapacityBytes},
+		Target:   volumeTargetXML{Format: volumeFormatXML{Type: v.Format}},
+	}
+	if v.BackingPath != "" {
+		if !validVolumeFormat(v.BackingFormat) {
+			return "", permanentf("volume XML for %q: unknown backing format %q (want %s or %s)", v.Name, v.BackingFormat, provider.KVMFormatQcow2, provider.KVMFormatRaw)
+		}
+		x.BackingStore = &backingStoreXML{Path: v.BackingPath, Format: volumeFormatXML{Type: v.BackingFormat}}
+	}
+	out, err := xml.MarshalIndent(x, "", "  ")
+	if err != nil {
+		return "", permanentf("volume XML for %q: %v", v.Name, err)
+	}
+	return string(out), nil
+}

internal/provider/kvm2/volume_test.go

@@ -0,0 +1,152 @@
+package kvm2
+
+import (
+	"encoding/xml"
+	"errors"
+	"regexp"
+	"testing"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+type checkedVolume struct {
+	Type     string `xml:"type,attr"`
+	Name     string `xml:"name"`
+	Capacity struct {
+		Unit  string `xml:"unit,attr"`
+		Value int64  `xml:",chardata"`
+	} `xml:"capacity"`
+	Target struct {
+		Format struct {
+			Type string `xml:"type,attr"`
+		} `xml:"format"`
+	} `xml:"target"`
+	BackingStores []struct {
+		Path   string `xml:"path"`
+		Format struct {
+			Type string `xml:"type,attr"`
+		} `xml:"format"`
+	} `xml:"backingStore"`
+}
+
+func mustBuildVolume(t *testing.T, v volumeSpec) checkedVolume {
+	t.Helper()
+	out, err := buildVolumeXML(v)
+	if err != nil {
+		t.Fatalf("buildVolumeXML: %v", err)
+	}
+	var c checkedVolume
+	if err := xml.Unmarshal([]byte(out), &c); err != nil {
+		t.Fatalf("unmarshalling produced XML: %v\n%s", err, out)
+	}
+	return c
+}
+
+func TestVolumeXML_overlayBackingStore(t *testing.T) {
+	t.Parallel()
+	c := mustBuildVolume(t, volumeSpec{
+		Name:          "proxy-abc.qcow2",
+		CapacityBytes: 10 << 30,
+		Format:        provider.KVMFormatQcow2,
+		BackingPath:   "/var/lib/libvirt/images/base-debian-13.qcow2",
+		BackingFormat: provider.KVMFormatQcow2,
+	})
+	if c.Type != "file" || c.Name != "proxy-abc.qcow2" {
+		t.Errorf("type=%q name=%q, want file/proxy-abc.qcow2", c.Type, c.Name)
+	}
+	if c.Capacity.Unit != "bytes" || c.Capacity.Value != 10<<30 {
+		t.Errorf("capacity = %d %s, want %d bytes", c.Capacity.Value, c.Capacity.Unit, 10<<30)
+	}
+	if c.Target.Format.Type != "qcow2" {
+		t.Errorf("target format = %q, want qcow2", c.Target.Format.Type)
+	}
+	if len(c.BackingStores) != 1 {
+		t.Fatalf("got %d backingStore elements, want 1", len(c.BackingStores))
+	}
+	bs := c.BackingStores[0]
+	if bs.Path != "/var/lib/libvirt/images/base-debian-13.qcow2" || bs.Format.Type != "qcow2" {
+		t.Errorf("backingStore = %+v, want the base path with qcow2 format", bs)
+	}
+}
+
+func TestVolumeXML_rawWithoutBacking(t *testing.T) {
+	t.Parallel()
+	c := mustBuildVolume(t, volumeSpec{Name: "proxy-abc-seed.iso", CapacityBytes: 385024, Format: provider.KVMFormatRaw})
+	if c.Target.Format.Type != "raw" {
+		t.Errorf("target format = %q, want raw", c.Target.Format.Type)
+	}
+	if c.Capacity.Value != 385024 || c.Capacity.Unit != "bytes" {
+		t.Errorf("capacity = %d %s, want 385024 bytes", c.Capacity.Value, c.Capacity.Unit)
+	}
+	if len(c.BackingStores) != 0 {
+		t.Errorf("got backingStore %+v, want none", c.BackingStores)
+	}
+}
+
+func TestVolumeXML_rejectsBadFormat(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		spec volumeSpec
+	}{
+		{name: "vmdk format", spec: volumeSpec{Name: "x", CapacityBytes: 1, Format: "vmdk"}},
+		{name: "empty format", spec: volumeSpec{Name: "x", CapacityBytes: 1}},
+		{name: "bad backing format", spec: volumeSpec{Name: "x", CapacityBytes: 1, Format: "qcow2", BackingPath: "/b", BackingFormat: "vdi"}},
+		{name: "backing path without format", spec: volumeSpec{Name: "x", CapacityBytes: 1, Format: "qcow2", BackingPath: "/b"}},
+		{name: "empty name", spec: volumeSpec{CapacityBytes: 1, Format: "qcow2"}},
+		{name: "zero capacity", spec: volumeSpec{Name: "x", Format: "qcow2"}},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			out, err := buildVolumeXML(tc.spec)
+			if !errors.Is(err, provider.ErrPermanent) {
+				t.Errorf("err = %v, want ErrPermanent", err)
+			}
+			if out != "" {
+				t.Errorf("got XML %q alongside the error", out)
+			}
+		})
+	}
+}
+
+func TestVolumeNames(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		got  string
+		want string
+	}{
+		{name: "overlay", got: overlayVolumeName("proxy-abc"), want: "proxy-abc.qcow2"},
+		{name: "seed", got: seedVolumeName("proxy-abc"), want: "proxy-abc-seed.iso"},
+		{name: "base qcow2", got: baseVolumeName("debian-13", provider.KVMFormatQcow2), want: "base-debian-13.qcow2"},
+		{name: "base raw", got: baseVolumeName("alpine", provider.KVMFormatRaw), want: "base-alpine.raw"},
+		{name: "marker", got: markerVolumeName("base-debian-13.qcow2"), want: "base-debian-13.qcow2.sha256"},
+		{name: "marker of url base", got: markerVolumeName(baseVolumeName(urlImageKey("https://x/y.img"), "qcow2")), want: "base-" + urlImageKey("https://x/y.img") + ".qcow2.sha256"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if tc.got != tc.want {
+				t.Errorf("got %q, want %q", tc.got, tc.want)
+			}
+		})
+	}
+
+	t.Run("url key", func(t *testing.T) {
+		t.Parallel()
+		hex16 := regexp.MustCompile(`^[0-9a-f]{16}$`)
+		a := urlImageKey("https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2")
+		b := urlImageKey("https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2")
+		c := urlImageKey("https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2")
+		if !hex16.MatchString(a) {
+			t.Errorf("urlImageKey = %q, want 16 lowercase hex chars", a)
+		}
+		if a != b {
+			t.Errorf("urlImageKey not deterministic: %q vs %q", a, b)
+		}
+		if a == c {
+			t.Errorf("urlImageKey gives %q for two different URLs", a)
+		}
+	})
+}

internal/provider/kvm2/xml.go

@@ -0,0 +1,256 @@
+package kvm2
+
+import (
+	"encoding/xml"
+	"time"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// domainParams is everything buildDomainXML needs, already resolved: the
+// paths are what StorageVolGetPath returned for the overlay and the seed.
+type domainParams struct {
+	Name       string
+	UID        string
+	CreatedAt  time.Time
+	VCPUs      int32
+	MemoryMiB  int64
+	Firmware   string
+	Bridge     string
+	DiskPath   string
+	SeedPath   string
+	DiskFormat string
+}
+
+const (
+	domainArch            = "x86_64"
+	domainMachine         = "q35"
+	guestAgentChannelName = "org.qemu.guest_agent.0"
+)
+
+type domainXML struct {
+	XMLName    xml.Name          `xml:"domain"`
+	Type       string            `xml:"type,attr"`
+	Name       string            `xml:"name"`
+	Memory     domainMemoryXML   `xml:"memory"`
+	VCPU       int32             `xml:"vcpu"`
+	Metadata   domainMetadataXML `xml:"metadata"`
+	OS         domainOSXML       `xml:"os"`
+	Features   domainFeaturesXML `xml:"features"`
+	CPU        domainCPUXML      `xml:"cpu"`
+	OnPoweroff string            `xml:"on_poweroff"`
+	OnReboot   string            `xml:"on_reboot"`
+	OnCrash    string            `xml:"on_crash"`
+	Devices    domainDevicesXML  `xml:"devices"`
+}
+
+type domainMemoryXML struct {
+	Unit  string `xml:"unit,attr"`
+	Value int64  `xml:",chardata"`
+}
+
+type domainMetadataXML struct {
+	Instance instanceMetadata `xml:"instance"`
+}
+
+type domainOSXML struct {
+	// Firmware is the attribute firmware="efi"; FirmwareFeatures is the
+	// <firmware> child that turns secure boot off. Both are absent for BIOS.
+	Firmware         string             `xml:"firmware,attr,omitempty"`
+	Type             domainOSTypeXML    `xml:"type"`
+	FirmwareFeatures *domainFirmwareXML `xml:"firmware"`
+}
+
+type domainOSTypeXML struct {
+	Arch    string `xml:"arch,attr"`
+	Machine string `xml:"machine,attr"`
+	Value   string `xml:",chardata"`
+}
+
+type domainFirmwareXML struct {
+	Features []domainFirmwareFeatureXML `xml:"feature"`
+}
+
+type domainFirmwareFeatureXML struct {
+	Enabled string `xml:"enabled,attr"`
+	Name    string `xml:"name,attr"`
+}
+
+type domainFeaturesXML struct {
+	ACPI *struct{} `xml:"acpi"`
+	APIC *struct{} `xml:"apic"`
+}
+
+type domainCPUXML struct {
+	Mode string `xml:"mode,attr"`
+}
+
+type domainDevicesXML struct {
+	Disks      []domainDiskXML      `xml:"disk"`
+	Interfaces []domainInterfaceXML `xml:"interface"`
+	Serials    []domainSerialXML    `xml:"serial"`
+	Consoles   []domainConsoleXML   `xml:"console"`
+	Channels   []domainChannelXML   `xml:"channel"`
+	RNGs       []domainRNGXML       `xml:"rng"`
+}
+
+type domainDiskXML struct {
+	Type     string              `xml:"type,attr"`
+	Device   string              `xml:"device,attr"`
+	Driver   domainDiskDriverXML `xml:"driver"`
+	Source   domainDiskSourceXML `xml:"source"`
+	Target   domainDiskTargetXML `xml:"target"`
+	ReadOnly *struct{}           `xml:"readonly"`
+}
+
+type domainDiskDriverXML struct {
+	Name string `xml:"name,attr"`
+	Type string `xml:"type,attr"`
+}
+
+type domainDiskSourceXML struct {
+	File string `xml:"file,attr"`
+}
+
+type domainDiskTargetXML struct {
+	Dev string `xml:"dev,attr"`
+	Bus string `xml:"bus,attr"`
+}
+
+type domainInterfaceXML struct {
+	Type   string                   `xml:"type,attr"`
+	Source domainInterfaceSourceXML `xml:"source"`
+	Model  domainModelXML           `xml:"model"`
+}
+
+type domainInterfaceSourceXML struct {
+	Bridge string `xml:"bridge,attr"`
+}
+
+type domainModelXML struct {
+	Type string `xml:"type,attr"`
+}
+
+type domainSerialXML struct {
+	Type   string        `xml:"type,attr"`
+	Target domainPortXML `xml:"target"`
+}
+
+type domainPortXML struct {
+	Port int `xml:"port,attr"`
+}
+
+type domainConsoleXML struct {
+	Type   string                 `xml:"type,attr"`
+	Target domainConsoleTargetXML `xml:"target"`
+}
+
+type domainConsoleTargetXML struct {
+	Type string `xml:"type,attr"`
+	Port int    `xml:"port,attr"`
+}
+
+type domainChannelXML struct {
+	Type   string                 `xml:"type,attr"`
+	Target domainChannelTargetXML `xml:"target"`
+}
+
+type domainChannelTargetXML struct {
+	Type string `xml:"type,attr"`
+	Name string `xml:"name,attr"`
+}
+
+type domainRNGXML struct {
+	Model   string              `xml:"model,attr"`
+	Backend domainRNGBackendXML `xml:"backend"`
+}
+
+type domainRNGBackendXML struct {
+	Model string `xml:"model,attr"`
+	Value string `xml:",chardata"`
+}
+
+// buildDomainXML renders the XML for DomainDefineXML. Both disks are
+// type="file" with <source file>: AppArmor derives QEMU's profile from those
+// paths, and type="volume" fails with permission denied on stock
+// Debian/Ubuntu hosts. UEFI boots with secure boot off; libvirt picks the
+// OVMF binary itself.
+func buildDomainXML(p domainParams) (string, error) {
+	if p.Name == "" {
+		return "", permanentf("domain XML: name is empty")
+	}
+	if p.DiskPath == "" {
+		return "", permanentf("domain XML for %q: disk path is empty", p.Name)
+	}
+	if p.SeedPath == "" {
+		return "", permanentf("domain XML for %q: seed path is empty", p.Name)
+	}
+	if p.DiskFormat == "" {
+		p.DiskFormat = provider.KVMFormatQcow2
+	}
+	if p.DiskFormat != provider.KVMFormatQcow2 && p.DiskFormat != provider.KVMFormatRaw {
+		return "", permanentf("domain XML for %q: unknown disk format %q", p.Name, p.DiskFormat)
+	}
+	osx := domainOSXML{
+		Type: domainOSTypeXML{Arch: domainArch, Machine: domainMachine, Value: "hvm"},
+	}
+	switch p.Firmware {
+	case provider.KVMFirmwareUEFI:
+		osx.Firmware = "efi"
+		osx.FirmwareFeatures = &domainFirmwareXML{
+			Features: []domainFirmwareFeatureXML{{Enabled: "no", Name: "secure-boot"}},
+		}
+	case provider.KVMFirmwareBIOS:
+	default:
+		return "", permanentf("domain XML for %q: unknown firmware %q (want %s or %s)", p.Name, p.Firmware, provider.KVMFirmwareUEFI, provider.KVMFirmwareBIOS)
+	}
+	d := domainXML{
+		Type:       "kvm",
+		Name:       p.Name,
+		Memory:     domainMemoryXML{Unit: "MiB", Value: p.MemoryMiB},
+		VCPU:       p.VCPUs,
+		Metadata:   domainMetadataXML{Instance: newInstanceMetadata(p.UID, p.CreatedAt)},
+		OS:         osx,
+		Features:   domainFeaturesXML{ACPI: &struct{}{}, APIC: &struct{}{}},
+		CPU:        domainCPUXML{Mode: "host-passthrough"},
+		OnPoweroff: "destroy",
+		OnReboot:   "restart",
+		OnCrash:    "destroy",
+		Devices: domainDevicesXML{
+			Disks: []domainDiskXML{
+				{
+					Type:   "file",
+					Device: "disk",
+					Driver: domainDiskDriverXML{Name: "qemu", Type: p.DiskFormat},
+					Source: domainDiskSourceXML{File: p.DiskPath},
+					Target: domainDiskTargetXML{Dev: "vda", Bus: "virtio"},
+				},
+				{
+					Type:     "file",
+					Device:   "cdrom",
+					Driver:   domainDiskDriverXML{Name: "qemu", Type: provider.KVMFormatRaw},
+					Source:   domainDiskSourceXML{File: p.SeedPath},
+					Target:   domainDiskTargetXML{Dev: "sda", Bus: "sata"},
+					ReadOnly: &struct{}{},
+				},
+			},
+			Interfaces: []domainInterfaceXML{{
+				Type:   "bridge",
+				Source: domainInterfaceSourceXML{Bridge: p.Bridge},
+				Model:  domainModelXML{Type: "virtio"},
+			}},
+			Serials:  []domainSerialXML{{Type: "pty", Target: domainPortXML{Port: 0}}},
+			Consoles: []domainConsoleXML{{Type: "pty", Target: domainConsoleTargetXML{Type: "serial", Port: 0}}},
+			Channels: []domainChannelXML{{
+				Type:   "unix",
+				Target: domainChannelTargetXML{Type: "virtio", Name: guestAgentChannelName},
+			}},
+			RNGs: []domainRNGXML{{Model: "virtio", Backend: domainRNGBackendXML{Model: "random", Value: "/dev/urandom"}}},
+		},
+	}
+	out, err := xml.MarshalIndent(d, "", "  ")
+	if err != nil {
+		return "", permanentf("domain XML for %q: %v", p.Name, err)
+	}
+	return string(out), nil
+}

internal/provider/kvm2/xml_test.go

@@ -0,0 +1,309 @@
+package kvm2
+
+import (
+	"encoding/xml"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// checkedDomain is an independent view of the marshalled XML, so the tests
+// assert on what libvirt would see rather than on the builder's own structs.
+type checkedDomain struct {
+	Type   string `xml:"type,attr"`
+	Name   string `xml:"name"`
+	Memory struct {
+		Unit  string `xml:"unit,attr"`
+		Value int64  `xml:",chardata"`
+	} `xml:"memory"`
+	VCPU     int32 `xml:"vcpu"`
+	Metadata struct {
+		Inner string `xml:",innerxml"`
+	} `xml:"metadata"`
+	OS struct {
+		Firmware string `xml:"firmware,attr"`
+		Type     struct {
+			Arch    string `xml:"arch,attr"`
+			Machine string `xml:"machine,attr"`
+			Value   string `xml:",chardata"`
+		} `xml:"type"`
+		FirmwareElems []struct {
+			Features []struct {
+				Enabled string `xml:"enabled,attr"`
+				Name    string `xml:"name,attr"`
+			} `xml:"feature"`
+		} `xml:"firmware"`
+	} `xml:"os"`
+	Features struct {
+		ACPI []struct{} `xml:"acpi"`
+		APIC []struct{} `xml:"apic"`
+	} `xml:"features"`
+	CPU struct {
+		Mode string `xml:"mode,attr"`
+	} `xml:"cpu"`
+	OnPoweroff string `xml:"on_poweroff"`
+	OnReboot   string `xml:"on_reboot"`
+	OnCrash    string `xml:"on_crash"`
+	Devices    struct {
+		Disks []struct {
+			Type   string `xml:"type,attr"`
+			Device string `xml:"device,attr"`
+			Driver struct {
+				Name string `xml:"name,attr"`
+				Type string `xml:"type,attr"`
+			} `xml:"driver"`
+			Source struct {
+				File string `xml:"file,attr"`
+				Pool string `xml:"pool,attr"`
+				Vol  string `xml:"volume,attr"`
+			} `xml:"source"`
+			Target struct {
+				Dev string `xml:"dev,attr"`
+				Bus string `xml:"bus,attr"`
+			} `xml:"target"`
+			ReadOnly []struct{} `xml:"readonly"`
+		} `xml:"disk"`
+		Interfaces []struct {
+			Type   string `xml:"type,attr"`
+			Source struct {
+				Bridge string `xml:"bridge,attr"`
+			} `xml:"source"`
+			Model struct {
+				Type string `xml:"type,attr"`
+			} `xml:"model"`
+		} `xml:"interface"`
+		Serials []struct {
+			Type string `xml:"type,attr"`
+		} `xml:"serial"`
+		Consoles []struct {
+			Type   string `xml:"type,attr"`
+			Target struct {
+				Type string `xml:"type,attr"`
+			} `xml:"target"`
+		} `xml:"console"`
+		Channels []struct {
+			Type   string `xml:"type,attr"`
+			Target struct {
+				Type string `xml:"type,attr"`
+				Name string `xml:"name,attr"`
+			} `xml:"target"`
+		} `xml:"channel"`
+		RNGs []struct {
+			Model   string `xml:"model,attr"`
+			Backend struct {
+				Model string `xml:"model,attr"`
+				Value string `xml:",chardata"`
+			} `xml:"backend"`
+		} `xml:"rng"`
+		Graphics []struct{} `xml:"graphics"`
+	} `xml:"devices"`
+}
+
+var testCreatedAt = time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)
+
+func testDomainParams() domainParams {
+	return domainParams{
+		Name:       "proxy-abc",
+		UID:        "0b1f9a2e-uid",
+		CreatedAt:  testCreatedAt,
+		VCPUs:      2,
+		MemoryMiB:  2048,
+		Firmware:   provider.KVMFirmwareUEFI,
+		Bridge:     "br0",
+		DiskPath:   "/var/lib/libvirt/images/proxy-abc.qcow2",
+		SeedPath:   "/var/lib/libvirt/images/proxy-abc-seed.iso",
+		DiskFormat: provider.KVMFormatQcow2,
+	}
+}
+
+func mustBuildDomain(t *testing.T, p domainParams) (string, checkedDomain) {
+	t.Helper()
+	out, err := buildDomainXML(p)
+	if err != nil {
+		t.Fatalf("buildDomainXML: %v", err)
+	}
+	var d checkedDomain
+	if err := xml.Unmarshal([]byte(out), &d); err != nil {
+		t.Fatalf("unmarshalling produced XML: %v\n%s", err, out)
+	}
+	return out, d
+}
+
+func TestDomainXML_fileDisksWithResolvedPaths(t *testing.T) {
+	t.Parallel()
+	p := testDomainParams()
+	_, d := mustBuildDomain(t, p)
+	if d.Type != "kvm" || d.Name != p.Name {
+		t.Errorf("domain type=%q name=%q, want kvm/%s", d.Type, d.Name, p.Name)
+	}
+	if d.Memory.Unit != "MiB" || d.Memory.Value != p.MemoryMiB || d.VCPU != p.VCPUs {
+		t.Errorf("memory=%d %s vcpu=%d, want %d MiB / %d", d.Memory.Value, d.Memory.Unit, d.VCPU, p.MemoryMiB, p.VCPUs)
+	}
+	if len(d.Devices.Disks) != 2 {
+		t.Fatalf("got %d disks, want 2", len(d.Devices.Disks))
+	}
+	overlay, seed := d.Devices.Disks[0], d.Devices.Disks[1]
+	if overlay.Type != "file" || overlay.Device != "disk" {
+		t.Errorf("overlay type=%q device=%q, want file/disk", overlay.Type, overlay.Device)
+	}
+	if overlay.Source.File != p.DiskPath || overlay.Source.Pool != "" || overlay.Source.Vol != "" {
+		t.Errorf("overlay source = %+v, want file=%q only", overlay.Source, p.DiskPath)
+	}
+	if overlay.Driver.Name != "qemu" || overlay.Driver.Type != "qcow2" {
+		t.Errorf("overlay driver = %+v, want qemu/qcow2", overlay.Driver)
+	}
+	if overlay.Target.Dev != "vda" || overlay.Target.Bus != "virtio" {
+		t.Errorf("overlay target = %+v, want vda/virtio", overlay.Target)
+	}
+	if len(overlay.ReadOnly) != 0 {
+		t.Error("overlay is readonly")
+	}
+	if seed.Type != "file" || seed.Device != "cdrom" {
+		t.Errorf("seed type=%q device=%q, want file/cdrom", seed.Type, seed.Device)
+	}
+	if seed.Source.File != p.SeedPath || seed.Source.Pool != "" || seed.Source.Vol != "" {
+		t.Errorf("seed source = %+v, want file=%q only", seed.Source, p.SeedPath)
+	}
+	if seed.Driver.Name != "qemu" || seed.Driver.Type != "raw" {
+		t.Errorf("seed driver = %+v, want qemu/raw", seed.Driver)
+	}
+	if seed.Target.Dev != "sda" || seed.Target.Bus != "sata" {
+		t.Errorf("seed target = %+v, want sda/sata", seed.Target)
+	}
+	if len(seed.ReadOnly) != 1 {
+		t.Error("seed is not readonly")
+	}
+	if d.OnPoweroff != "destroy" || d.OnReboot != "restart" || d.OnCrash != "destroy" {
+		t.Errorf("lifecycle actions = %q/%q/%q, want destroy/restart/destroy", d.OnPoweroff, d.OnReboot, d.OnCrash)
+	}
+}
+
+func TestDomainXML_uefiDefaultSecureBootOff(t *testing.T) {
+	t.Parallel()
+	_, d := mustBuildDomain(t, testDomainParams())
+	if d.OS.Firmware != "efi" {
+		t.Errorf(`<os firmware> = %q, want "efi"`, d.OS.Firmware)
+	}
+	if len(d.OS.FirmwareElems) != 1 {
+		t.Fatalf("got %d <firmware> elements, want 1", len(d.OS.FirmwareElems))
+	}
+	feats := d.OS.FirmwareElems[0].Features
+	if len(feats) != 1 || feats[0].Name != "secure-boot" || feats[0].Enabled != "no" {
+		t.Errorf("firmware features = %+v, want one secure-boot enabled=no", feats)
+	}
+	if d.OS.Type.Value != "hvm" {
+		t.Errorf("os type = %q, want hvm", d.OS.Type.Value)
+	}
+}
+
+func TestDomainXML_biosEscape(t *testing.T) {
+	t.Parallel()
+	p := testDomainParams()
+	p.Firmware = provider.KVMFirmwareBIOS
+	out, d := mustBuildDomain(t, p)
+	if d.OS.Firmware != "" {
+		t.Errorf(`<os firmware> = %q, want absent`, d.OS.Firmware)
+	}
+	if len(d.OS.FirmwareElems) != 0 {
+		t.Errorf("got %d <firmware> elements, want none", len(d.OS.FirmwareElems))
+	}
+	if strings.Contains(out, "firmware") {
+		t.Errorf("BIOS domain XML still mentions firmware:\n%s", out)
+	}
+	if d.OS.Type.Value != "hvm" || d.OS.Type.Machine != "q35" {
+		t.Errorf("os type = %+v, want hvm on q35", d.OS.Type)
+	}
+}
+
+func TestDomainXML_metadataElement(t *testing.T) {
+	t.Parallel()
+	p := testDomainParams()
+	out, d := mustBuildDomain(t, p)
+	if !strings.Contains(out, `<instance xmlns="`+metadataURI+`">`) {
+		t.Errorf("domain XML lacks the namespaced instance element:\n%s", out)
+	}
+	m, err := parseInstanceMetadata(d.Metadata.Inner)
+	if err != nil {
+		t.Fatalf("parseInstanceMetadata(%q): %v", d.Metadata.Inner, err)
+	}
+	if !m.managed() {
+		t.Error("managed() = false, want true")
+	}
+	if m.UID != p.UID {
+		t.Errorf("UID = %q, want %q", m.UID, p.UID)
+	}
+	if m.CreatedAt != "2026-09-16T12:00:00Z" {
+		t.Errorf("CreatedAt = %q, want RFC3339 UTC", m.CreatedAt)
+	}
+	if !m.createdAt().Equal(p.CreatedAt) {
+		t.Errorf("createdAt() = %v, want %v", m.createdAt(), p.CreatedAt)
+	}
+}
+
+func TestDomainXML_guestAgentChannel(t *testing.T) {
+	t.Parallel()
+	p := testDomainParams()
+	_, d := mustBuildDomain(t, p)
+	if len(d.Devices.Channels) != 1 || d.Devices.Channels[0].Type != "unix" ||
+		d.Devices.Channels[0].Target.Type != "virtio" || d.Devices.Channels[0].Target.Name != "org.qemu.guest_agent.0" {
+		t.Errorf("channels = %+v, want one unix/virtio org.qemu.guest_agent.0", d.Devices.Channels)
+	}
+	if len(d.Devices.Interfaces) != 1 || d.Devices.Interfaces[0].Type != "bridge" ||
+		d.Devices.Interfaces[0].Source.Bridge != p.Bridge || d.Devices.Interfaces[0].Model.Type != "virtio" {
+		t.Errorf("interfaces = %+v, want one bridge %s with virtio model", d.Devices.Interfaces, p.Bridge)
+	}
+	if len(d.Devices.Serials) != 1 || d.Devices.Serials[0].Type != "pty" {
+		t.Errorf("serials = %+v, want one pty", d.Devices.Serials)
+	}
+	if len(d.Devices.Consoles) != 1 || d.Devices.Consoles[0].Type != "pty" || d.Devices.Consoles[0].Target.Type != "serial" {
+		t.Errorf("consoles = %+v, want one pty/serial", d.Devices.Consoles)
+	}
+	if len(d.Devices.RNGs) != 1 || d.Devices.RNGs[0].Model != "virtio" ||
+		d.Devices.RNGs[0].Backend.Model != "random" || d.Devices.RNGs[0].Backend.Value != "/dev/urandom" {
+		t.Errorf("rngs = %+v, want one virtio random /dev/urandom", d.Devices.RNGs)
+	}
+	if len(d.Devices.Graphics) != 0 {
+		t.Error("domain has <graphics>, want none")
+	}
+	if d.OS.Type.Arch != "x86_64" || d.OS.Type.Machine != "q35" {
+		t.Errorf("os type = %+v, want x86_64/q35", d.OS.Type)
+	}
+	if d.CPU.Mode != "host-passthrough" {
+		t.Errorf("cpu mode = %q, want host-passthrough", d.CPU.Mode)
+	}
+	if len(d.Features.ACPI) != 1 || len(d.Features.APIC) != 1 {
+		t.Errorf("features acpi=%d apic=%d, want one each", len(d.Features.ACPI), len(d.Features.APIC))
+	}
+}
+
+func TestDomainXML_rejectsBadInput(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name   string
+		mutate func(*domainParams)
+	}{
+		{name: "unknown firmware", mutate: func(p *domainParams) { p.Firmware = "coreboot" }},
+		{name: "empty firmware", mutate: func(p *domainParams) { p.Firmware = "" }},
+		{name: "empty name", mutate: func(p *domainParams) { p.Name = "" }},
+		{name: "empty disk path", mutate: func(p *domainParams) { p.DiskPath = "" }},
+		{name: "empty seed path", mutate: func(p *domainParams) { p.SeedPath = "" }},
+		{name: "unknown disk format", mutate: func(p *domainParams) { p.DiskFormat = "vmdk" }},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			p := testDomainParams()
+			tc.mutate(&p)
+			out, err := buildDomainXML(p)
+			if !errors.Is(err, provider.ErrPermanent) {
+				t.Errorf("err = %v, want ErrPermanent", err)
+			}
+			if out != "" {
+				t.Errorf("got XML %q alongside the error", out)
+			}
+		})
+	}
+}