Record live and negative Tempo e2e runs in the execution log

Co-Authored-By: Claude <noreply@anthropic.com>

.claude/settings.json

@@ -84,12 +84,16 @@
       "Bash(kubectl -n egress-proxies-operator-system get deploy egress-proxies-operator-controller-manager -o jsonpath='{.spec.template.spec.containers[0].args}')",
       "Bash(kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager)",
       "Bash(python3 -c \"import json; d=json.load\\(open\\('docs/deploy/sa_key.json'\\)\\); print\\(d.get\\('type'\\), d.get\\('client_email'\\)\\)\")",
-      "Bash(git add *)",
-      "Bash(git commit -q -m 'Add plan: KVM/libvirt provider *)",
-      "Bash(go get *)",
-      "Bash(go build *)",
-      "Bash(go test *)",
-      "Bash(git commit *)"
+      "Bash(tea pr *)",
+      "Bash(git worktree *)",
+      "Bash(python3 -c \"import yaml; yaml.safe_load\\(open\\('.gitea/workflows/build.yaml'\\)\\); print\\('YAML OK'\\)\")",
+      "Bash(ruby -ryaml -e \"YAML.load_file\\('.gitea/workflows/build.yaml'\\); puts 'YAML OK'\")",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator status --short --branch)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator log --oneline -1)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator tag 0.01)",
+      "Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator push origin 0.01)",
+      "Bash(chmod +x docs/demo/run-demo.sh)",
+      "Bash(bash -n docs/demo/run-demo.sh)"
     ],
     "additionalDirectories": [
       "/Users/jan.novak/srv/go/egress-proxies-operator/.claude",

Makefile

@@ -90,7 +90,8 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist
 
 .PHONY: test-e2e
 test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
-	KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v
+	KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) TEMPO_URL=$(TEMPO_URL) OTLP_ENDPOINT=$(OTLP_ENDPOINT) \
+		go test -tags=e2e ./test/e2e/ -v -ginkgo.v -timeout 30m
 	$(MAKE) cleanup-test-e2e
 
 .PHONY: cleanup-test-e2e

README.md

@@ -136,6 +136,81 @@ Cloud-init from a Secret: the Secret **must** carry the label
 labelled Secrets, so an unlabelled one is invisible (the Proxy reports
 `CloudInitError`). Rotating the Secret's content triggers VM replacement.
 
+## Providers
+
+`Managed` proxies are provisioned by a provider — a small compute backend
+behind one minimal interface. Providers are configured in the
+`--providers-config` YAML file as **named instances**: `spec.provider` on a
+Proxy refers to an entry's `name`, not its `type`, so `gcp-eu` and `gcp-us`
+can be two differently-configured instances of the same `gcp` type (see
+[config/samples/providers-config.yaml](config/samples/providers-config.yaml)).
+
+### Implemented providers
+
+| Type | Creates | Per-instance config | Notes |
+| --- | --- | --- | --- |
+| `kubernetes` | A real Squid pod (`ubuntu/squid:6.6-24.04_edge` by default) in the same cluster the operator runs in | `image` (optional) | Needs no cloud account — local dev, CI, and the kind quickstart. Pods share the cluster's egress IP, so it exercises the full lifecycle but not distinct egress paths. |
+| `gcp` | A Compute Engine VM with an ephemeral external IP | `project` (required), `network`, `networkTag`, `diskSizeGb` | The real egress fleet. Deliberately uses only four API calls (Insert / Get / Delete / AggregatedList), all fire-and-forget: `Create` returns as soon as the operation is submitted and the reconciler discovers progress by polling `Get`. Auth is Application Default Credentials — workload identity in-cluster, `gcloud` ADC locally; no key-file plumbing. |
+
+### Adding a provider
+
+A new backend (Hetzner, AWS, ...) is four pieces; the contract lives in
+[internal/provider/provider.go](internal/provider/provider.go):
+
+1. **Implement the 4-method `Provider` interface** in a new
+   `internal/provider/<type>/` package:
+   - `Create` submits and returns — it never blocks until the VM runs, and
+     must be idempotent keyed on `req.Name` (a deterministic name derived
+     from the Proxy's UID), so a repeat call after a crash finds the
+     existing instance instead of duplicating it.
+   - `Get` returns `provider.ErrNotFound` as a *normal* outcome — the
+     reconciler branches on it for replacement and adoption, so don't
+     treat it as exceptional.
+   - `Delete` is idempotent: deleting an already-gone instance is not an
+     error.
+   - `ListByTag` returns every instance the operator ever tagged, for
+     orphan GC.
+2. **Tag every created resource** with `LabelManaged=true` and
+   `LabelUID=<Proxy UID>`, and report `CreatedAt` — orphan GC relies on
+   all three to find owned resources and skip in-flight creates.
+3. **Classify every returned error** with `provider.Wrap` into the
+   four-sentinel taxonomy in
+   [internal/provider/errors.go](internal/provider/errors.go)
+   (`ErrNotFound` / `ErrQuotaExceeded` / `ErrTransient` / `ErrPermanent`)
+   — the reconciler decides retry, slow backoff, or latching `Failed`
+   purely from that classification, never from provider-specific types.
+4. **Wire it up**: add a type-specific config block in
+   [internal/provider/config.go](internal/provider/config.go), and
+   register the constructor in the builtins map in
+   [cmd/main.go](cmd/main.go) (`"<type>": <pkg>.New`). The registry
+   ([internal/provider/registry](internal/provider/registry/registry.go))
+   handles named instances, and the metrics wrapper is applied
+   automatically.
+
+Test against a fake API seam rather than the real cloud — see the
+`instancesAPI` seam in
+[internal/provider/gcp/gcp.go](internal/provider/gcp/gcp.go) for the
+pattern.
+
+## Tracing
+
+OpenTelemetry tracing is built in but **off by default** — the operator
+only exports spans when pointed at an OTLP receiver:
+
+```bash
+# In config/manager/manager.yaml (commented-out block is already there):
+#   OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger-collector.observability:4318
+# Local dev: print spans to stdout instead
+OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml
+```
+
+Every reconcile, discovery API request, and GC sweep becomes a trace, with
+child spans for provider and Kubernetes API calls; log lines inside a traced
+operation carry matching `traceID`/`spanID` fields (point Grafana/Loki
+derived fields at `traceID`). Sampling and endpoints follow the standard
+`OTEL_*` env vars; `--trace-health-probes` additionally traces each health
+probe (high volume). Details in `docs/architecture.md` §10.
+
 ## Caveats — read these two
 
 **Changing a proxy changes its IP.** Proxies are immutable cattle: editing

cmd/main.go

@@ -42,6 +42,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -58,6 +59,7 @@ import (
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version"
 	// +kubebuilder:scaffold:imports
 )
@@ -93,6 +95,7 @@ func main() {
 	var leaseCooldown, maxLeaseTTL time.Duration
 	var showVersion bool
 	var gcpWireFullPayloads bool
+	var traceHealthProbes bool
 
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -133,6 +136,9 @@ func main() {
 		"Print the commit the binary was built from and exit.")
 	flag.BoolVar(&gcpWireFullPayloads, "gcp-wire-log-full-payloads", false,
 		"Log GCP V(5) wire payloads verbatim instead of eliding fields larger than 1KiB.")
+	flag.BoolVar(&traceHealthProbes, "trace-health-probes", false,
+		"Emit one trace span per health probe. Off by default: probes run about "+
+			"once per second per proxy and would dominate trace volume.")
 
 	opts := zap.Options{
 		Development: true,
@@ -148,6 +154,17 @@ func main() {
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 	setupLog.Info("Starting egress-proxies-operator",
 		"commit", version.Resolve(), "goVersion", goruntime.Version())
+
+	// Off (no-op spans, unchanged logs) unless OTEL_* env opts in; see
+	// internal/tracing. Shutdown is called explicitly after mgr.Start
+	// returns — the os.Exit paths below skip defers.
+	tracingShutdown, err := tracing.Setup(logf.IntoContext(context.Background(), setupLog),
+		"egress-proxies-operator", version.Resolve())
+	if err != nil {
+		setupLog.Error(err, "Failed to set up tracing")
+		os.Exit(1)
+	}
+
 	ctx := ctrl.SetupSignalHandler()
 
 	// Providers load first and fail fast: a manager that comes up without
@@ -162,7 +179,11 @@ func main() {
 		os.Exit(1)
 	}
 	providers, err := registry.Build(ctx, cfg, map[string]registry.Constructor{
-		"kubernetes": kubernetes.New,
+		"kubernetes": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
+			// The kubernetes provider builds its own uncached client;
+			// wrap its transport so its API calls join the caller's trace.
+			return kubernetes.NewWithTransportWrapper(ctx, pc, tracing.RestConfigWrapper())
+		},
 		"gcp": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
 			return gcp.NewWithWireOptions(ctx, pc, gcp.WireLogOptions{FullPayloads: gcpWireFullPayloads})
 		},
@@ -173,7 +194,8 @@ func main() {
 	}
 	m := metrics.New()
 	for name, p := range providers {
-		providers[name] = provider.WithMetrics(name, p, m)
+		// Tracing outermost: the span covers the metrics recording too.
+		providers[name] = provider.WithTracing(name, provider.WithMetrics(name, p, m))
 	}
 
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
@@ -236,7 +258,9 @@ func main() {
 		cacheOpts.DefaultNamespaces = map[string]cache.Config{proxyNamespace: {}}
 	}
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	restCfg := ctrl.GetConfigOrDie()
+	restCfg.Wrap(tracing.RestConfigWrapper())
+	mgr, err := ctrl.NewManager(restCfg, ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
@@ -258,6 +282,7 @@ func main() {
 	engine := health.NewEngine(mgr.GetClient())
 	engine.Workers = healthWorkers
 	engine.Metrics = m
+	engine.TraceProbes = traceHealthProbes
 	if err := mgr.Add(engine); err != nil {
 		setupLog.Error(err, "Failed to add health engine")
 		os.Exit(1)
@@ -323,8 +348,19 @@ func main() {
 	}
 
 	setupLog.Info("Starting manager")
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "Failed to run manager")
+	startErr := mgr.Start(ctx)
+
+	// Flush pending spans on the way out, error path included. Fresh
+	// context: the signal ctx is already cancelled by the time Start
+	// returns.
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	if err := tracingShutdown(shutdownCtx); err != nil {
+		setupLog.Error(err, "Failed to flush traces on shutdown")
+	}
+	cancel()
+
+	if startErr != nil {
+		setupLog.Error(startErr, "Failed to run manager")
 		os.Exit(1)
 	}
 }

config/manager/manager.yaml

@@ -64,6 +64,8 @@ spec:
           - --leader-elect
           - --health-probe-bind-address=:8081
           - --providers-config=/etc/proxy-operator/providers.yaml
+          # Add --trace-health-probes to emit one span per health probe
+          # (high volume: ~1/s per proxy).
         env:
           # Bearer token for the discovery API. Optional: without the
           # Secret the API serves unauthenticated (with a loud warning).
@@ -76,6 +78,33 @@ spec:
                 name: discovery-token
                 key: token
                 optional: true
+          # --- OpenTelemetry tracing ---
+          # Tracing is OFF until OTEL_EXPORTER_OTLP_ENDPOINT (or
+          # OTEL_TRACES_EXPORTER) is set; without it the operator runs
+          # exactly as before. The downward-API vars must stay listed
+          # before OTEL_RESOURCE_ATTRIBUTES: $(VAR) expansion only sees
+          # earlier-listed vars.
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: OTEL_SERVICE_NAME
+            value: egress-proxies-operator
+          - name: OTEL_RESOURCE_ATTRIBUTES
+            value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)
+          # Point at an OTLP receiver (Tempo, Jaeger, otel-collector) to
+          # enable tracing; http/protobuf (4318) is the default protocol,
+          # set OTEL_EXPORTER_OTLP_PROTOCOL=grpc for 4317.
+          # - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          #   value: http://jaeger-collector.observability:4318
+          # - name: OTEL_TRACES_SAMPLER
+          #   value: parentbased_traceidratio
+          # - name: OTEL_TRACES_SAMPLER_ARG
+          #   value: "0.1"
         image: controller:latest
         name: manager
         ports:

docs/architecture.md

@@ -300,6 +300,45 @@ Each consuming package defines its own small recorder interface
 `metrics.Metrics` satisfies all of them structurally, so no package other
 than `cmd/main.go` imports the metrics package.
 
+### 10. Tracing (`internal/tracing/`)
+
+OpenTelemetry tracing, integrated with — not replacing — the logr/zap
+logging. Everything hangs off standard `OTEL_*` env vars: with no
+`OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` set, no SDK is
+installed, spans are no-ops, and logs are byte-for-byte what they were.
+`OTEL_SDK_DISABLED=true` and `OTEL_TRACES_EXPORTER=none` force it off;
+`console` prints spans to stdout for local dev; sampling follows
+`OTEL_TRACES_SAMPLER(_ARG)`.
+
+Span topology — each unit of work is a new trace (watch events carry no
+incoming trace context):
+
+- `Reconcile Proxy` (root, via a `reconcile.Reconciler` decorator) →
+  `reconcile.managed` / `reconcile.replaceInstance` / `reconcile.delete`,
+  plus `status.patch` from the deferred flush. `reconcileExternal` does no
+  I/O and is unspanned.
+- `provider.create|get|delete|list` (client spans, `provider.WithTracing`
+  decorator wired outermost around `WithMetrics`); the GCP SDK's own
+  otelhttp transport contributes HTTP child spans automatically.
+- Kubernetes API calls become client child spans via a wrapped
+  `rest.Config` transport — gated on an existing parent span, so informer
+  list/watch long-polls and leader-election renewals never create root
+  spans. The kube-apiserver ignores incoming `traceparent` by design;
+  these spans are leaves.
+- Discovery API: one server span per request (named from the route
+  pattern), incoming W3C `traceparent` honored so clients' traces continue
+  into the operator; `/healthz` excluded.
+- `gc.sweep` per GC pass; `health.probe` per probe **only** with
+  `--trace-health-probes` (default off — probes run ~1/s per proxy).
+
+Log correlation: logr sinks never see a context, so trace IDs ride on the
+logger — `tracing.Start` re-derives the ctx logger from a captured base
+with `traceID`/`spanID` values (lowerCamel, matching `reconcileID`;
+Grafana/Loki derived-field regexes must match `traceID`, not `trace_id`).
+Re-deriving from the base rather than layering keeps zap from emitting
+duplicate keys on nested spans. GCP V(5) wire logs get the same keys from
+the slog handler's ctx.
+
 ## Decisions
 
 Judgment calls the spec left open, and deliberate deviations — recorded so
@@ -404,3 +443,24 @@ they read as choices, not accidents. Chronological by build step.
   logger to everything running under the manager — fighting that would
   mean two logging systems in one process. Noted as a deviation rather
   than silently ignored.
+- **Tracing is env-gated, not flag-gated:** it activates only when
+  `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` is set (user
+  decision). No collector configured means no SDK installed, no-op spans,
+  unchanged logs — the safe default for every existing deployment. The
+  one flag is `--trace-health-probes`, off by default, because probes at
+  ~1/s per proxy would dominate trace volume.
+- **Trace keys are `traceID`/`spanID`**, lowerCamel like `reconcileID` and
+  `providerID`, deliberately not the `trace_id` many Grafana derived-field
+  examples assume — configure the derived-field regex accordingly. logr
+  sinks can't read ctx, so the IDs ride on the ctx logger, re-derived from
+  a captured base per span so zap never emits duplicate keys.
+- **Probe→reconcile trace links are not attempted:** the health engine's
+  `GenericEvent` carries only namespace/name (no ctx), and the workqueue
+  coalesces events, so any link would be a guess. A health-triggered
+  reconcile starts a fresh trace; the probe that caused it is findable via
+  its own (opt-in) span and shared proxy attributes.
+- **No `traceparent` toward probe targets:** probe transports stay
+  uninstrumented so trace headers can never leak through a proxy to
+  external sites. The kube-apiserver ignores incoming `traceparent` by
+  design (public endpoint), so k8s client spans are leaves — in-process
+  traces, not cross-process ones.

docs/plans-executions/2026-08-24-1025-otel-tracing.md

@@ -0,0 +1,183 @@
+# Execution log: OpenTelemetry tracing integrated with logr/zap logging
+
+Plan: `docs/plans/2026-08-24-1025-otel-tracing.md`
+
+- [x] Step 1 — Dependencies
+- [x] Step 2 — New package `internal/tracing`
+- [x] Step 3 — `provider.WithTracing` decorator
+- [x] Step 4 — `cmd/main.go` wiring
+- [x] Step 5 — Reconciler spans
+- [x] Step 6 — Discovery server
+- [x] Step 7 — GC + health
+- [x] Step 8 — GCP wire-log enrichment
+- [x] Step 9 — Manifests + docs
+
+## Step 1 — Dependencies
+
+Aligned the pre-existing indirect skew (otel core v1.44.0 vs otlptrace exporters
+v1.40.0) and added the new direct deps in one shot:
+
+```bash
+go get go.opentelemetry.io/otel@v1.45.0 \
+  go.opentelemetry.io/otel/sdk@v1.45.0 \
+  go.opentelemetry.io/otel/trace@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp@v1.45.0 \
+  go.opentelemetry.io/otel/exporters/stdout/stdouttrace@v1.45.0 \
+  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0
+go mod tidy
+```
+
+MVS side-effects worth recording: `logr v1.4.3→v1.4.4`, `httpsnoop
+v1.0.4→v1.1.0`, `grpc-gateway/v2 v2.27.7→v2.29.0`, `proto/otlp v1.9.0→v1.11.0`,
+plus a `genproto/googleapis/api` pseudo-version bump. Full `make test` passed
+against the bumped graph (`google.golang.org/api v0.292.0` and k8s v0.36
+tolerate otelhttp v0.70.0).
+
+Worth noting: the first `go mod tidy` ran *before* any first-party code
+imported `otlptracehttp`/`stdouttrace`, so it silently dropped those two
+modules again; the Step 2 tidy re-added them. The plan's semconv question
+resolved to `semconv/v1.43.0` — that's what `sdk@v1.45.0/resource/builtin.go`
+imports, so first-party code uses the same version to avoid
+`ErrSchemaURLConflict` in the common path.
+
+## Step 2 — `internal/tracing` package
+
+Landed as planned: `tracing.go` (env-gated `Setup`, hand-rolled exporter
+selection), `logger.go` (`Start`/`StartSpan`/`ContextWithLogger` with the
+base-logger ctx key that prevents duplicate `traceID` zap fields on nested
+spans), `reconciler.go`, `transport.go`, `http.go`, `options.go`; tests for
+all of it (first use of `sdk/trace/tracetest` in the repo).
+
+Two deviations from the plan's letter:
+
+- **k8s transport gating is a custom RoundTripper, not `otelhttp.WithFilter`**
+  (`parentGatedTransport`): requests without a parent span bypass the otel
+  transport entirely, so the no-root-spans-from-informers guarantee doesn't
+  depend on otelhttp filter semantics for transports.
+- **`HTTPMiddleware` must copy the route pattern back.** The logger-injecting
+  inner handler wraps the request via `WithContext` (a shallow copy), so the
+  mux records the matched pattern on the copy while otelhttp's post-routing
+  span rename reads the original. Found by the middleware test (span named
+  `"GET"` instead of `"GET /v1/things/{id}"`); fixed with `r.Pattern =
+  r2.Pattern` after `next.ServeHTTP`.
+
+Also: both the transport and the middleware pass explicit W3C propagators
+instead of relying on the global, so behavior is deterministic under tests
+and when tracing is disabled. `Setup` tests reset the global provider to a
+fresh noop per case — restoring otel's own default delegate triggers a
+"Setting tracer provider to its current value" warning from the SDK.
+
+## Step 3 — `provider.WithTracing`
+
+`internal/provider/tracing.go` mirrors `metrics.go` exactly (same wrap shape,
+same `resultLabel` classification reused as the `provider.result` span
+attribute); tests reuse `staticProvider` from `metrics_test.go`. `ErrNotFound`
+maps to span status Ok as planned — the reconciler polls `Get` to NotFound
+during replacement/deletion, so it's an answer, not a failure.
+
+One small API addition to `internal/tracing` for this: exported
+`tracing.Tracer(opts...)`, since the decorator lives in package `provider`
+and couldn't reach the unexported option resolver. The `cmd/main.go` wiring
+(`WithTracing` outermost around `WithMetrics`) lands with Step 4's commit —
+same file, one commit.
+
+## Step 4 — `cmd/main.go` wiring
+
+As planned: `tracing.Setup` right after `ctrl.SetLogger`; manager rest.Config
+wrapped via `restCfg := ctrl.GetConfigOrDie(); restCfg.Wrap(...)` (the
+previous inline call left nowhere to wrap); providers decorated tracing-
+outermost; explicit trace flush after `mgr.Start` returns on both the error
+and clean paths, with a fresh 10s context since the signal ctx is already
+cancelled by then.
+
+The kubernetes provider grew `NewWithTransportWrapper(ctx, cfg, wrap)` (its
+client is built from its own `ctrl.GetConfig()`, invisible to the manager's
+wrapped config); `New` now delegates with a nil wrapper, so `newWithClient`
+tests stayed untouched, and main registers a closure — the same pattern the
+gcp constructor already used for wire-log options.
+
+Deviation: the plan put `--trace-health-probes` here, but the flag needs the
+`health.Engine.TraceProbes` field that Step 7 introduces — moved there to
+keep every commit compiling.
+
+## Step 5 — Reconciler spans
+
+`SetupWithManager` completes with `tracing.NewReconciler("Proxy", r)`;
+sub-spans `reconcile.managed` / `reconcile.replaceInstance` /
+`reconcile.delete` open at the top of each state machine, and `status.patch`
+opens inside the deferred flush closure so it stays within the root span
+while its error still folds into the recorded result. `reconcileExternal`
+left unspanned as planned (no I/O).
+
+Two small judgment calls: the sub-spans carry no extra attributes — the
+provider decorator already records `provider.id`, and the root span carries
+the object identity, so duplicating them was noise; and `status.patch` is
+emitted every reconcile even when nothing changed (the no-op compare is the
+span's content — a real PATCH shows up as its k8s HTTP child). Tests that
+call `r.Reconcile` directly bypass the wrapper; with no global tracer set
+they see no-op spans, so the existing fake-client and envtest suites run
+unchanged.
+
+## Step 6 — Discovery server
+
+Middleware chain is now recover → tracing (server span + request logger) →
+request-log → body cap → auth; the request-log middleware and the three
+handler error sites log via `logf.FromContext(r.Context())` — which is
+`s.log` enriched with the request's traceID/spanID by the tracing
+middleware, not a different logger (logr sinks can't read ctx at log time,
+so per-request values must ride on the logger instance in the ctx; user
+asked, answered in-session).
+
+Deviation from the plan's letter: `recoverMiddleware` keeps `s.log`. It sits
+*outside* the tracing layer, so its request ctx never has the enriched
+logger — switching it to `FromContext` would silently drop the `"discovery"`
+name and gain nothing.
+
+## Step 7 — GC + health
+
+`gc.sweep` runs in a root span with `gc.providers` / `gc.deleted` counters
+(a failed orphan delete no longer counts as deleted — the loop grew an
+explicit `continue`). The health engine got the deferred `--trace-health-probes`
+flag and `TraceProbes` field; the probe call moved into `runProbe`, which
+wraps it in a `health.probe` client span only when enabled, ending before
+`record` (which stays ctx-free by design). Probe transports remain
+uninstrumented so no traceparent can leak through a proxy to external
+targets. Probe→reconcile span links stay out of scope (GenericEvent carries
+no ctx; the workqueue coalesces events) — recorded in the architecture
+Decisions in Step 9.
+
+## Step 8 — GCP wire-log enrichment
+
+`wireFilterHandler.Handle` clones the record and appends `traceID`/`spanID`
+when the ctx carries a valid span — same keys as the logr enrichment, added
+before the elision pass (both values are well under the 1KiB threshold).
+The new `wirelog_test.go` covers the enrichment, that the JWT/token security
+filter still drops non-wire Debug records even with a span present, and that
+spanless records stay unchanged.
+
+## Step 9 — Manifests, docs, lint
+
+`config/manager/manager.yaml` gained the OTel env block (downward-API
+`POD_NAME`/`POD_NAMESPACE` deliberately listed *before*
+`OTEL_RESOURCE_ATTRIBUTES` — `$(VAR)` expansion only sees earlier vars),
+with the OTLP endpoint and sampler left as commented examples so deployments
+stay tracing-off by default. `docs/architecture.md` got "### 10. Tracing"
+plus five Decisions entries; README got a Tracing section.
+
+`make lint` surfaced 16 new-on-branch issues (checked with
+`golangci-lint run --new-from-rev main`; the other ~43 pre-date this work):
+
+- one gofmt slip, 13 goconst repeats — fixed with `env*`/`exporter*`
+  constants in `internal/tracing/tracing.go` shared by the tests;
+- two logcheck hits ("function takes both a context and a logger"):
+  `Setup` now reads its logger from ctx (`logf.FromContext`, caller seeds
+  via `logf.IntoContext` — the linter's sanctioned pattern), and
+  `ContextWithLogger` carries a justified `//nolint:logcheck`, since being
+  the IntoContext-analogue is its purpose.
+
+Worth noting: pre-existing lint debt (50 goconst etc. on `main`) was left
+untouched — `make lint` still fails overall; only branch-introduced issues
+were cleaned. The lint fixes changed `Setup`'s signature after Step 2/4 had
+landed, so those commits show the two-arg version; this commit is where it
+settles.

docs/plans-executions/2026-08-24-1224-tracing-e2e.md

@@ -0,0 +1,68 @@
+# Execution log: e2e test — OTel tracing against real Tempo
+
+Plan: `docs/plans/2026-08-24-1224-tracing-e2e.md`
+
+- [x] Step 1 — `test/e2e/tracing_test.go`
+- [x] Step 2 — Makefile
+- [x] Step 3 — Docs
+
+## Steps 1–3 — spec, Makefile, docs (one commit)
+
+The three steps landed together — the spec is one new file and the other
+two are its wiring. `Describe("OTel tracing", Ordered)` is fully
+self-contained (own ns create → `make install`/`make deploy` → teardown)
+because Ginkgo randomizes top-level container order, so it cannot share the
+Manager Describe's deployment. It reuses the package-level `namespace` /
+`managerImage` and the suite's idioms (`utils.Run`, curl-pod with the
+restricted-PSS overrides JSON, log-substring `Eventually`s).
+
+Judgment calls beyond the plan's letter:
+
+- The squid pre-pull (`docker pull` + `kind load`) is **best effort** — a
+  missing docker binary logs a note and continues rather than failing the
+  spec; the 5m Ready timeout still covers an in-cluster pull.
+- The OTLP preflight pod prints per-attempt HTTP codes and a final
+  `OTLP_OK`/`OTLP_UNREACHABLE` marker; the assertion quotes the pod's
+  output, so an unreachable endpoint names itself in the failure.
+- `kubectl set env` is passed the literal
+  `OTEL_RESOURCE_ATTRIBUTES=...$(POD_NAME)...` string via `exec.Command` —
+  no shell involved, kubectl stores `$()` verbatim, and the in-place update
+  keeps the var after the downward-API vars it references.
+- Tempo helpers are stdlib-only; `/api/traces/<id>` is decoded as
+  OTLP-JSON (`batches[].scopeSpans[].spans[].name`), which is Tempo's
+  actual shape (not Jaeger's).
+
+Verified so far: `go vet -tags=e2e ./...` clean.
+
+## Live run against homelab Tempo
+
+First attempt failed before the suite started — Docker Desktop wasn't
+running (`kind` could not create the cluster), and the failure was masked
+to exit 0 by a `| tail` pipe on the make invocation (no pipefail in that
+shell). Rerun with Docker started first and no pipe:
+
+```bash
+TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e
+```
+
+Result: **SUCCESS — 4/4 specs (Manager smoke + both tracing Its), 219 s**,
+kind cluster auto-deleted. Run id `e2e-1787567624494810000`; the traces are
+findable in Grafana with TraceQL
+`{resource.test.run.id="e2e-1787567624494810000"}`. Both proxies reached
+`Ready` on real squid pods; the create trace carried
+`Reconcile Proxy` / `reconcile.managed` / `provider.create` /
+`status.patch` and the expected resource attrs; the deletion trace carried
+`reconcile.delete` / `provider.delete`.
+
+Worth noting: the OTLP preflight and both Tempo polls succeeded on the
+in-cluster → LAN path (kind on macOS reaches 192.168.0.30 through Docker
+Desktop's NAT), so no extra networking setup is needed on this machine.
+
+The plan's negative verification also ran: with a deliberately wrong
+endpoint (`OTLP_ENDPOINT=http://192.168.0.30:9999`) the tracing spec
+failed in the BeforeAll preflight after ~52 s with
+`OTLP endpoint http://192.168.0.30:9999 is not reachable from inside the
+kind cluster; curl output: OTLP_UNREACHABLE` — a named, fast failure
+instead of a 2-minute opaque search timeout — and teardown still deleted
+the kind cluster (`make cleanup-test-e2e` run explicitly, since a failing
+`go test` skips the Makefile's cleanup step).

docs/plans-executions/2026-09-04-1404-kvm-provider.md

@@ -1,285 +0,0 @@
-# Execution: KVM/libvirt provider
-
-Plan: [2026-09-04-1404-kvm-provider.md](../plans/2026-09-04-1404-kvm-provider.md)
-
-- [x] Step 0 — Save plan + branch `feat/kvm-provider`
-- [x] Step 1 — Config (`KVMConfig` & friends, `validate()` case, tests)
-- [x] Step 2 — Pure builders (`domainxml.go`, `volume.go`, `seed.go`, `id.go`, `metadata.go`)
-- [x] Step 3 — Transport + errors (`conn.go`, `errors.go`)
-- [x] Step 3b — Image cache (`image.go`)
-- [x] Step 4 — Provider methods (`kvm.go`) over a fake API
-- [ ] Step 5 — Wiring, manifests, samples, docs, CHANGELOG
-- [ ] Step 6 — Integration test against a real hypervisor
-- [ ] Step 7 — Manual validation + MR
-
-## Step 0 — Save plan + branch
-
-Plan committed alone on `feat/proxy-operator`, then branched:
-
-```bash
-git checkout -b feat/kvm-provider
-```
-
-## Step 1 — Config
-
-The new structs (`KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage`) and
-`validateKVM` went into their own file, `internal/provider/config_kvm.go`,
-rather than being appended to `config.go` as the plan said: the KVM block has
-far more fields and rules than the other two providers, and keeping it
-separate leaves `config.go` readable. Tests likewise live in
-`config_kvm_test.go` instead of extending the table in `config_test.go`; the
-invalid-config table covers every required field, duplicate host names, image
-URL scheme, sha256 shape, and the disk format enum.
-
-While adding the `kvm` case to `validate()`, the pairwise "type is X but a Y
-block is set" checks (which would have needed six branches with three types)
-were replaced by a `setBlocks()` helper that lists the non-nil blocks and
-rejects any that don't match `type`. The error strings are unchanged, so the
-existing GCP/kubernetes tests still pass.
-
-Dependencies were fetched up front so the API signatures in the plan could be
-checked with `go doc`; they stay `// indirect` in `go.mod` until Step 2 imports
-them:
-
-```bash
-go get github.com/digitalocean/go-libvirt@latest github.com/kdomanski/iso9660@v0.4.0
-# go-libvirt resolved to v0.0.0-20260814190004-1a83157e1858 (no tagged release)
-# side effect: golang.org/x/crypto 0.54.0 → 0.56.0, x/mod, x/text, x/tools bumped
-```
-
-Worth noting: `go doc` showed that go-libvirt's `New(net.Conn)` is deprecated
-in favour of `NewWithDialer(socket.Dialer)`, where `Dialer` is a one-method
-`Dial() (net.Conn, error)` interface. Step 3 should wrap the SSH client's unix
-forward in that interface instead of using `New`; the plan's transport design
-is otherwise unaffected.
-
-## Step 2 — Pure builders
-
-`internal/provider/kvm/` now has the five builders from the plan plus
-`kvm.go` holding the package doc and `withDefaults` (disk size 10 GiB, the
-default vendor-data, `/var/run/libvirt/libvirt-sock`, pool `default`, image
-format `qcow2`). Everything is a pure function over `encoding/xml` structs or
-strings, unit-tested by unmarshalling the output back and checking fields.
-
-Two design choices differ from the plan text:
-
-- **Domain disks reference pool volumes, not host paths.** The plan had
-  `buildDomainXML` take the overlay and seed paths (needing two
-  `StorageVolGetPath` round-trips per Create). The domain XML now uses
-  `<disk type="volume"><source pool=... volume=.../>`, so only the *base*
-  image path is ever looked up (the overlay's `<backingStore>` needs a real
-  path). Fewer RPCs and the builder's inputs are all names.
-- **Capacities are always `unit="bytes"`.** libvirt's `G` means GiB but
-  `GB` means 10^9; spelling the byte count out removes the ambiguity from
-  both the overlay and the seed/base volumes.
-
-The metadata element is written with a default namespace
-(`<instance xmlns="…/kvm">`) because `encoding/xml` cannot emit prefixes on
-marshal. libvirt may hand it back prefixed; `parseInstanceMetadata` is
-tested against both spellings since `encoding/xml` matches on the namespace
-URI. Whether libvirt accepts the default-namespace form on `DomainDefineXML`
-is confirmed in Step 6 against a real host, not here.
-
-Checked before relying on `github.com/kdomanski/iso9660` for the seed: its
-filename mangling lowercases and keeps `-`, so `user-data`/`meta-data`/
-`vendor-data` come out as `user-data;1` etc. in the directory record, which
-the Linux isofs driver (and therefore cloud-init's NoCloud mount) presents as
-`user-data`. The seed test opens the ISO with the same library's reader and
-asserts label `cidata`, all three files, and byte-identical output for
-identical inputs. `NewWriter()` stages files in a temp directory, so
-`Cleanup()` is deferred.
-
-`go mod tidy` ran as part of this step; it promoted `kdomanski/iso9660` to a
-direct dependency and dropped `go-libvirt` again because nothing imports it
-yet — Step 3 brings it back.
-
-Worth noting: `withDefaults` copies the `Images` map and `Hosts` slice
-instead of mutating the config in place, so the registry's shared
-`ProviderConfig` stays untouched. The default vendor-data installs
-`qemu-guest-agent` via `packages:` + `runcmd`, which means the first boot
-needs network access to the distro mirrors; a Proxy on an isolated bridge
-would still work (ARP discovery), just without the agent.
-
-## Step 3 — Transport + errors
-
-`conn.go` holds the `libvirtAPI` seam (17 methods, signatures as in the
-plan), a per-host connection pool, and the SSH dialer; `errors.go` holds
-`classify`, `isLibvirtError`, `hasCode` and `logAPIError`. Both are
-unit-tested; the whole package still runs in about a second under `-race`.
-
-The one thing the plan did not anticipate: **go-libvirt RPCs take no
-context and never time out**. A hung hypervisor would pin a reconcile worker
-forever. So the pool exposes `call(ctx, host, fn)` rather than handing out
-raw clients: `fn` runs in a goroutine, and if `ctx` ends first the pool
-closes the SSH connection out from under it (which makes the blocked RPC
-return) and reports `ctx.Err()`. The same helper (`awaitOrCancel`) bounds
-`lv.Connect()` during dial. Two more liveness measures for the same reason:
-the SSH handshake gets a deadline (ctx's, or 30 s), and each connection runs
-an OpenSSH `keepalive@openssh.com` loop every 15 s that closes the client
-when a request errors or goes unanswered for one interval.
-
-Connection-drop policy in `call`: any error that is *not* a `libvirt.Error`
-(so io.EOF, "connection closed", SSH failures) drops the cached connection
-and the next call redials; a daemon-reported `libvirt.Error` keeps it. Dials
-are serialised per host (concurrent Creates on one host share a dial) but
-hosts don't block each other.
-
-As Step 1 noted, the dialer uses `libvirt.NewWithDialer` with a
-`socket.Dialer` that wraps `ssh.Client.Dial("unix", socket)` rather than the
-deprecated `New(net.Conn)`.
-
-Tests: the pool is exercised through a fake dialer (reuse, redial after
-`Disconnected`, drop on transport error, keep on libvirt error, cancel
-releases a blocked call, concurrent callers share one dial, hosts
-independent). The SSH layer is tested against an in-process
-`x/crypto/ssh` server that serves `direct-streamlocal@openssh.com` channels
-by dialing the requested unix path (an echo socket standing in for
-`libvirt-sock`), with keys generated per test and a real `known_hosts` file
-written via `knownhosts.Line`. Covered: forward round-trip and the socket
-path the server saw, unknown host key rejected with `knownhosts.KeyError`,
-`insecureIgnoreHostKey`, wrong client key, cancelled ctx, missing key or
-known_hosts file, and keepalive both closing a stalled client and leaving a
-healthy one alone. The libvirt handshake (`Connect()`) itself is *not*
-covered here — faking libvirt's XDR RPC just for that is not worth it; the
-Step 6 integration test covers it against a real host.
-
-`go mod tidy` promoted `go-libvirt` and `golang.org/x/crypto` to direct
-dependencies.
-
-Worth noting: `classify` maps `ErrOperationInvalid` (e.g. destroying a
-domain that is already off) to Transient; Delete in Step 4 must check that
-code itself via `hasCode` and treat it as success rather than relying on
-the classifier. Nothing maps to `ErrQuotaExceeded` — libvirt has no quota
-concept; an out-of-space pool surfaces as a Transient `ErrInternalError`.
-
-## Step 3b — Image cache
-
-`image.go` follows the plan closely: `resolveImage` maps `placement.image`
-to a configured key or a direct http(s) URL (else Permanent);
-`imageCache.ensure` checks base volume + marker (+ marker content when a
-checksum is configured), tears down and re-fetches when anything is off,
-streams the HTTP body through `StorageVolUpload` with a `sha256` tee,
-refreshes the pool, then writes the marker last. A per-`(host, volume)`
-mutex serialises concurrent callers; the pool itself is the only cache
-state.
-
-Errors from this file are pre-tagged with the taxonomy sentinel via
-`fmt.Errorf("%w: …", provider.ErrPermanent, …)` and `classify` now honours
-such tags before looking at libvirt codes. That keeps one `wrapErr` path in
-the provider instead of special-casing image errors.
-
-`fake_test.go` landed with this step rather than Step 4: an in-memory
-`libvirtAPI` with pools, volumes (content, format, capacity, backing path),
-domains (state, metadata, per-source interface addresses), per-method error
-injection (`fail` always / `failOnce`) and an `uploadHook` to break a stream
-mid-way. It models the libvirt error codes the provider branches on
-(`ErrStorageVolExist`, `ErrNoStorageVol`, `ErrNoDomain`,
-`ErrOperationInvalid`, `ErrNoDomainMetadata`, `ErrAgentUnresponsive`).
-
-Tests cover every case in the plan plus: no checksum configured (marker is
-still written with the computed digest; second call needs no marker read),
-403/500 alongside 404/503, missing pool → Permanent, cancelled ctx →
-Transient with nothing created, and different hosts fetching independently.
-
-One test-harness lesson worth keeping: `httptest`'s server drops
-`Content-Length` and switches to chunked encoding for bodies larger than
-its 2 KiB write buffer, so the fake mirror sets the header explicitly.
-Real distro mirrors send it; a server that doesn't is reported as
-Permanent with a message naming the header.
-
-Worth noting: the base volume is created with `capacity = Content-Length`
-and the *image's* format (`qcow2`), and the raw bytes are uploaded over it.
-That relies on libvirt's upload writing the file verbatim (which is how
-`virsh vol-upload` populates volumes); the following `StoragePoolRefresh`
-makes libvirt re-read the real virtual size from the qcow2 header. Step 6
-confirms this on a real host. Also, the whole download runs inside one
-`pool.call`, so cancellation aborts the HTTP body (ctx-bound) and the
-upload stream fails; the half-written base volume is deleted on the spot
-and, failing that, on the next attempt via the missing marker.
-
-## Step 4 — Provider methods
-
-`kvm.go` now holds `Provider`, `New` (loads the SSH key and known_hosts
-up front via `newSSHAuth`, so a bad path fails at startup), the
-`newWithDialer` seam, and `Create`/`Get`/`Delete`/`ListByTag`. Each
-method resolves its host from config, then runs the whole libvirt
-conversation inside one `hostPool.call`, so a cancelled reconcile drops
-the connection and the blocked RPC returns. Errors pass through `classify`
-in one `wrapErr`, as the Step 3b note intended.
-
-Behaviour that goes beyond the plan text, or that a future reader should
-know before touching this file:
-
-- **Create looks up the domain first.** If it exists, only the running
-  check runs and the volumes are never touched. If not, nothing can be
-  using the volumes yet: the overlay is reused when it exists (its
-  creation is atomic; `ErrStorageVolExist` → reuse) but the **seed is
-  always deleted and rebuilt**. The plan said "exists → skip upload"; the
-  reviewer pointed out that a reconcile cancelled mid-upload leaves a
-  truncated seed that the compensating delete cannot remove, because
-  `hostPool.call` has already closed the connection the delete would go
-  over. Skipping such a seed would boot the VM without cloud-init and no
-  retry would ever repair it. Rebuilding costs one ~40 KiB upload per
-  attempt before the domain exists, nothing afterwards. The table test
-  breaks Create after each step and re-runs it, and a separate test
-  starts from a leftover truncated seed.
-- **Get swallows address-source errors.** `DomainInterfaceAddresses` from
-  ARP returns nothing until the guest has sent traffic and the agent
-  source errors with `ErrAgentUnresponsive` until `qemu-guest-agent` is
-  up, so both are logged at V(2) and mean "no IP yet" → `Provisioning`.
-  Loopback, link-local and unspecified addresses are skipped when
-  choosing the first IPv4 (the agent reports `lo` too).
-- **State mapping:** `running`/`blocked` are active; `nostate` →
-  Provisioning (the reconciler answers Stopped with delete-and-recreate,
-  too drastic for a state libvirt could not determine); everything else
-  (paused, pmsuspended, shutdown, shutoff, crashed) → Stopped.
-- **Delete treats a missing pool as "nothing to clean"** rather than
-  Permanent: failing there would pin the Proxy's finalizer on a config
-  edit that renamed the pool. Domain removal is destroy (`ErrOperationInvalid`
-  on a stopped domain is fine) → undefine with managed-save, snapshot
-  metadata and nvram flags → overlay and seed deleted by name.
-- **ListByTag** reads each domain's metadata once (passed into the same
-  `inspect` that Get uses, so no second `DomainGetMetadata` round-trip)
-  and skips domains without the operator's element. A domain that cannot
-  be read at all — vanished mid-sweep, corrupt metadata, state call
-  failing — is *skipped*, not fatal: GC never deletes what it does not
-  see, so leaving one out is safe, whereas one bad domain blocking GC for
-  every other one (the first version's behaviour, caught in review) is
-  not. An unreachable host still fails the whole sweep.
-  `ConnectListAllDomains` is called with flags 0 (active and inactive).
-- A host named in a providerID but no longer in `kvm.hosts` is Permanent
-  for Get and Delete: the operator cannot reach it, and retrying forever
-  would hide the config problem.
-
-Tests run the provider over a `hostDialer` that maps host names to
-`fakeAPI` instances (an unmapped host is "unreachable"). The image is
-pre-seeded in the pool for most tests; one test runs the full first-use
-download through `httptest` and confirms a second Proxy on the same host
-does not download again.
-
-Found while running the package: `TestBuildSeedISO_deterministic` from
-Step 2 was wrong. `iso9660.ImageWriter` stamps `time.Now()` into the
-volume descriptor, so two builds in different seconds differ by six
-timestamp bytes; the test had only passed because both builds landed in
-the same second. Byte identity is nothing the provider depends on (an
-existing seed is never re-uploaded), so the test now compares label,
-files and size, and the comment on `buildSeedISO` no longer claims
-identical output.
-
-The `go-operator-reviewer` agent ran on the first version of this step
-and found the two bugs above (truncated seed, wedged GC sweep) plus the
-double metadata read and the `nostate` mapping; all were fixed before
-committing. One finding it raised is deliberately **not** fixed here
-because it lives in Step 3b's `image.go` and changes the volume naming
-the plan chose: re-fetching a base image (checksum edited in config, or
-upstream content changed) deletes `base-<key>.<fmt>` while existing
-overlays still back onto it, breaking those VMs' disks. Content-addressed
-base names would avoid it. Flagged for the user; until decided, Step 5's
-docs must warn that an image's `sha256`/`url` must not change while
-Proxies using it exist.
-
-Worth noting: `go test -cover` reports 88.2 % for the package; the
-uncovered lines are the real SSH/libvirt dial path and log-only
-branches. `go vet` and `-race` (three runs) are clean. Wiring into
-`cmd/main.go` is Step 5, so `New` is not yet reachable from a running
-operator.

docs/plans/2026-08-24-1025-otel-tracing.md

@@ -0,0 +1,107 @@
+# Plan: OpenTelemetry tracing integrated with the existing logr/zap logging
+**Created:** 2026-08-24 10:25
+
+## Context
+
+The operator has structured logging (controller-runtime zap → logr, `logf.FromContext(ctx)` everywhere) and Prometheus metrics, but no tracing. A single reconcile fans out into k8s API calls, provider calls (GCP VMs / squid Pods), and status patches; the discovery API serves lease requests; GC and health engines run on tickers. Today correlating one operation across those hops means grepping for `reconcileID` or `providerID`. Goal: proper OTel traces — a new trace per unit of work (reconcile, HTTP request, GC sweep, optionally health probe), child spans for k8s/provider/HTTP calls — **and every log line inside a traced operation enriched with the trace context**, without disturbing the existing logr/zap system.
+
+User decisions (confirmed):
+- **Enablement:** auto from standard `OTEL_*` env vars — no collector configured ⇒ tracing fully off, zero new flags for the common case.
+- **Health probes:** not traced by default; new `--trace-health-probes` flag enables one root span per probe.
+- **Backend:** none exists yet — verification uses the console exporter and a throwaway Jaeger all-in-one in kind; manifests ship commented-out OTLP env examples.
+
+## Key design facts (verified, non-obvious)
+
+- **logr can't read ctx:** `logr.LogSink` never sees `context.Context`, so trace IDs cannot be added inside the zap sink. They must be attached as `WithValues` at span-creation points via `logf.IntoContext`. zap does **not** dedupe repeated keys, so enrichment must happen exactly once per ctx chain (see `tracing.Start` below).
+- Log keys: **`traceID` / `spanID`** (lowerCamel matches `reconcileID`/`providerID` house style; note in docs that Grafana/Loki derived-field regexes must match `traceID`, not the `trace_id` default).
+- **GCP calls trace themselves:** `google.golang.org/api/transport/http` already wraps its transport in `otelhttp.NewTransport` — once a global TracerProvider is set, every Compute call emits an HTTP client span. Do not add HTTP-level spans in the gcp provider; the `provider.*` decorator span becomes their parent.
+- **kube-apiserver ignores incoming `traceparent`** by design; our k8s-API client spans are leaves. Don't promise cross-process traces into the apiserver.
+- The Go SDK does **not** implement `OTEL_SDK_DISABLED` — handle it ourselves.
+- `contrib/exporters/autoexport` drags in metric/log/prometheus exporters (binary + deps on a 128Mi pod) — hand-roll trace-exporter selection (~40 lines) instead.
+- Span events are deprecated (OTel spec, Mar 2026): use `span.SetStatus` + `error.type`-style attributes, not `RecordError`/`AddEvent`.
+- `rest.Config.Wrap` puts our transport innermost (after auth), so spans see the final request — correct; but informer list/watch + leader election run outside any span ⇒ the transport must **filter out requests with no parent span** or every watch becomes a long-lived root span.
+- Manager runnables receive no logger in ctx (only HTTP servers do) — `logf.FromContext` falls back to the global logger there, which is fine; the discovery server must keep seeding from `s.log`.
+
+## Steps
+
+### Step 1 — Dependencies
+
+`go get` (aligning the existing indirect skew: otel core v1.44 / otlptrace v1.40):
+`go.opentelemetry.io/otel@v1.45.0`, `otel/sdk@v1.45.0`, `otel/trace@v1.45.0`,
+`otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0`, `.../otlptracehttp@v1.45.0`,
+`otel/exporters/stdout/stdouttrace@v1.45.0`,
+`go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0`, plus the
+`semconv` version matching the SDK's `resource.Default()` (treat `ErrSchemaURLConflict` from `resource.New` as non-fatal if versions drift).
+`go mod tidy`; full build + `make test` to confirm `google.golang.org/api v0.292.0` / k8s v0.36 tolerate the bump (MVS picks the higher otelhttp).
+
+### Step 2 — New package `internal/tracing`
+
+- **`tracing.go` — `Setup(ctx, log logr.Logger, service, version string) (shutdown func(context.Context) error, err error)`**
+  - Disabled (returns no-op shutdown, installs nothing) when `OTEL_SDK_DISABLED=true`, or when neither `OTEL_TRACES_EXPORTER` nor `OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is set, or `OTEL_TRACES_EXPORTER=none`. One startup Info line either way stating the effective state/exporter.
+  - Exporter selection (hand-rolled, no autoexport): `OTEL_TRACES_EXPORTER` ∈ `otlp` (default) | `console`; for otlp, `OTEL_EXPORTER_OTLP_(TRACES_)PROTOCOL` ∈ `http/protobuf` (default) | `grpc` → `otlptracehttp` / `otlptracegrpc` (endpoint/TLS/headers via their native env handling).
+  - `sdktrace.NewTracerProvider` with batch processor (`WithMaxQueueSize(512)` — bounded on the 128Mi pod; sampler stays the SDK default so `OTEL_TRACES_SAMPLER(_ARG)` works), resource = `resource.New(WithAttributes(service.name, service.version=commit), WithFromEnv())` with env last so it wins.
+  - Globals: `otel.SetTracerProvider`, `otel.SetTextMapPropagator(TraceContext+Baggage)`, `otel.SetLogger(log.WithName("otel"))`, `otel.SetErrorHandler` → logr at `V(1)` (an unreachable collector errors every few seconds; Error level would spam).
+- **`logger.go` — the one logger-enrichment mechanism** (used by everything; prevents duplicate `traceID` keys):
+  - private ctx key holding the *base* (pre-enrichment) logr.Logger.
+  - `Start(ctx, spanName, opts...) (context.Context, trace.Span)`: starts a span from the global tracer; if the resulting span context is valid, capture base = stored base or `logf.FromContext(ctx)` on first call, then `logf.IntoContext(ctx, base.WithValues("traceID", ..., "spanID", ...))`. Nested `Start` re-derives from base — never stacks. Documented trade-off: `WithValues` pushed via `logf.IntoContext` *between* two `Start` calls is dropped (nothing first-party does that; controller-runtime's `reconcileID` logger is the base and survives).
+  - `ContextWithLogger(ctx, base)` seeds the base explicitly (discovery middleware).
+- **`reconciler.go` — `NewReconciler(kind string, inner reconcile.Reconciler, opts...) reconcile.Reconciler`**: root span `Reconcile <kind>` (SpanKind Internal) via `Start`; attrs `k8s.namespace.name`, `k8s.object.name`, `reconcile.id` (`controller.ReconcileIDFromContext`); on return records `reconcile.requeue_after`, sets Error status from err. `WithTracerProvider` option for tests.
+- **`transport.go` — `RestConfigWrapper() transport.WrapperFunc`**: `otelhttp.NewTransport` with `WithFilter` requiring `trace.SpanContextFromContext(r.Context()).IsValid()` — k8s API spans only under an existing trace (excludes informers, leader election, metrics authn).
+- **`http.go` — `HTTPMiddleware(operation string, base logr.Logger) func(http.Handler) http.Handler`**: `otelhttp.NewHandler` (filter `/healthz`; otelhttp renames the span from `r.Pattern` after routing — no custom naming needed) + inner layer calling `ContextWithLogger`-then-enrich so handler logs carry `traceID`. Accepts incoming `traceparent` (clients continue their traces).
+
+Tests (all with `sdktrace` + `tracetest.NewSpanRecorder` via the `WithTracerProvider` options, parallel; `Setup` env tests use `t.Setenv`, not parallel):
+- `Start`: nested spans → child span recorded; **funcr-captured log line contains `traceID` exactly once** (pattern: `internal/provider/gcp/gcp_test.go:277`); disabled tracer → no `traceID` in logs.
+- `Setup`: table over env combinations (unset→disabled, `none`→disabled, endpoint set→otlp, `console`, `OTEL_SDK_DISABLED`).
+- Reconciler decorator: fake inner returns result/err → span name/attrs/status.
+- Transport filter: request without parent span produces no span.
+- Middleware: `httptest` request → server span named from route pattern; `/healthz` unspanned; handler logger carries `traceID`.
+
+### Step 3 — `provider.WithTracing` decorator
+
+`internal/provider/tracing.go`, mirroring `metrics.go`: `WithTracing(name string, p Provider, opts...) Provider` — one Client span per call (`provider.create|get|delete|list`), attrs `provider.name`, `provider.id` (where present), result class attr from `resultLabel(err)`; status Error only for quota/permanent/transient — `ErrNotFound` stays Ok (expected during deletion polling). Returns the inner error **unmodified**. Uses `tracing.Start` so provider-internal logs (gcp `p.logger(ctx)`) inherit `traceID`. Tests in `internal/provider/tracing_test.go` reusing `staticProvider` from `metrics_test.go:19`.
+
+Wiring in `cmd/main.go:174-177` — tracing outermost: `provider.WithTracing(name, provider.WithMetrics(name, p, m))`.
+
+### Step 4 — `cmd/main.go` wiring
+
+- `tracing.Setup(...)` immediately after `ctrl.SetLogger` (cmd/main.go:148); on error: log + `os.Exit(1)`.
+- Explicit `shutdown(context.WithTimeout(context.Background(), 10*time.Second))` (fresh ctx — signal ctx is already cancelled) after `mgr.Start` returns, on **both** the error path before `os.Exit(1)` and the clean path. Earlier `os.Exit` sites have nothing to flush.
+- `cfg := ctrl.GetConfigOrDie(); cfg.Wrap(tracing.RestConfigWrapper())` before `ctrl.NewManager` (cmd/main.go:239).
+- kubernetes provider: add `kubernetes.NewWithTransportWrapper(ctx, cfg, wrap transport.WrapperFunc)` beside `New` (internal/provider/kubernetes/kubernetes.go:42) and register a closure in main's constructor map (same pattern as gcp at cmd/main.go:166) passing `tracing.RestConfigWrapper()` — keeps `internal/tracing` out of provider packages and `newWithClient` tests untouched.
+- New flag `--trace-health-probes` (default false) → `engine.TraceProbes`.
+
+### Step 5 — Reconciler spans
+
+- `SetupWithManager` (proxy_controller.go:412): `b.Complete(tracing.NewReconciler("Proxy", r))`.
+- Sub-spans via `tracing.Start`: `reconcile.managed` (:116), `reconcile.replaceInstance` (:226), `reconcile.delete` (:254), and `status.patch` opened **inside the deferred flush closure** (:98-104, runs within the root span; its error already folds into `err`). `reconcileExternal` stays unspanned — no I/O, pure noise.
+- Useful attrs where cheap: phase transitions, `providerID`.
+
+### Step 6 — Discovery server
+
+- `handler()` (internal/discovery/server.go:151-156): chain becomes recover → `tracing.HTTPMiddleware("discovery", s.log)` → log → maxBytes → auth (otelhttp must stay outside `maxBytesMiddleware`).
+- Handlers and `logMiddleware`/`recoverMiddleware` switch `s.log` → `logf.FromContext(r.Context())` (handlers.go:85, :141, :234; server.go:163, :191) so request logs carry `traceID`. `s.log` remains for startup lines.
+- k8s `List` calls in handlers already use `r.Context()` → child spans appear via the wrapped rest transport.
+
+### Step 7 — GC + health
+
+- `gc.Sweeper.sweep` (internal/gc/gc.go:83): wrap in root span `gc.sweep` via `tracing.Start` (before the `.WithName("orphan-gc")` derivation so it inherits), attrs: providers swept, orphans deleted. Provider/k8s child spans come free from Steps 3/4.
+- `health.Engine`: new field `TraceProbes bool`. In the worker loop (engine.go:158) when enabled: root span `health.probe` (SpanKind Client) around `probeFn`, attrs proxy key/ok/latency/error class, ended before `record` (which stays ctx-free). No transport wrap and no propagation — never inject `traceparent` toward external targets through the proxies. Probe→reconcile span links are **out of scope**: `GenericEvent` (engine.go:330) carries no ctx and the workqueue coalesces events; documented in architecture Decisions.
+- Lease store sweep: not traced (in-memory only).
+
+### Step 8 — GCP wire-log enrichment
+
+`wireFilterHandler.Handle` (internal/provider/gcp/wirelog.go:46) receives the real request ctx from the SDK's slog calls: append `traceID`/`spanID` attrs when `trace.SpanContextFromContext(ctx)` is valid — V(5) wire logs become correlatable with the `provider.*` span. Add `wirelog_test.go` case.
+
+### Step 9 — Manifests + docs
+
+- `config/manager/manager.yaml` env (after `DISCOVERY_TOKEN`): `POD_NAME`/`POD_NAMESPACE` via downward API **first**, then `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)` ($(VAR) expansion only sees earlier vars); commented-out `OTEL_EXPORTER_OTLP_ENDPOINT` + `OTEL_TRACES_SAMPLER` examples with a "tracing is off until an endpoint is set" comment. Note `--trace-health-probes` beside the args list.
+- `docs/architecture.md`: new "### 10. Tracing" section after Metrics (:281) — span topology, enablement, `traceID` log-key choice; Decisions entries (:303): env-gated enablement, probes off by default, no probe→reconcile links (why), `traceID` naming vs Grafana defaults, no traceparent into apiserver.
+- README ops note; CHANGELOG entry (dated via `date`) once the user confirms it works.
+- Run `make lint` (CI has no lint step; `lll`/`revive` will hit new files).
+
+## Verification
+
+1. `make test` (unit, includes all new `tracetest` assertions) and `make lint`.
+2. Console smoke: `OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml ...` against kind — create a Proxy, confirm (a) `Reconcile Proxy` span JSON on stdout with nested `provider.create` + k8s PATCH spans, (b) the reconcile log lines carry the same `traceID` as the span, (c) no spans from informer watches or leader election.
+3. End-to-end in kind: deploy Jaeger all-in-one (`jaegertracing/all-in-one`, OTLP 4318), set `OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318` on the manager Deployment; exercise a reconcile + a `POST /v1/leases` (with and without a client `traceparent`) + wait one GC interval; in the Jaeger UI: reconcile trace shows provider+k8s children; lease trace shows route-named server span with k8s child; grep manager logs for a `traceID` and resolve it in Jaeger.
+4. Negative: run with no `OTEL_*` env — startup line says tracing disabled, no export-error spam, logs unchanged (no `traceID` keys).

docs/plans/2026-08-24-1224-tracing-e2e.md

@@ -0,0 +1,150 @@
+# Plan: e2e test — OTel tracing against real Tempo
+**Created:** 2026-08-24 12:24
+
+## Context
+
+The tracing feature (MR #4, branch `feat/otel-tracing`) is covered by unit
+tests with an in-memory span recorder, but nothing proves the full pipeline:
+operator in a real cluster → OTLP export → Tempo ingest → queryable traces
+with the documented span topology. The homelab Tempo lives at
+`http://192.168.0.30:3200` (query API — verified live from this host:
+`/api/echo`, TraceQL `GET /api/search?q=…`, `GET /api/traces/<id>`), with
+OTLP ingest on `:4318` (HTTP, verified 200) and `:4317` (gRPC, open). User
+decisions: **Go Ginkgo e2e test** (not a shell script), **structural
+assertions** (span tree, not just trace-exists), flow per the user's sketch:
+deploy operator in kind with tracing on → create kubernetes-provider proxies
+→ delete them → shut down — with traces tagged for easy discovery per test
+run. This work continues on `feat/otel-tracing`; MR #4 stays the vehicle.
+
+## Design decisions (from review, with evidence)
+
+- **Proxy CRs go in the `default` namespace, not the operator namespace.**
+  The squid pods the kubernetes provider creates carry no securityContext
+  (internal/provider/kubernetes/pod.go:33-49) and are created in the CR's
+  namespace; the operator ns is labeled `pod-security enforce=restricted`
+  (test/e2e/e2e_test.go:61-62) and would reject them. Pod RBAC is
+  cluster-scoped (config/rbac/role.yaml:10), so `default` works. All
+  kubectl calls use explicit `-n`.
+- **Anchor assertions on `provider.create`, not the root-span search.** A
+  `name="Reconcile Proxy"` hit may be the finalizer-add or a drift
+  reconcile (no provider call inside). TraceQL matches span names anywhere
+  in a trace, so search `{resource.test.run.id="<runID>" && name="provider.create"}`,
+  fetch *that* traceID, then assert companion span names.
+- **Run identification with zero code changes:** OTel resource attribute
+  `test.run.id=<runID>` via `OTEL_RESOURCE_ATTRIBUTES` (env wins —
+  `resource.WithFromEnv()` merges last, internal/tracing/tracing.go). Tempo
+  vParquet searches arbitrary resource attrs without config; fallback query
+  documented: `{resource.service.name="egress-proxies-operator" && name="provider.create"}`
+  + start/end window (Unix **seconds**).
+- **OTLP preflight from inside the cluster:** host-reachability of
+  192.168.0.30 doesn't prove pod-reachability from kind on macOS, and
+  export failures log only at V(1) (invisible) — without a preflight, a
+  broken path is a 2-minute opaque timeout. Reuse the suite's curl-pod
+  pattern (e2e_test.go:218-248, incl. the restricted-PSS overrides JSON)
+  to POST to `<OTLP_ENDPOINT>/v1/traces` and fail fast.
+- **No homelab defaults baked into the Makefile.** The spec Skips unless
+  `TEMPO_URL`/`OTLP_ENDPOINT` are set (otherwise everyone without that LAN
+  host inherits a 2-minute failure). The copy-pasteable invocation lives in
+  docs/testing.md.
+- **Ginkgo ordering:** top-level container order is randomized, so the
+  tracing Describe is fully self-contained (own ns create → make install →
+  make deploy → teardown), same shape as the Manager Describe. `kubectl
+  delete ns` blocks until termination, so no create/delete race between
+  the two Describes.
+
+## Steps
+
+### Step 1 — `test/e2e/tracing_test.go` (new; `//go:build e2e`, package e2e)
+
+`Describe("OTel tracing", Ordered)`:
+
+- **BeforeAll:**
+  1. Read `TEMPO_URL` + `OTLP_ENDPOINT`; `Skip("TEMPO_URL/OTLP_ENDPOINT not set")` when empty.
+  2. `runID := "e2e-" + strconv.FormatInt(time.Now().UnixNano(), 10)`;
+     print it to `GinkgoWriter` so the run is findable in Grafana by hand.
+  3. Create operator ns + restricted-PSS label; `make install`;
+     `make deploy IMG=<managerImage>` (same commands as e2e_test.go:54-75,
+     via `utils.Run`).
+  4. Pre-pull the proxy image to kill the biggest flake source:
+     `docker pull ubuntu/squid:6.6-24.04_edge` + `kind load docker-image`
+     (via `utils.Run`, honoring `KIND`/`KIND_CLUSTER` env like
+     utils.LoadImageToKindClusterWithName).
+  5. OTLP preflight: curl pod POSTing `{}` to `<OTLP_ENDPOINT>/v1/traces`,
+     assert HTTP 200 in its logs (fail message names the endpoint).
+  6. `kubectl set env deployment/egress-proxies-operator-controller-manager
+     -n <ns> OTEL_EXPORTER_OTLP_ENDPOINT=<OTLP_ENDPOINT>
+     "OTEL_RESOURCE_ATTRIBUTES=k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE),test.run.id=<runID>"`
+     — in-place update keeps the var after the downward-API vars ($(VAR)
+     expansion needs that ordering); `exec.Command` means no shell mangling
+     of `$()`. Then `kubectl rollout status`.
+  7. `Eventually`: newest controller pod (sorted by creationTimestamp — a
+     terminating old pod may otherwise be picked) logs contain
+     `"tracing enabled"`.
+- **It "creates proxies and reports reconcile traces":**
+  - Write 2 Proxy manifests (names `proxy-tracing-e2e-1/2`,
+    `provider: kubernetes`, modeled on config/samples/proxy_kubernetes.yaml)
+    to `GinkgoT().TempDir()` (absolute paths — `utils.Run` chdirs the
+    process); `kubectl apply -n default`.
+  - `Eventually` (5m — first squid start) `.status.phase == "Ready"`
+    (confirmed field, api/v1alpha1/proxy_types.go:41).
+  - `Eventually`: Tempo search
+    `{resource.test.run.id="<runID>" && name="provider.create"}` ≥1 hit
+    (BSP flushes every ~5s; 2m default timeout is ample).
+  - Fetch that traceID via `/api/traces/<id>`; assert span names include
+    `Reconcile Proxy`, `reconcile.managed`, `provider.create`,
+    `status.patch`, and resource attr `service.name=egress-proxies-operator`.
+- **It "traces deletion":**
+  - `kubectl delete proxy -n default …`; `Eventually` CRs gone (finalizer
+    → `provider.delete`, DeletionPoll 10s).
+  - `Eventually`: Tempo search finds `provider.delete` for the runID; fetch
+    and assert `reconcile.delete` in the same trace.
+- **AfterAll** (= the sketch's "shut down operator"): delete leftover
+  proxies in `default`; `make undeploy` (SIGTERM → `tracingShutdown`
+  flushes); `make uninstall`; delete operator ns — errors discarded
+  (`_, _ = utils.Run(...)`), mirroring e2e_test.go:79-95.
+- **Tempo client helpers** (same file, stdlib only — the test process runs
+  on the host, which reaches Tempo directly):
+  - `tempoSearch(traceql string)` → `GET {TEMPO_URL}/api/search?q=…&start=…&end=…`
+    (Unix seconds, window = suite start − 5m → now); minimal struct
+    `{Traces []struct{ TraceID string }}`.
+  - `tempoTrace(id string)` → `/api/traces/<id>`, decoded as **OTLP-JSON**
+    (`batches[].scopeSpans[].spans[].name`, resource attrs as
+    `{key, value:{stringValue}}`) — not Jaeger's shape. Helper flattens to
+    a span-name set + resource attr map.
+
+### Step 2 — Makefile
+
+- `test-e2e`: pass `TEMPO_URL`/`OTLP_ENDPOINT` through to `go test` env
+  (no defaults) and add `-timeout 30m` — the suite already runs
+  docker-build + kind-load in BeforeSuite plus 3–5m Eventuallys, and this
+  adds a second full deploy cycle + image pulls; the 10m default will be
+  exceeded on cold caches.
+
+### Step 3 — Docs
+
+- `docs/testing.md` (§ e2e, currently "scaffold… only asserts manager runs
+  and serves metrics"): document the tracing spec, its Skip gate, and the
+  invocation:
+  `TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`.
+- Execution-log entries per step in `docs/plans-executions/2026-08-24-1224-tracing-e2e.md`.
+- CHANGELOG entry only after the user confirms a green run (house rule).
+
+## Not doing (and why)
+
+- No dedicated flush assertion on shutdown — teardown exercises the flush
+  path, but attributing a specific late span to it is guesswork.
+- No gRPC (4317) variant — http/protobuf is the operator default and the
+  verified path; a variant run is a one-env-var change if ever wanted.
+- No changes to `internal/` — the whole test works through public surface
+  (env vars, kubectl, Tempo API), which is the point of an e2e test.
+
+## Verification
+
+1. `TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`
+   → tracing spec passes (both Its); without the env vars → spec reports
+   Skipped, remainder of suite unaffected.
+2. In Grafana: TraceQL `{resource.test.run.id="<runID printed in test log>"}`
+   shows the run's traces with the expected tree.
+3. `go vet -tags=e2e ./...` clean (documented routine).
+4. A deliberately wrong `OTLP_ENDPOINT` (e.g. port 9) fails fast in the
+   preflight step with a clear message, not a 2-minute search timeout.

docs/plans/2026-09-04-1404-kvm-provider.md

@@ -1,313 +0,0 @@
-# Plan: KVM/libvirt provider
-
-**Created:** 2026-09-04 14:04
-
-## Context
-
-The operator provisions egress proxies through named provider instances (today
-`gcp` and `kubernetes`). The user has several KVM hypervisors on the home
-network and wants proxy VMs created there, using stock Debian/Ubuntu cloud
-images so the VM stays a normal machine they can customise through their own
-cloud-init user-data. Longer term the operator may become a generic "run a VM
-or container with a set of scripts" tool; the KVM provider should not
-foreclose that.
-
-The provider contract (`internal/provider/provider.go`: `Create`/`Get`/`Delete`/
-`ListByTag`, keyed by `NameFromUID`, cloud-init passed verbatim, GC-by-tag) fits
-libvirt with **no CRD or reconciler changes**. All work is a new provider
-package plus config/wiring/docs, following the GCP provider's shape.
-
-Decisions taken with the user (2026-09-04):
-
-- **Transport:** SSH to each hypervisor, forwarding the libvirt unix socket
-  (`ssh.Client.Dial("unix", "/var/run/libvirt/libvirt-sock")` → `libvirt.New(conn)`).
-  Pure Go, no cgo, no libvirt libs in the image. Verified: `x/crypto/ssh`
-  `Client.Dial` has a `"unix"` case (`dialStreamLocal`), go-libvirt `New(net.Conn)`.
-- **Topology:** one provider instance can span several hosts from day one
-  (config has a `hosts` list; providerID encodes the host), so adding hosts
-  later is a config edit with no providerID migration. With one host configured
-  `placement.zone` may be omitted.
-- **Networking:** VM NIC on a host bridge to the LAN, IP from the router's
-  DHCP. IP discovery via `DomainInterfaceAddresses` (ARP source first, then
-  guest agent). Provider-injected `vendor-data` installs `qemu-guest-agent`
-  so agent discovery works on stock images without touching user-data.
-- **Base image:** a stock Debian/Ubuntu cloud image (qcow2, cloud-init
-  preinstalled) that **the operator fetches itself** on first use per host:
-  `placement.image` names an entry in the provider's `images:` map (url +
-  optional sha256) or is an `https://` URL directly. The provider streams the
-  download through libvirt's `StorageVolUpload` into the host's pool, so
-  nothing is pre-placed and the operator needs no filesystem access on the
-  host. Per-VM disk is a qcow2 overlay with the cached base as backing store.
-- **Seed:** NoCloud ISO (`user-data`, `meta-data`, `vendor-data`) built in
-  pure Go with `github.com/kdomanski/iso9660` (`WriteTo(w, "cidata")`) and
-  streamed to the pool with `StorageVolUpload(vol, io.Reader, ...)`. Nothing
-  to install on the hypervisors.
-
-## Design
-
-### Config (`internal/provider/config.go`)
-
-```yaml
-- name: kvm-home
-  type: kvm
-  kvm:
-    sshKeyFile: /etc/proxy-operator/kvm/id_ed25519        # required
-    knownHostsFile: /etc/proxy-operator/kvm/known_hosts   # required unless insecureIgnoreHostKey
-    insecureIgnoreHostKey: false
-    diskSizeGb: 10                    # overlay capacity; cloud-init growpart expands root fs
-    vendorData: |                     # optional override; default installs qemu-guest-agent
-      #cloud-config
-      ...
-    images:                           # optional; placement.image names a key (or is an https URL)
-      debian-13:
-        url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
-        sha256: "…"                   # optional; verified after download, re-fetch when it changes
-        format: qcow2                 # default qcow2
-      ubuntu-24.04:
-        url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
-    machineTypes:                     # required, ≥1; placement.machineType names a key
-      small:  {vcpus: 1, memoryMiB: 1024}
-      medium: {vcpus: 2, memoryMiB: 2048}
-    hosts:                            # required, ≥1, unique names
-      - name: nas1                    # placement.zone value
-        address: nas1.home.lan:22     # required
-        user: virt                    # required; must be in the libvirt group
-        socket: /var/run/libvirt/libvirt-sock   # default
-        pool: default                 # default "default"
-        bridge: br0                   # required
-```
-
-Add `KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage` structs; a `"kvm"`
-case in `validate()` (cross-block checks like the others, required fields
-above, unique host names, image URLs must be `http(s)`, sha256 if set must be
-64 hex chars). Tests in `config_test.go` extend the existing table.
-
-### Placement mapping
-
-| Placement field | KVM meaning |
-|---|---|
-| `zone` | host name from `hosts`; empty → the single configured host, else `ErrPermanent` |
-| `machineType` | key in `machineTypes`; unknown → `ErrPermanent` |
-| `image` | key in `images:`, or an `https://` URL; anything else → `ErrPermanent`. Fetched into the host's pool on first use (see *Image cache*) |
-| `region` | ignored |
-
-### providerID
-
-`hosts/<host>/domains/<name>` with `formatProviderID`/`parseProviderID`
-(same self-contained style as GCP's `zones/<zone>/instances/<name>`).
-
-### GC contract without labels
-
-Domain XML gets a `<metadata>` child in a private namespace, e.g.
-`xmlns="https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm"`,
-carrying `<managed>true</managed><uid>…</uid><createdAt>RFC3339</createdAt>`.
-`Get` reads it via `DomainGetMetadata(dom, DomainMetadataElement, uri)`;
-`ListByTag` does `ConnectListAllDomains` on every host and keeps domains where
-that call succeeds and `managed == "true"`. `CreatedAt` comes from the metadata
-(libvirt has no creation time). A host that is unreachable makes `ListByTag`
-return an error (GC skips the sweep that round rather than acting on a partial view).
-
-### Connection management (`internal/provider/kvm/conn.go`)
-
-Per-host lazily dialed `*ssh.Client` + `*libvirt.Libvirt`, cached under a
-mutex, re-dialed when `Disconnected()` is closed or a call fails with a
-transport error. SSH auth: private key from `sshKeyFile`; host key check via
-`knownhosts.New(knownHostsFile)` unless `insecureIgnoreHostKey`. Dial timeout
-from `ctx`. The provider calls libvirt through a narrow interface:
-
-```go
-type libvirtAPI interface {
-    DomainLookupByName(name string) (libvirt.Domain, error)
-    DomainDefineXML(xml string) (libvirt.Domain, error)
-    DomainCreate(d libvirt.Domain) error
-    DomainDestroy(d libvirt.Domain) error
-    DomainUndefineFlags(d libvirt.Domain, f libvirt.DomainUndefineFlagsValues) error
-    DomainGetState(d libvirt.Domain, flags uint32) (int32, int32, error)
-    DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, f libvirt.DomainModificationImpact) (string, error)
-    DomainInterfaceAddresses(d libvirt.Domain, src uint32, flags uint32) ([]libvirt.DomainInterface, error)
-    ConnectListAllDomains(need int32, f libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error)
-    StoragePoolLookupByName(name string) (libvirt.StoragePool, error)
-    StorageVolLookupByName(p libvirt.StoragePool, name string) (libvirt.StorageVol, error)
-    StorageVolGetPath(v libvirt.StorageVol) (string, error)
-    StorageVolCreateXML(p libvirt.StoragePool, xml string, f libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error)
-    StorageVolUpload(v libvirt.StorageVol, r io.Reader, off, n uint64, f libvirt.StorageVolUploadFlags) error
-    StorageVolDownload(v libvirt.StorageVol, w io.Writer, off, n uint64, f libvirt.StorageVolDownloadFlags) error
-    StorageVolDelete(v libvirt.StorageVol, f libvirt.StorageVolDeleteFlags) error
-    StoragePoolRefresh(p libvirt.StoragePool, flags uint32) error
-}
-```
-
-(All signatures verified against go-libvirt `main` on 2026-09-04.) A
-`hostDialer func(ctx, KVMHost) (libvirtAPI, io.Closer, error)` is the test seam;
-`newWithDialer(cfg, dialer)` mirrors GCP's `newWithAPI`.
-
-### Image cache (`internal/provider/kvm/image.go`)
-
-`ensureImage(ctx, host, api, img) (basePath string, err error)` makes the base
-image present in the host's pool, once per host per image:
-
-- Volume names: `base-<key>.<format>` for an `images:` key, or
-  `base-<sha256(url)[:16]>.<format>` for a URL given directly in
-  `placement.image`. Marker volume `<base>.sha256` (raw, 64 bytes) holds the
-  digest of the verified upload.
-- **Ready check:** base volume exists *and* marker exists *and* (no `sha256`
-  configured, or marker content == configured `sha256`). Read the marker with
-  `StorageVolDownload`. Ready → return `StorageVolGetPath(base)`.
-- **Not ready:** delete both volumes if present (partial upload from a crash,
-  or a changed checksum), then `GET url` with the ctx, require a
-  `Content-Length` (`ErrPermanent` otherwise), `StorageVolCreateXML` the base
-  with `<capacity unit="bytes">Content-Length</capacity>` and the configured
-  format, `StorageVolUpload(base, io.TeeReader(body, sha256), 0, 0, 0)` (length
-  0 = until EOF, so nothing is buffered in the operator), then compare the
-  digest with the configured one (mismatch → delete base, `ErrPermanent`),
-  `StoragePoolRefresh` so libvirt re-reads the real qcow2 capacity, create the
-  marker volume and upload the digest last.
-- HTTP 4xx → `ErrPermanent`; 5xx, network, or upload transport errors →
-  `ErrTransient` (partial volumes are cleaned on the next attempt).
-- A per-`(host, image)` `sync.Mutex` map serialises concurrent Creates for the
-  same image on the same host; the second caller finds it ready.
-- **Judgment call:** the fetch runs synchronously inside `Create`, so the first
-  Proxy for a given image on a given host holds a reconcile worker for the
-  download (LAN speed, a few hundred MB, once). This keeps `Create` idempotent
-  and needs no background state. Async fetch with a `Provisioning`-style
-  progress status is a possible later improvement, noted in the docs.
-
-### Create (idempotent, fire-and-forget after the image is cached)
-
-1. Resolve host, machine type, pool, and image source (`images:` key or URL;
-   else `ErrPermanent`). `ensureImage` → base path.
-2. Overlay volume `<name>.qcow2`: `StorageVolCreateXML` with
-   `<capacity unit="G">diskSizeGb</capacity>`, `<target><format type="qcow2"/></target>`,
-   `<backingStore><path>base</path><format type="qcow2"/></backingStore>`.
-   `ErrStorageVolExist` → reuse.
-3. Seed volume `<name>-seed.iso`: build ISO in memory (`user-data` = req.CloudInit,
-   `meta-data` = `instance-id: <name>\nlocal-hostname: <name>\n`, `vendor-data` =
-   cfg or default), `StorageVolCreateXML` with exact capacity + `<format type="raw"/>`,
-   then `StorageVolUpload`. Exists → skip upload (content is deterministic per spec hash;
-   spec changes recreate the whole instance anyway).
-4. `DomainLookupByName(name)`; if `ErrNoDomain` → `DomainDefineXML(buildDomainXML(...))`.
-5. `DomainGetState`; if not running → `DomainCreate`. Return providerID.
-
-`buildDomainXML` is a pure function over `encoding/xml` structs: `type="kvm"`,
-q35 machine, `host-passthrough` CPU, vcpus/memory from the profile, virtio disk
-(overlay) + sata cdrom (seed), `interface type="bridge"` with virtio model,
-`org.qemu.guest_agent.0` virtio channel, serial console, virtio rng, no
-graphics, and the `<metadata>` block. Unit-tested field by field like
-`buildInsertRequest`.
-
-### Get
-
-`parseProviderID` → host → `DomainLookupByName` (`ErrNoDomain` → `ErrNotFound`)
-→ `DomainGetState` + `DomainGetMetadata` + `DomainInterfaceAddresses`
-(ARP, then agent; first IPv4 wins; agent errors while the agent isn't up yet
-are swallowed). Mapping:
-
-| libvirt state | IP | Instance.State |
-|---|---|---|
-| running | yes | Running |
-| running | no | Provisioning |
-| shutoff / crashed / paused / pmsuspended | – | Stopped |
-| no domain | – | `ErrNotFound` |
-
-### Delete (idempotent)
-
-Destroy if running (`ErrOperationInvalid` on a stopped domain is fine) →
-`DomainUndefineFlags(managed-save | snapshots-metadata | nvram)` → delete
-`<name>.qcow2` and `<name>-seed.iso` (`ErrNoStorageVol` fine). Volumes are
-deleted by name even when the domain is already gone, so a crash between
-volume creation and define still gets cleaned up on the next reconcile.
-Known gap (documented, not solved now): volumes orphaned with no domain *and*
-no CR are invisible to `ListByTag`.
-
-### Errors (`internal/provider/kvm/errors.go`)
-
-`classify(err)` on `libvirt.Error.Code`: `ErrNoDomain`, `ErrNoStorageVol` →
-`ErrNotFound`; `ErrNoStoragePool`, `ErrXMLError`, `ErrConfigUnsupported`,
-`ErrNoSupport`, `ErrAuthFailed`, `ErrInvalidArg` → `ErrPermanent`; SSH dial /
-host-key / transport errors and everything else → `ErrTransient`. Image
-download: HTTP 4xx, missing Content-Length, checksum mismatch → `ErrPermanent`;
-5xx and network errors → `ErrTransient`. Quota is not detectable from libvirt;
-left as transient (revisit with a capacity check later).
-
-### Wiring & manifests
-
-- `cmd/main.go`: add `"kvm": kvm.New` to the builtins map.
-- `config/manager/manager.yaml`: mount Secret `kvm-ssh` (optional) at
-  `/etc/proxy-operator/kvm` (keys `id_ed25519`, `known_hosts`); document the
-  `kubectl create secret generic kvm-ssh --from-file=...` command.
-- `config/samples/providers-config.yaml`: add the `kvm-home` block with the
-  `images:` map (Debian 13 and Ubuntu 24.04 official cloud-image URLs);
-  `config/samples/proxy_kvm.yaml`: Proxy with `placement: {zone: nas1,
-  machineType: small, image: debian-13}` and the same squid cloud-init as the
-  GCP sample plus `ssh_authorized_keys`.
-- `hack/providers-dev.yaml` stays kubernetes-only (needs no secrets).
-- Deps: `github.com/digitalocean/go-libvirt` (latest pseudo-version),
-  `github.com/kdomanski/iso9660 v0.4.0`, promote `golang.org/x/crypto` to direct.
-
-### Docs
-
-- New `docs/kvm-provider.md`: hypervisor prerequisites (user in `libvirt`
-  group, a bridge to the LAN, a storage pool with enough space), the config
-  reference including `images:`, how the image cache works (`base-*` and
-  `*.sha256` volumes in the pool, when a re-download happens, how to evict
-  with `virsh vol-delete`), the placement table, how to log into a VM
-  (`ssh_authorized_keys` in user-data, `virsh console` on the host), and the
-  known gaps (no quota detection, orphan volumes without a domain,
-  synchronous first-time image fetch).
-- Update the provider tables in `README.md`, `docs/architecture.md` §5,
-  `docs/testing.md` (integration test section).
-
-### Towards the generic runner (not built now, just kept open)
-
-Provider knows nothing about squid. Future hooks: `vendor-data` for injecting an
-agent, `QEMUDomainAgentCommand` for reading exit status/output, and a future
-`Placement`-independent "job" mode in the CRD. None of this changes the KVM
-provider's shape.
-
-## Steps
-
-0. **Save plan + branch.** Copy this file to `docs/plans/<ts>-kvm-provider.md`,
-   commit alone, then `git checkout -b feat/kvm-provider` off `feat/proxy-operator`
-   (the working main for this project).
-1. **Config.** `KVMConfig` & friends, `validate()` case, table tests.
-2. **Pure builders.** `internal/provider/kvm/{domainxml.go,volume.go,seed.go,id.go,metadata.go}`
-   + tests: domain XML fields, volume XML, seed contents (unpack the ISO with
-   the same library's reader and assert the three files + label `cidata`),
-   providerID round-trip, metadata marshal/parse.
-3. **Transport + errors.** `conn.go` (SSH dial, host-key policy, per-host cache,
-   reconnect), `errors.go` + classification table test. SSH dialer tested
-   against an in-process `x/crypto/ssh` server that serves a unix socket
-   forward (cheap, keeps the dial code covered).
-3b. **Image cache.** `image.go` + `image_test.go`: `httptest.Server` serving a
-   small fake image over the `fakeAPI` (which records uploaded bytes);
-   cases: fresh fetch, already ready, marker missing (partial upload → redo),
-   checksum changed → redo, checksum mismatch → Permanent + cleanup, 404 →
-   Permanent, 503 → Transient, two concurrent callers → one download.
-4. **Provider methods.** `kvm.go`: `New`, `newWithDialer`, `Create/Get/Delete/ListByTag`
-   over `libvirtAPI`; `kvm_test.go` with a hand-written `fakeAPI` (domains and
-   volumes in maps, per-call error injection) covering idempotent re-Create
-   after each partial step, state/IP mapping, delete-when-gone, ListByTag
-   across two hosts and with one host down.
-5. **Wiring, manifests, samples, docs, CHANGELOG.**
-6. **Integration test** (`//go:build integration`, `kvm_integration_test.go`):
-   skipped unless `KVM_TEST_HOST`, `KVM_TEST_USER`, `KVM_TEST_SSH_KEY`,
-   `KVM_TEST_BRIDGE`, `KVM_TEST_IMAGE_URL` are set; fetches the image (or finds
-   it cached), creates, waits for Running+IP, deletes, asserts per-VM volumes
-   gone and the base image still cached. Deviation from the testcontainers convention,
-   called out in `docs/testing.md`: libvirt+KVM can't run inside a container
-   without `/dev/kvm`, so this targets a real hypervisor.
-7. **Manual validation** on one real hypervisor, results in
-   `docs/plans-executions/` (like `docs/gcp-vm-validation.md`), then MR via `tea`.
-
-## Verification
-
-- `make test` green (`-race`, envtest suite untouched but must still pass).
-- `go test -short ./internal/provider/...` covers builders, fake-API provider, errors, config.
-- Integration: `KVM_TEST_HOST=nas1.home.lan … go test -tags=integration -run TestKVM ./internal/provider/kvm/`.
-- End to end: deploy with the `kvm-ssh` Secret and `kvm-home` config against a
-  host with no cached image, apply `config/samples/proxy_kvm.yaml`, watch the
-  operator log the image fetch and `virsh vol-list` show `base-debian-13.qcow2`
-  plus its `.sha256` marker, watch `kubectl get px -w` reach `Ready` with a LAN
-  IP, apply a second Proxy and confirm no second download, confirm the VM shows up in `GET /v1/proxies` and works with
-  `hack/` demo egress-IP check, delete the Proxy and confirm `virsh list --all`
-  and `virsh vol-list default` are clean.

docs/testing.md

@@ -92,8 +92,38 @@ compiles only under `-tags=e2e`, manages its own kind cluster
 (`make test-e2e` / `make cleanup-test-e2e`), and has been kept compiling
 (`go vet -tags=e2e ./...` is part of the routine) but is **not part of
 `make test` and was not used for the release verification** — the manual
-kind run below covers strictly more. Treat it as scaffold to grow into if
-CI wants an automated cluster smoke test.
+kind run below covers strictly more.
+
+### The OTel tracing spec (Tempo-gated)
+
+`test/e2e/tracing_test.go` proves the full tracing pipeline against a
+**real Tempo**: deploy in kind with tracing enabled, create two
+kubernetes-provider proxies, delete them, and assert in Tempo that the
+traces exist with the documented span topology (`Reconcile Proxy` →
+`reconcile.managed` / `provider.create` / `status.patch`, and
+`reconcile.delete` / `provider.delete` on the way out).
+
+It **skips unless both env vars are set** (so the rest of the suite runs
+anywhere). Homelab invocation:
+
+```bash
+TEMPO_URL=http://192.168.0.30:3200 \
+OTLP_ENDPOINT=http://192.168.0.30:4318 \
+make test-e2e
+```
+
+Worth knowing:
+
+- Every span of a run carries the resource attribute
+  `test.run.id=e2e-<nanos>` (injected via `OTEL_RESOURCE_ATTRIBUTES`, no
+  code involved); the run ID is printed in the test log, and
+  `{resource.test.run.id="<id>"}` in Grafana shows exactly that run.
+- The spec preflights the OTLP endpoint **from inside the cluster** with a
+  curl pod and fails fast with a clear message if it's unreachable —
+  export failures are otherwise only visible at `-zap-log-level=1`.
+- Proxy CRs are created in `default`, not the operator namespace: the
+  squid pods carry no securityContext and the operator namespace enforces
+  restricted PSS.
 
 ## The kind verification run (the real end-to-end)
 

go.mod

@@ -4,13 +4,17 @@ go 1.26.0
 
 require (
 	cloud.google.com/go/compute v1.65.0
-	github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858
-	github.com/go-logr/logr v1.4.3
-	github.com/kdomanski/iso9660 v0.4.0
+	github.com/go-logr/logr v1.4.4
 	github.com/onsi/ginkgo/v2 v2.27.4
 	github.com/onsi/gomega v1.39.0
 	github.com/prometheus/client_golang v1.23.2
-	golang.org/x/crypto v0.56.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
+	go.opentelemetry.io/otel v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0
+	go.opentelemetry.io/otel/sdk v1.45.0
+	go.opentelemetry.io/otel/trace v1.45.0
 	google.golang.org/api v0.292.0
 	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
 	k8s.io/api v0.36.0
@@ -34,7 +38,7 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/felixge/httpsnoop v1.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -51,7 +55,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect
 	github.com/googleapis/gax-go/v2 v2.23.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -69,31 +73,27 @@ require (
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
-	go.opentelemetry.io/otel v1.44.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.44.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.44.0 // indirect
-	go.opentelemetry.io/otel/trace v1.44.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
+	go.opentelemetry.io/otel/metric v1.45.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.54.0 // indirect
 	golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
-	golang.org/x/mod v0.38.0 // indirect
+	golang.org/x/mod v0.37.0 // indirect
 	golang.org/x/net v0.57.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sync v0.22.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect
 	golang.org/x/term v0.45.0 // indirect
-	golang.org/x/text v0.41.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
-	golang.org/x/tools v0.48.0 // indirect
+	golang.org/x/tools v0.47.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
 	google.golang.org/grpc v1.83.0 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect

go.sum

@@ -28,16 +28,14 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858 h1:8xCFt73OddCD5WXxoYidJBWp3bJMC+CpE7Zwf5tcwk8=
-github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858/go.mod h1:qb0Ofa71d3oXARQf633h2tNaeBxLsVxuDp+jcsVO2+4=
 github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
 github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
 github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
 github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
-github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
-github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
+github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
@@ -49,8 +47,8 @@ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6O
 github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
 github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
-github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
+github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
@@ -88,8 +86,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.19 h1:mMOE7DN2+p76/EdIrm
 github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k=
 github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE=
 github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -98,8 +96,6 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE
 github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg=
-github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -174,24 +170,28 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
-go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
-go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
-go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
-go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
-go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
-go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
-go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
-go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
-go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
-go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
-go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
-go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
+go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
+go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=
+go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
+go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
+go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
+go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
+go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
+go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
+go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
+go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
+go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
+go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -202,12 +202,12 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
 go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
-golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=
 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
-golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
-golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
 golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
 golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
@@ -218,12 +218,12 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
 golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
 golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
-golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
-golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
-golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
-golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
@@ -232,8 +232,8 @@ google.golang.org/api v0.292.0 h1:Ewiwo/GTtiaPZSNAZQUcWLh8AYDEoPmIXyJfeoTSMHU=
 google.golang.org/api v0.292.0/go.mod h1:07kjmMnFGm2RQuCza2EZM/5N68G/fVvFb1xKjWqoFA0=
 google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
 google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
-google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU=
-google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=

internal/controller/proxy_controller.go

@@ -41,6 +41,7 @@ import (
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // HealthSnapshotter provides the current probe verdict for a proxy. The
@@ -96,9 +97,14 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 	}
 	base := p.DeepCopy()
 	defer func() {
+		// Runs inside the root reconcile span (this defer fires before the
+		// tracing.NewReconciler wrapper sees the return), and its error is
+		// folded into err, which that wrapper records.
+		pctx, span := tracing.Start(ctx, "status.patch")
+		defer span.End()
 		// NotFound is expected when this reconcile just removed the last
 		// finalizer and the object is already gone.
-		if perr := r.patchStatusIfChanged(ctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
+		if perr := r.patchStatusIfChanged(pctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
 			err = errors.Join(err, perr)
 		}
 	}()
@@ -114,6 +120,8 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 }
 
 func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.managed")
+	defer span.End()
 	log := logf.FromContext(ctx)
 
 	if controllerutil.AddFinalizer(p, crawlv1alpha1.FinalizerName) {
@@ -224,6 +232,8 @@ func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1
 // exists" — hence: delete, poll to NotFound, only then advance the hash and
 // let the create branch run.
 func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.Proxy, prov provider.Provider, hash string) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.replaceInstance")
+	defer span.End()
 	_, err := prov.Get(ctx, p.Status.ProviderID)
 	if provider.Class(err) == provider.ErrNotFound {
 		// Old instance is gone. The Update inside setSpecHash refreshes p
@@ -252,6 +262,8 @@ func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.
 }
 
 func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
+	ctx, span := tracing.Start(ctx, "reconcile.delete")
+	defer span.End()
 	if !controllerutil.ContainsFinalizer(p, crawlv1alpha1.FinalizerName) {
 		return ctrl.Result{}, nil
 	}
@@ -409,7 +421,10 @@ func (r *ProxyReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	if r.HealthEvents != nil {
 		b = b.WatchesRawSource(source.Channel(r.HealthEvents, &handler.EnqueueRequestForObject{}))
 	}
-	return b.Complete(r)
+	// Root span per reconcile; sub-reconcilers and the status flush hang
+	// their spans off it. Tests calling r.Reconcile directly bypass the
+	// wrapper and see no-op spans — the global tracer is never set there.
+	return b.Complete(tracing.NewReconciler("Proxy", r))
 }
 
 func (r *ProxyReconciler) applyDefaults() {

internal/discovery/handlers.go

@@ -11,6 +11,7 @@ import (
 
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
@@ -82,7 +83,10 @@ func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
 
 	var list crawlv1alpha1.ProxyList
 	if err := s.Reader.List(r.Context(), &list); err != nil {
-		s.log.Error(err, "listing proxies")
+		// The ctx logger is s.log enriched with this request's
+		// traceID/spanID by the tracing middleware (logr sinks can't read
+		// ctx at log time, so per-request values ride on the logger).
+		logf.FromContext(r.Context()).Error(err, "listing proxies")
 		writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
 		return
 	}
@@ -138,7 +142,7 @@ func (s *Server) handleAcquireLease(w http.ResponseWriter, r *http.Request) {
 
 	var list crawlv1alpha1.ProxyList
 	if err := s.Reader.List(r.Context(), &list); err != nil {
-		s.log.Error(err, "listing proxies for lease")
+		logf.FromContext(r.Context()).Error(err, "listing proxies for lease")
 		writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
 		return
 	}
@@ -231,7 +235,7 @@ func (s *Server) handleReportLease(w http.ResponseWriter, r *http.Request) {
 			writeError(w, http.StatusNotFound, "unknown_lease", "no such lease")
 			return
 		}
-		s.log.Error(err, "reporting lease", "leaseID", r.PathValue("id"))
+		logf.FromContext(r.Context()).Error(err, "reporting lease", "leaseID", r.PathValue("id"))
 		writeError(w, http.StatusInternalServerError, "internal", "report failed")
 		return
 	}

internal/discovery/server.go

@@ -20,6 +20,7 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // LeaseStore is what the handlers need from a lease backend. Defined here,
@@ -136,7 +137,9 @@ func (s *Server) Start(ctx context.Context) error {
 }
 
 // handler assembles the mux and the middleware chain, outermost first:
-// recover → request-log → body-size cap → bearer auth.
+// recover → tracing (server span + request logger) → request-log →
+// body-size cap → bearer auth. Everything inside the tracing layer logs via
+// logf.FromContext(r.Context()) and so carries traceID/spanID.
 func (s *Server) handler() http.Handler {
 	mux := http.NewServeMux()
 	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
@@ -152,6 +155,7 @@ func (s *Server) handler() http.Handler {
 	h = s.authMiddleware(h)
 	h = maxBytesMiddleware(h)
 	h = s.logMiddleware(h)
+	h = tracing.HTTPMiddleware("discovery", s.log)(h)
 	h = s.recoverMiddleware(h)
 	return h
 }
@@ -188,7 +192,7 @@ func (s *Server) logMiddleware(next http.Handler) http.Handler {
 		rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
 		start := time.Now()
 		next.ServeHTTP(rec, r)
-		s.log.Info("request",
+		logf.FromContext(r.Context()).Info("request",
 			"method", r.Method, "path", r.URL.Path,
 			"status", rec.status, "duration", time.Since(start).String())
 	})

internal/gc/gc.go

@@ -12,8 +12,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
+	"go.opentelemetry.io/otel/attribute"
+
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
 	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // Sweeper is the manager Runnable running the sweep loop.
@@ -81,6 +84,8 @@ func (s *Sweeper) Start(ctx context.Context) error {
 // that deletion, and GC racing it would double-delete. A UID becomes
 // orphan-eligible only once the object is fully gone.
 func (s *Sweeper) sweep(ctx context.Context) {
+	ctx, span := tracing.Start(ctx, "gc.sweep")
+	defer span.End()
 	log := logf.FromContext(ctx).WithName("orphan-gc")
 
 	var list crawlv1alpha1.ProxyList
@@ -95,6 +100,7 @@ func (s *Sweeper) sweep(ctx context.Context) {
 		live[string(list.Items[i].UID)] = true
 	}
 
+	deleted := 0
 	for name, prov := range s.Providers {
 		instances, err := prov.ListByTag(ctx)
 		if err != nil {
@@ -119,7 +125,13 @@ func (s *Sweeper) sweep(ctx context.Context) {
 			if err := prov.Delete(ctx, inst.ID); err != nil {
 				log.Error(err, "deleting orphaned instance",
 					"provider", name, "providerID", inst.ID, "uid", inst.UID)
+				continue
 			}
+			deleted++
 		}
 	}
+	span.SetAttributes(
+		attribute.Int("gc.providers", len(s.Providers)),
+		attribute.Int("gc.deleted", deleted),
+	)
 }

internal/health/engine.go

@@ -10,6 +10,9 @@ import (
 	"sync"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -18,6 +21,7 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
 )
 
 // Snapshot is the engine's current verdict for one proxy, read by the
@@ -99,6 +103,10 @@ type Engine struct {
 	// transition-only by design, so metrics are where high-frequency
 	// signal (true probe recency, every latency sample) lives.
 	Metrics ProbeMetrics
+	// TraceProbes emits one root span per probe (--trace-health-probes).
+	// Off by default: probes run about once per second per proxy and would
+	// dominate trace volume.
+	TraceProbes bool
 
 	probeFn func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult
 
@@ -155,8 +163,7 @@ func (e *Engine) Start(ctx context.Context) error {
 				case <-ctx.Done():
 					return
 				case job := <-jobs:
-					res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
-					e.record(job, res, time.Now())
+					e.record(job, e.runProbe(ctx, job), time.Now())
 				}
 			}
 		})
@@ -264,6 +271,29 @@ func newState(p *crawlv1alpha1.Proxy, now time.Time, interval time.Duration) *st
 	return st
 }
 
+// runProbe executes one probe, inside its own root span when TraceProbes is
+// on. The span ends before record folds the result (record stays ctx-free),
+// and the probe transport is deliberately uninstrumented — no traceparent
+// must ever leak through a proxy toward external targets.
+func (e *Engine) runProbe(ctx context.Context, job probeJob) probeResult {
+	if !e.TraceProbes {
+		return e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
+	}
+	ctx, span := tracing.Start(ctx, "health.probe",
+		trace.WithSpanKind(trace.SpanKindClient),
+		trace.WithAttributes(attribute.String("proxy", job.key.String())))
+	defer span.End()
+	res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
+	span.SetAttributes(
+		attribute.Bool("probe.ok", res.ok),
+		attribute.Int64("probe.latency_ms", res.latency.Milliseconds()),
+	)
+	if !res.ok {
+		span.SetStatus(codes.Error, res.err.Error())
+	}
+	return res
+}
+
 // record folds one probe result into the proxy's threshold state and emits
 // an event when the result is status-affecting: a first-ever verdict, a
 // threshold-crossing flip, or a material latency change (beyond

internal/provider/config.go

@@ -21,23 +21,6 @@ type ProviderConfig struct {
 	Type       string            `json:"type"`
 	Kubernetes *KubernetesConfig `json:"kubernetes,omitempty"`
 	GCP        *GCPConfig        `json:"gcp,omitempty"`
-	KVM        *KVMConfig        `json:"kvm,omitempty"`
-}
-
-// setBlocks names the type-specific blocks that are non-nil, so validate can
-// reject a block that doesn't match Type without a pairwise check per type.
-func (p ProviderConfig) setBlocks() []string {
-	var out []string
-	if p.Kubernetes != nil {
-		out = append(out, "kubernetes")
-	}
-	if p.GCP != nil {
-		out = append(out, "gcp")
-	}
-	if p.KVM != nil {
-		out = append(out, "kvm")
-	}
-	return out
 }
 
 // KubernetesConfig configures the kubernetes-pod provider, which creates
@@ -98,28 +81,22 @@ func (c *Config) validate() error {
 		}
 		seen[p.Name] = true
 
-		prefix := fmt.Sprintf("providers[%d] %q", i, p.Name)
 		switch p.Type {
-		case "kubernetes", "gcp", "kvm":
-		case "":
-			return fmt.Errorf("%s: type is required", prefix)
-		default:
-			return fmt.Errorf("%s: unknown provider type %q", prefix, p.Type)
-		}
-		for _, block := range p.setBlocks() {
-			if block != p.Type {
-				return fmt.Errorf("%s: type is %s but a %s block is set", prefix, p.Type, block)
+		case "kubernetes":
+			if p.GCP != nil {
+				return fmt.Errorf("providers[%d] %q: type is kubernetes but a gcp block is set", i, p.Name)
 			}
-		}
-		switch p.Type {
 		case "gcp":
+			if p.Kubernetes != nil {
+				return fmt.Errorf("providers[%d] %q: type is gcp but a kubernetes block is set", i, p.Name)
+			}
 			if p.GCP == nil || p.GCP.Project == "" {
-				return fmt.Errorf("%s: gcp.project is required", prefix)
-			}
-		case "kvm":
-			if err := validateKVM(prefix, p.KVM); err != nil {
-				return err
+				return fmt.Errorf("providers[%d] %q: gcp.project is required", i, p.Name)
 			}
+		case "":
+			return fmt.Errorf("providers[%d] %q: type is required", i, p.Name)
+		default:
+			return fmt.Errorf("providers[%d] %q: unknown provider type %q", i, p.Name, p.Type)
 		}
 	}
 	return nil

internal/provider/config_kvm.go

@@ -1,138 +0,0 @@
-package provider
-
-import (
-	"fmt"
-	"net/url"
-	"regexp"
-)
-
-// KVMConfig configures a kvm provider instance: one or more libvirt/KVM
-// hypervisors reached over SSH (the libvirt unix socket is forwarded through
-// the SSH connection, so hosts need no libvirtd TCP/TLS listener).
-// placement.zone on a Proxy selects a host by name; with a single host it may
-// be omitted.
-type KVMConfig struct {
-	// SSHKeyFile is the path to the private key used for every host. Required.
-	SSHKeyFile string `json:"sshKeyFile"`
-	// KnownHostsFile is an OpenSSH known_hosts file used to verify host keys.
-	// Required unless InsecureIgnoreHostKey is set.
-	KnownHostsFile string `json:"knownHostsFile,omitempty"`
-	// InsecureIgnoreHostKey skips host-key verification entirely.
-	InsecureIgnoreHostKey bool `json:"insecureIgnoreHostKey,omitempty"`
-	// DiskSizeGB is the capacity of each VM's overlay disk. Default 10. The
-	// guest's cloud-init growpart expands the root filesystem into it.
-	DiskSizeGB int64 `json:"diskSizeGb,omitempty"`
-	// VendorData is cloud-init vendor-data placed on every seed next to the
-	// Proxy's own user-data. Default: installs and starts qemu-guest-agent so
-	// the provider can read the VM's IP even before it shows up in ARP.
-	VendorData string `json:"vendorData,omitempty"`
-	// Images are named base images; placement.image names a key here (or is
-	// an http(s) URL directly). The provider downloads each image into a
-	// host's storage pool on first use and caches it there.
-	Images map[string]KVMImage `json:"images,omitempty"`
-	// MachineTypes are named size profiles; placement.machineType names a
-	// key here. Required, at least one.
-	MachineTypes map[string]KVMMachineType `json:"machineTypes"`
-	// Hosts are the hypervisors. Required, at least one, unique names.
-	Hosts []KVMHost `json:"hosts"`
-}
-
-// KVMImage is one downloadable base image (a distro cloud image with
-// cloud-init preinstalled, e.g. Debian's generic qcow2).
-type KVMImage struct {
-	// URL is where to fetch the image from. Required, http or https.
-	URL string `json:"url"`
-	// SHA256 is the expected hex digest of the file. Optional; when set it is
-	// verified after download and a later change triggers a re-download.
-	SHA256 string `json:"sha256,omitempty"`
-	// Format is the image's disk format, "qcow2" (default) or "raw".
-	Format string `json:"format,omitempty"`
-}
-
-// KVMMachineType is a VM size profile.
-type KVMMachineType struct {
-	VCPUs     int32 `json:"vcpus"`
-	MemoryMiB int64 `json:"memoryMiB"`
-}
-
-// KVMHost is one hypervisor.
-type KVMHost struct {
-	// Name is the placement.zone value that selects this host. Required.
-	Name string `json:"name"`
-	// Address is the SSH endpoint, host:port. Required.
-	Address string `json:"address"`
-	// User is the SSH user; it must be allowed to use the libvirt socket
-	// (typically a member of the libvirt group). Required.
-	User string `json:"user"`
-	// Socket is the libvirt unix socket path on the host.
-	// Default /var/run/libvirt/libvirt-sock.
-	Socket string `json:"socket,omitempty"`
-	// Pool is the libvirt storage pool for base images, overlay disks and
-	// seed ISOs. Default "default".
-	Pool string `json:"pool,omitempty"`
-	// Bridge is the host bridge the VM's NIC attaches to; it must lead to a
-	// network the cluster can reach and that hands out DHCP leases. Required.
-	Bridge string `json:"bridge"`
-}
-
-var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
-
-func validateKVM(prefix string, c *KVMConfig) error {
-	if c == nil {
-		return fmt.Errorf("%s: kvm block is required", prefix)
-	}
-	if c.SSHKeyFile == "" {
-		return fmt.Errorf("%s: kvm.sshKeyFile is required", prefix)
-	}
-	if c.KnownHostsFile == "" && !c.InsecureIgnoreHostKey {
-		return fmt.Errorf("%s: kvm.knownHostsFile is required unless kvm.insecureIgnoreHostKey is true", prefix)
-	}
-	if c.DiskSizeGB < 0 {
-		return fmt.Errorf("%s: kvm.diskSizeGb must not be negative", prefix)
-	}
-	for name, img := range c.Images {
-		p := fmt.Sprintf("%s: kvm.images[%q]", prefix, name)
-		u, err := url.Parse(img.URL)
-		if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
-			return fmt.Errorf("%s.url must be an http(s) URL (got %q)", p, img.URL)
-		}
-		if img.SHA256 != "" && !sha256Hex.MatchString(img.SHA256) {
-			return fmt.Errorf("%s.sha256 must be 64 hex characters", p)
-		}
-		if img.Format != "" && img.Format != "qcow2" && img.Format != "raw" {
-			return fmt.Errorf("%s.format must be qcow2 or raw (got %q)", p, img.Format)
-		}
-	}
-	if len(c.MachineTypes) == 0 {
-		return fmt.Errorf("%s: kvm.machineTypes must define at least one profile", prefix)
-	}
-	for name, mt := range c.MachineTypes {
-		if mt.VCPUs <= 0 || mt.MemoryMiB <= 0 {
-			return fmt.Errorf("%s: kvm.machineTypes[%q] needs vcpus > 0 and memoryMiB > 0", prefix, name)
-		}
-	}
-	if len(c.Hosts) == 0 {
-		return fmt.Errorf("%s: kvm.hosts must list at least one host", prefix)
-	}
-	seen := make(map[string]bool, len(c.Hosts))
-	for i, h := range c.Hosts {
-		p := fmt.Sprintf("%s: kvm.hosts[%d]", prefix, i)
-		if h.Name == "" {
-			return fmt.Errorf("%s: name is required", p)
-		}
-		if seen[h.Name] {
-			return fmt.Errorf("%s: duplicate host name %q", p, h.Name)
-		}
-		seen[h.Name] = true
-		if h.Address == "" {
-			return fmt.Errorf("%s %q: address is required", p, h.Name)
-		}
-		if h.User == "" {
-			return fmt.Errorf("%s %q: user is required", p, h.Name)
-		}
-		if h.Bridge == "" {
-			return fmt.Errorf("%s %q: bridge is required", p, h.Name)
-		}
-	}
-	return nil
-}

internal/provider/config_kvm_test.go

@@ -1,208 +0,0 @@
-package provider
-
-import (
-	"strings"
-	"testing"
-)
-
-const validKVMYAML = `
-providers:
-  - name: kvm-home
-    type: kvm
-    kvm:
-      sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
-      knownHostsFile: /etc/proxy-operator/kvm/known_hosts
-      diskSizeGb: 20
-      images:
-        debian-13:
-          url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
-          sha256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-        ubuntu:
-          url: http://mirror.local/noble.img
-          format: raw
-      machineTypes:
-        small: {vcpus: 1, memoryMiB: 1024}
-        medium: {vcpus: 2, memoryMiB: 2048}
-      hosts:
-        - name: nas1
-          address: nas1.home.lan:22
-          user: virt
-          bridge: br0
-        - name: lab2
-          address: 10.0.0.2:2222
-          user: root
-          socket: /run/libvirt/virtqemud-sock
-          pool: fast
-          bridge: br1
-`
-
-func TestLoadConfig_kvmValid(t *testing.T) {
-	t.Parallel()
-	cfg, err := LoadConfig([]byte(validKVMYAML))
-	if err != nil {
-		t.Fatalf("LoadConfig() error = %v, want nil", err)
-	}
-	k := cfg.Providers[0].KVM
-	if k == nil {
-		t.Fatal("providers[0].kvm = nil")
-	}
-	if k.SSHKeyFile != "/etc/proxy-operator/kvm/id_ed25519" || k.DiskSizeGB != 20 {
-		t.Errorf("kvm = %+v, want sshKeyFile and diskSizeGb parsed", k)
-	}
-	if got := k.Images["debian-13"].SHA256; !strings.HasPrefix(got, "0123") {
-		t.Errorf("images[debian-13].sha256 = %q", got)
-	}
-	if k.Images["ubuntu"].Format != "raw" {
-		t.Errorf("images[ubuntu].format = %q, want raw", k.Images["ubuntu"].Format)
-	}
-	if mt := k.MachineTypes["medium"]; mt.VCPUs != 2 || mt.MemoryMiB != 2048 {
-		t.Errorf("machineTypes[medium] = %+v, want {2 2048}", mt)
-	}
-	if len(k.Hosts) != 2 || k.Hosts[1].Socket != "/run/libvirt/virtqemud-sock" || k.Hosts[1].Pool != "fast" {
-		t.Errorf("hosts = %+v, want two hosts with lab2's socket and pool parsed", k.Hosts)
-	}
-}
-
-func TestLoadConfig_kvmInsecureHostKeyNeedsNoKnownHosts(t *testing.T) {
-	t.Parallel()
-	yaml := strings.Replace(validKVMYAML,
-		"knownHostsFile: /etc/proxy-operator/kvm/known_hosts",
-		"insecureIgnoreHostKey: true", 1)
-	if _, err := LoadConfig([]byte(yaml)); err != nil {
-		t.Fatalf("LoadConfig() error = %v, want nil", err)
-	}
-}
-
-func TestLoadConfig_kvmInvalid(t *testing.T) {
-	t.Parallel()
-	minimal := func(kvmBody string) string {
-		return "providers:\n  - name: kvm-home\n    type: kvm\n    kvm:\n" + indent(kvmBody, 6)
-	}
-	hostsAndTypes := `
-machineTypes:
-  small: {vcpus: 1, memoryMiB: 1024}
-hosts:
-  - {name: nas1, address: nas1:22, user: virt, bridge: br0}
-`
-	tests := []struct {
-		name       string
-		yaml       string
-		wantErrSub string
-	}{
-		{
-			name:       "missing kvm block",
-			yaml:       "providers:\n  - name: kvm-home\n    type: kvm\n",
-			wantErrSub: "kvm block is required",
-		},
-		{
-			name:       "missing sshKeyFile",
-			yaml:       minimal("knownHostsFile: /kh\n" + hostsAndTypes),
-			wantErrSub: "kvm.sshKeyFile is required",
-		},
-		{
-			name:       "missing knownHostsFile without insecure flag",
-			yaml:       minimal("sshKeyFile: /key\n" + hostsAndTypes),
-			wantErrSub: "kvm.knownHostsFile is required unless",
-		},
-		{
-			name:       "negative disk size",
-			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\ndiskSizeGb: -1\n" + hostsAndTypes),
-			wantErrSub: "diskSizeGb must not be negative",
-		},
-		{
-			name:       "no machine types",
-			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nhosts:\n  - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
-			wantErrSub: "machineTypes must define at least one",
-		},
-		{
-			name:       "machine type with zero memory",
-			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1}\nhosts:\n  - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
-			wantErrSub: `machineTypes["small"] needs vcpus > 0 and memoryMiB > 0`,
-		},
-		{
-			name:       "no hosts",
-			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\n"),
-			wantErrSub: "kvm.hosts must list at least one host",
-		},
-		{
-			name:       "duplicate host name",
-			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, user: virt, bridge: br0}\n  - {name: nas1, address: b:22, user: virt, bridge: br0}\n"),
-			wantErrSub: `duplicate host name "nas1"`,
-		},
-		{
-			name:       "host missing address",
-			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, user: virt, bridge: br0}\n"),
-			wantErrSub: "address is required",
-		},
-		{
-			name:       "host missing user",
-			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, bridge: br0}\n"),
-			wantErrSub: "user is required",
-		},
-		{
-			name:       "host missing bridge",
-			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n  small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n  - {name: nas1, address: a:22, user: virt}\n"),
-			wantErrSub: "bridge is required",
-		},
-		{
-			name:       "image with non-http url",
-			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: file:///tmp/x.qcow2}\n" + hostsAndTypes),
-			wantErrSub: `images["d"].url must be an http(s) URL`,
-		},
-		{
-			name:       "image with bad sha256",
-			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.qcow2, sha256: abc}\n" + hostsAndTypes),
-			wantErrSub: `images["d"].sha256 must be 64 hex characters`,
-		},
-		{
-			name:       "image with unknown format",
-			yaml:       minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n  d: {url: https://x/y.vmdk, format: vmdk}\n" + hostsAndTypes),
-			wantErrSub: `images["d"].format must be qcow2 or raw`,
-		},
-		{
-			name: "kvm type with gcp block",
-			yaml: `
-providers:
-  - name: p1
-    type: kvm
-    gcp:
-      project: my-project`,
-			wantErrSub: "type is kvm but a gcp block is set",
-		},
-		{
-			name: "gcp type with kvm block",
-			yaml: `
-providers:
-  - name: p1
-    type: gcp
-    gcp:
-      project: my-project
-    kvm:
-      sshKeyFile: /key`,
-			wantErrSub: "type is gcp but a kvm block is set",
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			_, err := LoadConfig([]byte(tc.yaml))
-			if err == nil {
-				t.Fatalf("LoadConfig() error = nil, want error containing %q", tc.wantErrSub)
-			}
-			if !strings.Contains(err.Error(), tc.wantErrSub) {
-				t.Errorf("LoadConfig() error = %q, want substring %q", err.Error(), tc.wantErrSub)
-			}
-		})
-	}
-}
-
-func indent(s string, n int) string {
-	pad := strings.Repeat(" ", n)
-	lines := strings.Split(strings.TrimLeft(s, "\n"), "\n")
-	for i, l := range lines {
-		if l != "" {
-			lines[i] = pad + l
-		}
-	}
-	return strings.Join(lines, "\n")
-}

internal/provider/gcp/wirelog.go

@@ -6,6 +6,7 @@ import (
 	"log/slog"
 
 	"github.com/go-logr/logr"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // wireLogMaxFieldBytes is the elision threshold for string fields in wire
@@ -47,6 +48,15 @@ func (h *wireFilterHandler) Handle(ctx context.Context, rec slog.Record) error {
 	if rec.Level <= slog.LevelDebug && rec.Message != "api request" && rec.Message != "api response" {
 		return nil
 	}
+	// The SDK logs with the request ctx, so wire records can carry the
+	// surrounding provider span — the one place slog's ctx-aware handlers
+	// beat logr, and the same keys the logr enrichment uses.
+	if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
+		rec = rec.Clone()
+		rec.AddAttrs(
+			slog.String("traceID", sc.TraceID().String()),
+			slog.String("spanID", sc.SpanID().String()))
+	}
 	if h.fullPayloads {
 		return h.inner.Handle(ctx, rec)
 	}

internal/provider/gcp/wirelog_test.go

@@ -0,0 +1,50 @@
+package gcp
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/go-logr/logr/funcr"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+func TestWireFilterHandler_addsTraceContext(t *testing.T) {
+	t.Parallel()
+
+	var lines []string
+	base := funcr.New(func(prefix, args string) {
+		lines = append(lines, prefix+" "+args)
+	}, funcr.Options{Verbosity: 5})
+	log := wireLogger(base, WireLogOptions{})
+
+	tp := sdktrace.NewTracerProvider()
+	t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
+	ctx, span := tp.Tracer("test").Start(context.Background(), "provider.create")
+	defer span.End()
+
+	log.DebugContext(ctx, "api request", "url", "https://compute.googleapis.com/x")
+	if len(lines) != 1 {
+		t.Fatalf("got %d lines, want 1: %v", len(lines), lines)
+	}
+	traceID := span.SpanContext().TraceID().String()
+	if !strings.Contains(lines[0], "traceID") || !strings.Contains(lines[0], traceID) {
+		t.Errorf("wire record missing trace context %s: %s", traceID, lines[0])
+	}
+
+	// The security filter must still win: non-wire Debug records are
+	// dropped even when a span is present.
+	log.DebugContext(ctx, "token exchange", "assertion", "secret-jwt")
+	if len(lines) != 1 {
+		t.Fatalf("filtered record leaked: %v", lines[1:])
+	}
+
+	// No span in ctx: record passes through without trace keys.
+	log.DebugContext(context.Background(), "api response", "status", 200)
+	if len(lines) != 2 {
+		t.Fatalf("got %d lines, want 2", len(lines))
+	}
+	if strings.Contains(lines[1], "traceID") {
+		t.Errorf("spanless record must not carry traceID: %s", lines[1])
+	}
+}

internal/provider/kubernetes/kubernetes.go

@@ -14,6 +14,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/transport"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -39,11 +40,22 @@ type Provider struct {
 
 // New builds a kubernetes Provider from its config block. Satisfies
 // registry.Constructor.
-func New(_ context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
+func New(ctx context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
+	return NewWithTransportWrapper(ctx, cfg, nil)
+}
+
+// NewWithTransportWrapper is New with an optional transport wrapper applied
+// to the provider's own rest.Config (this client is built independently of
+// the manager's, so the composition root must wrap it separately for
+// tracing). A nil wrapper means a plain client.
+func NewWithTransportWrapper(_ context.Context, cfg provider.ProviderConfig, wrap transport.WrapperFunc) (provider.Provider, error) {
 	restCfg, err := ctrl.GetConfig()
 	if err != nil {
 		return nil, fmt.Errorf("kubernetes provider %q: loading kubeconfig: %w", cfg.Name, err)
 	}
+	if wrap != nil {
+		restCfg.Wrap(wrap)
+	}
 	scheme := runtime.NewScheme()
 	if err := corev1.AddToScheme(scheme); err != nil {
 		return nil, fmt.Errorf("kubernetes provider %q: %w", cfg.Name, err)

internal/provider/kvm/conn.go

@@ -1,301 +0,0 @@
-package kvm
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net"
-	"os"
-	"sync"
-	"time"
-
-	libvirt "github.com/digitalocean/go-libvirt"
-	"golang.org/x/crypto/ssh"
-	"golang.org/x/crypto/ssh/knownhosts"
-
-	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
-)
-
-const (
-	defaultHandshakeTimeout = 30 * time.Second
-	keepaliveInterval       = 15 * time.Second
-)
-
-// libvirtAPI is the slice of *libvirt.Libvirt the provider uses, and the
-// test seam: the fake in kvm_test.go implements exactly this.
-type libvirtAPI interface {
-	DomainLookupByName(name string) (libvirt.Domain, error)
-	DomainDefineXML(xml string) (libvirt.Domain, error)
-	DomainCreate(d libvirt.Domain) error
-	DomainDestroy(d libvirt.Domain) error
-	DomainUndefineFlags(d libvirt.Domain, flags libvirt.DomainUndefineFlagsValues) error
-	DomainGetState(d libvirt.Domain, flags uint32) (state int32, reason int32, err error)
-	DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, flags libvirt.DomainModificationImpact) (string, error)
-	DomainInterfaceAddresses(d libvirt.Domain, source uint32, flags uint32) ([]libvirt.DomainInterface, error)
-	ConnectListAllDomains(needResults int32, flags libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error)
-	StoragePoolLookupByName(name string) (libvirt.StoragePool, error)
-	StoragePoolRefresh(pool libvirt.StoragePool, flags uint32) error
-	StorageVolLookupByName(pool libvirt.StoragePool, name string) (libvirt.StorageVol, error)
-	StorageVolGetPath(vol libvirt.StorageVol) (string, error)
-	StorageVolCreateXML(pool libvirt.StoragePool, xml string, flags libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error)
-	StorageVolUpload(vol libvirt.StorageVol, r io.Reader, offset, length uint64, flags libvirt.StorageVolUploadFlags) error
-	StorageVolDownload(vol libvirt.StorageVol, w io.Writer, offset, length uint64, flags libvirt.StorageVolDownloadFlags) error
-	StorageVolDelete(vol libvirt.StorageVol, flags libvirt.StorageVolDeleteFlags) error
-}
-
-// hostClient is one live connection to a hypervisor.
-type hostClient struct {
-	api libvirtAPI
-	// done is closed once the connection is known to be lost. A nil channel
-	// never fires, which suits fakes.
-	done  <-chan struct{}
-	close func()
-}
-
-func (c *hostClient) alive() bool {
-	select {
-	case <-c.done:
-		return false
-	default:
-		return true
-	}
-}
-
-// hostDialer opens a connection to one host. The SSH implementation is the
-// real one; tests inject their own.
-type hostDialer func(ctx context.Context, host provider.KVMHost) (*hostClient, error)
-
-// hostPool caches one connection per host, dialing lazily and redialing
-// after the connection drops. Dials for different hosts don't block each
-// other; concurrent callers for the same host share a single dial.
-type hostPool struct {
-	dial hostDialer
-
-	mu      sync.Mutex
-	entries map[string]*hostEntry
-}
-
-type hostEntry struct {
-	mu     sync.Mutex
-	client *hostClient
-}
-
-func newHostPool(dial hostDialer) *hostPool {
-	return &hostPool{dial: dial, entries: make(map[string]*hostEntry)}
-}
-
-func (p *hostPool) entry(name string) *hostEntry {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	e, ok := p.entries[name]
-	if !ok {
-		e = &hostEntry{}
-		p.entries[name] = e
-	}
-	return e
-}
-
-func (p *hostPool) get(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
-	e := p.entry(host.Name)
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	if e.client != nil {
-		if e.client.alive() {
-			return e.client, nil
-		}
-		e.client.close()
-		e.client = nil
-	}
-	c, err := p.dial(ctx, host)
-	if err != nil {
-		return nil, fmt.Errorf("connecting to host %s (%s): %w", host.Name, host.Address, err)
-	}
-	e.client = c
-	return c, nil
-}
-
-// drop closes and forgets c if it is still the cached client for host, so a
-// connection that was already replaced by a redial is left alone.
-func (p *hostPool) drop(host string, c *hostClient) {
-	e := p.entry(host)
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	if e.client == c {
-		e.client = nil
-	}
-	c.close()
-}
-
-// call runs fn against host's connection. go-libvirt RPCs take no context
-// and block until the daemon answers, so when ctx ends first the connection
-// is closed — which unblocks fn with an error — and ctx.Err() is returned.
-// A transport error (anything that is not a libvirt.Error) also drops the
-// cached connection so the next call redials instead of reusing a dead
-// link.
-func (p *hostPool) call(ctx context.Context, host provider.KVMHost, fn func(api libvirtAPI) error) error {
-	c, err := p.get(ctx, host)
-	if err != nil {
-		return err
-	}
-	err = awaitOrCancel(ctx, func() error { return fn(c.api) }, func() { p.drop(host.Name, c) })
-	if err != nil && !isLibvirtError(err) {
-		p.drop(host.Name, c)
-	}
-	return err
-}
-
-// awaitOrCancel runs fn in the background and returns its result, unless ctx
-// ends first: then abort is invoked (it must make fn return eventually, e.g.
-// by closing the connection fn is blocked on) and ctx.Err() is returned. The
-// abandoned fn's result is discarded.
-func awaitOrCancel(ctx context.Context, fn func() error, abort func()) error {
-	errc := make(chan error, 1)
-	go func() { errc <- fn() }()
-	select {
-	case err := <-errc:
-		return err
-	case <-ctx.Done():
-		abort()
-		return ctx.Err()
-	}
-}
-
-// sshAuth holds the pieces of the SSH client config shared by every host:
-// the private key and the host-key policy, both loaded once at startup so a
-// missing or unreadable file fails the operator fast rather than on the
-// first Create.
-type sshAuth struct {
-	signer  ssh.Signer
-	hostKey ssh.HostKeyCallback
-}
-
-func newSSHAuth(cfg provider.KVMConfig) (*sshAuth, error) {
-	key, err := os.ReadFile(cfg.SSHKeyFile)
-	if err != nil {
-		return nil, fmt.Errorf("reading kvm.sshKeyFile: %w", err)
-	}
-	signer, err := ssh.ParsePrivateKey(key)
-	if err != nil {
-		return nil, fmt.Errorf("parsing kvm.sshKeyFile: %w", err)
-	}
-	hostKey := ssh.InsecureIgnoreHostKey() //nolint:gosec // opt-in via config, documented as insecure
-	if !cfg.InsecureIgnoreHostKey {
-		hostKey, err = knownhosts.New(cfg.KnownHostsFile)
-		if err != nil {
-			return nil, fmt.Errorf("loading kvm.knownHostsFile: %w", err)
-		}
-	}
-	return &sshAuth{signer: signer, hostKey: hostKey}, nil
-}
-
-// dialer returns the production hostDialer: SSH to the host, forward its
-// libvirt unix socket through the connection, and open a libvirt session
-// over that.
-func (a *sshAuth) dialer() hostDialer {
-	return func(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
-		client, err := a.dialSSH(ctx, host)
-		if err != nil {
-			return nil, err
-		}
-		hc, err := connectLibvirt(ctx, client, host.Socket)
-		if err != nil {
-			client.Close()
-			return nil, err
-		}
-		return hc, nil
-	}
-}
-
-// dialSSH establishes the SSH connection. The handshake is bounded by ctx's
-// deadline, or defaultHandshakeTimeout when ctx has none, since a
-// half-open hypervisor must not pin a reconcile worker indefinitely.
-func (a *sshAuth) dialSSH(ctx context.Context, host provider.KVMHost) (*ssh.Client, error) {
-	deadline, ok := ctx.Deadline()
-	if !ok {
-		deadline = time.Now().Add(defaultHandshakeTimeout)
-	}
-	var d net.Dialer
-	nc, err := d.DialContext(ctx, "tcp", host.Address)
-	if err != nil {
-		return nil, fmt.Errorf("ssh dial: %w", err)
-	}
-	_ = nc.SetDeadline(deadline)
-	conn, chans, reqs, err := ssh.NewClientConn(nc, host.Address, &ssh.ClientConfig{
-		User:            host.User,
-		Auth:            []ssh.AuthMethod{ssh.PublicKeys(a.signer)},
-		HostKeyCallback: a.hostKey,
-	})
-	if err != nil {
-		nc.Close()
-		return nil, fmt.Errorf("ssh handshake: %w", err)
-	}
-	_ = nc.SetDeadline(time.Time{})
-	return ssh.NewClient(conn, chans, reqs), nil
-}
-
-// unixForward adapts an SSH client to go-libvirt's socket.Dialer: each Dial
-// opens a direct-streamlocal channel to the libvirt socket on the host.
-type unixForward struct {
-	client *ssh.Client
-	path   string
-}
-
-func (u unixForward) Dial() (net.Conn, error) {
-	return u.client.Dial("unix", u.path)
-}
-
-// connectLibvirt opens the libvirt session over the SSH client and starts a
-// keepalive loop so a dead link is noticed within roughly two intervals
-// rather than at the TCP stack's leisure. Closing the returned client tears
-// down the SSH connection, which in turn ends the libvirt session.
-func connectLibvirt(ctx context.Context, client *ssh.Client, socket string) (*hostClient, error) {
-	lv := libvirt.NewWithDialer(unixForward{client: client, path: socket})
-	if err := awaitOrCancel(ctx, lv.Connect, func() { client.Close() }); err != nil {
-		return nil, fmt.Errorf("libvirt connect via %s: %w", socket, err)
-	}
-	stop := make(chan struct{})
-	var once sync.Once
-	go keepalive(client, keepaliveInterval, stop)
-	return &hostClient{
-		api:  lv,
-		done: lv.Disconnected(),
-		close: func() {
-			once.Do(func() {
-				close(stop)
-				client.Close()
-			})
-		},
-	}, nil
-}
-
-// keepalive sends OpenSSH keepalive requests until stop closes; a failed or
-// unanswered request closes the client, which surfaces as Disconnected on
-// the libvirt side and makes the pool redial.
-func keepalive(client *ssh.Client, interval time.Duration, stop <-chan struct{}) {
-	t := time.NewTicker(interval)
-	defer t.Stop()
-	for {
-		select {
-		case <-stop:
-			return
-		case <-t.C:
-		}
-		replied := make(chan error, 1)
-		go func() {
-			_, _, err := client.SendRequest("keepalive@openssh.com", true, nil)
-			replied <- err
-		}()
-		select {
-		case <-stop:
-			return
-		case err := <-replied:
-			if err != nil {
-				client.Close()
-				return
-			}
-		case <-time.After(interval):
-			client.Close()
-			return
-		}
-	}
-}

internal/provider/kvm/conn_test.go

@@ -1,569 +0,0 @@
-package kvm
-
-import (
-	"bytes"
-	"context"
-	"crypto/ed25519"
-	"crypto/rand"
-	"encoding/pem"
-	"errors"
-	"io"
-	"net"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	libvirt "github.com/digitalocean/go-libvirt"
-	"golang.org/x/crypto/ssh"
-	"golang.org/x/crypto/ssh/knownhosts"
-
-	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
-)
-
-// --- hostPool over a fake dialer ---
-
-type fakeClient struct {
-	done   chan struct{}
-	closed atomic.Bool
-}
-
-func (c *fakeClient) close() {
-	if c.closed.CompareAndSwap(false, true) {
-		close(c.done)
-	}
-}
-
-type fakeDialer struct {
-	mu      sync.Mutex
-	dials   int
-	err     error
-	delay   time.Duration
-	clients []*fakeClient
-}
-
-func (d *fakeDialer) dial(ctx context.Context, _ provider.KVMHost) (*hostClient, error) {
-	if d.delay > 0 {
-		select {
-		case <-time.After(d.delay):
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		}
-	}
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	d.dials++
-	if d.err != nil {
-		return nil, d.err
-	}
-	fc := &fakeClient{done: make(chan struct{})}
-	d.clients = append(d.clients, fc)
-	return &hostClient{api: nil, done: fc.done, close: fc.close}, nil
-}
-
-func (d *fakeDialer) dialCount() int {
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	return d.dials
-}
-
-var testHost = provider.KVMHost{Name: "nas1", Address: "nas1:22", User: "virt", Socket: defaultSocket, Pool: "default", Bridge: "br0"}
-
-func noop(libvirtAPI) error { return nil }
-
-func TestHostPool_call_reusesConnection(t *testing.T) {
-	t.Parallel()
-	d := &fakeDialer{}
-	p := newHostPool(d.dial)
-	for range 3 {
-		if err := p.call(context.Background(), testHost, noop); err != nil {
-			t.Fatalf("call() error = %v", err)
-		}
-	}
-	if d.dialCount() != 1 {
-		t.Errorf("dials = %d, want 1", d.dialCount())
-	}
-}
-
-func TestHostPool_call_redialsAfterDisconnect(t *testing.T) {
-	t.Parallel()
-	d := &fakeDialer{}
-	p := newHostPool(d.dial)
-	if err := p.call(context.Background(), testHost, noop); err != nil {
-		t.Fatalf("call() error = %v", err)
-	}
-	d.clients[0].close()
-	if err := p.call(context.Background(), testHost, noop); err != nil {
-		t.Fatalf("second call() error = %v", err)
-	}
-	if d.dialCount() != 2 {
-		t.Errorf("dials = %d, want 2 after the first connection dropped", d.dialCount())
-	}
-}
-
-func TestHostPool_call_transportErrorDropsConnection(t *testing.T) {
-	t.Parallel()
-	d := &fakeDialer{}
-	p := newHostPool(d.dial)
-	err := p.call(context.Background(), testHost, func(libvirtAPI) error { return io.ErrUnexpectedEOF })
-	if !errors.Is(err, io.ErrUnexpectedEOF) {
-		t.Fatalf("call() error = %v, want the transport error passed through", err)
-	}
-	if !d.clients[0].closed.Load() {
-		t.Error("connection not closed after a transport error")
-	}
-	if err := p.call(context.Background(), testHost, noop); err != nil {
-		t.Fatalf("call() after transport error = %v", err)
-	}
-	if d.dialCount() != 2 {
-		t.Errorf("dials = %d, want 2", d.dialCount())
-	}
-}
-
-func TestHostPool_call_libvirtErrorKeepsConnection(t *testing.T) {
-	t.Parallel()
-	d := &fakeDialer{}
-	p := newHostPool(d.dial)
-	err := p.call(context.Background(), testHost, func(libvirtAPI) error { return lvErr(libvirt.ErrNoDomain) })
-	if !hasCode(err, libvirt.ErrNoDomain) {
-		t.Fatalf("call() error = %v, want the libvirt error passed through", err)
-	}
-	if err := p.call(context.Background(), testHost, noop); err != nil {
-		t.Fatalf("call() error = %v", err)
-	}
-	if d.dialCount() != 1 {
-		t.Errorf("dials = %d, want 1: a daemon-reported error must not cost the connection", d.dialCount())
-	}
-	if d.clients[0].closed.Load() {
-		t.Error("connection closed after a libvirt error")
-	}
-}
-
-func TestHostPool_call_dialErrorPropagates(t *testing.T) {
-	t.Parallel()
-	d := &fakeDialer{err: errors.New("ssh: handshake failed")}
-	p := newHostPool(d.dial)
-	err := p.call(context.Background(), testHost, noop)
-	if err == nil || !strings.Contains(err.Error(), "nas1") || !strings.Contains(err.Error(), "handshake failed") {
-		t.Errorf("call() error = %v, want the dial error naming the host", err)
-	}
-}
-
-func TestHostPool_call_cancelAbandonsBlockedCall(t *testing.T) {
-	t.Parallel()
-	d := &fakeDialer{}
-	p := newHostPool(d.dial)
-	ctx, cancel := context.WithCancel(context.Background())
-	released := make(chan struct{})
-	go func() {
-		time.Sleep(20 * time.Millisecond)
-		cancel()
-	}()
-	err := p.call(ctx, testHost, func(libvirtAPI) error {
-		// Simulates an RPC stuck on a hung hypervisor: it only returns once
-		// the pool closes the connection out from under it.
-		<-d.clients[0].done
-		close(released)
-		return io.EOF
-	})
-	if !errors.Is(err, context.Canceled) {
-		t.Fatalf("call() error = %v, want context.Canceled", err)
-	}
-	select {
-	case <-released:
-	case <-time.After(time.Second):
-		t.Fatal("blocked call was not released by closing the connection")
-	}
-	if err := p.call(context.Background(), testHost, noop); err != nil {
-		t.Fatalf("call() after cancel = %v", err)
-	}
-	if d.dialCount() != 2 {
-		t.Errorf("dials = %d, want 2", d.dialCount())
-	}
-}
-
-func TestHostPool_call_concurrentCallersShareOneDial(t *testing.T) {
-	t.Parallel()
-	d := &fakeDialer{delay: 30 * time.Millisecond}
-	p := newHostPool(d.dial)
-	var wg sync.WaitGroup
-	for range 8 {
-		wg.Go(func() {
-			if err := p.call(context.Background(), testHost, noop); err != nil {
-				t.Errorf("call() error = %v", err)
-			}
-		})
-	}
-	wg.Wait()
-	if d.dialCount() != 1 {
-		t.Errorf("dials = %d, want 1", d.dialCount())
-	}
-}
-
-func TestHostPool_call_hostsAreIndependent(t *testing.T) {
-	t.Parallel()
-	d := &fakeDialer{}
-	p := newHostPool(d.dial)
-	other := testHost
-	other.Name = "lab2"
-	for _, h := range []provider.KVMHost{testHost, other, testHost, other} {
-		if err := p.call(context.Background(), h, noop); err != nil {
-			t.Fatalf("call(%s) error = %v", h.Name, err)
-		}
-	}
-	if d.dialCount() != 2 {
-		t.Errorf("dials = %d, want one per host", d.dialCount())
-	}
-}
-
-// --- SSH dial against an in-process server ---
-
-func newKeyPair(t *testing.T) (ssh.Signer, []byte) {
-	t.Helper()
-	_, priv, err := ed25519.GenerateKey(rand.Reader)
-	if err != nil {
-		t.Fatal(err)
-	}
-	signer, err := ssh.NewSignerFromKey(priv)
-	if err != nil {
-		t.Fatal(err)
-	}
-	block, err := ssh.MarshalPrivateKey(priv, "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	return signer, pem.EncodeToMemory(block)
-}
-
-// testSSHServer accepts one public key and serves
-// direct-streamlocal@openssh.com channels by dialing the requested unix
-// path, which is exactly what OpenSSH does for ssh.Client.Dial("unix").
-type testSSHServer struct {
-	addr    string
-	hostKey ssh.PublicKey
-
-	mu    sync.Mutex
-	paths []string
-}
-
-func startSSHServer(t *testing.T, clientKey ssh.PublicKey, answerGlobalRequests bool) *testSSHServer {
-	t.Helper()
-	hostSigner, _ := newKeyPair(t)
-	cfg := &ssh.ServerConfig{
-		PublicKeyCallback: func(_ ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
-			if bytes.Equal(key.Marshal(), clientKey.Marshal()) {
-				return nil, nil
-			}
-			return nil, errors.New("unknown public key")
-		},
-	}
-	cfg.AddHostKey(hostSigner)
-	ln, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Cleanup(func() { ln.Close() })
-	s := &testSSHServer{addr: ln.Addr().String(), hostKey: hostSigner.PublicKey()}
-	go func() {
-		for {
-			nc, err := ln.Accept()
-			if err != nil {
-				return
-			}
-			go s.serve(nc, cfg, answerGlobalRequests)
-		}
-	}()
-	return s
-}
-
-func (s *testSSHServer) serve(nc net.Conn, cfg *ssh.ServerConfig, answerGlobalRequests bool) {
-	_, chans, reqs, err := ssh.NewServerConn(nc, cfg)
-	if err != nil {
-		return
-	}
-	if answerGlobalRequests {
-		go ssh.DiscardRequests(reqs)
-	} else {
-		go func() {
-			for range reqs { //nolint:revive // deliberately never replies, to stall keepalives
-			}
-		}()
-	}
-	for newCh := range chans {
-		if newCh.ChannelType() != "direct-streamlocal@openssh.com" {
-			_ = newCh.Reject(ssh.UnknownChannelType, "test server only forwards unix sockets")
-			continue
-		}
-		var payload struct {
-			Path      string
-			Reserved0 string
-			Reserved1 uint32
-		}
-		if err := ssh.Unmarshal(newCh.ExtraData(), &payload); err != nil {
-			_ = newCh.Reject(ssh.ConnectionFailed, err.Error())
-			continue
-		}
-		s.mu.Lock()
-		s.paths = append(s.paths, payload.Path)
-		s.mu.Unlock()
-		target, err := net.Dial("unix", payload.Path)
-		if err != nil {
-			_ = newCh.Reject(ssh.ConnectionFailed, err.Error())
-			continue
-		}
-		ch, chReqs, err := newCh.Accept()
-		if err != nil {
-			target.Close()
-			continue
-		}
-		go ssh.DiscardRequests(chReqs)
-		go func() { _, _ = io.Copy(ch, target); ch.Close() }()
-		go func() { _, _ = io.Copy(target, ch); target.Close() }()
-	}
-}
-
-func (s *testSSHServer) requestedPaths() []string {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return append([]string(nil), s.paths...)
-}
-
-// startEchoSocket serves a unix socket that echoes everything back, standing
-// in for libvirt-sock.
-func startEchoSocket(t *testing.T) string {
-	t.Helper()
-	path := filepath.Join(t.TempDir(), "libvirt-sock")
-	ln, err := net.Listen("unix", path)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Cleanup(func() { ln.Close() })
-	go func() {
-		for {
-			c, err := ln.Accept()
-			if err != nil {
-				return
-			}
-			go func() { _, _ = io.Copy(c, c); c.Close() }()
-		}
-	}()
-	return path
-}
-
-type sshFixture struct {
-	server  *testSSHServer
-	cfg     provider.KVMConfig
-	host    provider.KVMHost
-	keyFile string
-}
-
-func newSSHFixture(t *testing.T, answerGlobalRequests bool) *sshFixture {
-	t.Helper()
-	clientSigner, clientPEM := newKeyPair(t)
-	server := startSSHServer(t, clientSigner.PublicKey(), answerGlobalRequests)
-	dir := t.TempDir()
-	keyFile := filepath.Join(dir, "id_ed25519")
-	if err := os.WriteFile(keyFile, clientPEM, 0o600); err != nil {
-		t.Fatal(err)
-	}
-	khFile := filepath.Join(dir, "known_hosts")
-	line := knownhosts.Line([]string{server.addr}, server.hostKey)
-	if err := os.WriteFile(khFile, []byte(line+"\n"), 0o600); err != nil {
-		t.Fatal(err)
-	}
-	return &sshFixture{
-		server:  server,
-		keyFile: keyFile,
-		cfg:     provider.KVMConfig{SSHKeyFile: keyFile, KnownHostsFile: khFile},
-		host:    provider.KVMHost{Name: "nas1", Address: server.addr, User: "virt", Socket: startEchoSocket(t)},
-	}
-}
-
-func TestSSHAuth_dialSSH_forwardsUnixSocket(t *testing.T) {
-	t.Parallel()
-	f := newSSHFixture(t, true)
-	auth, err := newSSHAuth(f.cfg)
-	if err != nil {
-		t.Fatalf("newSSHAuth() error = %v", err)
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	client, err := auth.dialSSH(ctx, f.host)
-	if err != nil {
-		t.Fatalf("dialSSH() error = %v", err)
-	}
-	defer client.Close()
-
-	conn, err := unixForward{client: client, path: f.host.Socket}.Dial()
-	if err != nil {
-		t.Fatalf("unixForward.Dial() error = %v", err)
-	}
-	defer conn.Close()
-	if _, err := conn.Write([]byte("ping")); err != nil {
-		t.Fatal(err)
-	}
-	buf := make([]byte, 4)
-	if _, err := io.ReadFull(conn, buf); err != nil || string(buf) != "ping" {
-		t.Fatalf("echo through the forward = %q, %v; want ping", buf, err)
-	}
-	if paths := f.server.requestedPaths(); len(paths) != 1 || paths[0] != f.host.Socket {
-		t.Errorf("server saw forward requests %v, want [%s]", paths, f.host.Socket)
-	}
-}
-
-func TestSSHAuth_dialSSH_rejectsUnknownHostKey(t *testing.T) {
-	t.Parallel()
-	f := newSSHFixture(t, true)
-	impostor, _ := newKeyPair(t)
-	line := knownhosts.Line([]string{f.server.addr}, impostor.PublicKey())
-	if err := os.WriteFile(f.cfg.KnownHostsFile, []byte(line+"\n"), 0o600); err != nil {
-		t.Fatal(err)
-	}
-	auth, err := newSSHAuth(f.cfg)
-	if err != nil {
-		t.Fatalf("newSSHAuth() error = %v", err)
-	}
-	client, err := auth.dialSSH(context.Background(), f.host)
-	if err == nil {
-		client.Close()
-		t.Fatal("dialSSH() succeeded against a host whose key is not in known_hosts")
-	}
-	var keyErr *knownhosts.KeyError
-	if !errors.As(err, &keyErr) {
-		t.Errorf("dialSSH() error = %v, want a knownhosts.KeyError", err)
-	}
-}
-
-func TestSSHAuth_dialSSH_insecureIgnoreHostKey(t *testing.T) {
-	t.Parallel()
-	f := newSSHFixture(t, true)
-	f.cfg.KnownHostsFile = ""
-	f.cfg.InsecureIgnoreHostKey = true
-	auth, err := newSSHAuth(f.cfg)
-	if err != nil {
-		t.Fatalf("newSSHAuth() error = %v", err)
-	}
-	client, err := auth.dialSSH(context.Background(), f.host)
-	if err != nil {
-		t.Fatalf("dialSSH() error = %v", err)
-	}
-	client.Close()
-}
-
-func TestSSHAuth_dialSSH_rejectsWrongClientKey(t *testing.T) {
-	t.Parallel()
-	f := newSSHFixture(t, true)
-	_, otherPEM := newKeyPair(t)
-	if err := os.WriteFile(f.keyFile, otherPEM, 0o600); err != nil {
-		t.Fatal(err)
-	}
-	auth, err := newSSHAuth(f.cfg)
-	if err != nil {
-		t.Fatalf("newSSHAuth() error = %v", err)
-	}
-	client, err := auth.dialSSH(context.Background(), f.host)
-	if err == nil {
-		client.Close()
-		t.Fatal("dialSSH() succeeded with a key the server does not know")
-	}
-	if !strings.Contains(err.Error(), "ssh handshake") {
-		t.Errorf("dialSSH() error = %v, want an auth failure from the handshake", err)
-	}
-}
-
-func TestSSHAuth_dialSSH_cancelledContext(t *testing.T) {
-	t.Parallel()
-	f := newSSHFixture(t, true)
-	auth, err := newSSHAuth(f.cfg)
-	if err != nil {
-		t.Fatalf("newSSHAuth() error = %v", err)
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	if client, err := auth.dialSSH(ctx, f.host); err == nil {
-		client.Close()
-		t.Fatal("dialSSH() succeeded with an already-cancelled context")
-	}
-}
-
-func TestNewSSHAuth_missingFiles(t *testing.T) {
-	t.Parallel()
-	_, keyPEM := newKeyPair(t)
-	keyFile := filepath.Join(t.TempDir(), "key")
-	if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil {
-		t.Fatal(err)
-	}
-	tests := []struct {
-		name    string
-		cfg     provider.KVMConfig
-		wantSub string
-	}{
-		{"key file missing", provider.KVMConfig{SSHKeyFile: "/nonexistent/key", InsecureIgnoreHostKey: true}, "sshKeyFile"},
-		{"key file garbage", provider.KVMConfig{SSHKeyFile: keyFile + ".bad", InsecureIgnoreHostKey: true}, "sshKeyFile"},
-		{"known_hosts missing", provider.KVMConfig{SSHKeyFile: keyFile, KnownHostsFile: "/nonexistent/kh"}, "knownHostsFile"},
-	}
-	if err := os.WriteFile(keyFile+".bad", []byte("not a key"), 0o600); err != nil {
-		t.Fatal(err)
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			_, err := newSSHAuth(tc.cfg)
-			if err == nil || !strings.Contains(err.Error(), tc.wantSub) {
-				t.Errorf("newSSHAuth() error = %v, want one mentioning %s", err, tc.wantSub)
-			}
-		})
-	}
-}
-
-func TestKeepalive_closesClientWhenServerStalls(t *testing.T) {
-	t.Parallel()
-	f := newSSHFixture(t, false)
-	auth, err := newSSHAuth(f.cfg)
-	if err != nil {
-		t.Fatalf("newSSHAuth() error = %v", err)
-	}
-	client, err := auth.dialSSH(context.Background(), f.host)
-	if err != nil {
-		t.Fatalf("dialSSH() error = %v", err)
-	}
-	defer client.Close()
-	stop := make(chan struct{})
-	defer close(stop)
-	go keepalive(client, 20*time.Millisecond, stop)
-
-	closed := make(chan struct{})
-	go func() { _ = client.Wait(); close(closed) }()
-	select {
-	case <-closed:
-	case <-time.After(2 * time.Second):
-		t.Fatal("client still open after keepalives went unanswered")
-	}
-}
-
-func TestKeepalive_keepsHealthyClientOpen(t *testing.T) {
-	t.Parallel()
-	f := newSSHFixture(t, true)
-	auth, err := newSSHAuth(f.cfg)
-	if err != nil {
-		t.Fatalf("newSSHAuth() error = %v", err)
-	}
-	client, err := auth.dialSSH(context.Background(), f.host)
-	if err != nil {
-		t.Fatalf("dialSSH() error = %v", err)
-	}
-	defer client.Close()
-	stop := make(chan struct{})
-	defer close(stop)
-	go keepalive(client, 10*time.Millisecond, stop)
-	time.Sleep(60 * time.Millisecond)
-	fwd := unixForward{client: client, path: f.host.Socket}
-	conn, err := fwd.Dial()
-	if err != nil {
-		t.Fatalf("forward after several keepalives failed: %v", err)
-	}
-	conn.Close()
-}

internal/provider/kvm/domainxml.go

@@ -1,224 +0,0 @@
-package kvm
-
-import (
-	"encoding/xml"
-	"fmt"
-	"time"
-)
-
-const guestAgentChannel = "org.qemu.guest_agent.0"
-
-// domainParams is everything buildDomainXML needs, already resolved from the
-// config and placement so the builder stays a pure function.
-type domainParams struct {
-	Name      string
-	UID       string
-	CreatedAt time.Time
-	VCPUs     int32
-	MemoryMiB int64
-	Pool      string
-	Bridge    string
-	DiskVol   string
-	SeedVol   string
-}
-
-// buildDomainXML renders a q35 KVM guest: host-passthrough CPU, a virtio
-// overlay disk and a SATA cdrom for the seed (both referenced as pool
-// volumes so no host paths are needed), one virtio NIC on the host bridge,
-// a serial console, the guest-agent channel, a virtio RNG, no graphics, and
-// the operator's metadata element.
-func buildDomainXML(p domainParams) (string, error) {
-	d := domainXML{
-		Type:     "kvm",
-		Name:     p.Name,
-		Metadata: domainMetadataXML{Instance: newInstanceMetadata(p.UID, p.CreatedAt)},
-		Memory:   sizeXML{Unit: "MiB", Value: p.MemoryMiB},
-		VCPU:     p.VCPUs,
-		OS: osXML{
-			Type: osTypeXML{Arch: "x86_64", Machine: "q35", Value: "hvm"},
-			Boot: bootXML{Dev: "hd"},
-		},
-		Features:   featuresXML{ACPI: &struct{}{}, APIC: &struct{}{}},
-		CPU:        cpuXML{Mode: "host-passthrough"},
-		Clock:      clockXML{Offset: "utc"},
-		OnPoweroff: "destroy",
-		OnReboot:   "restart",
-		OnCrash:    "destroy",
-		Devices: devicesXML{
-			Disks: []diskXML{
-				{
-					Type:   "volume",
-					Device: "disk",
-					Driver: driverXML{Name: "qemu", Type: formatQcow2},
-					Source: diskSourceXML{Pool: p.Pool, Volume: p.DiskVol},
-					Target: targetXML{Dev: "vda", Bus: "virtio"},
-				},
-				{
-					Type:     "volume",
-					Device:   "cdrom",
-					Driver:   driverXML{Name: "qemu", Type: formatRaw},
-					Source:   diskSourceXML{Pool: p.Pool, Volume: p.SeedVol},
-					Target:   targetXML{Dev: "sda", Bus: "sata"},
-					ReadOnly: &struct{}{},
-				},
-			},
-			Interfaces: []interfaceXML{{
-				Type:   "bridge",
-				Source: ifaceSourceXML{Bridge: p.Bridge},
-				Model:  modelXML{Type: "virtio"},
-			}},
-			Serials:  []serialXML{{Type: "pty", Target: portXML{Port: 0}}},
-			Consoles: []consoleXML{{Type: "pty", Target: consoleTargetXML{Type: "serial", Port: 0}}},
-			Channels: []channelXML{{
-				Type:   "unix",
-				Target: channelTargetXML{Type: "virtio", Name: guestAgentChannel},
-			}},
-			RNG: rngXML{Model: "virtio", Backend: rngBackendXML{Model: "random", Value: "/dev/urandom"}},
-		},
-	}
-	out, err := xml.Marshal(d)
-	if err != nil {
-		return "", fmt.Errorf("marshalling domain xml: %w", err)
-	}
-	return string(out), nil
-}
-
-type domainXML struct {
-	XMLName    xml.Name          `xml:"domain"`
-	Type       string            `xml:"type,attr"`
-	Name       string            `xml:"name"`
-	Metadata   domainMetadataXML `xml:"metadata"`
-	Memory     sizeXML           `xml:"memory"`
-	VCPU       int32             `xml:"vcpu"`
-	OS         osXML             `xml:"os"`
-	Features   featuresXML       `xml:"features"`
-	CPU        cpuXML            `xml:"cpu"`
-	Clock      clockXML          `xml:"clock"`
-	OnPoweroff string            `xml:"on_poweroff"`
-	OnReboot   string            `xml:"on_reboot"`
-	OnCrash    string            `xml:"on_crash"`
-	Devices    devicesXML        `xml:"devices"`
-}
-
-type domainMetadataXML struct {
-	Instance instanceMetadata
-}
-
-type sizeXML struct {
-	Unit  string `xml:"unit,attr"`
-	Value int64  `xml:",chardata"`
-}
-
-type osXML struct {
-	Type osTypeXML `xml:"type"`
-	Boot bootXML   `xml:"boot"`
-}
-
-type osTypeXML struct {
-	Arch    string `xml:"arch,attr"`
-	Machine string `xml:"machine,attr"`
-	Value   string `xml:",chardata"`
-}
-
-type bootXML struct {
-	Dev string `xml:"dev,attr"`
-}
-
-type featuresXML struct {
-	ACPI *struct{} `xml:"acpi"`
-	APIC *struct{} `xml:"apic"`
-}
-
-type cpuXML struct {
-	Mode string `xml:"mode,attr"`
-}
-
-type clockXML struct {
-	Offset string `xml:"offset,attr"`
-}
-
-type devicesXML struct {
-	Disks      []diskXML      `xml:"disk"`
-	Interfaces []interfaceXML `xml:"interface"`
-	Serials    []serialXML    `xml:"serial"`
-	Consoles   []consoleXML   `xml:"console"`
-	Channels   []channelXML   `xml:"channel"`
-	RNG        rngXML         `xml:"rng"`
-}
-
-type diskXML struct {
-	Type     string        `xml:"type,attr"`
-	Device   string        `xml:"device,attr"`
-	Driver   driverXML     `xml:"driver"`
-	Source   diskSourceXML `xml:"source"`
-	Target   targetXML     `xml:"target"`
-	ReadOnly *struct{}     `xml:"readonly"`
-}
-
-type driverXML struct {
-	Name string `xml:"name,attr"`
-	Type string `xml:"type,attr"`
-}
-
-type diskSourceXML struct {
-	Pool   string `xml:"pool,attr"`
-	Volume string `xml:"volume,attr"`
-}
-
-type targetXML struct {
-	Dev string `xml:"dev,attr"`
-	Bus string `xml:"bus,attr"`
-}
-
-type interfaceXML struct {
-	Type   string         `xml:"type,attr"`
-	Source ifaceSourceXML `xml:"source"`
-	Model  modelXML       `xml:"model"`
-}
-
-type ifaceSourceXML struct {
-	Bridge string `xml:"bridge,attr"`
-}
-
-type modelXML struct {
-	Type string `xml:"type,attr"`
-}
-
-type serialXML struct {
-	Type   string  `xml:"type,attr"`
-	Target portXML `xml:"target"`
-}
-
-type portXML struct {
-	Port int `xml:"port,attr"`
-}
-
-type consoleXML struct {
-	Type   string           `xml:"type,attr"`
-	Target consoleTargetXML `xml:"target"`
-}
-
-type consoleTargetXML struct {
-	Type string `xml:"type,attr"`
-	Port int    `xml:"port,attr"`
-}
-
-type channelXML struct {
-	Type   string           `xml:"type,attr"`
-	Target channelTargetXML `xml:"target"`
-}
-
-type channelTargetXML struct {
-	Type string `xml:"type,attr"`
-	Name string `xml:"name,attr"`
-}
-
-type rngXML struct {
-	Model   string        `xml:"model,attr"`
-	Backend rngBackendXML `xml:"backend"`
-}
-
-type rngBackendXML struct {
-	Model string `xml:"model,attr"`
-	Value string `xml:",chardata"`
-}

internal/provider/kvm/domainxml_test.go

@@ -1,113 +0,0 @@
-package kvm
-
-import (
-	"encoding/xml"
-	"strings"
-	"testing"
-	"time"
-)
-
-func testDomainParams() domainParams {
-	return domainParams{
-		Name:      "proxy-abc123def456ghij",
-		UID:       "11111111-2222-3333-4444-555555555555",
-		CreatedAt: time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC),
-		VCPUs:     2,
-		MemoryMiB: 2048,
-		Pool:      "default",
-		Bridge:    "br0",
-		DiskVol:   "proxy-abc123def456ghij.qcow2",
-		SeedVol:   "proxy-abc123def456ghij-seed.iso",
-	}
-}
-
-func TestBuildDomainXML_fieldByField(t *testing.T) {
-	t.Parallel()
-	raw, err := buildDomainXML(testDomainParams())
-	if err != nil {
-		t.Fatalf("buildDomainXML() error = %v", err)
-	}
-	var d domainXML
-	if err := xml.Unmarshal([]byte(raw), &d); err != nil {
-		t.Fatalf("output is not well-formed XML: %v\n%s", err, raw)
-	}
-
-	if d.Type != "kvm" || d.Name != "proxy-abc123def456ghij" {
-		t.Errorf("type/name = %s/%s", d.Type, d.Name)
-	}
-	if d.VCPU != 2 || d.Memory.Value != 2048 || d.Memory.Unit != "MiB" {
-		t.Errorf("vcpu/memory = %d/%d %s, want 2/2048 MiB", d.VCPU, d.Memory.Value, d.Memory.Unit)
-	}
-	if d.OS.Type.Value != "hvm" || d.OS.Type.Arch != "x86_64" || d.OS.Type.Machine != "q35" || d.OS.Boot.Dev != "hd" {
-		t.Errorf("os = %+v", d.OS)
-	}
-	if d.Features.ACPI == nil || d.Features.APIC == nil {
-		t.Errorf("features = %+v, want acpi and apic", d.Features)
-	}
-	if d.CPU.Mode != "host-passthrough" {
-		t.Errorf("cpu mode = %q", d.CPU.Mode)
-	}
-	if d.OnPoweroff != "destroy" || d.OnReboot != "restart" || d.OnCrash != "destroy" {
-		t.Errorf("lifecycle = %s/%s/%s", d.OnPoweroff, d.OnReboot, d.OnCrash)
-	}
-
-	m := d.Metadata.Instance
-	if !m.managed() || m.UID != "11111111-2222-3333-4444-555555555555" || m.CreatedAt != "2026-09-04T12:00:00Z" {
-		t.Errorf("metadata = %+v", m)
-	}
-	if !strings.Contains(raw, `xmlns="`+metadataURI+`"`) {
-		t.Errorf("metadata element is not namespaced; libvirt rejects that:\n%s", raw)
-	}
-
-	if len(d.Devices.Disks) != 2 {
-		t.Fatalf("disks = %d, want 2", len(d.Devices.Disks))
-	}
-	disk, seed := d.Devices.Disks[0], d.Devices.Disks[1]
-	if disk.Type != "volume" || disk.Device != "disk" || disk.Driver.Type != "qcow2" ||
-		disk.Source.Pool != "default" || disk.Source.Volume != "proxy-abc123def456ghij.qcow2" ||
-		disk.Target.Dev != "vda" || disk.Target.Bus != "virtio" || disk.ReadOnly != nil {
-		t.Errorf("overlay disk = %+v", disk)
-	}
-	if seed.Type != "volume" || seed.Device != "cdrom" || seed.Driver.Type != "raw" ||
-		seed.Source.Pool != "default" || seed.Source.Volume != "proxy-abc123def456ghij-seed.iso" ||
-		seed.Target.Bus != "sata" || seed.ReadOnly == nil {
-		t.Errorf("seed cdrom = %+v", seed)
-	}
-
-	if len(d.Devices.Interfaces) != 1 {
-		t.Fatalf("interfaces = %d, want 1", len(d.Devices.Interfaces))
-	}
-	if nic := d.Devices.Interfaces[0]; nic.Type != "bridge" || nic.Source.Bridge != "br0" || nic.Model.Type != "virtio" {
-		t.Errorf("interface = %+v", nic)
-	}
-
-	if len(d.Devices.Channels) != 1 || d.Devices.Channels[0].Target.Name != guestAgentChannel || d.Devices.Channels[0].Target.Type != "virtio" {
-		t.Errorf("channels = %+v, want one virtio %s channel", d.Devices.Channels, guestAgentChannel)
-	}
-	if len(d.Devices.Serials) != 1 || len(d.Devices.Consoles) != 1 || d.Devices.Consoles[0].Target.Type != "serial" {
-		t.Errorf("serial/console = %+v/%+v", d.Devices.Serials, d.Devices.Consoles)
-	}
-	if d.Devices.RNG.Model != "virtio" || d.Devices.RNG.Backend.Value != "/dev/urandom" {
-		t.Errorf("rng = %+v", d.Devices.RNG)
-	}
-	if strings.Contains(raw, "<graphics") {
-		t.Errorf("domain has a graphics device; headless VMs must not:\n%s", raw)
-	}
-}
-
-func TestBuildDomainXML_escapesUserValues(t *testing.T) {
-	t.Parallel()
-	p := testDomainParams()
-	p.Bridge = `br0" onload="x`
-	raw, err := buildDomainXML(p)
-	if err != nil {
-		t.Fatalf("buildDomainXML() error = %v", err)
-	}
-	var d domainXML
-	if err := xml.Unmarshal([]byte(raw), &d); err != nil {
-		t.Fatalf("output is not well-formed XML: %v", err)
-	}
-	if d.Devices.Interfaces[0].Source.Bridge != p.Bridge {
-		t.Errorf("bridge = %q, want the raw value back", d.Devices.Interfaces[0].Source.Bridge)
-	}
-}

internal/provider/kvm/errors.go

@@ -1,72 +0,0 @@
-package kvm
-
-import (
-	"errors"
-
-	libvirt "github.com/digitalocean/go-libvirt"
-	"github.com/go-logr/logr"
-
-	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
-)
-
-// classify maps an error onto the provider taxonomy. Errors the package
-// already tagged with a sentinel (image download failures, placement
-// errors) keep their class. For libvirt errors: missing domain or volume →
-// NotFound; anything the host will keep rejecting (bad XML, unsupported
-// config, denied access, missing pool) → Permanent; every other libvirt
-// code, plus SSH/transport failures and cancellation, → Transient, because
-// retrying is safer than latching Failed on an error nobody taught this
-// function to recognise. libvirt exposes no quota signal, so nothing maps
-// to QuotaExceeded.
-func classify(err error) error {
-	for _, class := range []error{provider.ErrNotFound, provider.ErrQuotaExceeded, provider.ErrPermanent, provider.ErrTransient} {
-		if errors.Is(err, class) {
-			return class
-		}
-	}
-	var lerr libvirt.Error
-	if !errors.As(err, &lerr) {
-		return provider.ErrTransient
-	}
-	switch libvirt.ErrorNumber(lerr.Code) {
-	case libvirt.ErrNoDomain, libvirt.ErrNoStorageVol:
-		return provider.ErrNotFound
-	case libvirt.ErrNoStoragePool,
-		libvirt.ErrXMLError, libvirt.ErrXMLDetail, libvirt.ErrXMLInvalidSchema,
-		libvirt.ErrConfigUnsupported, libvirt.ErrNoSupport,
-		libvirt.ErrOperationUnsupported, libvirt.ErrArgumentUnsupported,
-		libvirt.ErrInvalidArg,
-		libvirt.ErrAuthFailed, libvirt.ErrAccessDenied, libvirt.ErrOperationDenied:
-		return provider.ErrPermanent
-	default:
-		return provider.ErrTransient
-	}
-}
-
-// isLibvirtError reports whether err came back from the libvirt daemon (as
-// opposed to the SSH or RPC transport), which is what decides whether the
-// cached connection is still trustworthy.
-func isLibvirtError(err error) bool {
-	var lerr libvirt.Error
-	return errors.As(err, &lerr)
-}
-
-func hasCode(err error, code libvirt.ErrorNumber) bool {
-	var lerr libvirt.Error
-	return errors.As(err, &lerr) && libvirt.ErrorNumber(lerr.Code) == code
-}
-
-// logAPIError records the raw libvirt code and message at V(1); classify
-// collapses them onto the coarser taxonomy, so this is the only place the
-// original detail survives.
-func logAPIError(log logr.Logger, op string, err error) {
-	if !log.V(1).Enabled() {
-		return
-	}
-	kv := []any{"op", op, "error", err.Error()}
-	var lerr libvirt.Error
-	if errors.As(err, &lerr) {
-		kv = append(kv, "libvirtCode", lerr.Code, "libvirtMessage", lerr.Message)
-	}
-	log.V(1).Info("libvirt call failed", kv...)
-}

internal/provider/kvm/errors_test.go

@@ -1,69 +0,0 @@
-package kvm
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"testing"
-
-	libvirt "github.com/digitalocean/go-libvirt"
-
-	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
-)
-
-func lvErr(code libvirt.ErrorNumber) error {
-	return libvirt.Error{Code: uint32(code), Message: code.String()}
-}
-
-func TestClassify(t *testing.T) {
-	t.Parallel()
-	tests := []struct {
-		name string
-		err  error
-		want error
-	}{
-		{"no domain", lvErr(libvirt.ErrNoDomain), provider.ErrNotFound},
-		{"no storage vol", lvErr(libvirt.ErrNoStorageVol), provider.ErrNotFound},
-		{"no storage pool", lvErr(libvirt.ErrNoStoragePool), provider.ErrPermanent},
-		{"xml error", lvErr(libvirt.ErrXMLError), provider.ErrPermanent},
-		{"xml detail", lvErr(libvirt.ErrXMLDetail), provider.ErrPermanent},
-		{"config unsupported", lvErr(libvirt.ErrConfigUnsupported), provider.ErrPermanent},
-		{"no support", lvErr(libvirt.ErrNoSupport), provider.ErrPermanent},
-		{"invalid arg", lvErr(libvirt.ErrInvalidArg), provider.ErrPermanent},
-		{"auth failed", lvErr(libvirt.ErrAuthFailed), provider.ErrPermanent},
-		{"access denied", lvErr(libvirt.ErrAccessDenied), provider.ErrPermanent},
-		{"operation invalid", lvErr(libvirt.ErrOperationInvalid), provider.ErrTransient},
-		{"operation failed", lvErr(libvirt.ErrOperationFailed), provider.ErrTransient},
-		{"internal error", lvErr(libvirt.ErrInternalError), provider.ErrTransient},
-		{"storage vol exists", lvErr(libvirt.ErrStorageVolExist), provider.ErrTransient},
-		{"wrapped libvirt error", fmt.Errorf("defining: %w", lvErr(libvirt.ErrNoDomain)), provider.ErrNotFound},
-		{"pre-tagged permanent", fmt.Errorf("%w: image returned HTTP 404", provider.ErrPermanent), provider.ErrPermanent},
-		{"pre-tagged transient wrapping libvirt not-found", fmt.Errorf("%w: %w", provider.ErrTransient, lvErr(libvirt.ErrNoDomain)), provider.ErrTransient},
-		{"transport eof", io.EOF, provider.ErrTransient},
-		{"context cancelled", context.Canceled, provider.ErrTransient},
-		{"plain error", errors.New("ssh: handshake failed"), provider.ErrTransient},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			if got := classify(tc.err); !errors.Is(got, tc.want) {
-				t.Errorf("classify(%v) = %v, want %v", tc.err, got, tc.want)
-			}
-		})
-	}
-}
-
-func TestHasCode(t *testing.T) {
-	t.Parallel()
-	err := fmt.Errorf("lookup: %w", lvErr(libvirt.ErrNoDomain))
-	if !hasCode(err, libvirt.ErrNoDomain) {
-		t.Error("hasCode(ErrNoDomain) = false through a wrap")
-	}
-	if hasCode(err, libvirt.ErrNoStorageVol) {
-		t.Error("hasCode(ErrNoStorageVol) = true for an ErrNoDomain error")
-	}
-	if hasCode(io.EOF, libvirt.ErrNoDomain) || isLibvirtError(io.EOF) {
-		t.Error("io.EOF classified as a libvirt error")
-	}
-}

internal/provider/kvm/fake_test.go

@@ -1,408 +0,0 @@
-package kvm
-
-import (
-	"bytes"
-	"encoding/xml"
-	"fmt"
-	"io"
-	"slices"
-	"sync"
-
-	libvirt "github.com/digitalocean/go-libvirt"
-)
-
-// fakeAPI is an in-memory libvirtAPI: pools of volumes and a set of
-// domains, with per-method error injection. It models just enough libvirt
-// semantics for the provider's decisions to be observable — ErrStorageVolExist
-// on duplicate create, ErrNoDomain/ErrNoStorageVol on missing lookups,
-// ErrOperationInvalid on destroying a stopped domain, and so on.
-type fakeAPI struct {
-	mu       sync.Mutex
-	poolPath string
-	pools    map[string]map[string]*fakeVol
-	domains  map[string]*fakeDomain
-	calls    []string
-	fail     map[string]error
-	failOnce map[string]error
-	// uploadHook, when set, sees every StorageVolUpload before the fake
-	// reads the stream; it can return an error to simulate a broken upload.
-	uploadHook func(vol string, r io.Reader) error
-}
-
-type fakeVol struct {
-	name     string
-	format   string
-	capacity int64
-	backing  string
-	data     []byte
-}
-
-type fakeDomain struct {
-	name     string
-	xml      string
-	metadata string
-	running  bool
-	// addrs is what DomainInterfaceAddresses returns per source.
-	addrs map[uint32][]libvirt.DomainInterface
-}
-
-func newFakeAPI(pools ...string) *fakeAPI {
-	f := &fakeAPI{
-		poolPath: "/var/lib/libvirt/images",
-		pools:    map[string]map[string]*fakeVol{},
-		domains:  map[string]*fakeDomain{},
-		fail:     map[string]error{},
-		failOnce: map[string]error{},
-	}
-	for _, p := range pools {
-		f.pools[p] = map[string]*fakeVol{}
-	}
-	return f
-}
-
-// enter records the call and returns any injected error for method.
-func (f *fakeAPI) enter(method string) error {
-	f.calls = append(f.calls, method)
-	if err, ok := f.failOnce[method]; ok {
-		delete(f.failOnce, method)
-		return err
-	}
-	return f.fail[method]
-}
-
-func (f *fakeAPI) callCount(method string) int {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	n := 0
-	for _, c := range f.calls {
-		if c == method {
-			n++
-		}
-	}
-	return n
-}
-
-func (f *fakeAPI) hasCall(method string) bool { return f.callCount(method) > 0 }
-
-func (f *fakeAPI) vol(pool, name string) *fakeVol {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return f.pools[pool][name]
-}
-
-func (f *fakeAPI) putVol(pool, name, format string, data []byte) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.pools[pool][name] = &fakeVol{name: name, format: format, capacity: int64(len(data)), data: data}
-}
-
-func (f *fakeAPI) volNames(pool string) []string {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	var out []string
-	for n := range f.pools[pool] {
-		out = append(out, n)
-	}
-	slices.Sort(out)
-	return out
-}
-
-func (f *fakeAPI) domain(name string) *fakeDomain {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return f.domains[name]
-}
-
-func (f *fakeAPI) putDomain(d *fakeDomain) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.domains[d.name] = d
-}
-
-func lvError(code libvirt.ErrorNumber, format string, args ...any) error {
-	return libvirt.Error{Code: uint32(code), Message: fmt.Sprintf(format, args...)}
-}
-
-// --- domains ---
-
-func (f *fakeAPI) DomainLookupByName(name string) (libvirt.Domain, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("DomainLookupByName"); err != nil {
-		return libvirt.Domain{}, err
-	}
-	if _, ok := f.domains[name]; !ok {
-		return libvirt.Domain{}, lvError(libvirt.ErrNoDomain, "Domain not found: no domain with matching name '%s'", name)
-	}
-	return libvirt.Domain{Name: name}, nil
-}
-
-func (f *fakeAPI) DomainDefineXML(raw string) (libvirt.Domain, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("DomainDefineXML"); err != nil {
-		return libvirt.Domain{}, err
-	}
-	var d domainXML
-	if err := xml.Unmarshal([]byte(raw), &d); err != nil {
-		return libvirt.Domain{}, lvError(libvirt.ErrXMLError, "%v", err)
-	}
-	meta, err := xml.Marshal(d.Metadata.Instance)
-	if err != nil {
-		return libvirt.Domain{}, lvError(libvirt.ErrXMLError, "%v", err)
-	}
-	existing := f.domains[d.Name]
-	nd := &fakeDomain{name: d.Name, xml: raw, metadata: string(meta)}
-	if existing != nil {
-		nd.running = existing.running
-		nd.addrs = existing.addrs
-	}
-	f.domains[d.Name] = nd
-	return libvirt.Domain{Name: d.Name}, nil
-}
-
-func (f *fakeAPI) DomainCreate(d libvirt.Domain) error {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("DomainCreate"); err != nil {
-		return err
-	}
-	dom, ok := f.domains[d.Name]
-	if !ok {
-		return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
-	}
-	if dom.running {
-		return lvError(libvirt.ErrOperationInvalid, "domain is already running")
-	}
-	dom.running = true
-	return nil
-}
-
-func (f *fakeAPI) DomainDestroy(d libvirt.Domain) error {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("DomainDestroy"); err != nil {
-		return err
-	}
-	dom, ok := f.domains[d.Name]
-	if !ok {
-		return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
-	}
-	if !dom.running {
-		return lvError(libvirt.ErrOperationInvalid, "domain is not running")
-	}
-	dom.running = false
-	return nil
-}
-
-func (f *fakeAPI) DomainUndefineFlags(d libvirt.Domain, _ libvirt.DomainUndefineFlagsValues) error {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("DomainUndefineFlags"); err != nil {
-		return err
-	}
-	if _, ok := f.domains[d.Name]; !ok {
-		return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
-	}
-	delete(f.domains, d.Name)
-	return nil
-}
-
-func (f *fakeAPI) DomainGetState(d libvirt.Domain, _ uint32) (int32, int32, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("DomainGetState"); err != nil {
-		return 0, 0, err
-	}
-	dom, ok := f.domains[d.Name]
-	if !ok {
-		return 0, 0, lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
-	}
-	if dom.running {
-		return int32(libvirt.DomainRunning), 0, nil
-	}
-	return int32(libvirt.DomainShutoff), 0, nil
-}
-
-func (f *fakeAPI) DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, _ libvirt.DomainModificationImpact) (string, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("DomainGetMetadata"); err != nil {
-		return "", err
-	}
-	dom, ok := f.domains[d.Name]
-	if !ok {
-		return "", lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
-	}
-	if typ != int32(libvirt.DomainMetadataElement) || len(uri) != 1 || uri[0] != metadataURI || dom.metadata == "" {
-		return "", lvError(libvirt.ErrNoDomainMetadata, "Requested metadata element is not present")
-	}
-	return dom.metadata, nil
-}
-
-func (f *fakeAPI) DomainInterfaceAddresses(d libvirt.Domain, source uint32, _ uint32) ([]libvirt.DomainInterface, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("DomainInterfaceAddresses"); err != nil {
-		return nil, err
-	}
-	dom, ok := f.domains[d.Name]
-	if !ok {
-		return nil, lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
-	}
-	if !dom.running {
-		return nil, lvError(libvirt.ErrOperationInvalid, "domain is not running")
-	}
-	if source == uint32(libvirt.DomainInterfaceAddressesSrcAgent) && dom.addrs[source] == nil {
-		return nil, lvError(libvirt.ErrAgentUnresponsive, "Guest agent is not responding")
-	}
-	return dom.addrs[source], nil
-}
-
-func (f *fakeAPI) ConnectListAllDomains(_ int32, _ libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("ConnectListAllDomains"); err != nil {
-		return nil, 0, err
-	}
-	names := make([]string, 0, len(f.domains))
-	for n := range f.domains {
-		names = append(names, n)
-	}
-	slices.Sort(names)
-	out := make([]libvirt.Domain, 0, len(names))
-	for _, n := range names {
-		out = append(out, libvirt.Domain{Name: n})
-	}
-	return out, uint32(len(out)), nil
-}
-
-// --- storage ---
-
-func (f *fakeAPI) StoragePoolLookupByName(name string) (libvirt.StoragePool, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("StoragePoolLookupByName"); err != nil {
-		return libvirt.StoragePool{}, err
-	}
-	if _, ok := f.pools[name]; !ok {
-		return libvirt.StoragePool{}, lvError(libvirt.ErrNoStoragePool, "Storage pool not found: no storage pool with matching name '%s'", name)
-	}
-	return libvirt.StoragePool{Name: name}, nil
-}
-
-func (f *fakeAPI) StoragePoolRefresh(pool libvirt.StoragePool, _ uint32) error {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("StoragePoolRefresh"); err != nil {
-		return err
-	}
-	if _, ok := f.pools[pool.Name]; !ok {
-		return lvError(libvirt.ErrNoStoragePool, "no pool %s", pool.Name)
-	}
-	return nil
-}
-
-func (f *fakeAPI) StorageVolLookupByName(pool libvirt.StoragePool, name string) (libvirt.StorageVol, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("StorageVolLookupByName"); err != nil {
-		return libvirt.StorageVol{}, err
-	}
-	if _, ok := f.pools[pool.Name][name]; !ok {
-		return libvirt.StorageVol{}, lvError(libvirt.ErrNoStorageVol, "Storage volume not found: no storage vol with matching name '%s'", name)
-	}
-	return libvirt.StorageVol{Pool: pool.Name, Name: name, Key: f.poolPath + "/" + name}, nil
-}
-
-func (f *fakeAPI) StorageVolGetPath(vol libvirt.StorageVol) (string, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("StorageVolGetPath"); err != nil {
-		return "", err
-	}
-	if _, ok := f.pools[vol.Pool][vol.Name]; !ok {
-		return "", lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
-	}
-	return f.poolPath + "/" + vol.Name, nil
-}
-
-func (f *fakeAPI) StorageVolCreateXML(pool libvirt.StoragePool, raw string, _ libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("StorageVolCreateXML"); err != nil {
-		return libvirt.StorageVol{}, err
-	}
-	vols, ok := f.pools[pool.Name]
-	if !ok {
-		return libvirt.StorageVol{}, lvError(libvirt.ErrNoStoragePool, "no pool %s", pool.Name)
-	}
-	var v volumeXML
-	if err := xml.Unmarshal([]byte(raw), &v); err != nil {
-		return libvirt.StorageVol{}, lvError(libvirt.ErrXMLError, "%v", err)
-	}
-	if _, exists := vols[v.Name]; exists {
-		return libvirt.StorageVol{}, lvError(libvirt.ErrStorageVolExist, "storage volume '%s' exists already", v.Name)
-	}
-	fv := &fakeVol{name: v.Name, format: v.Target.Format.Type, capacity: v.Capacity.Value}
-	if v.BackingStore != nil {
-		fv.backing = v.BackingStore.Path
-	}
-	vols[v.Name] = fv
-	return libvirt.StorageVol{Pool: pool.Name, Name: v.Name, Key: f.poolPath + "/" + v.Name}, nil
-}
-
-func (f *fakeAPI) StorageVolUpload(vol libvirt.StorageVol, r io.Reader, _, _ uint64, _ libvirt.StorageVolUploadFlags) error {
-	f.mu.Lock()
-	if err := f.enter("StorageVolUpload"); err != nil {
-		f.mu.Unlock()
-		return err
-	}
-	fv, ok := f.pools[vol.Pool][vol.Name]
-	hook := f.uploadHook
-	f.mu.Unlock()
-	if !ok {
-		return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
-	}
-	if hook != nil {
-		if err := hook(vol.Name, r); err != nil {
-			return err
-		}
-	}
-	// Read outside the lock: the stream may be a slow HTTP body.
-	data, err := io.ReadAll(r)
-	if err != nil {
-		return fmt.Errorf("reading upload stream: %w", err)
-	}
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	fv.data = data
-	return nil
-}
-
-func (f *fakeAPI) StorageVolDownload(vol libvirt.StorageVol, w io.Writer, _, _ uint64, _ libvirt.StorageVolDownloadFlags) error {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("StorageVolDownload"); err != nil {
-		return err
-	}
-	fv, ok := f.pools[vol.Pool][vol.Name]
-	if !ok {
-		return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
-	}
-	_, err := io.Copy(w, bytes.NewReader(fv.data))
-	return err
-}
-
-func (f *fakeAPI) StorageVolDelete(vol libvirt.StorageVol, _ libvirt.StorageVolDeleteFlags) error {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if err := f.enter("StorageVolDelete"); err != nil {
-		return err
-	}
-	if _, ok := f.pools[vol.Pool][vol.Name]; !ok {
-		return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
-	}
-	delete(f.pools[vol.Pool], vol.Name)
-	return nil
-}

internal/provider/kvm/id.go

@@ -1,21 +0,0 @@
-package kvm
-
-import (
-	"fmt"
-	"strings"
-)
-
-// The providerID carries the host, so Get/Delete stay correct after a
-// placement.zone edit mid-replacement — re-reading the spec then would look
-// at the wrong hypervisor.
-func formatProviderID(host, name string) string {
-	return fmt.Sprintf("hosts/%s/domains/%s", host, name)
-}
-
-func parseProviderID(id string) (host, name string, err error) {
-	parts := strings.Split(id, "/")
-	if len(parts) != 4 || parts[0] != "hosts" || parts[2] != "domains" || parts[1] == "" || parts[3] == "" {
-		return "", "", fmt.Errorf("malformed kvm providerID %q, want hosts/<host>/domains/<name>", id)
-	}
-	return parts[1], parts[3], nil
-}

internal/provider/kvm/id_test.go

@@ -1,41 +0,0 @@
-package kvm
-
-import "testing"
-
-func TestParseProviderID_roundTrip(t *testing.T) {
-	t.Parallel()
-	id := formatProviderID("nas1", "proxy-abc123def456ghij")
-	if id != "hosts/nas1/domains/proxy-abc123def456ghij" {
-		t.Fatalf("formatProviderID() = %q", id)
-	}
-	host, name, err := parseProviderID(id)
-	if err != nil {
-		t.Fatalf("parseProviderID(%q) error = %v", id, err)
-	}
-	if host != "nas1" || name != "proxy-abc123def456ghij" {
-		t.Errorf("parseProviderID(%q) = %q, %q", id, host, name)
-	}
-}
-
-func TestParseProviderID_malformed(t *testing.T) {
-	t.Parallel()
-	tests := []struct {
-		name string
-		id   string
-	}{
-		{"empty", ""},
-		{"gcp style", "zones/europe-west1-b/instances/proxy-x"},
-		{"missing name", "hosts/nas1/domains/"},
-		{"missing host", "hosts//domains/proxy-x"},
-		{"too many segments", "hosts/nas1/domains/proxy-x/extra"},
-		{"bare name", "proxy-x"},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			if _, _, err := parseProviderID(tc.id); err == nil {
-				t.Errorf("parseProviderID(%q) error = nil, want error", tc.id)
-			}
-		})
-	}
-}

internal/provider/kvm/image.go

@@ -1,232 +0,0 @@
-package kvm
-
-import (
-	"bytes"
-	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strings"
-	"sync"
-	"time"
-
-	libvirt "github.com/digitalocean/go-libvirt"
-	"github.com/go-logr/logr"
-
-	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
-)
-
-// imageSource is a resolved placement.image: either an entry from the
-// provider's images map (Key set) or a URL given directly.
-type imageSource struct {
-	Key    string
-	URL    string
-	SHA256 string
-	Format string
-}
-
-// resolveImage maps placement.image to a source. Anything that is neither a
-// configured key nor an http(s) URL is a spec error the reconciler should
-// surface, not retry.
-func resolveImage(cfg provider.KVMConfig, image string) (imageSource, error) {
-	if img, ok := cfg.Images[image]; ok {
-		return imageSource{Key: image, URL: img.URL, SHA256: img.SHA256, Format: img.Format}, nil
-	}
-	u, err := url.Parse(image)
-	if err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != "" {
-		return imageSource{URL: image, Format: formatQcow2}, nil
-	}
-	return imageSource{}, fmt.Errorf("%w: placement.image %q is neither a key in kvm.images nor an http(s) URL",
-		provider.ErrPermanent, image)
-}
-
-// volumeName is the cached base image's name in the pool: readable for
-// configured keys, a URL hash for ad-hoc URLs.
-func (s imageSource) volumeName() string {
-	if s.Key != "" {
-		return "base-" + s.Key + "." + s.Format
-	}
-	sum := sha256.Sum256([]byte(s.URL))
-	return "base-" + hex.EncodeToString(sum[:8]) + "." + s.Format
-}
-
-// markerName is the sidecar volume holding the sha256 of the verified
-// upload. Its presence is what says "the base volume is complete".
-func (s imageSource) markerName() string {
-	return s.volumeName() + ".sha256"
-}
-
-// imageCache fetches base images into a host's pool on first use. It keeps
-// no state about what is cached — the pool is the source of truth, checked
-// on every call — only a per-(host, image) lock so concurrent Creates for
-// the same image on the same host result in one download.
-type imageCache struct {
-	http *http.Client
-
-	mu    sync.Mutex
-	locks map[string]*sync.Mutex
-}
-
-func newImageCache(client *http.Client) *imageCache {
-	if client == nil {
-		client = http.DefaultClient
-	}
-	return &imageCache{http: client, locks: make(map[string]*sync.Mutex)}
-}
-
-func (c *imageCache) lockFor(key string) *sync.Mutex {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	m, ok := c.locks[key]
-	if !ok {
-		m = &sync.Mutex{}
-		c.locks[key] = m
-	}
-	return m
-}
-
-// ensure makes src present in pool and returns the base volume's path on
-// the host. The image is downloaded synchronously through the libvirt
-// stream when the pool does not already hold a verified copy.
-func (c *imageCache) ensure(ctx context.Context, log logr.Logger, api libvirtAPI, hostName string, pool libvirt.StoragePool, src imageSource) (string, error) {
-	m := c.lockFor(hostName + "/" + src.volumeName())
-	m.Lock()
-	defer m.Unlock()
-
-	base, ready, err := c.lookup(api, pool, src)
-	if err != nil {
-		return "", err
-	}
-	if ready {
-		log.V(1).Info("base image already cached", "host", hostName, "volume", src.volumeName())
-		return api.StorageVolGetPath(base)
-	}
-	if err := deleteIfExists(api, pool, src.markerName()); err != nil {
-		return "", err
-	}
-	if err := deleteIfExists(api, pool, src.volumeName()); err != nil {
-		return "", err
-	}
-	base, err = c.fetch(ctx, log, api, hostName, pool, src)
-	if err != nil {
-		return "", err
-	}
-	return api.StorageVolGetPath(base)
-}
-
-// lookup reports whether the pool holds a complete, verified base volume:
-// the volume and its marker both exist and, when a checksum is configured,
-// the marker records that checksum.
-func (c *imageCache) lookup(api libvirtAPI, pool libvirt.StoragePool, src imageSource) (libvirt.StorageVol, bool, error) {
-	base, err := api.StorageVolLookupByName(pool, src.volumeName())
-	if hasCode(err, libvirt.ErrNoStorageVol) {
-		return libvirt.StorageVol{}, false, nil
-	}
-	if err != nil {
-		return libvirt.StorageVol{}, false, fmt.Errorf("looking up base volume: %w", err)
-	}
-	marker, err := api.StorageVolLookupByName(pool, src.markerName())
-	if hasCode(err, libvirt.ErrNoStorageVol) {
-		return base, false, nil
-	}
-	if err != nil {
-		return libvirt.StorageVol{}, false, fmt.Errorf("looking up image marker: %w", err)
-	}
-	if src.SHA256 == "" {
-		return base, true, nil
-	}
-	var buf bytes.Buffer
-	if err := api.StorageVolDownload(marker, &buf, 0, 0, 0); err != nil {
-		return libvirt.StorageVol{}, false, fmt.Errorf("reading image marker: %w", err)
-	}
-	return base, strings.EqualFold(strings.TrimSpace(buf.String()), src.SHA256), nil
-}
-
-// fetch streams the image from its URL straight into a new base volume,
-// hashing on the way, then writes the marker. Nothing is buffered in the
-// operator beyond the HTTP body's read buffer.
-func (c *imageCache) fetch(ctx context.Context, log logr.Logger, api libvirtAPI, hostName string, pool libvirt.StoragePool, src imageSource) (libvirt.StorageVol, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, src.URL, nil)
-	if err != nil {
-		return libvirt.StorageVol{}, fmt.Errorf("%w: building image request: %v", provider.ErrPermanent, err)
-	}
-	resp, err := c.http.Do(req)
-	if err != nil {
-		return libvirt.StorageVol{}, fmt.Errorf("%w: fetching image %s: %v", provider.ErrTransient, src.URL, err)
-	}
-	defer resp.Body.Close()
-	switch {
-	case resp.StatusCode >= 200 && resp.StatusCode < 300:
-	case resp.StatusCode >= 400 && resp.StatusCode < 500:
-		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s returned HTTP %d", provider.ErrPermanent, src.URL, resp.StatusCode)
-	default:
-		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s returned HTTP %d", provider.ErrTransient, src.URL, resp.StatusCode)
-	}
-	if resp.ContentLength < 0 {
-		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s has no Content-Length; the base volume needs a known size", provider.ErrPermanent, src.URL)
-	}
-
-	log.Info("fetching base image into host pool",
-		"host", hostName, "pool", pool.Name, "volume", src.volumeName(),
-		"url", src.URL, "bytes", resp.ContentLength)
-	start := time.Now()
-
-	baseXML, err := buildVolumeXML(volumeSpec{Name: src.volumeName(), SizeBytes: resp.ContentLength, Format: src.Format})
-	if err != nil {
-		return libvirt.StorageVol{}, err
-	}
-	base, err := api.StorageVolCreateXML(pool, baseXML, 0)
-	if err != nil {
-		return libvirt.StorageVol{}, fmt.Errorf("creating base volume: %w", err)
-	}
-	hasher := sha256.New()
-	if err := api.StorageVolUpload(base, io.TeeReader(resp.Body, hasher), 0, 0, 0); err != nil {
-		_ = api.StorageVolDelete(base, 0)
-		return libvirt.StorageVol{}, fmt.Errorf("uploading image to base volume: %w", err)
-	}
-	digest := hex.EncodeToString(hasher.Sum(nil))
-	if src.SHA256 != "" && !strings.EqualFold(digest, src.SHA256) {
-		_ = api.StorageVolDelete(base, 0)
-		return libvirt.StorageVol{}, fmt.Errorf("%w: image %s sha256 is %s, config expects %s",
-			provider.ErrPermanent, src.URL, digest, src.SHA256)
-	}
-	// libvirt sized the volume from Content-Length; a refresh makes it
-	// re-read the real virtual size from the uploaded qcow2 header.
-	if err := api.StoragePoolRefresh(pool, 0); err != nil {
-		return libvirt.StorageVol{}, fmt.Errorf("refreshing pool after upload: %w", err)
-	}
-
-	markerXML, err := buildVolumeXML(rawVolume(src.markerName(), int64(len(digest))))
-	if err != nil {
-		return libvirt.StorageVol{}, err
-	}
-	marker, err := api.StorageVolCreateXML(pool, markerXML, 0)
-	if err != nil {
-		return libvirt.StorageVol{}, fmt.Errorf("creating image marker: %w", err)
-	}
-	if err := api.StorageVolUpload(marker, strings.NewReader(digest), 0, 0, 0); err != nil {
-		_ = api.StorageVolDelete(marker, 0)
-		return libvirt.StorageVol{}, fmt.Errorf("writing image marker: %w", err)
-	}
-	log.Info("base image cached",
-		"host", hostName, "volume", src.volumeName(), "sha256", digest,
-		"bytes", resp.ContentLength, "duration", time.Since(start).Round(time.Second))
-	return base, nil
-}
-
-func deleteIfExists(api libvirtAPI, pool libvirt.StoragePool, name string) error {
-	vol, err := api.StorageVolLookupByName(pool, name)
-	if hasCode(err, libvirt.ErrNoStorageVol) {
-		return nil
-	}
-	if err != nil {
-		return fmt.Errorf("looking up volume %s: %w", name, err)
-	}
-	if err := api.StorageVolDelete(vol, 0); err != nil && !hasCode(err, libvirt.ErrNoStorageVol) {
-		return fmt.Errorf("deleting volume %s: %w", name, err)
-	}
-	return nil
-}

internal/provider/kvm/image_test.go

@@ -1,385 +0,0 @@
-package kvm
-
-import (
-	"bytes"
-	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"errors"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/go-logr/logr"
-
-	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
-)
-
-var testImage = bytes.Repeat([]byte("QFI\xfb fake qcow2 payload "), 1024)
-
-func testImageSHA() string {
-	sum := sha256.Sum256(testImage)
-	return hex.EncodeToString(sum[:])
-}
-
-// imageServer serves testImage and counts hits. Status overrides the
-// response, noLength forces chunked encoding (no Content-Length), delay
-// slows each response so concurrent callers overlap.
-type imageServer struct {
-	*httptest.Server
-	hits     atomic.Int32
-	status   int
-	noLength bool
-	delay    time.Duration
-	body     []byte
-}
-
-func newImageServer(t *testing.T) *imageServer {
-	t.Helper()
-	s := &imageServer{status: http.StatusOK, body: testImage}
-	s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		s.hits.Add(1)
-		time.Sleep(s.delay)
-		if s.status != http.StatusOK {
-			w.WriteHeader(s.status)
-			return
-		}
-		// Bodies larger than the server's write buffer would otherwise go
-		// out chunked without a Content-Length, unlike real image mirrors.
-		if s.noLength {
-			w.(http.Flusher).Flush()
-		} else {
-			w.Header().Set("Content-Length", strconv.Itoa(len(s.body)))
-		}
-		_, _ = w.Write(s.body)
-	}))
-	t.Cleanup(s.Close)
-	return s
-}
-
-func testSource(srv *imageServer, sha string) imageSource {
-	return imageSource{Key: "debian-13", URL: srv.URL + "/debian.qcow2", SHA256: sha, Format: formatQcow2}
-}
-
-func ensureTest(t *testing.T, api *fakeAPI, src imageSource) (string, error) {
-	t.Helper()
-	c := newImageCache(nil)
-	pool, err := api.StoragePoolLookupByName("default")
-	if err != nil {
-		t.Fatal(err)
-	}
-	return c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, src)
-}
-
-func TestResolveImage(t *testing.T) {
-	t.Parallel()
-	cfg := withDefaults(provider.KVMConfig{Images: map[string]provider.KVMImage{
-		"debian-13": {URL: "https://cloud.debian.org/d.qcow2", SHA256: "abc"},
-	}})
-	tests := []struct {
-		name    string
-		image   string
-		want    imageSource
-		wantErr bool
-	}{
-		{"configured key", "debian-13", imageSource{Key: "debian-13", URL: "https://cloud.debian.org/d.qcow2", SHA256: "abc", Format: "qcow2"}, false},
-		{"direct https url", "https://x.example/img.qcow2", imageSource{URL: "https://x.example/img.qcow2", Format: "qcow2"}, false},
-		{"direct http url", "http://mirror/img.qcow2", imageSource{URL: "http://mirror/img.qcow2", Format: "qcow2"}, false},
-		{"unknown key", "ubuntu", imageSource{}, true},
-		{"file url", "file:///tmp/x.qcow2", imageSource{}, true},
-		{"empty", "", imageSource{}, true},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			got, err := resolveImage(cfg, tc.image)
-			if tc.wantErr {
-				if !errors.Is(err, provider.ErrPermanent) {
-					t.Fatalf("resolveImage(%q) error = %v, want ErrPermanent", tc.image, err)
-				}
-				return
-			}
-			if err != nil {
-				t.Fatalf("resolveImage(%q) error = %v", tc.image, err)
-			}
-			if got != tc.want {
-				t.Errorf("resolveImage(%q) = %+v, want %+v", tc.image, got, tc.want)
-			}
-		})
-	}
-}
-
-func TestImageSource_volumeNames(t *testing.T) {
-	t.Parallel()
-	keyed := imageSource{Key: "debian-13", URL: "https://a/b.qcow2", Format: "qcow2"}
-	if keyed.volumeName() != "base-debian-13.qcow2" || keyed.markerName() != "base-debian-13.qcow2.sha256" {
-		t.Errorf("keyed names = %s / %s", keyed.volumeName(), keyed.markerName())
-	}
-	direct := imageSource{URL: "https://a/b.qcow2", Format: "qcow2"}
-	name := direct.volumeName()
-	if !strings.HasPrefix(name, "base-") || !strings.HasSuffix(name, ".qcow2") || len(name) != len("base-")+16+len(".qcow2") {
-		t.Errorf("direct-url name = %q, want base-<16 hex>.qcow2", name)
-	}
-	other := imageSource{URL: "https://a/c.qcow2", Format: "qcow2"}
-	if other.volumeName() == name {
-		t.Error("different URLs produced the same volume name")
-	}
-}
-
-func TestImageCache_ensure_freshFetch(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	api := newFakeAPI("default")
-	path, err := ensureTest(t, api, testSource(srv, testImageSHA()))
-	if err != nil {
-		t.Fatalf("ensure() error = %v", err)
-	}
-	if path != "/var/lib/libvirt/images/base-debian-13.qcow2" {
-		t.Errorf("path = %q", path)
-	}
-	base := api.vol("default", "base-debian-13.qcow2")
-	if base == nil || !bytes.Equal(base.data, testImage) {
-		t.Fatalf("base volume missing or content differs")
-	}
-	if base.format != "qcow2" || base.capacity != int64(len(testImage)) {
-		t.Errorf("base volume format/capacity = %s/%d, want qcow2/%d", base.format, base.capacity, len(testImage))
-	}
-	marker := api.vol("default", "base-debian-13.qcow2.sha256")
-	if marker == nil || string(marker.data) != testImageSHA() {
-		t.Fatalf("marker missing or wrong: %+v", marker)
-	}
-	if !api.hasCall("StoragePoolRefresh") {
-		t.Error("pool was not refreshed after the upload")
-	}
-	if srv.hits.Load() != 1 {
-		t.Errorf("http hits = %d, want 1", srv.hits.Load())
-	}
-}
-
-func TestImageCache_ensure_noChecksumConfigured(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	api := newFakeAPI("default")
-	if _, err := ensureTest(t, api, testSource(srv, "")); err != nil {
-		t.Fatalf("ensure() error = %v", err)
-	}
-	if marker := api.vol("default", "base-debian-13.qcow2.sha256"); marker == nil || string(marker.data) != testImageSHA() {
-		t.Errorf("marker should still record the computed digest, got %+v", marker)
-	}
-	// Second call: ready without a checksum to compare, so no marker read.
-	if _, err := ensureTest(t, api, testSource(srv, "")); err != nil {
-		t.Fatalf("second ensure() error = %v", err)
-	}
-	if srv.hits.Load() != 1 {
-		t.Errorf("http hits = %d, want 1", srv.hits.Load())
-	}
-}
-
-func TestImageCache_ensure_alreadyCached(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	api := newFakeAPI("default")
-	api.putVol("default", "base-debian-13.qcow2", "qcow2", testImage)
-	api.putVol("default", "base-debian-13.qcow2.sha256", "raw", []byte(testImageSHA()))
-	path, err := ensureTest(t, api, testSource(srv, testImageSHA()))
-	if err != nil {
-		t.Fatalf("ensure() error = %v", err)
-	}
-	if path != "/var/lib/libvirt/images/base-debian-13.qcow2" {
-		t.Errorf("path = %q", path)
-	}
-	if srv.hits.Load() != 0 {
-		t.Errorf("http hits = %d, want 0 for a cached image", srv.hits.Load())
-	}
-	if api.hasCall("StorageVolCreateXML") || api.hasCall("StorageVolDelete") {
-		t.Errorf("cached image was touched: calls = %v", api.calls)
-	}
-}
-
-func TestImageCache_ensure_markerMissingRedownloads(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	api := newFakeAPI("default")
-	api.putVol("default", "base-debian-13.qcow2", "qcow2", []byte("half an image"))
-	if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
-		t.Fatalf("ensure() error = %v", err)
-	}
-	if srv.hits.Load() != 1 {
-		t.Errorf("http hits = %d, want 1: a base without its marker is a partial upload", srv.hits.Load())
-	}
-	if base := api.vol("default", "base-debian-13.qcow2"); base == nil || !bytes.Equal(base.data, testImage) {
-		t.Error("partial base volume was not replaced")
-	}
-}
-
-func TestImageCache_ensure_checksumChangedRedownloads(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	api := newFakeAPI("default")
-	api.putVol("default", "base-debian-13.qcow2", "qcow2", []byte("an older release"))
-	api.putVol("default", "base-debian-13.qcow2.sha256", "raw", []byte(strings.Repeat("0", 64)))
-	if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
-		t.Fatalf("ensure() error = %v", err)
-	}
-	if srv.hits.Load() != 1 {
-		t.Errorf("http hits = %d, want 1 after the configured checksum changed", srv.hits.Load())
-	}
-	if marker := api.vol("default", "base-debian-13.qcow2.sha256"); marker == nil || string(marker.data) != testImageSHA() {
-		t.Errorf("marker not updated: %+v", marker)
-	}
-}
-
-func TestImageCache_ensure_checksumMismatchIsPermanent(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	api := newFakeAPI("default")
-	_, err := ensureTest(t, api, testSource(srv, strings.Repeat("a", 64)))
-	if !errors.Is(err, provider.ErrPermanent) {
-		t.Fatalf("ensure() error = %v, want ErrPermanent", err)
-	}
-	if names := api.volNames("default"); len(names) != 0 {
-		t.Errorf("volumes left behind after a checksum mismatch: %v", names)
-	}
-}
-
-func TestImageCache_ensure_httpErrors(t *testing.T) {
-	t.Parallel()
-	tests := []struct {
-		name   string
-		status int
-		want   error
-	}{
-		{"404 is permanent", http.StatusNotFound, provider.ErrPermanent},
-		{"403 is permanent", http.StatusForbidden, provider.ErrPermanent},
-		{"503 is transient", http.StatusServiceUnavailable, provider.ErrTransient},
-		{"500 is transient", http.StatusInternalServerError, provider.ErrTransient},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			srv := newImageServer(t)
-			srv.status = tc.status
-			api := newFakeAPI("default")
-			_, err := ensureTest(t, api, testSource(srv, ""))
-			if !errors.Is(err, tc.want) {
-				t.Fatalf("ensure() error = %v, want %v", err, tc.want)
-			}
-			if names := api.volNames("default"); len(names) != 0 {
-				t.Errorf("volumes created despite HTTP %d: %v", tc.status, names)
-			}
-		})
-	}
-}
-
-func TestImageCache_ensure_noContentLengthIsPermanent(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	srv.noLength = true
-	api := newFakeAPI("default")
-	_, err := ensureTest(t, api, testSource(srv, ""))
-	if !errors.Is(err, provider.ErrPermanent) || !strings.Contains(err.Error(), "Content-Length") {
-		t.Fatalf("ensure() error = %v, want ErrPermanent about Content-Length", err)
-	}
-}
-
-func TestImageCache_ensure_uploadFailureCleansUp(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	api := newFakeAPI("default")
-	api.uploadHook = func(_ string, r io.Reader) error {
-		_, _ = io.CopyN(io.Discard, r, 100)
-		return io.ErrUnexpectedEOF
-	}
-	_, err := ensureTest(t, api, testSource(srv, testImageSHA()))
-	if !errors.Is(err, provider.ErrTransient) && classify(err) != provider.ErrTransient {
-		t.Fatalf("ensure() error = %v, want a transient failure", err)
-	}
-	if names := api.volNames("default"); len(names) != 0 {
-		t.Errorf("volumes left behind after a failed upload: %v", names)
-	}
-	// Next attempt starts clean and succeeds.
-	api.uploadHook = nil
-	if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
-		t.Fatalf("retry ensure() error = %v", err)
-	}
-}
-
-func TestImageCache_ensure_missingPoolIsPermanent(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	api := newFakeAPI("default")
-	c := newImageCache(nil)
-	pool, _ := api.StoragePoolLookupByName("default")
-	delete(api.pools, "default")
-	_, err := c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, testSource(srv, ""))
-	if classify(err) != provider.ErrPermanent {
-		t.Fatalf("ensure() error = %v, want one classifying as Permanent", err)
-	}
-}
-
-func TestImageCache_ensure_concurrentCallersDownloadOnce(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	srv.delay = 50 * time.Millisecond
-	api := newFakeAPI("default")
-	c := newImageCache(nil)
-	pool, _ := api.StoragePoolLookupByName("default")
-	src := testSource(srv, testImageSHA())
-	var wg sync.WaitGroup
-	paths := make([]string, 4)
-	errs := make([]error, 4)
-	for i := range 4 {
-		wg.Go(func() {
-			paths[i], errs[i] = c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, src)
-		})
-	}
-	wg.Wait()
-	for i := range 4 {
-		if errs[i] != nil || paths[i] != "/var/lib/libvirt/images/base-debian-13.qcow2" {
-			t.Errorf("caller %d: path=%q err=%v", i, paths[i], errs[i])
-		}
-	}
-	if srv.hits.Load() != 1 {
-		t.Errorf("http hits = %d, want 1 for four concurrent callers", srv.hits.Load())
-	}
-}
-
-func TestImageCache_ensure_differentHostsFetchIndependently(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	c := newImageCache(nil)
-	src := testSource(srv, "")
-	for _, host := range []string{"nas1", "lab2"} {
-		api := newFakeAPI("default")
-		pool, _ := api.StoragePoolLookupByName("default")
-		if _, err := c.ensure(context.Background(), logr.Discard(), api, host, pool, src); err != nil {
-			t.Fatalf("ensure(%s) error = %v", host, err)
-		}
-	}
-	if srv.hits.Load() != 2 {
-		t.Errorf("http hits = %d, want one per host", srv.hits.Load())
-	}
-}
-
-func TestImageCache_ensure_cancelledContextIsTransient(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	api := newFakeAPI("default")
-	c := newImageCache(nil)
-	pool, _ := api.StoragePoolLookupByName("default")
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	_, err := c.ensure(ctx, logr.Discard(), api, "nas1", pool, testSource(srv, ""))
-	if classify(err) != provider.ErrTransient {
-		t.Fatalf("ensure() error = %v, want transient", err)
-	}
-	if names := api.volNames("default"); len(names) != 0 {
-		t.Errorf("volumes created despite cancelled context: %v", names)
-	}
-}

internal/provider/kvm/kvm.go

@@ -1,560 +0,0 @@
-// Package kvm implements the provider contract on libvirt/KVM hypervisors.
-// Each configured host is reached over SSH with the libvirt unix socket
-// forwarded through the connection, so the hypervisors need no libvirtd TCP
-// or TLS listener and the operator image needs no libvirt libraries. VMs
-// boot a stock cloud image (fetched into the host's storage pool on first
-// use) through a qcow2 overlay, with the Proxy's cloud-init delivered on a
-// NoCloud seed ISO built in-process.
-package kvm
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"net"
-	"net/http"
-	"time"
-
-	libvirt "github.com/digitalocean/go-libvirt"
-	"github.com/go-logr/logr"
-	logf "sigs.k8s.io/controller-runtime/pkg/log"
-
-	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
-)
-
-const (
-	defaultDiskSizeGB = 10
-	defaultSocket     = "/var/run/libvirt/libvirt-sock"
-	defaultPool       = "default"
-	formatQcow2       = "qcow2"
-	formatRaw         = "raw"
-
-	// undefineFlags removes everything libvirt may keep alongside a domain
-	// definition, so an undefine never fails on leftover state.
-	undefineFlags = libvirt.DomainUndefineManagedSave |
-		libvirt.DomainUndefineSnapshotsMetadata |
-		libvirt.DomainUndefineNvram
-)
-
-func withDefaults(cfg provider.KVMConfig) provider.KVMConfig {
-	if cfg.DiskSizeGB == 0 {
-		cfg.DiskSizeGB = defaultDiskSizeGB
-	}
-	if cfg.VendorData == "" {
-		cfg.VendorData = defaultVendorData
-	}
-	images := make(map[string]provider.KVMImage, len(cfg.Images))
-	for k, img := range cfg.Images {
-		if img.Format == "" {
-			img.Format = formatQcow2
-		}
-		images[k] = img
-	}
-	cfg.Images = images
-	hosts := make([]provider.KVMHost, len(cfg.Hosts))
-	for i, h := range cfg.Hosts {
-		if h.Socket == "" {
-			h.Socket = defaultSocket
-		}
-		if h.Pool == "" {
-			h.Pool = defaultPool
-		}
-		hosts[i] = h
-	}
-	cfg.Hosts = hosts
-	return cfg
-}
-
-// Provider implements provider.Provider on one or more libvirt hosts.
-type Provider struct {
-	name   string
-	cfg    provider.KVMConfig
-	hosts  *hostPool
-	images *imageCache
-	now    func() time.Time
-}
-
-// New builds a Provider that reaches its hosts over SSH. The key and
-// known_hosts files are read here so a bad path fails at startup rather
-// than on the first Create. Satisfies registry.Constructor.
-func New(_ context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
-	cfg := provider.KVMConfig{}
-	if pc.KVM != nil {
-		cfg = *pc.KVM
-	}
-	auth, err := newSSHAuth(cfg)
-	if err != nil {
-		return nil, fmt.Errorf("kvm provider %q: %w", pc.Name, err)
-	}
-	return newWithDialer(pc, auth.dialer(), nil), nil
-}
-
-// newWithDialer is the test seam: the dialer decides what libvirtAPI each
-// host resolves to, and the HTTP client is what image downloads go through.
-func newWithDialer(pc provider.ProviderConfig, dial hostDialer, httpClient *http.Client) *Provider {
-	cfg := provider.KVMConfig{}
-	if pc.KVM != nil {
-		cfg = *pc.KVM
-	}
-	return &Provider{
-		name:   pc.Name,
-		cfg:    withDefaults(cfg),
-		hosts:  newHostPool(dial),
-		images: newImageCache(httpClient),
-		now:    time.Now,
-	}
-}
-
-func (p *Provider) logger(ctx context.Context) logr.Logger {
-	return logf.FromContext(ctx).WithName("kvm").WithValues("provider", p.name)
-}
-
-func (p *Provider) wrapErr(op, id string, err error) error {
-	return provider.Wrap(classify(err), op, p.name, id, err)
-}
-
-// host resolves placement.zone (or a providerID's host segment) to a
-// configured host. An empty zone is allowed only when exactly one host is
-// configured; anything else is a spec error.
-func (p *Provider) host(zone string) (provider.KVMHost, error) {
-	if zone == "" {
-		switch len(p.cfg.Hosts) {
-		case 0:
-			return provider.KVMHost{}, fmt.Errorf("%w: kvm.hosts is empty", provider.ErrPermanent)
-		case 1:
-			return p.cfg.Hosts[0], nil
-		default:
-			return provider.KVMHost{}, fmt.Errorf("%w: placement.zone is required when kvm.hosts lists more than one host",
-				provider.ErrPermanent)
-		}
-	}
-	for _, h := range p.cfg.Hosts {
-		if h.Name == zone {
-			return h, nil
-		}
-	}
-	return provider.KVMHost{}, fmt.Errorf("%w: host %q is not in kvm.hosts", provider.ErrPermanent, zone)
-}
-
-// Create makes the VM exist and running on the placed host, and is
-// idempotent: a defined domain is only started if needed, its volumes
-// untouched. Before the domain exists nothing can be using the volumes,
-// so an existing overlay is reused (its creation is atomic) but the seed
-// is rebuilt from scratch — a previous attempt may have been cut off mid
-// upload, and a truncated seed would boot a VM without cloud-init. The
-// first Create for an image on a host also downloads that image into the
-// host's pool, synchronously.
-func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
-	pl := req.Placement
-	host, err := p.host(pl.Zone)
-	if err != nil {
-		return "", p.wrapErr("create", "", err)
-	}
-	mt, ok := p.cfg.MachineTypes[pl.MachineType]
-	if !ok {
-		return "", p.wrapErr("create", "", fmt.Errorf("%w: placement.machineType %q is not in kvm.machineTypes",
-			provider.ErrPermanent, pl.MachineType))
-	}
-	src, err := resolveImage(p.cfg, pl.Image)
-	if err != nil {
-		return "", p.wrapErr("create", "", err)
-	}
-	id := formatProviderID(host.Name, req.Name)
-	log := p.logger(ctx).WithValues("host", host.Name, "name", req.Name)
-	log.V(2).Info("KVM create resolved",
-		"pool", host.Pool, "bridge", host.Bridge, "machineType", pl.MachineType,
-		"vcpus", mt.VCPUs, "memoryMiB", mt.MemoryMiB, "diskSizeGb", p.cfg.DiskSizeGB,
-		"image", src.volumeName(), "cloudInitBytes", len(req.CloudInit))
-
-	err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
-		dom, err := api.DomainLookupByName(req.Name)
-		if err == nil {
-			log.V(1).Info("domain already defined")
-			return p.ensureRunning(log, api, dom)
-		}
-		if !hasCode(err, libvirt.ErrNoDomain) {
-			return fmt.Errorf("looking up domain: %w", err)
-		}
-		pool, err := api.StoragePoolLookupByName(host.Pool)
-		if err != nil {
-			return fmt.Errorf("looking up pool %s: %w", host.Pool, err)
-		}
-		basePath, err := p.images.ensure(ctx, log, api, host.Name, pool, src)
-		if err != nil {
-			return err
-		}
-		if err := p.ensureOverlay(log, api, pool, req.Name, basePath, src.Format); err != nil {
-			return err
-		}
-		if err := p.writeSeed(log, api, pool, req.Name, req.CloudInit); err != nil {
-			return err
-		}
-		dom, err = p.defineDomain(log, api, host, req, mt)
-		if err != nil {
-			return err
-		}
-		return p.ensureRunning(log, api, dom)
-	})
-	if err != nil {
-		logAPIError(log, "create", err)
-		return "", p.wrapErr("create", id, err)
-	}
-	log.V(1).Info("KVM domain ensured running", "id", id)
-	return id, nil
-}
-
-func (p *Provider) ensureOverlay(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, basePath, baseFormat string) error {
-	xml, err := buildVolumeXML(overlayVolume(name, p.cfg.DiskSizeGB, basePath, baseFormat))
-	if err != nil {
-		return err
-	}
-	_, err = api.StorageVolCreateXML(pool, xml, 0)
-	switch {
-	case err == nil:
-		log.V(1).Info("overlay volume created", "volume", overlayVolumeName(name), "base", basePath)
-	case hasCode(err, libvirt.ErrStorageVolExist):
-		log.V(1).Info("overlay volume already exists", "volume", overlayVolumeName(name))
-	default:
-		return fmt.Errorf("creating overlay volume: %w", err)
-	}
-	return nil
-}
-
-// writeSeed replaces the seed volume with a freshly built NoCloud ISO.
-// Only called while no domain references it.
-func (p *Provider) writeSeed(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, userData string) error {
-	seedName := seedVolumeName(name)
-	if err := deleteIfExists(api, pool, seedName); err != nil {
-		return err
-	}
-	iso, err := buildSeedISO(name, userData, p.cfg.VendorData)
-	if err != nil {
-		return fmt.Errorf("%w: %v", provider.ErrPermanent, err)
-	}
-	xml, err := buildVolumeXML(rawVolume(seedName, int64(len(iso))))
-	if err != nil {
-		return err
-	}
-	vol, err := api.StorageVolCreateXML(pool, xml, 0)
-	if err != nil {
-		return fmt.Errorf("creating seed volume: %w", err)
-	}
-	if err := api.StorageVolUpload(vol, bytes.NewReader(iso), 0, 0, 0); err != nil {
-		return fmt.Errorf("uploading seed volume: %w", err)
-	}
-	log.V(1).Info("seed volume uploaded", "volume", seedName, "bytes", len(iso))
-	return nil
-}
-
-func (p *Provider) defineDomain(log logr.Logger, api libvirtAPI, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType) (libvirt.Domain, error) {
-	xml, err := buildDomainXML(domainParams{
-		Name:      req.Name,
-		UID:       req.UID,
-		CreatedAt: p.now(),
-		VCPUs:     mt.VCPUs,
-		MemoryMiB: mt.MemoryMiB,
-		Pool:      host.Pool,
-		Bridge:    host.Bridge,
-		DiskVol:   overlayVolumeName(req.Name),
-		SeedVol:   seedVolumeName(req.Name),
-	})
-	if err != nil {
-		return libvirt.Domain{}, err
-	}
-	dom, err := api.DomainDefineXML(xml)
-	if err != nil {
-		return libvirt.Domain{}, fmt.Errorf("defining domain: %w", err)
-	}
-	log.V(1).Info("domain defined", "uid", req.UID)
-	return dom, nil
-}
-
-func (p *Provider) ensureRunning(log logr.Logger, api libvirtAPI, dom libvirt.Domain) error {
-	state, _, err := api.DomainGetState(dom, 0)
-	if err != nil {
-		return fmt.Errorf("reading domain state: %w", err)
-	}
-	if isActive(libvirt.DomainState(state)) {
-		return nil
-	}
-	err = api.DomainCreate(dom)
-	switch {
-	case err == nil:
-		log.V(1).Info("domain started")
-	case hasCode(err, libvirt.ErrOperationInvalid):
-		log.V(1).Info("domain already running, start treated as success")
-	default:
-		return fmt.Errorf("starting domain: %w", err)
-	}
-	return nil
-}
-
-// Get reads the domain's state, metadata and addresses. The providerID
-// carries the host, so this looks at the right hypervisor even after a
-// placement.zone edit.
-func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instance, error) {
-	hostName, name, err := parseProviderID(providerID)
-	if err != nil {
-		return nil, provider.Wrap(provider.ErrPermanent, "get", p.name, providerID, err)
-	}
-	host, err := p.host(hostName)
-	if err != nil {
-		return nil, p.wrapErr("get", providerID, err)
-	}
-	log := p.logger(ctx).WithValues("host", host.Name, "name", name)
-	var inst *provider.Instance
-	err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
-		dom, err := api.DomainLookupByName(name)
-		if err != nil {
-			return fmt.Errorf("looking up domain: %w", err)
-		}
-		meta, err := p.metadata(api, dom)
-		if err != nil {
-			return err
-		}
-		inst, err = p.inspect(log, api, host.Name, dom, meta)
-		return err
-	})
-	if err != nil {
-		logAPIError(log, "get", err)
-		return nil, p.wrapErr("get", providerID, err)
-	}
-	log.V(1).Info("KVM domain fetched", "state", inst.State, "ip", inst.IP, "uid", inst.UID)
-	return inst, nil
-}
-
-// inspect assembles an Instance from a domain that is known to exist and
-// whose metadata has already been read (the zero value for a domain
-// without the operator's element, which reports an empty UID as the
-// contract requires).
-func (p *Provider) inspect(log logr.Logger, api libvirtAPI, hostName string, dom libvirt.Domain, meta instanceMetadata) (*provider.Instance, error) {
-	state, _, err := api.DomainGetState(dom, 0)
-	if err != nil {
-		return nil, fmt.Errorf("reading domain state: %w", err)
-	}
-	var ip string
-	if isActive(libvirt.DomainState(state)) {
-		ip = p.address(log, api, dom)
-	}
-	return &provider.Instance{
-		ID:        formatProviderID(hostName, dom.Name),
-		IP:        ip,
-		State:     mapState(libvirt.DomainState(state), ip),
-		UID:       meta.UID,
-		CreatedAt: meta.createdAt(),
-	}, nil
-}
-
-// metadata returns the operator's element, or the zero value when the
-// domain has none (it was not created by this operator).
-func (p *Provider) metadata(api libvirtAPI, dom libvirt.Domain) (instanceMetadata, error) {
-	raw, err := api.DomainGetMetadata(dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, libvirt.DomainAffectCurrent)
-	if hasCode(err, libvirt.ErrNoDomainMetadata) {
-		return instanceMetadata{}, nil
-	}
-	if err != nil {
-		return instanceMetadata{}, fmt.Errorf("reading domain metadata: %w", err)
-	}
-	meta, err := parseInstanceMetadata(raw)
-	if err != nil {
-		return instanceMetadata{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
-	}
-	return meta, nil
-}
-
-// address returns the domain's first routable IPv4, asking the host's ARP
-// table first and the guest agent second. Both sources fail routinely while
-// the guest is still booting (no traffic seen yet, agent not installed
-// yet), so their errors mean "no address yet", not "Get failed".
-func (p *Provider) address(log logr.Logger, api libvirtAPI, dom libvirt.Domain) string {
-	for _, source := range []libvirt.DomainInterfaceAddressesSource{
-		libvirt.DomainInterfaceAddressesSrcArp,
-		libvirt.DomainInterfaceAddressesSrcAgent,
-	} {
-		ifaces, err := api.DomainInterfaceAddresses(dom, uint32(source), 0)
-		if err != nil {
-			log.V(2).Info("no addresses from source", "source", source.String(), "error", err.Error())
-			continue
-		}
-		if ip := firstIPv4(ifaces); ip != "" {
-			return ip
-		}
-	}
-	return ""
-}
-
-func firstIPv4(ifaces []libvirt.DomainInterface) string {
-	for _, iface := range ifaces {
-		for _, a := range iface.Addrs {
-			if a.Type != int32(libvirt.IPAddrTypeIpv4) {
-				continue
-			}
-			ip := net.ParseIP(a.Addr)
-			if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsUnspecified() {
-				continue
-			}
-			return a.Addr
-		}
-	}
-	return ""
-}
-
-// isActive reports whether the guest is executing (or blocked on the
-// hypervisor, which is still "up" from the outside).
-func isActive(s libvirt.DomainState) bool {
-	return s == libvirt.DomainRunning || s == libvirt.DomainBlocked
-}
-
-// mapState collapses libvirt's states onto the provider's. An active
-// domain without an address is still Provisioning: an empty IP must never
-// be published as Running. "No state" is also Provisioning — the
-// reconciler answers Stopped with delete-and-recreate, too drastic for a
-// state libvirt could not determine; the next poll decides. Everything
-// else — paused, suspended, shut off, crashed, shutting down — is Stopped.
-func mapState(s libvirt.DomainState, ip string) provider.InstanceState {
-	if s == libvirt.DomainNostate {
-		return provider.StateProvisioning
-	}
-	if !isActive(s) {
-		return provider.StateStopped
-	}
-	if ip == "" {
-		return provider.StateProvisioning
-	}
-	return provider.StateRunning
-}
-
-// Delete tears the VM down and is idempotent at every step: a stopped
-// domain is not destroyed again, a missing domain is not undefined, and
-// the per-VM volumes are removed by name even when the domain is already
-// gone, so a crash between volume creation and define is cleaned up on the
-// next reconcile.
-func (p *Provider) Delete(ctx context.Context, providerID string) error {
-	hostName, name, err := parseProviderID(providerID)
-	if err != nil {
-		return provider.Wrap(provider.ErrPermanent, "delete", p.name, providerID, err)
-	}
-	host, err := p.host(hostName)
-	if err != nil {
-		return p.wrapErr("delete", providerID, err)
-	}
-	log := p.logger(ctx).WithValues("host", host.Name, "name", name)
-	err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
-		if err := p.removeDomain(log, api, name); err != nil {
-			return err
-		}
-		return p.removeVolumes(log, api, host.Pool, name)
-	})
-	if err != nil {
-		logAPIError(log, "delete", err)
-		return p.wrapErr("delete", providerID, err)
-	}
-	log.V(1).Info("KVM domain deleted")
-	return nil
-}
-
-func (p *Provider) removeDomain(log logr.Logger, api libvirtAPI, name string) error {
-	dom, err := api.DomainLookupByName(name)
-	if hasCode(err, libvirt.ErrNoDomain) {
-		log.V(1).Info("domain already gone")
-		return nil
-	}
-	if err != nil {
-		return fmt.Errorf("looking up domain: %w", err)
-	}
-	err = api.DomainDestroy(dom)
-	switch {
-	case err == nil:
-		log.V(1).Info("domain destroyed")
-	case hasCode(err, libvirt.ErrOperationInvalid):
-		log.V(1).Info("domain not running, destroy skipped")
-	case hasCode(err, libvirt.ErrNoDomain):
-		return nil
-	default:
-		return fmt.Errorf("destroying domain: %w", err)
-	}
-	if err := api.DomainUndefineFlags(dom, undefineFlags); err != nil && !hasCode(err, libvirt.ErrNoDomain) {
-		return fmt.Errorf("undefining domain: %w", err)
-	}
-	log.V(1).Info("domain undefined")
-	return nil
-}
-
-// removeVolumes deletes the overlay and seed. A missing pool means there
-// is nothing left to clean, not a failure: refusing to finish a Delete
-// over a pool that no longer exists would pin the Proxy's finalizer.
-func (p *Provider) removeVolumes(log logr.Logger, api libvirtAPI, poolName, name string) error {
-	pool, err := api.StoragePoolLookupByName(poolName)
-	if hasCode(err, libvirt.ErrNoStoragePool) {
-		log.Info("pool not found, no volumes to delete", "pool", poolName)
-		return nil
-	}
-	if err != nil {
-		return fmt.Errorf("looking up pool %s: %w", poolName, err)
-	}
-	for _, vol := range []string{overlayVolumeName(name), seedVolumeName(name)} {
-		if err := deleteIfExists(api, pool, vol); err != nil {
-			return err
-		}
-	}
-	log.V(1).Info("per-VM volumes deleted")
-	return nil
-}
-
-// ListByTag inspects every domain on every host and keeps those carrying
-// the operator's managed metadata. One unreachable host fails the whole
-// sweep, so orphan GC never acts on a partial view of the fleet; a single
-// domain that cannot be read is skipped instead, which is safe in the
-// same direction — a domain missing from the list is never collected.
-func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
-	log := p.logger(ctx)
-	var out []provider.Instance
-	for _, host := range p.cfg.Hosts {
-		hlog := log.WithValues("host", host.Name)
-		err := p.hosts.call(ctx, host, func(api libvirtAPI) error {
-			doms, _, err := api.ConnectListAllDomains(1, 0)
-			if err != nil {
-				return fmt.Errorf("listing domains: %w", err)
-			}
-			for _, dom := range doms {
-				if inst := p.managedInstance(hlog, api, host.Name, dom); inst != nil {
-					out = append(out, *inst)
-				}
-			}
-			return nil
-		})
-		if err != nil {
-			logAPIError(hlog, "list", err)
-			return nil, p.wrapErr("list", "", err)
-		}
-	}
-	log.V(1).Info("KVM domains listed", "hosts", len(p.cfg.Hosts), "managed", len(out))
-	for _, inst := range out {
-		log.V(2).Info("KVM listed instance", "id", inst.ID, "state", inst.State, "uid", inst.UID, "createdAt", inst.CreatedAt)
-	}
-	return out, nil
-}
-
-// managedInstance returns nil for domains the operator does not own, for
-// domains that vanished between the listing and the inspection, and for
-// domains whose metadata or state cannot be read (corrupt element, mid
-// teardown): one bad domain must not block GC for every other one. A
-// transport failure would also land here, but it fails the next RPC in
-// the same sweep too, so the host still counts as unreachable.
-func (p *Provider) managedInstance(log logr.Logger, api libvirtAPI, hostName string, dom libvirt.Domain) *provider.Instance {
-	log = log.WithValues("name", dom.Name)
-	meta, err := p.metadata(api, dom)
-	if err != nil {
-		log.V(1).Info("skipping domain with unreadable metadata", "error", err.Error())
-		return nil
-	}
-	if !meta.managed() {
-		return nil
-	}
-	inst, err := p.inspect(log, api, hostName, dom, meta)
-	if err != nil {
-		log.V(1).Info("skipping domain that could not be inspected", "error", err.Error())
-		return nil
-	}
-	return inst
-}

internal/provider/kvm/kvm_test.go

@@ -1,880 +0,0 @@
-package kvm
-
-import (
-	"context"
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"io"
-	"strings"
-	"testing"
-	"time"
-
-	libvirt "github.com/digitalocean/go-libvirt"
-
-	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
-)
-
-func TestWithDefaults(t *testing.T) {
-	t.Parallel()
-	cfg := withDefaults(provider.KVMConfig{
-		Images: map[string]provider.KVMImage{
-			"debian": {URL: "https://x/d.qcow2"},
-			"raw":    {URL: "https://x/r.img", Format: "raw"},
-		},
-		Hosts: []provider.KVMHost{
-			{Name: "a", Address: "a:22", User: "virt", Bridge: "br0"},
-			{Name: "b", Address: "b:22", User: "virt", Bridge: "br1", Socket: "/run/s", Pool: "fast"},
-		},
-	})
-	if cfg.DiskSizeGB != defaultDiskSizeGB {
-		t.Errorf("diskSizeGb = %d, want %d", cfg.DiskSizeGB, defaultDiskSizeGB)
-	}
-	if cfg.VendorData != defaultVendorData {
-		t.Errorf("vendorData = %q, want the default", cfg.VendorData)
-	}
-	if cfg.Images["debian"].Format != "qcow2" || cfg.Images["raw"].Format != "raw" {
-		t.Errorf("image formats = %q/%q, want qcow2 default and raw kept", cfg.Images["debian"].Format, cfg.Images["raw"].Format)
-	}
-	if a := cfg.Hosts[0]; a.Socket != defaultSocket || a.Pool != defaultPool {
-		t.Errorf("host a = %+v, want default socket and pool", a)
-	}
-	if b := cfg.Hosts[1]; b.Socket != "/run/s" || b.Pool != "fast" {
-		t.Errorf("host b = %+v, want explicit socket and pool kept", b)
-	}
-}
-
-func TestWithDefaults_keepsExplicitValues(t *testing.T) {
-	t.Parallel()
-	cfg := withDefaults(provider.KVMConfig{DiskSizeGB: 42, VendorData: "#cloud-config\n"})
-	if cfg.DiskSizeGB != 42 || cfg.VendorData != "#cloud-config\n" {
-		t.Errorf("cfg = %+v, want explicit values kept", cfg)
-	}
-}
-
-// --- provider over fake hosts ---
-
-const (
-	testName      = "proxy-abc123def456ghij"
-	testUID       = "11111111-2222-3333-4444-555555555555"
-	testImageURL  = "https://images.example/debian-13.qcow2"
-	testCloudInit = "#cloud-config\npackages: [squid]\n"
-)
-
-var (
-	testNow    = time.Date(2026, 9, 8, 10, 0, 0, 0, time.UTC)
-	errHostOff = errors.New("ssh dial: connection refused")
-)
-
-// hostFakes routes each configured host name to its own fakeAPI; a host
-// with no entry is unreachable.
-type hostFakes map[string]*fakeAPI
-
-func (h hostFakes) dial(_ context.Context, host provider.KVMHost) (*hostClient, error) {
-	api, ok := h[host.Name]
-	if !ok {
-		return nil, errHostOff
-	}
-	return &hostClient{api: api, close: func() {}}, nil
-}
-
-func testConfig(hosts ...string) provider.ProviderConfig {
-	cfg := &provider.KVMConfig{
-		SSHKeyFile:     "/k",
-		KnownHostsFile: "/kh",
-		Images:         map[string]provider.KVMImage{"debian-13": {URL: testImageURL}},
-		MachineTypes:   map[string]provider.KVMMachineType{"small": {VCPUs: 2, MemoryMiB: 2048}},
-	}
-	for _, h := range hosts {
-		cfg.Hosts = append(cfg.Hosts, provider.KVMHost{Name: h, Address: h + ":22", User: "virt", Bridge: "br0"})
-	}
-	return provider.ProviderConfig{Name: "kvm-home", Type: "kvm", KVM: cfg}
-}
-
-func newTestProvider(fakes hostFakes, hosts ...string) *Provider {
-	p := newWithDialer(testConfig(hosts...), fakes.dial, nil)
-	p.now = func() time.Time { return testNow }
-	return p
-}
-
-// cacheImage pre-populates the pool with the configured image and its
-// marker so Create never touches the network.
-func cacheImage(api *fakeAPI) {
-	src := imageSource{Key: "debian-13", URL: testImageURL, Format: formatQcow2}
-	api.putVol("default", src.volumeName(), formatQcow2, []byte("qcow2"))
-	api.putVol("default", src.markerName(), formatRaw, []byte(testImageSHA()))
-}
-
-func readyHost() *fakeAPI {
-	api := newFakeAPI("default")
-	cacheImage(api)
-	return api
-}
-
-func testRequest() provider.CreateRequest {
-	return provider.CreateRequest{
-		Name:      testName,
-		UID:       testUID,
-		Namespace: "default",
-		ProxyName: "home-proxy-1",
-		Placement: provider.Placement{Zone: "nas1", MachineType: "small", Image: "debian-13"},
-		CloudInit: testCloudInit,
-		Port:      3128,
-	}
-}
-
-func metadataXML(t *testing.T, uid string, created time.Time) string {
-	t.Helper()
-	out, err := xml.Marshal(newInstanceMetadata(uid, created))
-	if err != nil {
-		t.Fatal(err)
-	}
-	return string(out)
-}
-
-func ipv4(addr string) libvirt.DomainIPAddr {
-	return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv4), Addr: addr, Prefix: 24}
-}
-
-func ipv6(addr string) libvirt.DomainIPAddr {
-	return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv6), Addr: addr, Prefix: 64}
-}
-
-// managedDomain seeds a domain as Create would have left it.
-func managedDomain(t *testing.T, api *fakeAPI, name, uid string, running bool, addrs map[uint32][]libvirt.DomainInterface) {
-	t.Helper()
-	api.putDomain(&fakeDomain{name: name, metadata: metadataXML(t, uid, testNow), running: running, addrs: addrs})
-	api.putVol("default", overlayVolumeName(name), formatQcow2, nil)
-	api.putVol("default", seedVolumeName(name), formatRaw, []byte("iso"))
-}
-
-func wantClass(t *testing.T, err error, class error) {
-	t.Helper()
-	if err == nil {
-		t.Fatalf("got nil error, want %v", class)
-	}
-	if !errors.Is(err, class) {
-		t.Fatalf("error class = %v, want %v (err: %v)", provider.Class(err), class, err)
-	}
-}
-
-func assertCreated(t *testing.T, api *fakeAPI) {
-	t.Helper()
-	dom := api.domain(testName)
-	if dom == nil {
-		t.Fatal("domain not defined")
-	}
-	if !dom.running {
-		t.Error("domain not running")
-	}
-	overlay := api.vol("default", overlayVolumeName(testName))
-	if overlay == nil {
-		t.Fatal("overlay volume missing")
-	}
-	if overlay.backing != "/var/lib/libvirt/images/base-debian-13.qcow2" || overlay.format != formatQcow2 {
-		t.Errorf("overlay = %+v, want qcow2 backed by the cached base", overlay)
-	}
-	if overlay.capacity != defaultDiskSizeGB*bytesPerGiB {
-		t.Errorf("overlay capacity = %d, want %d", overlay.capacity, defaultDiskSizeGB*bytesPerGiB)
-	}
-	seed := api.vol("default", seedVolumeName(testName))
-	if seed == nil {
-		t.Fatal("seed volume missing")
-	}
-	if seed.format != formatRaw || len(seed.data) == 0 || int64(len(seed.data)) != seed.capacity {
-		t.Errorf("seed = format %s, %d bytes, capacity %d; want raw with capacity == uploaded size", seed.format, len(seed.data), seed.capacity)
-	}
-}
-
-func TestCreate_freshHost(t *testing.T) {
-	t.Parallel()
-	api := readyHost()
-	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-
-	id, err := p.Create(context.Background(), testRequest())
-	if err != nil {
-		t.Fatalf("Create: %v", err)
-	}
-	if want := "hosts/nas1/domains/" + testName; id != want {
-		t.Errorf("providerID = %s, want %s", id, want)
-	}
-	assertCreated(t, api)
-
-	var d domainXML
-	if err := xml.Unmarshal([]byte(api.domain(testName).xml), &d); err != nil {
-		t.Fatal(err)
-	}
-	if d.VCPU != 2 || d.Memory.Value != 2048 || d.Memory.Unit != "MiB" {
-		t.Errorf("domain size = %d vcpu / %d %s, want 2 / 2048 MiB", d.VCPU, d.Memory.Value, d.Memory.Unit)
-	}
-	if d.Devices.Interfaces[0].Source.Bridge != "br0" {
-		t.Errorf("bridge = %q, want br0", d.Devices.Interfaces[0].Source.Bridge)
-	}
-	if m := d.Metadata.Instance; m.UID != testUID || !m.managed() || !m.createdAt().Equal(testNow) {
-		t.Errorf("metadata = %+v, want managed, uid %s, createdAt %s", m, testUID, testNow)
-	}
-	if api.hasCall("StorageVolDownload") || api.callCount("StorageVolUpload") != 1 {
-		t.Errorf("calls = %v, want no image download and exactly one upload (the seed)", api.calls)
-	}
-}
-
-func TestCreate_fetchesImageOnFirstUse(t *testing.T) {
-	t.Parallel()
-	srv := newImageServer(t)
-	api := newFakeAPI("default")
-	cfg := testConfig("nas1")
-	cfg.KVM.Images["debian-13"] = provider.KVMImage{URL: srv.URL + "/debian.qcow2", SHA256: testImageSHA()}
-	p := newWithDialer(cfg, hostFakes{"nas1": api}.dial, nil)
-
-	if _, err := p.Create(context.Background(), testRequest()); err != nil {
-		t.Fatalf("Create: %v", err)
-	}
-	if srv.hits.Load() != 1 {
-		t.Errorf("image server hits = %d, want 1", srv.hits.Load())
-	}
-	base := api.vol("default", "base-debian-13.qcow2")
-	if base == nil || string(base.data) != string(testImage) {
-		t.Fatal("base image not cached in the pool")
-	}
-	if api.domain(testName) == nil || !api.domain(testName).running {
-		t.Error("domain not running after image fetch")
-	}
-
-	// A second Proxy with the same image on the same host reuses the cache.
-	req := testRequest()
-	req.Name, req.UID = "proxy-second", "uid-2"
-	if _, err := p.Create(context.Background(), req); err != nil {
-		t.Fatalf("second Create: %v", err)
-	}
-	if srv.hits.Load() != 1 {
-		t.Errorf("image server hits after second Create = %d, want still 1", srv.hits.Load())
-	}
-}
-
-func TestCreate_isIdempotentWhenComplete(t *testing.T) {
-	t.Parallel()
-	api := readyHost()
-	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-	if _, err := p.Create(context.Background(), testRequest()); err != nil {
-		t.Fatal(err)
-	}
-	before := len(api.calls)
-
-	id, err := p.Create(context.Background(), testRequest())
-	if err != nil {
-		t.Fatalf("second Create: %v", err)
-	}
-	if id != "hosts/nas1/domains/"+testName {
-		t.Errorf("providerID = %s", id)
-	}
-	for _, m := range []string{"DomainDefineXML", "DomainCreate", "StorageVolUpload"} {
-		if n := api.callCount(m); n != 1 {
-			t.Errorf("%s called %d times across two Creates, want 1", m, n)
-		}
-	}
-	if len(api.calls) == before {
-		t.Error("second Create made no calls at all; expected lookups")
-	}
-}
-
-// Each case breaks Create after a different step, then re-runs it and
-// expects the second attempt to pick up where the first stopped without
-// duplicating what already exists.
-func TestCreate_resumesAfterPartialFailure(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name     string
-		sabotage func(api *fakeAPI)
-		class    error
-		check    func(t *testing.T, api *fakeAPI)
-	}{
-		{
-			name: "seed upload fails after overlay exists",
-			sabotage: func(api *fakeAPI) {
-				api.uploadHook = func(vol string, _ io.Reader) error {
-					api.uploadHook = nil
-					return io.ErrUnexpectedEOF
-				}
-			},
-			class: provider.ErrTransient,
-			check: func(t *testing.T, api *fakeAPI) {
-				if n := api.callCount("StorageVolUpload"); n != 2 {
-					t.Errorf("seed uploads = %d, want 2 (failed, then redone)", n)
-				}
-				if n := api.callCount("StorageVolCreateXML"); n != 4 {
-					t.Errorf("volume creates = %d, want 4 (overlay, seed; overlay rejected as existing, seed again)", n)
-				}
-				if names := api.volNames("default"); len(names) != 4 {
-					t.Errorf("pool holds %v, want base, marker, one overlay, one seed", names)
-				}
-			},
-		},
-		{
-			name:     "define fails after volumes exist",
-			sabotage: func(api *fakeAPI) { api.failOnce["DomainDefineXML"] = lvError(libvirt.ErrInternalError, "boom") },
-			class:    provider.ErrTransient,
-			check: func(t *testing.T, api *fakeAPI) {
-				if n := api.callCount("StorageVolUpload"); n != 2 {
-					t.Errorf("seed uploads = %d, want 2 (seed rebuilt while no domain exists)", n)
-				}
-				if n := api.callCount("DomainDefineXML"); n != 2 {
-					t.Errorf("defines = %d, want 2", n)
-				}
-			},
-		},
-		{
-			name:     "start fails after define",
-			sabotage: func(api *fakeAPI) { api.failOnce["DomainCreate"] = lvError(libvirt.ErrInternalError, "boom") },
-			class:    provider.ErrTransient,
-			check: func(t *testing.T, api *fakeAPI) {
-				if n := api.callCount("DomainDefineXML"); n != 1 {
-					t.Errorf("defines = %d, want 1 (existing domain reused)", n)
-				}
-				if n := api.callCount("DomainCreate"); n != 2 {
-					t.Errorf("starts = %d, want 2", n)
-				}
-			},
-		},
-		{
-			name:     "transport drops mid-create",
-			sabotage: func(api *fakeAPI) { api.failOnce["DomainLookupByName"] = io.EOF },
-			class:    provider.ErrTransient,
-			check: func(t *testing.T, api *fakeAPI) {
-				if n := api.callCount("DomainDefineXML"); n != 1 {
-					t.Errorf("defines = %d, want 1", n)
-				}
-			},
-		},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			api := readyHost()
-			tc.sabotage(api)
-			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-
-			_, err := p.Create(context.Background(), testRequest())
-			wantClass(t, err, tc.class)
-			if _, err := p.Create(context.Background(), testRequest()); err != nil {
-				t.Fatalf("retry Create: %v", err)
-			}
-			assertCreated(t, api)
-			tc.check(t, api)
-		})
-	}
-}
-
-// A reconcile cancelled mid-upload leaves a seed volume whose content is
-// short; the compensating delete cannot run because the pool has already
-// closed the connection. The next Create must not trust that volume.
-func TestCreate_replacesLeftoverSeed(t *testing.T) {
-	t.Parallel()
-	api := readyHost()
-	api.putVol("default", seedVolumeName(testName), formatRaw, []byte("trunc"))
-	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-
-	if _, err := p.Create(context.Background(), testRequest()); err != nil {
-		t.Fatal(err)
-	}
-	assertCreated(t, api)
-	if seed := api.vol("default", seedVolumeName(testName)); string(seed.data) == "trunc" {
-		t.Error("leftover seed was kept")
-	}
-	if n := api.callCount("StorageVolUpload"); n != 1 {
-		t.Errorf("uploads = %d, want 1 (the rebuilt seed)", n)
-	}
-}
-
-func TestCreate_startsStoppedDomain(t *testing.T) {
-	t.Parallel()
-	api := readyHost()
-	managedDomain(t, api, testName, testUID, false, nil)
-	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-
-	if _, err := p.Create(context.Background(), testRequest()); err != nil {
-		t.Fatal(err)
-	}
-	if !api.domain(testName).running {
-		t.Error("domain still stopped")
-	}
-	if api.hasCall("DomainDefineXML") || api.hasCall("StorageVolUpload") || api.hasCall("StorageVolDelete") {
-		t.Errorf("calls = %v, want no define, upload or volume delete for an existing domain", api.calls)
-	}
-}
-
-func TestCreate_singleHostNeedsNoZone(t *testing.T) {
-	t.Parallel()
-	api := readyHost()
-	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-	req := testRequest()
-	req.Placement.Zone = ""
-
-	id, err := p.Create(context.Background(), req)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if id != "hosts/nas1/domains/"+testName {
-		t.Errorf("providerID = %s, want the single host filled in", id)
-	}
-}
-
-func TestCreate_placementErrors(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name  string
-		hosts []string
-		edit  func(*provider.CreateRequest)
-		want  string
-	}{
-		{"unknown host", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Zone = "nas9" }, "not in kvm.hosts"},
-		{"empty zone with two hosts", []string{"nas1", "nas2"}, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "placement.zone is required"},
-		{"empty zone with no hosts", nil, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "kvm.hosts is empty"},
-		{"unknown machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "huge" }, "not in kvm.machineTypes"},
-		{"empty machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "" }, "not in kvm.machineTypes"},
-		{"unknown image", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Image = "windows" }, "neither a key"},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			fakes := hostFakes{}
-			for _, h := range tc.hosts {
-				fakes[h] = readyHost()
-			}
-			p := newTestProvider(fakes, tc.hosts...)
-			req := testRequest()
-			tc.edit(&req)
-
-			_, err := p.Create(context.Background(), req)
-			wantClass(t, err, provider.ErrPermanent)
-			if !strings.Contains(err.Error(), tc.want) {
-				t.Errorf("error = %q, want it to mention %q", err, tc.want)
-			}
-			for _, api := range fakes {
-				if len(api.calls) != 0 {
-					t.Errorf("host was called: %v", api.calls)
-				}
-			}
-		})
-	}
-}
-
-func TestCreate_hostFailures(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name  string
-		setup func() (hostFakes, string)
-		class error
-	}{
-		{
-			name:  "host unreachable",
-			setup: func() (hostFakes, string) { return hostFakes{}, "nas1" },
-			class: provider.ErrTransient,
-		},
-		{
-			name: "pool missing",
-			setup: func() (hostFakes, string) {
-				api := newFakeAPI("other")
-				return hostFakes{"nas1": api}, "nas1"
-			},
-			class: provider.ErrPermanent,
-		},
-		{
-			name: "domain xml rejected",
-			setup: func() (hostFakes, string) {
-				api := readyHost()
-				api.fail["DomainDefineXML"] = lvError(libvirt.ErrConfigUnsupported, "no q35 here")
-				return hostFakes{"nas1": api}, "nas1"
-			},
-			class: provider.ErrPermanent,
-		},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			fakes, host := tc.setup()
-			p := newTestProvider(fakes, host)
-			_, err := p.Create(context.Background(), testRequest())
-			wantClass(t, err, tc.class)
-		})
-	}
-}
-
-func TestCreate_cancelledContext(t *testing.T) {
-	t.Parallel()
-	api := readyHost()
-	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-
-	_, err := p.Create(ctx, testRequest())
-	wantClass(t, err, provider.ErrTransient)
-	if !errors.Is(err, context.Canceled) {
-		t.Errorf("error = %v, want it to unwrap to context.Canceled", err)
-	}
-}
-
-func TestGet(t *testing.T) {
-	t.Parallel()
-	arp := map[uint32][]libvirt.DomainInterface{
-		uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Name: "vnet0", Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.50")}}},
-	}
-	agentOnly := map[uint32][]libvirt.DomainInterface{
-		uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
-			{Name: "lo", Addrs: []libvirt.DomainIPAddr{ipv4("127.0.0.1"), ipv6("::1")}},
-			{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv6("fe80::1"), ipv4("192.168.1.51")}},
-		},
-	}
-	linkLocalOnly := map[uint32][]libvirt.DomainInterface{
-		uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
-			{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv4("169.254.10.10")}},
-		},
-	}
-	cases := []struct {
-		name    string
-		running bool
-		addrs   map[uint32][]libvirt.DomainInterface
-		state   provider.InstanceState
-		ip      string
-	}{
-		{"running with arp address", true, arp, provider.StateRunning, "192.168.1.50"},
-		{"running, agent only, skips loopback and v6", true, agentOnly, provider.StateRunning, "192.168.1.51"},
-		{"running, no address yet", true, nil, provider.StateProvisioning, ""},
-		{"running, only link-local", true, linkLocalOnly, provider.StateProvisioning, ""},
-		{"shut off", false, arp, provider.StateStopped, ""},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			api := readyHost()
-			managedDomain(t, api, testName, testUID, tc.running, tc.addrs)
-			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-
-			inst, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
-			if err != nil {
-				t.Fatalf("Get: %v", err)
-			}
-			want := provider.Instance{ID: "hosts/nas1/domains/" + testName, IP: tc.ip, State: tc.state, UID: testUID, CreatedAt: testNow}
-			if *inst != want {
-				t.Errorf("instance = %+v, want %+v", *inst, want)
-			}
-		})
-	}
-}
-
-func TestGet_unmanagedDomainHasNoUID(t *testing.T) {
-	t.Parallel()
-	api := readyHost()
-	api.putDomain(&fakeDomain{name: "hand-made", running: true})
-	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-
-	inst, err := p.Get(context.Background(), "hosts/nas1/domains/hand-made")
-	if err != nil {
-		t.Fatalf("Get: %v", err)
-	}
-	if inst.UID != "" || !inst.CreatedAt.IsZero() {
-		t.Errorf("instance = %+v, want empty UID and zero CreatedAt", inst)
-	}
-}
-
-func TestGet_errors(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name  string
-		id    string
-		setup func(t *testing.T, api *fakeAPI)
-		class error
-	}{
-		{"missing domain", "hosts/nas1/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrNotFound},
-		{"malformed id", "zones/x/instances/y", func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
-		{"host not configured", "hosts/nas9/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
-		{
-			"state read fails", "hosts/nas1/domains/" + testName,
-			func(t *testing.T, api *fakeAPI) {
-				managedDomain(t, api, testName, testUID, true, nil)
-				api.fail["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
-			},
-			provider.ErrTransient,
-		},
-		{
-			"corrupt metadata", "hosts/nas1/domains/" + testName,
-			func(_ *testing.T, api *fakeAPI) {
-				api.putDomain(&fakeDomain{name: testName, metadata: "<instance", running: true})
-			},
-			provider.ErrPermanent,
-		},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			api := readyHost()
-			tc.setup(t, api)
-			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-			_, err := p.Get(context.Background(), tc.id)
-			wantClass(t, err, tc.class)
-		})
-	}
-}
-
-func TestGet_hostUnreachable(t *testing.T) {
-	t.Parallel()
-	p := newTestProvider(hostFakes{}, "nas1")
-	_, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
-	wantClass(t, err, provider.ErrTransient)
-}
-
-func assertGone(t *testing.T, api *fakeAPI) {
-	t.Helper()
-	if api.domain(testName) != nil {
-		t.Error("domain still defined")
-	}
-	if names := api.volNames("default"); len(names) != 2 {
-		t.Errorf("pool holds %v, want only the cached base image and its marker", names)
-	}
-}
-
-func TestDelete(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name  string
-		setup func(t *testing.T, api *fakeAPI)
-		check func(t *testing.T, api *fakeAPI)
-	}{
-		{
-			name:  "running domain",
-			setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, true, nil) },
-			check: func(t *testing.T, api *fakeAPI) {
-				if !api.hasCall("DomainDestroy") || !api.hasCall("DomainUndefineFlags") {
-					t.Errorf("calls = %v, want destroy and undefine", api.calls)
-				}
-			},
-		},
-		{
-			name:  "stopped domain",
-			setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, false, nil) },
-			check: func(t *testing.T, api *fakeAPI) {
-				if !api.hasCall("DomainUndefineFlags") {
-					t.Errorf("calls = %v, want undefine despite destroy being invalid", api.calls)
-				}
-			},
-		},
-		{
-			name: "domain gone, volumes left",
-			setup: func(_ *testing.T, api *fakeAPI) {
-				api.putVol("default", overlayVolumeName(testName), formatQcow2, nil)
-				api.putVol("default", seedVolumeName(testName), formatRaw, nil)
-			},
-			check: func(t *testing.T, api *fakeAPI) {
-				if api.hasCall("DomainDestroy") || api.hasCall("DomainUndefineFlags") {
-					t.Errorf("calls = %v, want no domain operations", api.calls)
-				}
-			},
-		},
-		{
-			name:  "nothing exists",
-			setup: func(*testing.T, *fakeAPI) {},
-			check: func(t *testing.T, api *fakeAPI) {
-				if api.hasCall("StorageVolDelete") {
-					t.Errorf("calls = %v, want no deletes", api.calls)
-				}
-			},
-		},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			api := readyHost()
-			tc.setup(t, api)
-			p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-
-			if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
-				t.Fatalf("Delete: %v", err)
-			}
-			assertGone(t, api)
-			tc.check(t, api)
-			if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
-				t.Fatalf("second Delete: %v", err)
-			}
-		})
-	}
-}
-
-func TestDelete_missingPoolIsNotAnError(t *testing.T) {
-	t.Parallel()
-	api := newFakeAPI("other")
-	api.putDomain(&fakeDomain{name: testName, metadata: metadataXML(t, testUID, testNow), running: true})
-	p := newTestProvider(hostFakes{"nas1": api}, "nas1")
-
-	if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
-		t.Fatalf("Delete: %v", err)
-	}
-	if api.domain(testName) != nil {
-		t.Error("domain still defined")
-	}
-}
-
-func TestDelete_errors(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name  string
-		id    string
-		fakes func(t *testing.T) hostFakes
-		class error
-	}{
-		{"malformed id", "nope", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
-		{"host not configured", "hosts/nas9/domains/x", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
-		{"host unreachable", "hosts/nas1/domains/x", func(*testing.T) hostFakes { return hostFakes{} }, provider.ErrTransient},
-		{
-			"undefine fails", "hosts/nas1/domains/" + testName,
-			func(t *testing.T) hostFakes {
-				api := readyHost()
-				managedDomain(t, api, testName, testUID, true, nil)
-				api.fail["DomainUndefineFlags"] = lvError(libvirt.ErrInternalError, "boom")
-				return hostFakes{"nas1": api}
-			},
-			provider.ErrTransient,
-		},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			p := newTestProvider(tc.fakes(t), "nas1")
-			wantClass(t, p.Delete(context.Background(), tc.id), tc.class)
-		})
-	}
-}
-
-func TestListByTag_acrossHosts(t *testing.T) {
-	t.Parallel()
-	nas1, nas2 := readyHost(), readyHost()
-	managedDomain(t, nas1, "proxy-one", "uid-1", true, map[uint32][]libvirt.DomainInterface{
-		uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.10")}}},
-	})
-	nas1.putDomain(&fakeDomain{name: "hand-made", running: true})
-	managedDomain(t, nas2, "proxy-two", "uid-2", false, nil)
-	nas2.putDomain(&fakeDomain{name: "foreign", metadata: `<instance xmlns="` + metadataURI + `"><managed>false</managed></instance>`})
-	p := newTestProvider(hostFakes{"nas1": nas1, "nas2": nas2}, "nas1", "nas2")
-
-	got, err := p.ListByTag(context.Background())
-	if err != nil {
-		t.Fatalf("ListByTag: %v", err)
-	}
-	want := []provider.Instance{
-		{ID: "hosts/nas1/domains/proxy-one", IP: "192.168.1.10", State: provider.StateRunning, UID: "uid-1", CreatedAt: testNow},
-		{ID: "hosts/nas2/domains/proxy-two", State: provider.StateStopped, UID: "uid-2", CreatedAt: testNow},
-	}
-	if len(got) != len(want) {
-		t.Fatalf("got %d instances %+v, want %d", len(got), got, len(want))
-	}
-	for i := range want {
-		if got[i] != want[i] {
-			t.Errorf("instance[%d] = %+v, want %+v", i, got[i], want[i])
-		}
-	}
-}
-
-func TestListByTag_oneHostDownFailsSweep(t *testing.T) {
-	t.Parallel()
-	nas1 := readyHost()
-	managedDomain(t, nas1, "proxy-one", "uid-1", true, nil)
-	p := newTestProvider(hostFakes{"nas1": nas1}, "nas1", "nas2")
-
-	got, err := p.ListByTag(context.Background())
-	wantClass(t, err, provider.ErrTransient)
-	if got != nil {
-		t.Errorf("got %+v alongside the error, want nil (no partial view)", got)
-	}
-}
-
-// A domain that cannot be read is left out of the sweep rather than
-// failing it: GC never deletes what it does not see, so skipping is the
-// safe direction, while one bad domain blocking GC for all others is not.
-func TestListByTag_skipsUnreadableDomain(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name  string
-		setup func(t *testing.T, api *fakeAPI)
-	}{
-		{
-			"deleted mid-sweep",
-			func(t *testing.T, api *fakeAPI) {
-				managedDomain(t, api, "proxy-one", "uid-1", true, nil)
-				api.failOnce["DomainGetMetadata"] = lvError(libvirt.ErrNoDomain, "gone")
-			},
-		},
-		{
-			"corrupt metadata",
-			func(_ *testing.T, api *fakeAPI) {
-				api.putDomain(&fakeDomain{name: "proxy-one", metadata: "<instance", running: true})
-			},
-		},
-		{
-			"state unreadable",
-			func(t *testing.T, api *fakeAPI) {
-				managedDomain(t, api, "proxy-one", "uid-1", true, nil)
-				api.failOnce["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
-			},
-		},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			nas1 := readyHost()
-			tc.setup(t, nas1)
-			managedDomain(t, nas1, "proxy-two", "uid-2", false, nil)
-			p := newTestProvider(hostFakes{"nas1": nas1}, "nas1")
-
-			got, err := p.ListByTag(context.Background())
-			if err != nil {
-				t.Fatalf("ListByTag: %v", err)
-			}
-			if len(got) != 1 || got[0].UID != "uid-2" {
-				t.Errorf("got %+v, want only proxy-two", got)
-			}
-		})
-	}
-}
-
-func TestListByTag_noHosts(t *testing.T) {
-	t.Parallel()
-	p := newTestProvider(hostFakes{})
-	got, err := p.ListByTag(context.Background())
-	if err != nil || len(got) != 0 {
-		t.Errorf("got %v, %v; want empty, nil", got, err)
-	}
-}
-
-func TestMapState(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		state libvirt.DomainState
-		ip    string
-		want  provider.InstanceState
-	}{
-		{libvirt.DomainRunning, "10.0.0.1", provider.StateRunning},
-		{libvirt.DomainRunning, "", provider.StateProvisioning},
-		{libvirt.DomainBlocked, "10.0.0.1", provider.StateRunning},
-		{libvirt.DomainPaused, "10.0.0.1", provider.StateStopped},
-		{libvirt.DomainShutdown, "", provider.StateStopped},
-		{libvirt.DomainShutoff, "", provider.StateStopped},
-		{libvirt.DomainCrashed, "", provider.StateStopped},
-		{libvirt.DomainPmsuspended, "", provider.StateStopped},
-		{libvirt.DomainNostate, "", provider.StateProvisioning},
-	}
-	for _, tc := range cases {
-		t.Run(fmt.Sprintf("%s/%q", tc.state, tc.ip), func(t *testing.T) {
-			t.Parallel()
-			if got := mapState(tc.state, tc.ip); got != tc.want {
-				t.Errorf("mapState(%s, %q) = %s, want %s", tc.state, tc.ip, got, tc.want)
-			}
-		})
-	}
-}
-
-func TestNew_badKeyPathFailsFast(t *testing.T) {
-	t.Parallel()
-	cfg := testConfig("nas1")
-	cfg.KVM.SSHKeyFile = t.TempDir() + "/missing"
-	if _, err := New(context.Background(), cfg); err == nil {
-		t.Fatal("New succeeded with a missing key file")
-	}
-}

internal/provider/kvm/metadata.go

@@ -1,57 +0,0 @@
-package kvm
-
-import (
-	"encoding/xml"
-	"fmt"
-	"time"
-
-	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
-)
-
-// metadataURI namespaces the operator's element inside the domain's
-// <metadata>. libvirt has no labels or tags, but it stores any namespaced XML
-// there and hands it back through DomainGetMetadata, which is how the GC
-// contract (managed flag + owning UID) and the creation time survive across
-// operator restarts.
-const metadataURI = "https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm"
-
-type instanceMetadata struct {
-	XMLName   xml.Name `xml:"https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm instance"`
-	Managed   string   `xml:"managed"`
-	UID       string   `xml:"uid"`
-	CreatedAt string   `xml:"createdAt"`
-}
-
-func newInstanceMetadata(uid string, createdAt time.Time) instanceMetadata {
-	return instanceMetadata{
-		Managed:   provider.LabelManagedYes,
-		UID:       uid,
-		CreatedAt: createdAt.UTC().Format(time.RFC3339),
-	}
-}
-
-// parseInstanceMetadata decodes the element as DomainGetMetadata returns it.
-// libvirt may re-serialise it with a namespace prefix rather than the default
-// namespace it was defined with; encoding/xml matches on namespace URI, so
-// both spellings decode.
-func parseInstanceMetadata(s string) (instanceMetadata, error) {
-	var m instanceMetadata
-	if err := xml.Unmarshal([]byte(s), &m); err != nil {
-		return instanceMetadata{}, fmt.Errorf("parsing domain metadata: %w", err)
-	}
-	return m, nil
-}
-
-func (m instanceMetadata) managed() bool {
-	return m.Managed == provider.LabelManagedYes
-}
-
-// createdAt returns the zero time on a malformed timestamp: orphan GC treats
-// zero as "old", so a corrupt value can never protect a domain forever.
-func (m instanceMetadata) createdAt() time.Time {
-	t, err := time.Parse(time.RFC3339, m.CreatedAt)
-	if err != nil {
-		return time.Time{}
-	}
-	return t
-}

internal/provider/kvm/metadata_test.go

@@ -1,79 +0,0 @@
-package kvm
-
-import (
-	"encoding/xml"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestInstanceMetadata_roundTrip(t *testing.T) {
-	t.Parallel()
-	created := time.Date(2026, 9, 4, 12, 30, 0, 0, time.FixedZone("CEST", 2*3600))
-	raw, err := xml.Marshal(newInstanceMetadata("uid-1", created))
-	if err != nil {
-		t.Fatalf("Marshal() error = %v", err)
-	}
-	if !strings.Contains(string(raw), `xmlns="`+metadataURI+`"`) {
-		t.Errorf("marshalled metadata lacks the namespace declaration: %s", raw)
-	}
-	m, err := parseInstanceMetadata(string(raw))
-	if err != nil {
-		t.Fatalf("parseInstanceMetadata() error = %v", err)
-	}
-	if !m.managed() || m.UID != "uid-1" {
-		t.Errorf("parsed = %+v, want managed with uid-1", m)
-	}
-	if got := m.createdAt(); !got.Equal(created) || got.Location() != time.UTC {
-		t.Errorf("createdAt() = %v, want %v in UTC", got, created)
-	}
-}
-
-func TestParseInstanceMetadata_libvirtPrefixedForm(t *testing.T) {
-	t.Parallel()
-	// libvirt re-serialises the stored element with whatever prefix it
-	// chooses; only the namespace URI is contractual.
-	in := `<op:instance xmlns:op="` + metadataURI + `">` +
-		`<op:managed>true</op:managed><op:uid>uid-2</op:uid>` +
-		`<op:createdAt>2026-09-04T10:00:00Z</op:createdAt></op:instance>`
-	m, err := parseInstanceMetadata(in)
-	if err != nil {
-		t.Fatalf("parseInstanceMetadata() error = %v", err)
-	}
-	if !m.managed() || m.UID != "uid-2" || m.createdAt().Hour() != 10 {
-		t.Errorf("parsed = %+v", m)
-	}
-}
-
-func TestParseInstanceMetadata_notManaged(t *testing.T) {
-	t.Parallel()
-	tests := []struct {
-		name    string
-		in      string
-		wantErr bool
-	}{
-		{"managed false", `<instance xmlns="` + metadataURI + `"><managed>false</managed><uid>u</uid></instance>`, false},
-		{"foreign element", `<foo xmlns="urn:other"><managed>true</managed></foo>`, true},
-		{"garbage", `not xml`, true},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			m, err := parseInstanceMetadata(tc.in)
-			if (err != nil) != tc.wantErr {
-				t.Fatalf("error = %v, wantErr %v", err, tc.wantErr)
-			}
-			if m.managed() {
-				t.Errorf("managed() = true for %q", tc.in)
-			}
-		})
-	}
-}
-
-func TestInstanceMetadata_createdAtMalformedIsZero(t *testing.T) {
-	t.Parallel()
-	m := instanceMetadata{CreatedAt: "yesterday"}
-	if !m.createdAt().IsZero() {
-		t.Errorf("createdAt() = %v, want zero time", m.createdAt())
-	}
-}

internal/provider/kvm/seed.go

@@ -1,53 +0,0 @@
-package kvm
-
-import (
-	"bytes"
-	"fmt"
-	"strings"
-
-	"github.com/kdomanski/iso9660"
-)
-
-// seedLabel is the volume label cloud-init's NoCloud datasource looks for.
-const seedLabel = "cidata"
-
-// defaultVendorData is the cloud-init vendor-data shipped on every seed
-// unless the config overrides it. cloud-init merges it under the Proxy's own
-// user-data, so stock images grow a guest agent (which lets the provider
-// read the VM's IP before ARP sees traffic) without the user's cloud-init
-// having to know about it.
-const defaultVendorData = `#cloud-config
-packages:
-  - qemu-guest-agent
-runcmd:
-  - [systemctl, enable, --now, qemu-guest-agent]
-`
-
-// buildSeedISO renders a NoCloud seed: user-data verbatim, vendor-data as
-// given, and a meta-data whose instance-id is the deterministic VM name, so
-// cloud-init never mistakes a reboot for a new instance. The ISO's volume
-// descriptor carries the build time, so two builds differ in a few
-// timestamp bytes; the files and label are identical.
-func buildSeedISO(name, userData, vendorData string) ([]byte, error) {
-	w, err := iso9660.NewWriter()
-	if err != nil {
-		return nil, fmt.Errorf("creating iso writer: %w", err)
-	}
-	defer w.Cleanup() //nolint:errcheck // temp-dir cleanup; nothing to do on failure
-
-	files := []struct{ name, body string }{
-		{"user-data", userData},
-		{"meta-data", fmt.Sprintf("instance-id: %s\nlocal-hostname: %s\n", name, name)},
-		{"vendor-data", vendorData},
-	}
-	for _, f := range files {
-		if err := w.AddFile(strings.NewReader(f.body), f.name); err != nil {
-			return nil, fmt.Errorf("adding %s to seed iso: %w", f.name, err)
-		}
-	}
-	var buf bytes.Buffer
-	if err := w.WriteTo(&buf, seedLabel); err != nil {
-		return nil, fmt.Errorf("writing seed iso: %w", err)
-	}
-	return buf.Bytes(), nil
-}

internal/provider/kvm/seed_test.go

@@ -1,100 +0,0 @@
-package kvm
-
-import (
-	"bytes"
-	"io"
-	"maps"
-	"strings"
-	"testing"
-
-	"github.com/kdomanski/iso9660"
-)
-
-func readSeed(t *testing.T, iso []byte) (label string, files map[string]string) {
-	t.Helper()
-	img, err := iso9660.OpenImage(bytes.NewReader(iso))
-	if err != nil {
-		t.Fatalf("OpenImage() error = %v", err)
-	}
-	label, err = img.Label()
-	if err != nil {
-		t.Fatalf("Label() error = %v", err)
-	}
-	root, err := img.RootDir()
-	if err != nil {
-		t.Fatalf("RootDir() error = %v", err)
-	}
-	children, err := root.GetChildren()
-	if err != nil {
-		t.Fatalf("GetChildren() error = %v", err)
-	}
-	files = make(map[string]string, len(children))
-	for _, c := range children {
-		body, err := io.ReadAll(c.Reader())
-		if err != nil {
-			t.Fatalf("reading %s: %v", c.Name(), err)
-		}
-		files[c.Name()] = string(body)
-	}
-	return label, files
-}
-
-func TestBuildSeedISO_contents(t *testing.T) {
-	t.Parallel()
-	userData := "#cloud-config\npackages: [squid]\n"
-	iso, err := buildSeedISO("proxy-abc123def456ghij", userData, defaultVendorData)
-	if err != nil {
-		t.Fatalf("buildSeedISO() error = %v", err)
-	}
-	label, files := readSeed(t, iso)
-	if !strings.EqualFold(label, seedLabel) {
-		t.Errorf("label = %q, want %q", label, seedLabel)
-	}
-	if len(files) != 3 {
-		t.Errorf("files = %v, want exactly user-data, meta-data, vendor-data", files)
-	}
-	if files["user-data"] != userData {
-		t.Errorf("user-data = %q, want verbatim cloud-init", files["user-data"])
-	}
-	if files["vendor-data"] != defaultVendorData {
-		t.Errorf("vendor-data = %q", files["vendor-data"])
-	}
-	if got, want := files["meta-data"], "instance-id: proxy-abc123def456ghij\nlocal-hostname: proxy-abc123def456ghij\n"; got != want {
-		t.Errorf("meta-data = %q, want %q", got, want)
-	}
-}
-
-// The writer stamps the build time into the volume descriptor, so two
-// builds are not byte-identical; what must not change between them is
-// what cloud-init reads: the label and the three files.
-func TestBuildSeedISO_deterministicContent(t *testing.T) {
-	t.Parallel()
-	a, err := buildSeedISO("proxy-x", "#cloud-config\n", "")
-	if err != nil {
-		t.Fatalf("first buildSeedISO() error = %v", err)
-	}
-	b, err := buildSeedISO("proxy-x", "#cloud-config\n", "")
-	if err != nil {
-		t.Fatalf("second buildSeedISO() error = %v", err)
-	}
-	labelA, filesA := readSeed(t, a)
-	labelB, filesB := readSeed(t, b)
-	if labelA != labelB || !maps.Equal(filesA, filesB) {
-		t.Errorf("two builds for the same inputs differ: %q %v vs %q %v", labelA, filesA, labelB, filesB)
-	}
-	if len(a) != len(b) {
-		t.Errorf("sizes differ: %d vs %d; the seed volume is created with the ISO's exact size", len(a), len(b))
-	}
-}
-
-func TestBuildSeedISO_emptyUserData(t *testing.T) {
-	t.Parallel()
-	iso, err := buildSeedISO("proxy-x", "", defaultVendorData)
-	if err != nil {
-		t.Fatalf("buildSeedISO() error = %v", err)
-	}
-	_, files := readSeed(t, iso)
-	if body, ok := files["user-data"]; !ok || body != "" {
-		t.Errorf("user-data = %q (present=%v), want an empty file to still exist", body, ok)
-	}
-}

internal/provider/kvm/volume.go

@@ -1,78 +0,0 @@
-package kvm
-
-import (
-	"encoding/xml"
-	"fmt"
-)
-
-const bytesPerGiB = 1 << 30
-
-func overlayVolumeName(name string) string { return name + ".qcow2" }
-
-func seedVolumeName(name string) string { return name + "-seed.iso" }
-
-// volumeSpec describes a storage volume to create. A non-empty BackingPath
-// makes it a copy-on-write overlay of that file.
-type volumeSpec struct {
-	Name          string
-	SizeBytes     int64
-	Format        string
-	BackingPath   string
-	BackingFormat string
-}
-
-func overlayVolume(name string, sizeGB int64, basePath, baseFormat string) volumeSpec {
-	return volumeSpec{
-		Name:          overlayVolumeName(name),
-		SizeBytes:     sizeGB * bytesPerGiB,
-		Format:        formatQcow2,
-		BackingPath:   basePath,
-		BackingFormat: baseFormat,
-	}
-}
-
-func rawVolume(name string, sizeBytes int64) volumeSpec {
-	return volumeSpec{Name: name, SizeBytes: sizeBytes, Format: formatRaw}
-}
-
-// buildVolumeXML renders the volume for StorageVolCreateXML. Capacity is
-// always given in bytes so no libvirt unit-suffix ambiguity (G vs GiB) can
-// creep in.
-func buildVolumeXML(v volumeSpec) (string, error) {
-	x := volumeXML{
-		Type:     "file",
-		Name:     v.Name,
-		Capacity: sizeXML{Unit: "bytes", Value: v.SizeBytes},
-		Target:   volumeTargetXML{Format: formatXML{Type: v.Format}},
-	}
-	if v.BackingPath != "" {
-		x.BackingStore = &backingStoreXML{Path: v.BackingPath, Format: formatXML{Type: v.BackingFormat}}
-	}
-	out, err := xml.Marshal(x)
-	if err != nil {
-		return "", fmt.Errorf("marshalling volume xml: %w", err)
-	}
-	return string(out), nil
-}
-
-type volumeXML struct {
-	XMLName      xml.Name         `xml:"volume"`
-	Type         string           `xml:"type,attr"`
-	Name         string           `xml:"name"`
-	Capacity     sizeXML          `xml:"capacity"`
-	Target       volumeTargetXML  `xml:"target"`
-	BackingStore *backingStoreXML `xml:"backingStore"`
-}
-
-type volumeTargetXML struct {
-	Format formatXML `xml:"format"`
-}
-
-type formatXML struct {
-	Type string `xml:"type,attr"`
-}
-
-type backingStoreXML struct {
-	Path   string    `xml:"path"`
-	Format formatXML `xml:"format"`
-}

internal/provider/kvm/volume_test.go

@@ -1,62 +0,0 @@
-package kvm
-
-import (
-	"encoding/xml"
-	"strings"
-	"testing"
-)
-
-func TestVolumeNames(t *testing.T) {
-	t.Parallel()
-	if got := overlayVolumeName("proxy-x"); got != "proxy-x.qcow2" {
-		t.Errorf("overlayVolumeName = %q", got)
-	}
-	if got := seedVolumeName("proxy-x"); got != "proxy-x-seed.iso" {
-		t.Errorf("seedVolumeName = %q", got)
-	}
-}
-
-func TestBuildVolumeXML_overlay(t *testing.T) {
-	t.Parallel()
-	raw, err := buildVolumeXML(overlayVolume("proxy-x", 10, "/var/lib/libvirt/images/base-debian-13.qcow2", "qcow2"))
-	if err != nil {
-		t.Fatalf("buildVolumeXML() error = %v", err)
-	}
-	var v volumeXML
-	if err := xml.Unmarshal([]byte(raw), &v); err != nil {
-		t.Fatalf("output is not well-formed XML: %v", err)
-	}
-	if v.Type != "file" || v.Name != "proxy-x.qcow2" {
-		t.Errorf("type/name = %s/%s", v.Type, v.Name)
-	}
-	if v.Capacity.Unit != "bytes" || v.Capacity.Value != 10*bytesPerGiB {
-		t.Errorf("capacity = %d %s, want %d bytes", v.Capacity.Value, v.Capacity.Unit, 10*bytesPerGiB)
-	}
-	if v.Target.Format.Type != "qcow2" {
-		t.Errorf("format = %q", v.Target.Format.Type)
-	}
-	if v.BackingStore == nil {
-		t.Fatalf("backingStore missing:\n%s", raw)
-	}
-	if v.BackingStore.Path != "/var/lib/libvirt/images/base-debian-13.qcow2" || v.BackingStore.Format.Type != "qcow2" {
-		t.Errorf("backingStore = %+v", v.BackingStore)
-	}
-}
-
-func TestBuildVolumeXML_rawHasNoBackingStore(t *testing.T) {
-	t.Parallel()
-	raw, err := buildVolumeXML(rawVolume("proxy-x-seed.iso", 372736))
-	if err != nil {
-		t.Fatalf("buildVolumeXML() error = %v", err)
-	}
-	if strings.Contains(raw, "backingStore") {
-		t.Errorf("raw volume must not declare a backing store:\n%s", raw)
-	}
-	var v volumeXML
-	if err := xml.Unmarshal([]byte(raw), &v); err != nil {
-		t.Fatalf("output is not well-formed XML: %v", err)
-	}
-	if v.Name != "proxy-x-seed.iso" || v.Capacity.Value != 372736 || v.Target.Format.Type != "raw" {
-		t.Errorf("volume = %+v", v)
-	}
-}

internal/provider/metrics.go

@@ -50,17 +50,25 @@ func (i *instrumented) record(op string, err error) {
 	i.rec.ProviderRequest(i.name, op, resultLabel(err))
 }
 
+const (
+	resultOK            = "ok"
+	resultNotFound      = "not_found"
+	resultQuotaExceeded = "quota_exceeded"
+	resultPermanent     = "permanent"
+	resultTransient     = "transient"
+)
+
 func resultLabel(err error) string {
 	switch Class(err) {
 	case nil:
-		return "ok"
+		return resultOK
 	case ErrNotFound:
-		return "not_found"
+		return resultNotFound
 	case ErrQuotaExceeded:
-		return "quota_exceeded"
+		return resultQuotaExceeded
 	case ErrPermanent:
-		return "permanent"
+		return resultPermanent
 	default:
-		return "transient"
+		return resultTransient
 	}
 }

internal/provider/tracing.go

@@ -0,0 +1,80 @@
+package provider
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+// WithTracing wraps a Provider so every call runs in a client span carrying
+// the classified result, mirroring WithMetrics. It goes through
+// tracing.StartSpan, so provider-internal logs (e.g. the gcp provider's
+// context logger) inherit traceID/spanID. Errors pass through unmodified.
+func WithTracing(name string, p Provider, opts ...tracing.Option) Provider {
+	return &traced{name: name, inner: p, tracer: tracing.Tracer(opts...)}
+}
+
+type traced struct {
+	name   string
+	inner  Provider
+	tracer trace.Tracer
+}
+
+func (t *traced) Create(ctx context.Context, req CreateRequest) (string, error) {
+	ctx, span := t.start(ctx, "provider.create")
+	id, err := t.inner.Create(ctx, req)
+	if id != "" {
+		span.SetAttributes(attribute.String("provider.id", id))
+	}
+	t.end(span, err)
+	return id, err
+}
+
+func (t *traced) Get(ctx context.Context, providerID string) (*Instance, error) {
+	ctx, span := t.start(ctx, "provider.get")
+	span.SetAttributes(attribute.String("provider.id", providerID))
+	inst, err := t.inner.Get(ctx, providerID)
+	t.end(span, err)
+	return inst, err
+}
+
+func (t *traced) Delete(ctx context.Context, providerID string) error {
+	ctx, span := t.start(ctx, "provider.delete")
+	span.SetAttributes(attribute.String("provider.id", providerID))
+	err := t.inner.Delete(ctx, providerID)
+	t.end(span, err)
+	return err
+}
+
+func (t *traced) ListByTag(ctx context.Context) ([]Instance, error) {
+	ctx, span := t.start(ctx, "provider.list")
+	instances, err := t.inner.ListByTag(ctx)
+	span.SetAttributes(attribute.Int("provider.instances", len(instances)))
+	t.end(span, err)
+	return instances, err
+}
+
+func (t *traced) start(ctx context.Context, op string) (context.Context, trace.Span) {
+	return tracing.StartSpan(ctx, t.tracer, op,
+		trace.WithSpanKind(trace.SpanKindClient),
+		trace.WithAttributes(attribute.String("provider.name", t.name)))
+}
+
+// end records the taxonomy class and closes the span. ErrNotFound stays Ok:
+// the reconciler polls Get to NotFound during replacement/deletion, so it is
+// an expected answer, not a failure — same reasoning as resultLabel's
+// distinct "not_found" bucket.
+func (t *traced) end(span trace.Span, err error) {
+	span.SetAttributes(attribute.String("provider.result", resultLabel(err)))
+	switch Class(err) {
+	case nil, ErrNotFound:
+		span.SetStatus(codes.Ok, "")
+	default:
+		span.SetStatus(codes.Error, err.Error())
+	}
+	span.End()
+}

internal/provider/tracing_test.go

@@ -0,0 +1,118 @@
+package provider
+
+import (
+	"context"
+	"errors"
+	"slices"
+	"testing"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	"go.opentelemetry.io/otel/trace"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
+)
+
+func TestWithTracing_spanPerCall(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name       string
+		inner      *staticProvider
+		call       func(p Provider) error
+		wantSpan   string
+		wantStatus codes.Code
+		wantResult string
+		wantAttr   attribute.KeyValue
+	}{
+		{
+			name:       "successful create",
+			inner:      &staticProvider{},
+			call:       func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
+			wantSpan:   "provider.create",
+			wantStatus: codes.Ok,
+			wantResult: resultOK,
+			wantAttr:   attribute.String("provider.id", "id-1"),
+		},
+		{
+			name:       "get NotFound is not a span error",
+			inner:      &staticProvider{getErr: Wrap(ErrNotFound, "get", "x", "id-1", nil)},
+			call:       func(p Provider) error { _, err := p.Get(context.Background(), "id-1"); return err },
+			wantSpan:   "provider.get",
+			wantStatus: codes.Ok,
+			wantResult: resultNotFound,
+			wantAttr:   attribute.String("provider.id", "id-1"),
+		},
+		{
+			name:       "delete transient error",
+			inner:      &staticProvider{deleteErr: Wrap(ErrTransient, "delete", "x", "id-1", errors.New("503"))},
+			call:       func(p Provider) error { return p.Delete(context.Background(), "id-1") },
+			wantSpan:   "provider.delete",
+			wantStatus: codes.Error,
+			wantResult: resultTransient,
+			wantAttr:   attribute.String("provider.id", "id-1"),
+		},
+		{
+			name:       "quota exceeded create",
+			inner:      &staticProvider{createErr: Wrap(ErrQuotaExceeded, "create", "x", "", nil)},
+			call:       func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
+			wantSpan:   "provider.create",
+			wantStatus: codes.Error,
+			wantResult: resultQuotaExceeded,
+			wantAttr:   attribute.String("provider.name", "x"),
+		},
+		{
+			name:       "list records instance count",
+			inner:      &staticProvider{},
+			call:       func(p Provider) error { _, err := p.ListByTag(context.Background()); return err },
+			wantSpan:   "provider.list",
+			wantStatus: codes.Ok,
+			wantResult: resultOK,
+			wantAttr:   attribute.Int("provider.instances", 0),
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			sr := tracetest.NewSpanRecorder()
+			tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+			t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
+
+			p := WithTracing("x", tc.inner, tracing.WithTracerProvider(tp))
+			err := tc.call(p)
+
+			wantErr := errors.Join(tc.inner.createErr, tc.inner.getErr, tc.inner.deleteErr, tc.inner.listErr)
+			if (wantErr == nil) != (err == nil) {
+				t.Fatalf("decorator changed the error: got %v", err)
+			}
+
+			ended := sr.Ended()
+			if len(ended) != 1 {
+				t.Fatalf("got %d spans, want 1", len(ended))
+			}
+			span := ended[0]
+			if span.Name() != tc.wantSpan {
+				t.Errorf("span name = %q, want %q", span.Name(), tc.wantSpan)
+			}
+			if span.SpanKind() != trace.SpanKindClient {
+				t.Errorf("span kind = %v, want client", span.SpanKind())
+			}
+			if span.Status().Code != tc.wantStatus {
+				t.Errorf("status = %v, want %v", span.Status().Code, tc.wantStatus)
+			}
+			attrs := span.Attributes()
+			hasAttr := func(want attribute.KeyValue) bool {
+				return slices.Contains(attrs, want)
+			}
+			if !hasAttr(attribute.String("provider.result", tc.wantResult)) {
+				t.Errorf("provider.result %q missing in %v", tc.wantResult, attrs)
+			}
+			if !hasAttr(tc.wantAttr) {
+				t.Errorf("attribute %v missing in %v", tc.wantAttr, attrs)
+			}
+		})
+	}
+}

internal/tracing/http.go

@@ -0,0 +1,41 @@
+package tracing
+
+import (
+	"net/http"
+
+	"github.com/go-logr/logr"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/propagation"
+)
+
+// HTTPMiddleware returns middleware that opens a server span per request
+// (named from the mux route pattern once routing has happened) and hands
+// handlers a request context whose logf.FromContext logger is base enriched
+// with the trace context. Incoming W3C traceparent headers are honored, so
+// clients see their own trace continue into the operator. /healthz is never
+// traced. With tracing disabled the middleware degrades to injecting base
+// unenriched — handlers can rely on logf.FromContext either way.
+func HTTPMiddleware(operation string, base logr.Logger, opts ...Option) func(http.Handler) http.Handler {
+	o := newOptions(opts)
+	return func(next http.Handler) http.Handler {
+		inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			r2 := r.WithContext(ContextWithLogger(r.Context(), base))
+			next.ServeHTTP(w, r2)
+			// WithContext copies the request, so the mux recorded the matched
+			// route on r2; surface it to otelhttp, which renames the span
+			// from r.Pattern after the handler returns.
+			r.Pattern = r2.Pattern
+		})
+		otelOpts := []otelhttp.Option{
+			otelhttp.WithFilter(func(r *http.Request) bool { return r.URL.Path != "/healthz" }),
+			// Explicit propagators: deterministic regardless of whether the
+			// global propagator has been installed (tests, disabled tracing).
+			otelhttp.WithPropagators(propagation.NewCompositeTextMapPropagator(
+				propagation.TraceContext{}, propagation.Baggage{})),
+		}
+		if o.tp != nil {
+			otelOpts = append(otelOpts, otelhttp.WithTracerProvider(o.tp))
+		}
+		return otelhttp.NewHandler(inner, operation, otelOpts...)
+	}
+}

internal/tracing/http_test.go

@@ -0,0 +1,82 @@
+package tracing
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+const sampleTraceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
+
+func TestHTTPMiddleware(t *testing.T) {
+	t.Parallel()
+
+	newServer := func(lines *[]string, tp *sdktrace.TracerProvider) http.Handler {
+		mux := http.NewServeMux()
+		handler := func(w http.ResponseWriter, r *http.Request) {
+			logf.FromContext(r.Context()).Info("handling")
+			w.WriteHeader(http.StatusOK)
+		}
+		mux.HandleFunc("GET /healthz", handler)
+		mux.HandleFunc("GET /v1/things/{id}", handler)
+		return HTTPMiddleware("discovery", captureLogger(lines), WithTracerProvider(tp))(mux)
+	}
+
+	t.Run("route span, enriched handler logs, traceparent continuation", func(t *testing.T) {
+		t.Parallel()
+		sr := tracetest.NewSpanRecorder()
+		tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+		t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
+
+		var lines []string
+		rec := httptest.NewRecorder()
+		req := httptest.NewRequest(http.MethodGet, "/v1/things/42", nil)
+		req.Header.Set("traceparent", sampleTraceparent)
+		newServer(&lines, tp).ServeHTTP(rec, req)
+
+		ended := sr.Ended()
+		if len(ended) != 1 {
+			t.Fatalf("got %d spans, want 1", len(ended))
+		}
+		if got := ended[0].Name(); got != "GET /v1/things/{id}" {
+			t.Errorf("span name = %q, want route pattern", got)
+		}
+		wantTrace := "4bf92f3577b34da6a3ce929d0e0e4736"
+		if got := ended[0].SpanContext().TraceID().String(); got != wantTrace {
+			t.Errorf("span traceID = %s, want continuation of client trace %s", got, wantTrace)
+		}
+		if len(lines) != 1 {
+			t.Fatalf("got %d log lines, want 1: %v", len(lines), lines)
+		}
+		if n := strings.Count(lines[0], `"traceID"`); n != 1 {
+			t.Errorf("traceID appears %d times, want 1: %s", n, lines[0])
+		}
+		if !strings.Contains(lines[0], wantTrace) {
+			t.Errorf("handler log missing traceID %s: %s", wantTrace, lines[0])
+		}
+	})
+
+	t.Run("healthz is not traced but still gets a logger", func(t *testing.T) {
+		t.Parallel()
+		sr := tracetest.NewSpanRecorder()
+		tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+		t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
+
+		var lines []string
+		rec := httptest.NewRecorder()
+		newServer(&lines, tp).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+
+		if len(sr.Ended()) != 0 {
+			t.Fatalf("healthz produced %d spans, want 0", len(sr.Ended()))
+		}
+		if len(lines) != 1 || strings.Contains(lines[0], "traceID") {
+			t.Fatalf("want one unenriched log line, got %v", lines)
+		}
+	})
+}

internal/tracing/logger.go

@@ -0,0 +1,65 @@
+package tracing
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+	"go.opentelemetry.io/otel/trace"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+// baseLoggerKey stores the pre-enrichment logger. logr sinks never see a
+// context, so trace IDs must ride on the logger as WithValues — and zap does
+// not dedupe repeated keys, so each nested span must re-derive from the same
+// base instead of stacking traceID/spanID onto an already-enriched logger.
+type baseLoggerKey struct{}
+
+// Start begins a span from the global tracer and returns a context whose
+// logf.FromContext logger carries the span's traceID/spanID. With tracing
+// disabled the span is a no-op with an invalid span context and the logger
+// is left untouched.
+//
+// Trade-off, documented: values pushed via logf.IntoContext *between* two
+// Start calls are dropped by the inner Start's re-derivation. Nothing
+// first-party does that; controller-runtime's per-reconcile logger
+// (reconcileID etc.) is captured as the base and survives.
+func Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
+	return StartSpan(ctx, newOptions(nil).tracer(), name, opts...)
+}
+
+// StartSpan is Start with an explicit tracer, for decorators that carry
+// their own (test-injected) TracerProvider.
+func StartSpan(ctx context.Context, tracer trace.Tracer, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
+	ctx, span := tracer.Start(ctx, name, opts...)
+	sc := span.SpanContext()
+	if !sc.IsValid() {
+		return ctx, span
+	}
+	base, ok := ctx.Value(baseLoggerKey{}).(logr.Logger)
+	if !ok {
+		base = logf.FromContext(ctx)
+		ctx = context.WithValue(ctx, baseLoggerKey{}, base)
+	}
+	return logf.IntoContext(ctx, withSpanValues(base, sc)), span
+}
+
+// ContextWithLogger seeds ctx with base as both the current logger and the
+// base for span enrichment. If ctx already carries a valid span (the HTTP
+// middleware calls this inside the otelhttp handler), the logger is
+// enriched immediately.
+//
+//nolint:logcheck // IntoContext-analogue: taking ctx and the logger to seed it with is the point.
+func ContextWithLogger(ctx context.Context, base logr.Logger) context.Context {
+	ctx = context.WithValue(ctx, baseLoggerKey{}, base)
+	if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
+		return logf.IntoContext(ctx, withSpanValues(base, sc))
+	}
+	return logf.IntoContext(ctx, base)
+}
+
+// withSpanValues uses lowerCamel keys to match the house style
+// (reconcileID, providerID); Grafana/Loki derived fields must match
+// "traceID", not the trace_id default.
+func withSpanValues(base logr.Logger, sc trace.SpanContext) logr.Logger {
+	return base.WithValues("traceID", sc.TraceID().String(), "spanID", sc.SpanID().String())
+}

internal/tracing/logger_test.go

@@ -0,0 +1,136 @@
+package tracing
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/go-logr/logr"
+	"github.com/go-logr/logr/funcr"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/noop"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+// captureLogger records every emitted line so tests can assert on the
+// rendered key/value output — the only place duplicate zap-style keys
+// would show up.
+func captureLogger(lines *[]string) logr.Logger {
+	return funcr.New(func(prefix, args string) {
+		*lines = append(*lines, prefix+" "+args)
+	}, funcr.Options{})
+}
+
+func recordingTracer(t *testing.T) (trace.Tracer, *tracetest.SpanRecorder) {
+	t.Helper()
+	sr := tracetest.NewSpanRecorder()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+	t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
+	return tp.Tracer("test"), sr
+}
+
+func TestStartSpan_enrichesLoggerOncePerNesting(t *testing.T) {
+	t.Parallel()
+	tracer, sr := recordingTracer(t)
+
+	var lines []string
+	ctx := logf.IntoContext(context.Background(), captureLogger(&lines))
+
+	ctx1, span1 := StartSpan(ctx, tracer, "outer")
+	logf.FromContext(ctx1).Info("outer work")
+
+	ctx2, span2 := StartSpan(ctx1, tracer, "inner")
+	logf.FromContext(ctx2).Info("inner work")
+
+	span2.End()
+	span1.End()
+
+	if len(lines) != 2 {
+		t.Fatalf("got %d log lines, want 2: %v", len(lines), lines)
+	}
+	traceID := span1.SpanContext().TraceID().String()
+	for i, want := range []string{span1.SpanContext().SpanID().String(), span2.SpanContext().SpanID().String()} {
+		if n := strings.Count(lines[i], `"traceID"`); n != 1 {
+			t.Errorf("line %d: traceID appears %d times, want exactly 1: %s", i, n, lines[i])
+		}
+		if n := strings.Count(lines[i], `"spanID"`); n != 1 {
+			t.Errorf("line %d: spanID appears %d times, want exactly 1: %s", i, n, lines[i])
+		}
+		if !strings.Contains(lines[i], traceID) {
+			t.Errorf("line %d: missing traceID %s: %s", i, traceID, lines[i])
+		}
+		if !strings.Contains(lines[i], want) {
+			t.Errorf("line %d: missing spanID %s: %s", i, want, lines[i])
+		}
+	}
+
+	ended := sr.Ended()
+	if len(ended) != 2 {
+		t.Fatalf("got %d spans, want 2", len(ended))
+	}
+	// Ended in LIFO order: inner first.
+	if got := ended[0].Parent().SpanID(); got != span1.SpanContext().SpanID() {
+		t.Errorf("inner span parent = %s, want %s", got, span1.SpanContext().SpanID())
+	}
+}
+
+func TestStartSpan_noopTracerLeavesLoggerUntouched(t *testing.T) {
+	t.Parallel()
+
+	var lines []string
+	ctx := logf.IntoContext(context.Background(), captureLogger(&lines))
+
+	ctx, span := StartSpan(ctx, noop.NewTracerProvider().Tracer("test"), "op")
+	defer span.End()
+	logf.FromContext(ctx).Info("work")
+
+	if len(lines) != 1 {
+		t.Fatalf("got %d log lines, want 1", len(lines))
+	}
+	if strings.Contains(lines[0], "traceID") {
+		t.Errorf("disabled tracing must not add traceID: %s", lines[0])
+	}
+}
+
+func TestContextWithLogger(t *testing.T) {
+	t.Parallel()
+	tracer, _ := recordingTracer(t)
+
+	t.Run("no span injects base as-is", func(t *testing.T) {
+		t.Parallel()
+		var lines []string
+		ctx := ContextWithLogger(context.Background(), captureLogger(&lines))
+		logf.FromContext(ctx).Info("plain")
+		if len(lines) != 1 || strings.Contains(lines[0], "traceID") {
+			t.Fatalf("want one line without traceID, got %v", lines)
+		}
+	})
+
+	t.Run("existing span enriches immediately and nested Start does not stack", func(t *testing.T) {
+		t.Parallel()
+		var lines []string
+		ctx, outer := tracer.Start(context.Background(), "server")
+		defer outer.End()
+
+		ctx = ContextWithLogger(ctx, captureLogger(&lines))
+		logf.FromContext(ctx).Info("handler")
+
+		ctx, inner := StartSpan(ctx, tracer, "child")
+		defer inner.End()
+		logf.FromContext(ctx).Info("nested")
+
+		if len(lines) != 2 {
+			t.Fatalf("got %d lines, want 2: %v", len(lines), lines)
+		}
+		for i, line := range lines {
+			if n := strings.Count(line, `"traceID"`); n != 1 {
+				t.Errorf("line %d: traceID appears %d times, want 1: %s", i, n, line)
+			}
+		}
+		if !strings.Contains(lines[1], inner.SpanContext().SpanID().String()) {
+			t.Errorf("nested line should carry the child spanID: %s", lines[1])
+		}
+	})
+}

internal/tracing/options.go

@@ -0,0 +1,48 @@
+package tracing
+
+import (
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// tracerName is the instrumentation scope for every span this module emits.
+const tracerName = "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator"
+
+type options struct {
+	tp trace.TracerProvider
+}
+
+// Option configures the tracing decorators. The zero configuration uses the
+// global TracerProvider installed by Setup; tests inject their own recorder
+// via WithTracerProvider so they can run in parallel without touching
+// process-global state.
+type Option func(*options)
+
+// WithTracerProvider overrides the global TracerProvider.
+func WithTracerProvider(tp trace.TracerProvider) Option {
+	return func(o *options) { o.tp = tp }
+}
+
+func newOptions(opts []Option) options {
+	var o options
+	for _, opt := range opts {
+		opt(&o)
+	}
+	return o
+}
+
+// Tracer resolves a tracer from the given options, for decorators outside
+// this package (provider.WithTracing).
+func Tracer(opts ...Option) trace.Tracer {
+	return newOptions(opts).tracer()
+}
+
+// tracer resolves the configured tracer. The global path goes through
+// otel.Tracer, which delegates to whatever provider Setup installs later —
+// construction order between decorators and Setup does not matter.
+func (o options) tracer() trace.Tracer {
+	if o.tp != nil {
+		return o.tp.Tracer(tracerName)
+	}
+	return otel.Tracer(tracerName)
+}

internal/tracing/reconciler.go

@@ -0,0 +1,52 @@
+package tracing
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+// NewReconciler wraps inner so every reconcile runs in a root span named
+// "Reconcile <kind>" and every log line under it carries the trace context.
+// Reconciles are triggered by watch events with no incoming trace to
+// continue, so each one starts a new trace.
+func NewReconciler(kind string, inner reconcile.Reconciler, opts ...Option) reconcile.Reconciler {
+	return &tracedReconciler{
+		spanName: "Reconcile " + kind,
+		inner:    inner,
+		tracer:   newOptions(opts).tracer(),
+	}
+}
+
+type tracedReconciler struct {
+	spanName string
+	inner    reconcile.Reconciler
+	tracer   trace.Tracer
+}
+
+func (t *tracedReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
+	ctx, span := StartSpan(ctx, t.tracer, t.spanName,
+		trace.WithAttributes(
+			attribute.String("k8s.namespace.name", req.Namespace),
+			attribute.String("k8s.object.name", req.Name),
+		))
+	defer span.End()
+	if id := controller.ReconcileIDFromContext(ctx); id != "" {
+		span.SetAttributes(attribute.String("reconcile.id", string(id)))
+	}
+
+	res, err := t.inner.Reconcile(ctx, req)
+	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
+		return res, err
+	}
+	span.SetStatus(codes.Ok, "")
+	if res.RequeueAfter > 0 {
+		span.SetAttributes(attribute.Int64("reconcile.requeue_after_ms", res.RequeueAfter.Milliseconds()))
+	}
+	return res, nil
+}

internal/tracing/reconciler_test.go

@@ -0,0 +1,86 @@
+package tracing
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+type fakeReconciler struct {
+	res reconcile.Result
+	err error
+}
+
+func (f *fakeReconciler) Reconcile(context.Context, reconcile.Request) (reconcile.Result, error) {
+	return f.res, f.err
+}
+
+func TestNewReconciler_spanPerReconcile(t *testing.T) {
+	t.Parallel()
+
+	req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "p1"}}
+
+	tests := []struct {
+		name       string
+		inner      *fakeReconciler
+		wantStatus codes.Code
+		wantAttr   attribute.KeyValue
+	}{
+		{
+			name:       "success records requeue",
+			inner:      &fakeReconciler{res: reconcile.Result{RequeueAfter: 10 * time.Second}},
+			wantStatus: codes.Ok,
+			wantAttr:   attribute.Int64("reconcile.requeue_after_ms", 10_000),
+		},
+		{
+			name:       "error sets error status",
+			inner:      &fakeReconciler{err: errors.New("boom")},
+			wantStatus: codes.Error,
+			wantAttr:   attribute.String("k8s.object.name", "p1"),
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			sr := tracetest.NewSpanRecorder()
+			tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+			t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
+
+			r := NewReconciler("Proxy", tc.inner, WithTracerProvider(tp))
+			res, err := r.Reconcile(context.Background(), req)
+			if res != tc.inner.res || !errors.Is(err, tc.inner.err) {
+				t.Fatalf("decorator changed the result: (%v, %v)", res, err)
+			}
+
+			ended := sr.Ended()
+			if len(ended) != 1 {
+				t.Fatalf("got %d spans, want 1", len(ended))
+			}
+			span := ended[0]
+			if span.Name() != "Reconcile Proxy" {
+				t.Errorf("span name = %q, want %q", span.Name(), "Reconcile Proxy")
+			}
+			if span.Status().Code != tc.wantStatus {
+				t.Errorf("status = %v, want %v", span.Status().Code, tc.wantStatus)
+			}
+			found := false
+			for _, a := range span.Attributes() {
+				if a == tc.wantAttr {
+					found = true
+				}
+			}
+			if !found {
+				t.Errorf("attribute %v missing in %v", tc.wantAttr, span.Attributes())
+			}
+		})
+	}
+}

internal/tracing/tracing.go

@@ -0,0 +1,157 @@
+// Package tracing wires OpenTelemetry tracing into the operator: SDK setup
+// gated on standard OTEL_* environment variables, span helpers that keep the
+// logr/zap logging enriched with trace context, and decorators for the
+// reconciler, providers, HTTP server, and Kubernetes API transport.
+package tracing
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+const (
+	envSDKDisabled        = "OTEL_SDK_DISABLED"
+	envTracesExporter     = "OTEL_TRACES_EXPORTER"
+	envOTLPEndpoint       = "OTEL_EXPORTER_OTLP_ENDPOINT"
+	envOTLPTracesEndpoint = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
+	envOTLPProtocol       = "OTEL_EXPORTER_OTLP_PROTOCOL"
+	envOTLPTracesProtocol = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"
+
+	exporterOTLP    = "otlp"
+	exporterConsole = "console"
+	exporterNone    = "none"
+)
+
+// maxQueueSize bounds the batch processor's span buffer; the default 2048
+// is oversized for this process (3 concurrent reconciles, tickers) and the
+// manager pod runs with a 128Mi memory limit.
+const maxQueueSize = 512
+
+// Setup initializes the global tracing pipeline from OTEL_* environment
+// variables and returns a shutdown func that flushes pending spans.
+//
+// Tracing stays fully off — no exporter, no global provider, spans are
+// no-ops, logs carry no traceID — unless the environment opts in by setting
+// OTEL_TRACES_EXPORTER or an OTLP endpoint. OTEL_SDK_DISABLED=true and
+// OTEL_TRACES_EXPORTER=none force it off (the Go SDK does not implement
+// OTEL_SDK_DISABLED itself). Sampling, endpoints, TLS, and headers follow
+// the standard SDK/exporter env vars (OTEL_TRACES_SAMPLER, OTEL_EXPORTER_OTLP_*).
+//
+// Setup logs via the logger in ctx (logf.FromContext) rather than a
+// parameter — the caller seeds it with logf.IntoContext.
+func Setup(ctx context.Context, service, version string) (func(context.Context) error, error) {
+	log := logf.FromContext(ctx)
+	noop := func(context.Context) error { return nil }
+
+	exporterEnv := strings.ToLower(strings.TrimSpace(os.Getenv(envTracesExporter)))
+	endpointSet := os.Getenv(envOTLPEndpoint) != "" ||
+		os.Getenv(envOTLPTracesEndpoint) != ""
+	sdkDisabled := strings.EqualFold(strings.TrimSpace(os.Getenv(envSDKDisabled)), "true")
+
+	if sdkDisabled || exporterEnv == exporterNone || (exporterEnv == "" && !endpointSet) {
+		log.Info("tracing disabled",
+			"reason", disabledReason(sdkDisabled, exporterEnv))
+		return noop, nil
+	}
+
+	exp, expName, err := newExporter(ctx, exporterEnv)
+	if err != nil {
+		return noop, fmt.Errorf("tracing setup: %w", err)
+	}
+
+	res, err := buildResource(ctx, service, version)
+	if err != nil {
+		if res == nil {
+			return noop, fmt.Errorf("tracing setup: building resource: %w", err)
+		}
+		// Schema-URL conflicts between semconv versions still yield a usable
+		// merged resource; keep it rather than losing tracing over metadata.
+		log.V(1).Info("tracing resource merge conflict; continuing", "err", err.Error())
+	}
+
+	tp := sdktrace.NewTracerProvider(
+		sdktrace.WithBatcher(exp, sdktrace.WithMaxQueueSize(maxQueueSize)),
+		sdktrace.WithResource(res),
+	)
+	otel.SetTracerProvider(tp)
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{}, propagation.Baggage{}))
+	otelLog := log.WithName("otel")
+	otel.SetLogger(otelLog)
+	// V(1), not Error: an unreachable collector fails every export cycle and
+	// would otherwise spam the error log every few seconds.
+	otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
+		otelLog.V(1).Info("otel error", "err", err.Error())
+	}))
+
+	log.Info("tracing enabled", "exporter", expName, "service", service)
+	return tp.Shutdown, nil
+}
+
+func disabledReason(sdkDisabled bool, exporterEnv string) string {
+	switch {
+	case sdkDisabled:
+		return "OTEL_SDK_DISABLED=true"
+	case exporterEnv == exporterNone:
+		return "OTEL_TRACES_EXPORTER=none"
+	default:
+		return "no OTEL_TRACES_EXPORTER or OTLP endpoint configured"
+	}
+}
+
+// newExporter hand-rolls the OTEL_TRACES_EXPORTER / OTEL_EXPORTER_OTLP_*
+// protocol selection instead of pulling in contrib's autoexport, which drags
+// metric/log/prometheus exporters into a trace-only binary.
+func newExporter(ctx context.Context, kind string) (sdktrace.SpanExporter, string, error) {
+	switch kind {
+	case "", exporterOTLP:
+		proto := strings.ToLower(strings.TrimSpace(os.Getenv(envOTLPTracesProtocol)))
+		if proto == "" {
+			proto = strings.ToLower(strings.TrimSpace(os.Getenv(envOTLPProtocol)))
+		}
+		switch proto {
+		case "", "http/protobuf":
+			exp, err := otlptracehttp.New(ctx)
+			return exp, "otlp/http", err
+		case "grpc":
+			exp, err := otlptracegrpc.New(ctx)
+			return exp, "otlp/grpc", err
+		default:
+			return nil, "", fmt.Errorf("unsupported OTLP protocol %q (supported: http/protobuf, grpc)", proto)
+		}
+	case exporterConsole:
+		exp, err := stdouttrace.New()
+		return exp, exporterConsole, err
+	default:
+		return nil, "", fmt.Errorf("unsupported OTEL_TRACES_EXPORTER %q (supported: otlp, console, none)", kind)
+	}
+}
+
+// buildResource layers defaults < service identity < environment, so
+// OTEL_SERVICE_NAME / OTEL_RESOURCE_ATTRIBUTES always win.
+func buildResource(ctx context.Context, service, version string) (*resource.Resource, error) {
+	base, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL,
+		semconv.ServiceName(service),
+		semconv.ServiceVersion(version),
+	))
+	if err != nil {
+		return base, err
+	}
+	env, err := resource.New(ctx, resource.WithFromEnv())
+	if err != nil {
+		return base, err
+	}
+	return resource.Merge(base, env)
+}

internal/tracing/tracing_test.go

@@ -0,0 +1,130 @@
+package tracing
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go.opentelemetry.io/otel"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace/noop"
+)
+
+const testOTLPEndpoint = "http://localhost:4318"
+
+// clearOTelEnv pins every env var Setup reads, so developer shells with
+// OTEL_* set can't change test outcomes. Not parallel-safe by design
+// (t.Setenv forbids t.Parallel).
+func clearOTelEnv(t *testing.T) {
+	t.Helper()
+	for _, k := range []string{
+		envSDKDisabled,
+		envTracesExporter,
+		envOTLPEndpoint,
+		envOTLPTracesEndpoint,
+		envOTLPProtocol,
+		envOTLPTracesProtocol,
+	} {
+		t.Setenv(k, "")
+	}
+}
+
+func TestSetup_envGating(t *testing.T) {
+	tests := []struct {
+		name        string
+		env         map[string]string
+		wantEnabled bool
+		wantErr     bool
+	}{
+		{name: "no env means disabled", env: nil, wantEnabled: false},
+		{
+			name:        "endpoint enables otlp",
+			env:         map[string]string{envOTLPEndpoint: testOTLPEndpoint},
+			wantEnabled: true,
+		},
+		{
+			name:        "traces endpoint enables otlp",
+			env:         map[string]string{envOTLPTracesEndpoint: testOTLPEndpoint},
+			wantEnabled: true,
+		},
+		{
+			name:        "explicit console exporter",
+			env:         map[string]string{envTracesExporter: exporterConsole},
+			wantEnabled: true,
+		},
+		{
+			name:        "grpc protocol",
+			env:         map[string]string{envTracesExporter: exporterOTLP, envOTLPProtocol: "grpc"},
+			wantEnabled: true,
+		},
+		{
+			name: "exporter none wins over endpoint",
+			env: map[string]string{
+				envTracesExporter: exporterNone,
+				envOTLPEndpoint:   testOTLPEndpoint,
+			},
+			wantEnabled: false,
+		},
+		{
+			name: "OTEL_SDK_DISABLED wins over everything",
+			env: map[string]string{
+				envSDKDisabled:  "true",
+				envOTLPEndpoint: testOTLPEndpoint,
+			},
+			wantEnabled: false,
+		},
+		{
+			name:    "unsupported exporter errors",
+			env:     map[string]string{envTracesExporter: "jaeger"},
+			wantErr: true,
+		},
+		{
+			name: "unsupported protocol errors",
+			env: map[string]string{
+				envTracesExporter: exporterOTLP,
+				envOTLPProtocol:   "http/json",
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			clearOTelEnv(t)
+			for k, v := range tc.env {
+				t.Setenv(k, v)
+			}
+			// Fresh noop global per case: disabled paths must leave it
+			// untouched, and resetting avoids leaking one case's SDK
+			// provider into the next (or warning-prone restores of the
+			// process-default delegate).
+			before := noop.NewTracerProvider()
+			otel.SetTracerProvider(before)
+
+			shutdown, err := Setup(context.Background(), "test-svc", "abc123")
+			if tc.wantErr {
+				if err == nil {
+					t.Fatal("want error, got nil")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("Setup: %v", err)
+			}
+
+			_, isSDK := otel.GetTracerProvider().(*sdktrace.TracerProvider)
+			if isSDK != tc.wantEnabled {
+				t.Errorf("global provider is SDK = %v, want %v", isSDK, tc.wantEnabled)
+			}
+			if tc.wantEnabled && otel.GetTracerProvider() == before {
+				t.Error("enabled Setup must install a new global provider")
+			}
+
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+			if err := shutdown(ctx); err != nil {
+				t.Errorf("shutdown: %v", err)
+			}
+		})
+	}
+}

internal/tracing/transport.go

@@ -0,0 +1,51 @@
+package tracing
+
+import (
+	"net/http"
+
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
+	"k8s.io/client-go/transport"
+)
+
+// RestConfigWrapper returns a rest.Config.Wrap-compatible wrapper that adds
+// a client span to every Kubernetes API request already running inside a
+// trace. Requests with no parent span pass through untouched: informer
+// list/watch long-polls, leader-election renewals, and metrics authn would
+// otherwise each become meaningless (and in the watch case, minutes-long)
+// root spans. client-go applies Wrap innermost, so the span sees the final
+// authenticated request.
+func RestConfigWrapper(opts ...Option) transport.WrapperFunc {
+	o := newOptions(opts)
+	return func(rt http.RoundTripper) http.RoundTripper {
+		otelOpts := []otelhttp.Option{
+			// Explicit propagators: deterministic regardless of global
+			// state. The apiserver ignores incoming traceparent (by
+			// design), so injection is harmless there.
+			otelhttp.WithPropagators(propagation.NewCompositeTextMapPropagator(
+				propagation.TraceContext{}, propagation.Baggage{})),
+		}
+		if o.tp != nil {
+			otelOpts = append(otelOpts, otelhttp.WithTracerProvider(o.tp))
+		}
+		return &parentGatedTransport{
+			traced: otelhttp.NewTransport(rt, otelOpts...),
+			plain:  rt,
+		}
+	}
+}
+
+// parentGatedTransport enforces the parent-span requirement itself rather
+// than relying on otelhttp filter semantics for transports.
+type parentGatedTransport struct {
+	traced http.RoundTripper
+	plain  http.RoundTripper
+}
+
+func (t *parentGatedTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	if trace.SpanContextFromContext(r.Context()).IsValid() {
+		return t.traced.RoundTrip(r)
+	}
+	return t.plain.RoundTrip(r)
+}

internal/tracing/transport_test.go

@@ -0,0 +1,73 @@
+package tracing
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	"go.opentelemetry.io/otel/trace"
+)
+
+func TestRestConfigWrapper_parentGated(t *testing.T) {
+	t.Parallel()
+
+	var gotTraceparent string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotTraceparent = r.Header.Get("traceparent")
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(srv.Close)
+
+	sr := tracetest.NewSpanRecorder()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+	t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
+
+	rt := RestConfigWrapper(WithTracerProvider(tp))(http.DefaultTransport)
+	do := func(ctx context.Context) {
+		t.Helper()
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		resp, err := rt.RoundTrip(req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}
+
+	// No parent span: informer watches, leader election. Must not trace.
+	do(context.Background())
+	if len(sr.Ended()) != 0 {
+		t.Fatalf("request without parent span produced %d spans, want 0", len(sr.Ended()))
+	}
+	if gotTraceparent != "" {
+		t.Fatalf("request without parent span injected traceparent %q", gotTraceparent)
+	}
+
+	// Under a parent span: one client child span, traceparent injected.
+	ctx, parent := tp.Tracer("test").Start(context.Background(), "reconcile")
+	do(ctx)
+	parent.End()
+
+	var client sdktrace.ReadOnlySpan
+	for _, s := range sr.Ended() {
+		if s.SpanKind() == trace.SpanKindClient {
+			client = s
+		}
+	}
+	if client == nil {
+		t.Fatalf("no client span recorded, got %d spans", len(sr.Ended()))
+	}
+	if got := client.Parent().SpanID(); got != parent.SpanContext().SpanID() {
+		t.Errorf("client span parent = %s, want %s", got, parent.SpanContext().SpanID())
+	}
+	if gotTraceparent == "" {
+		t.Error("traceparent header not injected under a parent span")
+	}
+}

test/e2e/tracing_test.go

@@ -0,0 +1,424 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/test/utils"
+)
+
+// The tracing e2e proves the full pipeline against a real Tempo: operator in
+// kind → OTLP export → Tempo ingest → traces queryable with the documented
+// span topology. It is gated on TEMPO_URL (Tempo query API, e.g.
+// http://192.168.0.30:3200) and OTLP_ENDPOINT (OTLP HTTP ingest, e.g.
+// http://192.168.0.30:4318) and skips when either is unset, so the rest of
+// the suite runs anywhere.
+//
+// Every span of a run carries the resource attribute test.run.id=<runID>
+// (injected via OTEL_RESOURCE_ATTRIBUTES — no code changes), so one TraceQL
+// query finds exactly this run's traces, in the test and in Grafana alike.
+var _ = Describe("OTel tracing", Ordered, func() {
+	// Proxy CRs live in default, not the operator namespace: the squid pods
+	// the kubernetes provider creates carry no securityContext and would be
+	// rejected by the operator namespace's restricted PSS label.
+	const proxyNS = "default"
+	const deploymentName = "egress-proxies-operator-controller-manager"
+	const squidImage = "ubuntu/squid:6.6-24.04_edge"
+
+	proxyNames := []string{"proxy-tracing-e2e-1", "proxy-tracing-e2e-2"}
+
+	var (
+		tempoURL     string
+		otlpEndpoint string
+		runID        string
+		suiteStart   time.Time
+	)
+
+	BeforeAll(func() {
+		tempoURL = os.Getenv("TEMPO_URL")
+		otlpEndpoint = os.Getenv("OTLP_ENDPOINT")
+		if tempoURL == "" || otlpEndpoint == "" {
+			Skip("TEMPO_URL / OTLP_ENDPOINT not set — skipping the Tempo-backed tracing e2e")
+		}
+		suiteStart = time.Now()
+		runID = "e2e-" + strconv.FormatInt(suiteStart.UnixNano(), 10)
+		_, _ = fmt.Fprintf(GinkgoWriter,
+			"tracing e2e run id: %s — find this run in Grafana with TraceQL {resource.test.run.id=%q}\n",
+			runID, runID)
+
+		By("creating manager namespace")
+		cmd := exec.Command("kubectl", "create", "ns", namespace)
+		_, err := utils.Run(cmd)
+		Expect(err).NotTo(HaveOccurred(), "Failed to create namespace")
+
+		By("labeling the namespace to enforce the restricted security policy")
+		cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace,
+			"pod-security.kubernetes.io/enforce=restricted")
+		_, err = utils.Run(cmd)
+		Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy")
+
+		By("installing CRDs")
+		cmd = exec.Command("make", "install")
+		_, err = utils.Run(cmd)
+		Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs")
+
+		By("deploying the controller-manager")
+		cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", managerImage))
+		_, err = utils.Run(cmd)
+		Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager")
+
+		By("pre-pulling the squid image into kind (best effort, kills the biggest flake source)")
+		if _, err := utils.Run(exec.Command("docker", "pull", squidImage)); err == nil {
+			if err := utils.LoadImageToKindClusterWithName(squidImage); err != nil {
+				_, _ = fmt.Fprintf(GinkgoWriter, "kind load of %s failed (continuing): %v\n", squidImage, err)
+			}
+		} else {
+			_, _ = fmt.Fprintf(GinkgoWriter, "docker pull %s failed (continuing): %v\n", squidImage, err)
+		}
+
+		By("preflighting the OTLP endpoint from inside the cluster")
+		// Host-reachability of the OTLP endpoint does not prove
+		// pod-reachability from inside kind, and the operator logs export
+		// failures only at V(1) — without this, a broken path is a slow,
+		// opaque search timeout instead of a clear failure.
+		preflightOTLP(otlpEndpoint)
+
+		By("pointing the operator at the OTLP endpoint and tagging the test run")
+		// OTEL_RESOURCE_ATTRIBUTES is replaced in place, which keeps it
+		// listed after the downward-API POD_NAME/POD_NAMESPACE vars —
+		// $(VAR) expansion only sees earlier-listed vars. exec.Command
+		// passes $(...) through without shell mangling.
+		cmd = exec.Command("kubectl", "set", "env",
+			"deployment/"+deploymentName, "-n", namespace,
+			"OTEL_EXPORTER_OTLP_ENDPOINT="+otlpEndpoint,
+			fmt.Sprintf(
+				"OTEL_RESOURCE_ATTRIBUTES=k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE),test.run.id=%s",
+				runID))
+		_, err = utils.Run(cmd)
+		Expect(err).NotTo(HaveOccurred(), "Failed to set OTel env on the deployment")
+
+		cmd = exec.Command("kubectl", "rollout", "status",
+			"deployment/"+deploymentName, "-n", namespace, "--timeout=3m")
+		_, err = utils.Run(cmd)
+		Expect(err).NotTo(HaveOccurred(), "Rollout after set env did not finish")
+
+		By("verifying the operator reports tracing enabled")
+		Eventually(func(g Gomega) {
+			pod := newestControllerPod(g)
+			out, err := utils.Run(exec.Command("kubectl", "logs", pod, "-n", namespace))
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(out).To(ContainSubstring("tracing enabled"),
+				"operator did not log 'tracing enabled' after rollout")
+		}, 2*time.Minute).Should(Succeed())
+	})
+
+	AfterAll(func() {
+		if tempoURL == "" || otlpEndpoint == "" {
+			return // spec was skipped; nothing was deployed
+		}
+		By("cleaning up test proxies")
+		args := append([]string{"delete", "proxy", "-n", proxyNS, "--ignore-not-found"}, proxyNames...)
+		_, _ = utils.Run(exec.Command("kubectl", args...))
+
+		By("cleaning up the OTLP preflight pod")
+		_, _ = utils.Run(exec.Command("kubectl", "delete", "pod", otlpProbePodName, "-n", namespace,
+			"--ignore-not-found"))
+
+		By("undeploying the controller-manager")
+		_, _ = utils.Run(exec.Command("make", "undeploy"))
+
+		By("uninstalling CRDs")
+		_, _ = utils.Run(exec.Command("make", "uninstall"))
+
+		By("removing manager namespace")
+		_, _ = utils.Run(exec.Command("kubectl", "delete", "ns", namespace))
+	})
+
+	It("creates proxies and reports reconcile traces to Tempo", func() {
+		By("applying two kubernetes-provider proxies")
+		// Absolute paths on purpose: utils.Run chdirs the whole process.
+		dir := GinkgoT().TempDir()
+		for _, name := range proxyNames {
+			manifest := fmt.Sprintf(`apiVersion: crawl.example.com/v1alpha1
+kind: Proxy
+metadata:
+  name: %s
+  namespace: %s
+spec:
+  mode: Managed
+  provider: kubernetes
+  attributes:
+    purpose: tracing-e2e
+`, name, proxyNS)
+			path := filepath.Join(dir, name+".yaml")
+			Expect(os.WriteFile(path, []byte(manifest), 0o644)).To(Succeed())
+			_, err := utils.Run(exec.Command("kubectl", "apply", "-f", path))
+			Expect(err).NotTo(HaveOccurred(), "Failed to apply %s", name)
+		}
+
+		By("waiting for the proxies to become Ready")
+		Eventually(func(g Gomega) {
+			for _, name := range proxyNames {
+				out, err := utils.Run(exec.Command("kubectl", "get", "proxy", name,
+					"-n", proxyNS, "-o", "jsonpath={.status.phase}"))
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(out).To(Equal("Ready"), "proxy %s not Ready", name)
+			}
+		}, 5*time.Minute).Should(Succeed())
+
+		By("finding a provider.create trace for this run in Tempo")
+		// Anchored on provider.create, not the root span name: a
+		// "Reconcile Proxy" hit could be the finalizer-add or a drift
+		// reconcile, which contain no provider call.
+		traceID := eventuallyFindTrace(tempoURL, runID, "provider.create", suiteStart)
+
+		By("asserting the reconcile trace structure")
+		spanNames, resAttrs, err := tempoTrace(tempoURL, traceID)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(spanNames).To(ContainElements(
+			"Reconcile Proxy", "reconcile.managed", "provider.create", "status.patch"),
+			"trace %s is missing expected spans; got: %v", traceID, spanNames)
+		Expect(resAttrs["service.name"]).To(Equal("egress-proxies-operator"))
+		Expect(resAttrs["test.run.id"]).To(Equal(runID))
+	})
+
+	It("traces proxy deletion", func() {
+		By("deleting the proxies")
+		args := append([]string{"delete", "proxy", "-n", proxyNS, "--wait=false"}, proxyNames...)
+		_, err := utils.Run(exec.Command("kubectl", args...))
+		Expect(err).NotTo(HaveOccurred(), "Failed to delete proxies")
+
+		By("waiting for the proxies to be gone (finalizer ran provider.delete)")
+		Eventually(func(g Gomega) {
+			out, err := utils.Run(exec.Command("kubectl", "get", "proxy", "-n", proxyNS, "-o", "name"))
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(out).NotTo(ContainSubstring("proxy-tracing-e2e"))
+		}, 3*time.Minute).Should(Succeed())
+
+		By("finding a provider.delete trace for this run in Tempo")
+		traceID := eventuallyFindTrace(tempoURL, runID, "provider.delete", suiteStart)
+
+		By("asserting the deletion trace structure")
+		spanNames, _, err := tempoTrace(tempoURL, traceID)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(spanNames).To(ContainElements("Reconcile Proxy", "reconcile.delete", "provider.delete"),
+			"trace %s is missing expected spans; got: %v", traceID, spanNames)
+	})
+})
+
+// preflightOTLP runs a one-shot curl pod inside the cluster POSTing to the
+// OTLP HTTP ingest, and fails with a clear message when it is unreachable.
+// The pod runs in the restricted-PSS operator namespace, hence the full
+// securityContext (same shape as the curl-metrics pod).
+func preflightOTLP(otlpEndpoint string) {
+	script := fmt.Sprintf(
+		"for i in $(seq 1 10); do "+
+			"code=$(curl -sS -o /dev/null -w '%%{http_code}' -X POST "+
+			"-H 'Content-Type: application/json' -d '{}' %s/v1/traces); "+
+			"echo \"attempt $i: HTTP $code\"; "+
+			"[ \"$code\" = \"200\" ] && echo OTLP_OK && exit 0; sleep 2; "+
+			"done; echo OTLP_UNREACHABLE; exit 1",
+		otlpEndpoint)
+	cmd := exec.Command("kubectl", "run", otlpProbePodName, "--restart=Never",
+		"--namespace", namespace,
+		"--image=curlimages/curl:latest",
+		"--overrides",
+		fmt.Sprintf(`{
+			"spec": {
+				"containers": [{
+					"name": "curl",
+					"image": "curlimages/curl:latest",
+					"command": ["/bin/sh", "-c"],
+					"args": [%q],
+					"securityContext": {
+						"readOnlyRootFilesystem": true,
+						"allowPrivilegeEscalation": false,
+						"capabilities": {
+							"drop": ["ALL"]
+						},
+						"runAsNonRoot": true,
+						"runAsUser": 1000,
+						"seccompProfile": {
+							"type": "RuntimeDefault"
+						}
+					}
+				}]
+			}
+		}`, script))
+	_, err := utils.Run(cmd)
+	Expect(err).NotTo(HaveOccurred(), "Failed to create the OTLP preflight pod")
+
+	Eventually(func(g Gomega) {
+		out, err := utils.Run(exec.Command("kubectl", "get", "pod", otlpProbePodName,
+			"-n", namespace, "-o", "jsonpath={.status.phase}"))
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(out).To(BeElementOf("Succeeded", "Failed"), "preflight pod still running")
+	}, 2*time.Minute).Should(Succeed())
+
+	logs, _ := utils.Run(exec.Command("kubectl", "logs", otlpProbePodName, "-n", namespace))
+	Expect(logs).To(ContainSubstring("OTLP_OK"),
+		"OTLP endpoint %s is not reachable from inside the kind cluster; curl output:\n%s",
+		otlpEndpoint, logs)
+}
+
+// otlpProbePodName mirrors the Describe-local constant for the helpers below.
+const otlpProbePodName = "curl-otlp"
+
+// newestControllerPod returns the most recently created controller pod —
+// right after a rollout, an unsorted lookup may pick the terminating one.
+func newestControllerPod(g Gomega) string {
+	out, err := utils.Run(exec.Command("kubectl", "get", "pods",
+		"-l", "control-plane=controller-manager", "-n", namespace,
+		"--sort-by=.metadata.creationTimestamp", "-o", "name"))
+	g.Expect(err).NotTo(HaveOccurred())
+	lines := utils.GetNonEmptyLines(out)
+	g.Expect(lines).NotTo(BeEmpty(), "no controller pods found")
+	return strings.TrimPrefix(lines[len(lines)-1], "pod/")
+}
+
+// eventuallyFindTrace polls Tempo until a trace containing a span with the
+// given name exists for this run, and returns its trace ID. The batch span
+// processor flushes every ~5s, so a couple of polls is normal.
+func eventuallyFindTrace(tempoURL, runID, spanName string, since time.Time) string {
+	var traceID string
+	query := fmt.Sprintf(`{resource.test.run.id=%q && name=%q}`, runID, spanName)
+	Eventually(func(g Gomega) {
+		ids, err := tempoSearch(tempoURL, query, since)
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(ids).NotTo(BeEmpty(), "no trace for %s yet (query: %s)", spanName, query)
+		traceID = ids[0]
+	}, 2*time.Minute).Should(Succeed())
+	return traceID
+}
+
+// tempoSearch runs a TraceQL query against Tempo's search API and returns
+// the matching trace IDs. start/end are Unix seconds.
+func tempoSearch(tempoURL, traceql string, since time.Time) ([]string, error) {
+	u, err := url.Parse(tempoURL + "/api/search")
+	if err != nil {
+		return nil, err
+	}
+	q := u.Query()
+	q.Set("q", traceql)
+	q.Set("start", strconv.FormatInt(since.Add(-5*time.Minute).Unix(), 10))
+	q.Set("end", strconv.FormatInt(time.Now().Add(time.Minute).Unix(), 10))
+	q.Set("limit", "20")
+	u.RawQuery = q.Encode()
+
+	body, err := tempoGet(u.String())
+	if err != nil {
+		return nil, err
+	}
+	var result struct {
+		Traces []struct {
+			TraceID string `json:"traceID"`
+		} `json:"traces"`
+	}
+	if err := json.Unmarshal(body, &result); err != nil {
+		return nil, fmt.Errorf("decoding Tempo search response: %w", err)
+	}
+	ids := make([]string, 0, len(result.Traces))
+	for _, t := range result.Traces {
+		ids = append(ids, t.TraceID)
+	}
+	return ids, nil
+}
+
+// tempoTrace fetches one trace and flattens it to a span-name list plus the
+// resource attributes. Tempo returns OTLP-JSON (batches → scopeSpans →
+// spans), not Jaeger's shape.
+func tempoTrace(tempoURL, traceID string) ([]string, map[string]string, error) {
+	body, err := tempoGet(tempoURL + "/api/traces/" + traceID)
+	if err != nil {
+		return nil, nil, err
+	}
+	var trace struct {
+		Batches []struct {
+			Resource struct {
+				Attributes []struct {
+					Key   string `json:"key"`
+					Value struct {
+						StringValue string `json:"stringValue"`
+					} `json:"value"`
+				} `json:"attributes"`
+			} `json:"resource"`
+			ScopeSpans []struct {
+				Spans []struct {
+					Name string `json:"name"`
+				} `json:"spans"`
+			} `json:"scopeSpans"`
+		} `json:"batches"`
+	}
+	if err := json.Unmarshal(body, &trace); err != nil {
+		return nil, nil, fmt.Errorf("decoding Tempo trace %s: %w", traceID, err)
+	}
+	var spanNames []string
+	resAttrs := map[string]string{}
+	for _, b := range trace.Batches {
+		for _, a := range b.Resource.Attributes {
+			if a.Value.StringValue != "" {
+				resAttrs[a.Key] = a.Value.StringValue
+			}
+		}
+		for _, ss := range b.ScopeSpans {
+			for _, s := range ss.Spans {
+				spanNames = append(spanNames, s.Name)
+			}
+		}
+	}
+	return spanNames, resAttrs, nil
+}
+
+func tempoGet(rawURL string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("querying Tempo: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("tempo returned %d for %s: %s", resp.StatusCode, rawURL, string(body))
+	}
+	return body, nil
+}