diff --git a/Makefile b/Makefile index b7ab1ce..fb685df 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,10 @@ test: manifests generate fmt vet setup-envtest ## Run tests. test-integration-kvm: ## Run the KVM provider test against a real hypervisor. Needs KVM_TEST_HOST/USER/SSH_KEY/BRIDGE/IMAGE_URL; see docs/testing.md. go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/ +.PHONY: test-integration-kvm2 +test-integration-kvm2: ## Run the kvm2 provider test against a real hypervisor. Needs KVM2_TEST_HOST/USER/SSH_KEY/BRIDGE/IMAGE_URL; see docs/testing.md. + go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM2 ./internal/provider/kvm2/ + # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. # kubectl kuberc is disabled by default for test isolation; enable with: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 8b3c5ee..2dd089d 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -149,9 +149,10 @@ spec: - name: providers-config mountPath: /etc/proxy-operator readOnly: true - # SSH material for kvm providers (sshKeyFile / knownHostsFile in the - # providers config point here). Optional: the mount is empty when the - # Secret does not exist, and only kvm entries need it. Create it with: + # SSH material for the kvm and kvm2 providers (sshKeyFile / + # knownHostsFile in the providers config point here). One Secret + # serves both types. Optional: the mount is empty when the Secret + # does not exist, and only kvm and kvm2 entries need it. Create it with: # kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \ # --from-file=id_ed25519= \ # --from-file=known_hosts= @@ -167,6 +168,7 @@ spec: - name: providers-config configMap: name: providers-config + # Shared by the kvm and kvm2 providers; see the mount above. - name: kvm-ssh secret: secretName: kvm-ssh diff --git a/config/manager/providers_config.yaml b/config/manager/providers_config.yaml index 311a96d..4ae1361 100644 --- a/config/manager/providers_config.yaml +++ b/config/manager/providers_config.yaml @@ -51,4 +51,35 @@ data: bridge: br0 # to a LAN with DHCP; the VM's IP comes from there # socket: /var/run/libvirt/libvirt-sock # pool: default + # kvm2 takes the same block shape as kvm: an entry ports by changing + # type to kvm2 and the block name to kvm2. kvm2 does not adopt domains + # made by kvm, so delete those Proxies first. See docs/kvm2-provider.md. + # - name: kvm2-home + # type: kvm2 + # kvm2: + # sshKeyFile: /etc/proxy-operator/kvm/id_ed25519 + # knownHostsFile: /etc/proxy-operator/kvm/known_hosts + # # insecureIgnoreHostKey: true + # # diskSizeGb: 10 + # # installGuestAgent: false + # # vendorData: | + # # #cloud-config + # # ... + # images: + # debian-13: + # url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2 + # # sha256: "<64 hex>" + # # firmware: uefi + # ubuntu-24.04: + # url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img + # machineTypes: + # small: {vcpus: 1, memoryMiB: 1024} + # medium: {vcpus: 2, memoryMiB: 2048} + # hosts: + # - name: beelink + # address: beelink.tail8c69c.ts.net:22 + # user: novakj + # bridge: br0 + # # socket: /var/run/libvirt/libvirt-sock + # # pool: default diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 4545440..c252b84 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -5,5 +5,6 @@ resources: - proxy_kubernetes.yaml - proxy_gcp.yaml - proxy_kvm.yaml +- proxy_kvm2.yaml - proxy_external.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/providers-config.yaml b/config/samples/providers-config.yaml index e7d57b9..14a0bfb 100644 --- a/config/samples/providers-config.yaml +++ b/config/samples/providers-config.yaml @@ -55,3 +55,34 @@ providers: bridge: br0 # to a LAN with DHCP; the VM's IP comes from there # socket: /var/run/libvirt/libvirt-sock # pool: default + # kvm2 takes the same block shape as kvm: an entry ports by changing type + # to kvm2 and the block name to kvm2. kvm2 does not adopt domains made by + # kvm, so delete those Proxies first. See docs/kvm2-provider.md. + # - name: kvm2-home + # type: kvm2 + # kvm2: + # sshKeyFile: /etc/proxy-operator/kvm/id_ed25519 + # knownHostsFile: /etc/proxy-operator/kvm/known_hosts + # # insecureIgnoreHostKey: true + # # diskSizeGb: 10 + # # installGuestAgent: false + # # vendorData: | + # # #cloud-config + # # ... + # images: + # debian-13: + # url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2 + # # sha256: "<64 hex>" + # # firmware: uefi + # ubuntu-24.04: + # url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img + # machineTypes: + # small: {vcpus: 1, memoryMiB: 1024} + # medium: {vcpus: 2, memoryMiB: 2048} + # hosts: + # - name: beelink + # address: beelink:22 + # user: novakj + # bridge: br0 + # # socket: /var/run/libvirt/libvirt-sock + # # pool: default diff --git a/config/samples/proxy_kvm2.yaml b/config/samples/proxy_kvm2.yaml new file mode 100644 index 0000000..0267d34 --- /dev/null +++ b/config/samples/proxy_kvm2.yaml @@ -0,0 +1,44 @@ +# A Managed proxy backed by a libvirt/KVM hypervisor through the kvm2 +# provider: the operator boots a stock Debian cloud image as a VM on the +# named host, with a NIC on the host's bridge (IP from the LAN's DHCP), and +# installs Squid via cloud-init. Requires a providers-config entry named +# "kvm2-home" (see providers-config.yaml) and the kvm-ssh Secret, which the +# kvm and kvm2 providers share (docs/kvm2-provider.md). +# +# placement.zone is the host name from kvm2.hosts (optional when only one +# host is configured); machineType and image are keys in the provider's +# machineTypes and images maps. The first Proxy for an image on a host +# waits for the image download; later ones reuse the cached copy. +apiVersion: crawl.example.com/v1alpha1 +kind: Proxy +metadata: + name: proxy-kvm2-sample +spec: + mode: Managed + provider: kvm2-home + placement: + zone: beelink + machineType: small + image: debian-13 + port: 3128 + cloudInit: + inline: | + #cloud-config + packages: + - squid + write_files: + - path: /etc/squid/conf.d/proxy-operator.conf + content: | + http_port 3128 + http_access allow all + via off + forwarded_for off + runcmd: + - systemctl restart squid + # Optional: log in with `ssh debian@` (the Debian image's default + # user) to look around; `virsh console ` on the host works too. + # ssh_authorized_keys: + # - ssh-ed25519 AAAA... you@laptop + attributes: + geo: home + purpose: crawl diff --git a/docs/plans-executions/2026-09-16-1333-kvm2-provider.md b/docs/plans-executions/2026-09-16-1333-kvm2-provider.md index 0d84110..98db9da 100644 --- a/docs/plans-executions/2026-09-16-1333-kvm2-provider.md +++ b/docs/plans-executions/2026-09-16-1333-kvm2-provider.md @@ -10,7 +10,7 @@ Plan: [docs/plans/2026-09-16-1333-kvm2-provider.md](../plans/2026-09-16-1333-kvm - [x] Step 5 — Image cache and pool-space precheck - [x] Step 6 — The four contract methods - [x] Step 7 — Checker and the CRD comment -- [ ] Step 8 — Deployment files, Makefile, integration test +- [x] Step 8 — Deployment files, Makefile, integration test - [ ] Step 9 — Documentation - [ ] Step 10 — Milestone review - [ ] Step 11 — Real-host validation, changelog, MR @@ -281,3 +281,40 @@ so it built its own locked capture; the orchestrator fixed `captureContext` with a mutex in this commit and re-ran the package at `-race -count=3`. Attestation: "files read outside the allowed set: none". + +## Step 8 — Deployment files, Makefile, integration test + +2026-09-16 19:39. Seventh subagent step. `kvm2_integration_test.go` +(`//go:build integration`, `KVM2_TEST_*` variables, the eleven ordered +assertions of spec entry 21 plus `TestKVM2Check_realHost`), +`config/samples/proxy_kvm2.yaml` (zone `beelink`, so the sample matches the +shipped config), a commented `kvm2-home` entry in both providers configs +with the porting note, comment-only edits on the shared `kvm-ssh` mount, the +`test-integration-kvm2` Makefile target and the kvm2 rows and section in +`docs/testing.md`. The integration test was not run; that is Step 11 with +the user's go-ahead. + +The agent found that `New` builds the production dialer without tracing +options, so in production the dialer and decorator use the global tracer +provider (correct: `cmd/main.go` sets it up), while a test that wants every +span family in one recorder sets the dialer's option itself. The test does +so and asserts that `kvm2.DomainDefineXML` and `kvm2.dial` are children of +`provider.create`. + +Decisions the agent made, accepted: the UID is generated from `crypto/rand` +rather than promoting `google/uuid` to a direct dependency; the sentinel +check looks for the full cloud-init string because the provider name also +appears in attributes; `-run TestKVM2` matches both tests, as the kvm +target's pattern does. Forward links to `docs/kvm2-provider.md` dangle until +Step 9. + +Commands: + +```sh +go vet ./... && go vet -tags=integration ./... && go vet -tags=e2e ./... +bin/kustomize build config/default > /dev/null && bin/kustomize build config/samples > /dev/null +make -n test-integration-kvm2 +# go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM2 ./internal/provider/kvm2/ +``` + +All clean. Attestation: "files read outside the allowed set: none". diff --git a/docs/testing.md b/docs/testing.md index cfa94d9..0adf88b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -62,6 +62,11 @@ manual runs too. | | `trace_test.go`, `conn_test.go`, `errors_test.go` | every `libvirtAPI` method traced exactly once (guarded by reflection on the interface), results and errors pass through, libvirt error fields, detail keys only at V2, upload content never logged; pool connect/drop lines, no wrapping below V1 | `fakeAPI` + `funcr` capture | | | `check_test.go`, `conn_test.go` | health check: details, per-host verdicts in config order, hosts checked concurrently, a timeout that keeps the connection and refuses a second check while the first still waits, no waiting behind a reconcile's dial, the reason table; `probe` abandons without closing, but still drops on a transport error; waiting for a host's slot ends with the caller's deadline; an inactive or missing pool and daemon errors keep the connection; every dial stage tagged through the real SSH server, with classification unchanged | `fakeAPI`, fake dialers, in-process SSH server | | | `kvm_integration_test.go` (`-tags=integration`) | one real VM: image fetch, Create, idempotent re-Create, Running with IP, ListByTag, Delete, pool state afterwards; `TestKVMCheck_realHost` runs the health check against the host and creates nothing | a **real hypervisor** — see below | +| `internal/provider/kvm2` | `xml_test.go`, `volume_test.go`, `seed_test.go`, `id_test.go`, `metadata_test.go`, `state_test.go` | pure builders: domain XML (file disks with resolved paths, UEFI with secure boot off, BIOS escape, the metadata element, the guest-agent channel), volume XML with and without a backing store, seed ISO reopened (files, label, user-data byte for byte, temp dir removed) and the vendor-data script vs. multipart rules, providerID round-trip and malformed forms, metadata in both namespace spellings, state mapping and the IPv4 filter | pure | +| | `errors_test.go`, `backend_test.go`, `hosts_test.go`, `ssh_test.go` | classification table (libvirt codes, SSH auth text, `knownhosts.KeyError`, missing files); the per-call budget adapter (deadline kills the connection and is Transient, cancel unblocks, killed by another caller, close idempotent) against a mocked libvirt socket; the host table (one dial per host under concurrency, redial after drop, `dialFresh` never cached, waiting ends with ctx); the real dialer end to end: socket forwarding, unknown and mismatched host keys, insecure mode, missing vs. unparsable key and known_hosts, wrong key, handshake timeout, cancel, every stage tagged, key material never in errors or logs | fake dialer; a **real in-process `x/crypto/ssh` server** forwarding a unix socket | +| | `image_test.go`, `create_test.go`, `get_test.go`, `delete_test.go`, `list_test.go`, `kvm2_test.go`, `provider_test.go` | image cache (fetch with the marker written last, cached, marker missing or without a digest, sha256 change or mismatch, 4xx/5xx, no Content-Length, short read, concurrent callers fetch once, cancel mid-stream, streamed without buffering, upload on a dedicated connection) and the pool-space quota precheck; Create resume after every partial step, returns only when running, adoption guard on UID, seed always rebuilt, cloud-init sentinel never logged; Get state and address mapping incl. agent-before-ARP; every Delete already-gone case, NVRAM flag, per-name lock; ListByTag across hosts, one down fails the whole call, unreadable domain skipped; `New` never dials and warns on missing files; every error wrapped with the configured name; logging tiers V0/V1/V2 | `httptest` image server + in-memory `fakeBackend` behind the `backend` seam | +| | `observe_test.go`, `check_test.go` | every seam method traced and logged exactly once (guarded by reflection), child spans under the parent, detail only at V2, upload bodies never logged, error fields; health check: one entry per host in config order, hosts checked concurrently, expired ctx returns promptly, never touches the pooled connection, the reason table, detail keys | `fakeBackend` + `funcr` capture + span recorder | +| | `kvm2_integration_test.go` (`-tags=integration`) | one real VM: `New` without a dial, `Check` Ready with details, Create, idempotent re-Create with an unchanged domain count, Running with IP and the metadata round-trip, ListByTag, Delete, NotFound, pool state afterwards, `kvm2.*` spans under `provider.create`; `TestKVM2Check_realHost` runs the health check against the host and creates nothing | a **real hypervisor** — see below | | `internal/health` | `probe_test.go` | probes through a **real CONNECT-capable proxy stub** to a real TLS server: tunnel success, refused CONNECT, wrong status, dead proxy, plain-http forward | `httptest` + hijacked bidirectional tunnel | | | `engine_test.go` | thresholds, first-verdict, latency suppression matrix, dropped-event retry, stale-probe UID guard, tick scheduling/seeding/pruning, `Start` end-to-end | fake `client.Reader` + injected `probeFn` | | `internal/lease` | `store_test.go` | capacity, `MaxLeases=0`, all three selection tie-breaks, target-scoped vs global cooldowns, expiry via fake clock, report-on-retained-lease, idempotent release, **40 concurrent acquires never exceeding capacity** | injectable clock; the concurrency case is why `-race` matters | @@ -93,12 +98,12 @@ envtest suite the scaffold generated. bug (below). - **Live GCP**: no test talks to Google. The seam boundary (`buildInsertRequest` + classification) is tested exhaustively instead. -- **Live libvirt, by default**: the KVM unit tests run over an in-memory - fake of the 20 libvirt calls the provider uses. Whether libvirt accepts - the domain XML, whether a raw qcow2 upload plus pool refresh yields a - usable backing image, and whether the guest actually boots and reports - an address are provable only on a real host — that is the opt-in - integration test below. +- **Live libvirt, by default**: the KVM and kvm2 unit tests run over an + in-memory fake of the 20 libvirt calls each provider uses. Whether + libvirt accepts the domain XML, whether a raw qcow2 upload plus pool + refresh yields a usable backing image, and whether the guest actually + boots and reports an address are provable only on a real host — that is + the opt-in integration test below (one per provider). ## The KVM integration test (real hypervisor) @@ -162,6 +167,95 @@ go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/ The test is not part of `make test` and not run in CI. Keep it compiling with `go vet -tags=integration ./...` when the provider's internals move. +## The kvm2 integration test (real hypervisor) + +`internal/provider/kvm2/kvm2_integration_test.go` compiles only under +`-tags=integration` and skips itself unless the environment names a +host. Like the kvm one it deviates from the "testcontainers for real +infra" convention on purpose: libvirt needs `/dev/kvm` and a bridge to a +DHCP network, and a container has neither. It targets whichever +hypervisor you point it at and leaves that host's pool with one more +cached base image (`base-test-image.qcow2` and its `.sha256` marker) than +it had. The kvm and kvm2 tests can share a host: kvm2 marks its domains +in its own metadata namespace and never lists or adopts the other's. + +```sh +export KVM2_TEST_HOST=beelink:22 # SSH endpoint +export KVM2_TEST_USER=novakj # must be in the libvirt group +export KVM2_TEST_SSH_KEY=$HOME/.ssh/id_ed25519 # private key for that user +export KVM2_TEST_KNOWN_HOSTS=$HOME/.ssh/kvm_known_hosts # the hypervisors' host keys +export KVM2_TEST_BRIDGE=br0 # bridge to a network with DHCP +export KVM2_TEST_IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2 +make test-integration-kvm2 +``` + +`KVM2_TEST_KNOWN_HOSTS` is technically optional, but leaving it out turns +host-key verification **off** for the run (the log says so); keep a +`kvm_known_hosts` holding just the hypervisors rather than pointing the +test at your everyday `known_hosts`, so a re-keyed hypervisor is a +one-line fix in a file nothing else uses. + +Also optional: `KVM2_TEST_IMAGE_SHA256` (verified after download and +compared with the cached marker), `KVM2_TEST_POOL` (default `default`), +`KVM2_TEST_SOCKET` (default `/var/run/libvirt/libvirt-sock`), +`KVM2_TEST_TIMEOUT` (default `20m`, the budget for the whole lifecycle; +it must cover the first-time image download), `KVM2_TEST_KEEP` to leave +the VM behind for `virsh console` when something goes wrong (the log then +prints the `virsh undefine --nvram` and `virsh vol-delete` lines that +clean it up), and `KVM2_TEST_LOG_LEVEL` (default `1`) for how much of the +provider's logging the `-v` output shows: at `1` every libvirt RPC is one +compact `libvirt call` line (`HH:MM:SS.mmm kvm2 "level"=1 "msg"="libvirt +call" "rpc"="DomainLookupByName" "domain"=... "duration"=...`, plus the +per-operation outcome lines and connection events), interleaved with the +test's own `t.Log` lines; `2` adds the domain and volume XML, metadata +and addresses; `0` leaves only `t.Log`. The per-run constants (provider, +host, domain name) are stated once in the first line and left off every +provider line. See the Logging section of +[kvm2-provider.md](kvm2-provider.md). + +What one run asserts, in order, each step logged with its duration: + +1. `New` returns within 100 ms and without dialling (no `libvirt host + connected` line before the first call). +2. `Check` reports the one host `Ready` with reason `Reachable` and the + `hostname`, `libvirtVersion`, `poolState` (`running`) and + `poolAvailableGiB` details, which are logged. +3. Every later call goes through `provider.WithTracing` with an in-memory + span recorder that the package's own spans also land in. +4. `Create` returns `hosts/kvm2-test/domains/` (downloading the + image on a cold pool — the log prints how long it took). +5. A second `Create` returns the same ID and the host's domain count, + read over a dedicated connection, is unchanged. +6. `Get`, polled every 5 s, reaches `Running` with an IPv4 within the + budget; its `UID` is the one Create wrote and `CreatedAt` is non-zero + and within a few minutes of now. The time from Create's return to the + first address is logged. +7. `ListByTag` contains the ID with that UID. +8. A TCP connect to port 22 on the guest, logged only — the machine + running the test may not route to the bridge network. +9. `Delete` returns nil, `Get` is `ErrNotFound`, a second `Delete` is a + no-op. +10. The pool, inspected over a dedicated connection, holds the base image + and its marker but no `.qcow2` or `-seed.iso`. +11. Among the recorded spans, a `provider.create` span has at least one + `kvm2.DomainDefineXML` and one `kvm2.dial` child, and no span + attribute carries the cloud-init text. + +With `KVM2_TEST_KEEP` steps 9 and 10 are skipped; otherwise a `t.Cleanup` +attempts the Delete even when an earlier assertion failed. + +`TestKVM2Check_realHost` runs the provider health check +([kvm2-provider.md](kvm2-provider.md)) against the same host. It must +find the host reachable and the pool running, and it creates nothing. It +needs the same variables; to run only this test: + +```sh +go test -tags=integration -count=1 -v -run TestKVM2Check ./internal/provider/kvm2/ +``` + +The test is not part of `make test` and not run in CI. Keep it compiling +with `go vet -tags=integration ./...` when the provider's internals move. + ## The scaffolded `make test-e2e` suite `test/e2e/` is the kubebuilder-generated smoke suite (build image → kind diff --git a/internal/provider/kvm2/kvm2_integration_test.go b/internal/provider/kvm2/kvm2_integration_test.go new file mode 100644 index 0000000..625f16e --- /dev/null +++ b/internal/provider/kvm2/kvm2_integration_test.go @@ -0,0 +1,590 @@ +//go:build integration + +package kvm2 + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "net" + "os" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/digitalocean/go-libvirt" + "github.com/go-logr/logr" + "github.com/go-logr/logr/funcr" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "k8s.io/apimachinery/pkg/types" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider" + "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing" +) + +// The real-hypervisor test of spec entry 21. It runs only with +// -tags=integration and skips itself unless KVM2_TEST_* names a host; see +// docs/testing.md for the variables and what one run asserts. +const ( + integrationProvider = "kvm2-integration" + integrationHost = "kvm2-test" + integrationImage = "test-image" + integrationMachine = "small" + integrationPort = 3128 + + // integrationCloudInit is the user-data the VM boots with. Nothing + // asserts on the guest running it; it exists so the sentinel check can + // prove that no span attribute carries the cloud-init text. + integrationCloudInit = "#cloud-config\nruncmd:\n - echo kvm2-integration\n" + + defaultIntegrationTimeout = 20 * time.Minute + defaultIntegrationLogLvl = 1 + getPollInterval = 5 * time.Second + cleanupBudget = 5 * time.Minute +) + +type integrationEnv struct { + host, user, sshKey, knownHosts, bridge, imageURL string + imageSHA256, pool, socket string + timeout time.Duration + keep bool + logLevel int +} + +// loadIntegrationEnv reads the KVM2_TEST_* variables, skipping the test +// under -short or when a required one is missing. +func loadIntegrationEnv(t *testing.T) integrationEnv { + t.Helper() + if testing.Short() { + t.Skip("real-hypervisor test skipped under -short") + } + env := integrationEnv{timeout: defaultIntegrationTimeout, logLevel: defaultIntegrationLogLvl} + required := []struct { + name string + dst *string + }{ + {"KVM2_TEST_HOST", &env.host}, + {"KVM2_TEST_USER", &env.user}, + {"KVM2_TEST_SSH_KEY", &env.sshKey}, + {"KVM2_TEST_BRIDGE", &env.bridge}, + {"KVM2_TEST_IMAGE_URL", &env.imageURL}, + } + for _, r := range required { + v := os.Getenv(r.name) + if v == "" { + t.Skipf("%s is not set; the kvm2 integration test needs a real hypervisor (see docs/testing.md)", r.name) + } + *r.dst = v + } + env.knownHosts = os.Getenv("KVM2_TEST_KNOWN_HOSTS") + env.imageSHA256 = os.Getenv("KVM2_TEST_IMAGE_SHA256") + env.pool = os.Getenv("KVM2_TEST_POOL") + env.socket = os.Getenv("KVM2_TEST_SOCKET") + env.keep = os.Getenv("KVM2_TEST_KEEP") != "" + if v := os.Getenv("KVM2_TEST_TIMEOUT"); v != "" { + d, err := time.ParseDuration(v) + if err != nil { + t.Fatalf("KVM2_TEST_TIMEOUT=%q: %v", v, err) + } + env.timeout = d + } + if v := os.Getenv("KVM2_TEST_LOG_LEVEL"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + t.Fatalf("KVM2_TEST_LOG_LEVEL=%q: want a non-negative integer", v) + } + env.logLevel = n + } + return env +} + +// providerConfig is what the operator would load for this host: one host, +// one image, one machine type. Host keys are verified only when +// KVM2_TEST_KNOWN_HOSTS is set. +func (e integrationEnv) providerConfig() provider.ProviderConfig { + return provider.ProviderConfig{ + Name: integrationProvider, + Type: "kvm2", + KVM2: &provider.KVMConfig{ + SSHKeyFile: e.sshKey, + KnownHostsFile: e.knownHosts, + InsecureIgnoreHostKey: e.knownHosts == "", + Images: map[string]provider.KVMImage{ + integrationImage: {URL: e.imageURL, SHA256: e.imageSHA256}, + }, + MachineTypes: map[string]provider.KVMMachineType{ + integrationMachine: {VCPUs: 1, MemoryMiB: 1024}, + }, + Hosts: []provider.KVMHost{{ + Name: integrationHost, + Address: e.host, + User: e.user, + Socket: e.socket, + Pool: e.pool, + Bridge: e.bridge, + }}, + }, + } +} + +// integrationLog routes the provider's logr output into t.Log through funcr +// at the chosen verbosity, keeps every line for assertions, and strips the +// key/value pairs that are constant for the run (stated once up front) so +// each line stays short. Lines arriving after the test ended are dropped: +// t.Log panics then. +type integrationLog struct { + t *testing.T + + mu sync.Mutex + drop []string + lines []string + done bool +} + +func newIntegrationLog(t *testing.T, level int, drop ...string) (*integrationLog, logr.Logger) { + t.Helper() + sink := &integrationLog{t: t, drop: drop} + t.Cleanup(func() { + sink.mu.Lock() + defer sink.mu.Unlock() + sink.done = true + }) + log := funcr.New(func(prefix, args string) { sink.write(prefix, args) }, funcr.Options{Verbosity: level}) + return sink, log +} + +// dropValue adds a rendered pair (`"key"="value"`) to strip from later lines. +func (l *integrationLog) dropValue(key, value string) { + l.mu.Lock() + defer l.mu.Unlock() + l.drop = append(l.drop, fmt.Sprintf("%q=%q", key, value)) +} + +func (l *integrationLog) write(prefix, args string) { + l.mu.Lock() + defer l.mu.Unlock() + l.lines = append(l.lines, args) + if l.done { + return + } + for _, d := range l.drop { + args = strings.ReplaceAll(args, " "+d, "") + } + l.t.Logf("%s %s %s", time.Now().Format("15:04:05.000"), prefix, args) +} + +func (l *integrationLog) matching(substr string) []string { + l.mu.Lock() + defer l.mu.Unlock() + var out []string + for _, line := range l.lines { + if strings.Contains(line, substr) { + out = append(out, line) + } + } + return out +} + +// newIntegrationProvider builds the provider the way New does, plus the span +// recorder's tracer on both places the package starts spans: the Provider +// (kvm2.download) and the dialer (kvm2.dial and, through observe, every +// kvm2.). newLibvirtDialer has no option for it, so the field is set +// directly; that is what makes the kvm2.* spans land in the same recorder +// as provider.WithTracing's provider.* spans. +func newIntegrationProvider(pc provider.ProviderConfig, tp *sdktrace.TracerProvider) *Provider { + cfg := withDefaults(*pc.KVM2) + opt := tracing.WithTracerProvider(tp) + d := newLibvirtDialer(pc.Name, cfg) + d.tracing = []tracing.Option{opt} + return newProvider(pc.Name, cfg, d, withTracing(opt)) +} + +// randomUID is a UUID-shaped string, so the run's domain never collides with +// a previous run's leftovers or with a real Proxy's. +func randomUID(t *testing.T) string { + t.Helper() + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("reading random bytes: %v", err) + } + h := hex.EncodeToString(b[:]) + return h[0:8] + "-" + h[8:12] + "-" + h[12:16] + "-" + h[16:20] + "-" + h[20:32] +} + +// stepDone logs one lifecycle step with its duration; the log is the record +// of how long a cold image download or a boot took on this host. +func stepDone(t *testing.T, step string, start time.Time) { + t.Helper() + t.Logf("step %s: %s", step, time.Since(start).Round(time.Millisecond)) +} + +// waitUntil polls cond every interval until it is true or ctx ends. +func waitUntil(ctx context.Context, t *testing.T, interval time.Duration, what string, cond func() bool) { + t.Helper() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for !cond() { + select { + case <-ticker.C: + case <-ctx.Done(): + t.Fatalf("gave up waiting for %s: %v", what, ctx.Err()) + } + } +} + +// domainCount lists every domain on the host over a dedicated connection. +func domainCount(ctx context.Context, t *testing.T, p *Provider, host provider.KVMHost) int { + t.Helper() + be, err := p.hosts.dialFresh(ctx, host) + if err != nil { + t.Fatalf("dialling %s to count domains: %v", host.Name, err) + } + defer func() { _ = be.Close() }() + doms, _, err := be.ConnectListAllDomains(ctx, 1, libvirt.ConnectListDomainsActive|libvirt.ConnectListDomainsInactive) + if err != nil { + t.Fatalf("listing domains on %s: %v", host.Name, err) + } + return len(doms) +} + +// poolVolumes reports, for each name, whether the host's pool holds a volume +// of that name, over a dedicated connection. Any error other than +// ErrNoStorageVol fails the test. +func poolVolumes(ctx context.Context, t *testing.T, p *Provider, host provider.KVMHost, names ...string) map[string]bool { + t.Helper() + be, err := p.hosts.dialFresh(ctx, host) + if err != nil { + t.Fatalf("dialling %s to inspect the pool: %v", host.Name, err) + } + defer func() { _ = be.Close() }() + pool, err := be.StoragePoolLookupByName(ctx, host.Pool) + if err != nil { + t.Fatalf("looking up pool %s: %v", host.Pool, err) + } + out := make(map[string]bool, len(names)) + for _, name := range names { + _, err := be.StorageVolLookupByName(ctx, pool, name) + switch { + case err == nil: + out[name] = true + case hasCode(err, libvirt.ErrNoStorageVol): + out[name] = false + default: + t.Fatalf("looking up volume %s in pool %s: %v", name, host.Pool, err) + } + } + return out +} + +func TestKVM2_lifecycle(t *testing.T) { + env := loadIntegrationEnv(t) + pc := env.providerConfig() + sink, log := newIntegrationLog(t, env.logLevel, + fmt.Sprintf("%q=%q", "provider", integrationProvider), + fmt.Sprintf("%q=%q", "host", integrationHost)) + ctx, cancel := context.WithTimeout(logf.IntoContext(context.Background(), log), env.timeout) + defer cancel() + + t.Logf("provider=%s host=%s address=%s user=%s bridge=%s image=%s budget=%s log level %d; provider, host and name are left off every provider log line", + integrationProvider, integrationHost, env.host, env.user, env.bridge, env.imageURL, env.timeout, env.logLevel) + if env.knownHosts == "" { + t.Log("KVM2_TEST_KNOWN_HOSTS is not set: host keys are NOT verified for this run") + } + + // 1. New returns without dialling. + start := time.Now() + built, err := New(ctx, pc) + elapsed := time.Since(start) + if err != nil { + t.Fatalf("New() error = %v", err) + } + if elapsed > 100*time.Millisecond { + t.Errorf("New() took %s, want under 100ms: it must not dial", elapsed) + } + if lines := sink.matching("libvirt host connected"); len(lines) > 0 { + t.Errorf("New() dialled the host before any call: %q", lines) + } + stepDone(t, "1 New", start) + + // 2. Check reports the host Ready with details. + checker, ok := built.(provider.Checker) + if !ok { + t.Fatalf("New() returned %T, which is not a provider.Checker", built) + } + start = time.Now() + targets := checker.Check(ctx) + if len(targets) != 1 { + t.Fatalf("Check() returned %d targets, want 1: %+v", len(targets), targets) + } + ts := targets[0] + if !ts.Ready || ts.Reason != provider.ReasonReachable { + t.Fatalf("Check() = ready %v reason %q message %q, want Ready/%s (details %v)", ts.Ready, ts.Reason, ts.Message, provider.ReasonReachable, ts.Details) + } + for _, key := range []string{"hostname", "libvirtVersion", "poolState", "poolAvailableGiB"} { + if ts.Details[key] == "" { + t.Errorf("Check() details lack %q: %v", key, ts.Details) + } + } + if ts.Details["poolState"] != "running" { + t.Errorf("Check() poolState = %q, want running", ts.Details["poolState"]) + } + t.Logf("host %s: hostname=%s libvirt=%s pool=%s (%s) available=%s GiB capacity=%s GiB latency=%s", + ts.Name, ts.Details["hostname"], ts.Details["libvirtVersion"], ts.Details["pool"], ts.Details["poolState"], + ts.Details["poolAvailableGiB"], ts.Details["poolCapacityGiB"], ts.Latency.Round(time.Millisecond)) + stepDone(t, "2 Check", start) + + // 3. Every later call goes through the tracing decorator, with the + // package's own spans recorded alongside. + sr := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr)) + t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + p := newIntegrationProvider(pc, tp) + traced := provider.WithTracing(pc.Name, p, tracing.WithTracerProvider(tp)) + host := p.cfg.Hosts[0] + + // 4. Create. + uid := randomUID(t) + name := provider.NameFromUID(types.UID(uid)) + sink.dropValue("name", name) + id := formatProviderID(integrationHost, name) + req := provider.CreateRequest{ + Name: name, + UID: uid, + Namespace: "integration", + ProxyName: "kvm2-lifecycle", + Placement: provider.Placement{Zone: integrationHost, MachineType: integrationMachine, Image: integrationImage}, + CloudInit: integrationCloudInit, + Port: integrationPort, + } + t.Logf("domain name %s, uid %s, id %s", name, uid, id) + + deleted := false + if env.keep { + t.Logf("KVM2_TEST_KEEP is set: the VM stays behind. Clean up on the host with:\n"+ + " virsh undefine --nvram %s\n virsh vol-delete --pool %s %s\n virsh vol-delete --pool %s %s", + name, host.Pool, overlayVolumeName(name), host.Pool, seedVolumeName(name)) + } else { + t.Cleanup(func() { + if deleted { + return + } + cctx, ccancel := context.WithTimeout(logf.IntoContext(context.Background(), log), cleanupBudget) + defer ccancel() + if err := p.Delete(cctx, id); err != nil { + t.Errorf("cleanup Delete(%s) error = %v; remove the domain and its volumes by hand", id, err) + } + }) + } + + start = time.Now() + gotID, err := traced.Create(ctx, req) + if err != nil { + t.Fatalf("Create() error = %v", err) + } + if gotID != id { + t.Fatalf("Create() = %q, want %q", gotID, id) + } + createReturned := time.Now() + stepDone(t, "4 Create (includes the image download on a cold pool)", start) + + // 5. Create again: same ID, nothing new on the host. + start = time.Now() + before := domainCount(ctx, t, p, host) + again, err := traced.Create(ctx, req) + if err != nil { + t.Fatalf("second Create() error = %v", err) + } + if again != id { + t.Errorf("second Create() = %q, want %q", again, id) + } + if after := domainCount(ctx, t, p, host); after != before { + t.Errorf("second Create() changed the host's domain count from %d to %d", before, after) + } + stepDone(t, "5 Create again", start) + + // 6. Get reaches Running with an IPv4. + start = time.Now() + var inst *provider.Instance + waitUntil(ctx, t, getPollInterval, "the domain to be Running with an IPv4", func() bool { + var err error + inst, err = traced.Get(ctx, id) + if err != nil { + t.Fatalf("Get(%s) error = %v", id, err) + } + return inst.State == provider.StateRunning && inst.IP != "" + }) + if inst.UID != uid { + t.Errorf("Get() UID = %q, want %q", inst.UID, uid) + } + if inst.CreatedAt.IsZero() { + t.Error("Get() CreatedAt is zero; the metadata mark did not round-trip") + } else if skew := time.Since(inst.CreatedAt).Abs(); skew > 5*time.Minute { + t.Errorf("Get() CreatedAt = %s, %s from now; want within a few minutes", inst.CreatedAt, skew) + } + t.Logf("domain running at %s, %s after Create returned (createdAt %s)", inst.IP, time.Since(createReturned).Round(time.Millisecond), inst.CreatedAt.Format(time.RFC3339)) + stepDone(t, "6 Get until Running", start) + + // 7. ListByTag contains it. + start = time.Now() + list, err := traced.ListByTag(ctx) + if err != nil { + t.Fatalf("ListByTag() error = %v", err) + } + found := false + for _, li := range list { + if li.ID == id { + found = true + if li.UID != uid { + t.Errorf("ListByTag() entry %s has UID %q, want %q", id, li.UID, uid) + } + } + } + if !found { + t.Errorf("ListByTag() = %d instances, none with ID %s", len(list), id) + } + stepDone(t, "7 ListByTag", start) + + // 8. Reachability, logged only: this machine may not route to the bridge. + start = time.Now() + addr := net.JoinHostPort(inst.IP, "22") + if conn, err := net.DialTimeout("tcp", addr, 3*time.Second); err != nil { + t.Logf("TCP connect to %s failed (informational; the test host may not route to the bridge): %v", addr, err) + } else { + _ = conn.Close() + t.Logf("TCP connect to %s succeeded", addr) + } + stepDone(t, "8 TCP connect", start) + + if env.keep { + t.Log("KVM2_TEST_KEEP is set: skipping Delete and the pool inspection") + } else { + // 9. Delete, Get is NotFound, Delete again. + start = time.Now() + if err := traced.Delete(ctx, id); err != nil { + t.Fatalf("Delete() error = %v", err) + } + deleted = true + if _, err := traced.Get(ctx, id); !errors.Is(err, provider.ErrNotFound) { + t.Errorf("Get() after Delete error = %v, want ErrNotFound", err) + } + if err := traced.Delete(ctx, id); err != nil { + t.Errorf("second Delete() error = %v, want nil", err) + } + stepDone(t, "9 Delete, Get, Delete", start) + + // 10. The pool keeps the shared base image and its marker, and holds + // no per-VM volume. + start = time.Now() + base := baseVolumeName(integrationImage, provider.KVMFormatQcow2) + marker := markerVolumeName(base) + overlay, seed := overlayVolumeName(name), seedVolumeName(name) + present := poolVolumes(ctx, t, p, host, base, marker, overlay, seed) + for _, want := range []string{base, marker} { + if !present[want] { + t.Errorf("pool %s lacks %s after Delete; the base image cache must stay", host.Pool, want) + } + } + for _, gone := range []string{overlay, seed} { + if present[gone] { + t.Errorf("pool %s still holds %s after Delete", host.Pool, gone) + } + } + stepDone(t, "10 pool inspection", start) + } + + // 11. Spans: the package's spans are children of the decorator's. + assertLifecycleSpans(t, sr.Ended()) +} + +// assertLifecycleSpans checks that at least one provider.create span has a +// kvm2.DomainDefineXML and a kvm2.dial child (the first Create dials the +// pooled connection and defines the domain), and that no attribute of any +// span carries the cloud-init text. +func assertLifecycleSpans(t *testing.T, spans []sdktrace.ReadOnlySpan) { + t.Helper() + var creates []sdktrace.ReadOnlySpan + for _, s := range spans { + if s.Name() == "provider.create" { + creates = append(creates, s) + } + } + if len(creates) == 0 { + t.Fatalf("no provider.create span among %d spans: %v", len(spans), spanNames(spans)) + } + satisfied := false + for _, create := range creates { + parent := create.SpanContext().SpanID() + var children, define, dial int + for _, s := range spans { + if s.Parent().SpanID() != parent || !strings.HasPrefix(s.Name(), "kvm2.") { + continue + } + children++ + switch s.Name() { + case "kvm2.DomainDefineXML": + define++ + case "kvm2.dial": + dial++ + } + } + t.Logf("provider.create span %s: %d kvm2.* children (%d DomainDefineXML, %d dial)", parent, children, define, dial) + if define > 0 && dial > 0 { + satisfied = true + } + } + if !satisfied { + t.Errorf("no provider.create span has both a kvm2.DomainDefineXML and a kvm2.dial child; spans: %v", spanNames(spans)) + } + for _, s := range spans { + for _, a := range s.Attributes() { + v := a.Value.Emit() + if strings.Contains(v, integrationCloudInit) || strings.Contains(v, "#cloud-config") { + t.Errorf("span %s attribute %s carries the cloud-init text", s.Name(), a.Key) + } + } + } +} + +// TestKVM2Check_realHost runs the health check against the host and creates +// nothing. +func TestKVM2Check_realHost(t *testing.T) { + env := loadIntegrationEnv(t) + pc := env.providerConfig() + _, log := newIntegrationLog(t, env.logLevel, + fmt.Sprintf("%q=%q", "provider", integrationProvider), + fmt.Sprintf("%q=%q", "host", integrationHost)) + ctx, cancel := context.WithTimeout(logf.IntoContext(context.Background(), log), env.timeout) + defer cancel() + if env.knownHosts == "" { + t.Log("KVM2_TEST_KNOWN_HOSTS is not set: host keys are NOT verified for this run") + } + + built, err := New(ctx, pc) + if err != nil { + t.Fatalf("New() error = %v", err) + } + checker, ok := built.(provider.Checker) + if !ok { + t.Fatalf("New() returned %T, which is not a provider.Checker", built) + } + targets := checker.Check(ctx) + if len(targets) != 1 { + t.Fatalf("Check() returned %d targets, want 1: %+v", len(targets), targets) + } + ts := targets[0] + if !ts.Ready || ts.Reason != provider.ReasonReachable { + t.Fatalf("Check() = ready %v reason %q message %q, want Ready/%s (details %v)", ts.Ready, ts.Reason, ts.Message, provider.ReasonReachable, ts.Details) + } + if ts.Details["poolState"] != "running" { + t.Errorf("Check() poolState = %q, want running", ts.Details["poolState"]) + } + t.Logf("host %s (%s): hostname=%s libvirt=%s pool=%s (%s) available=%s GiB latency=%s", + ts.Name, ts.Address, ts.Details["hostname"], ts.Details["libvirtVersion"], ts.Details["pool"], ts.Details["poolState"], + ts.Details["poolAvailableGiB"], ts.Latency.Round(time.Millisecond)) +}