kvm2: integration test, sample Proxy, providers-config entries and Makefile target
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5.1
@@ -70,6 +70,10 @@ test: manifests generate fmt vet setup-envtest ## Run tests.
|
|||||||
test-integration-kvm: ## Run the KVM provider test against a real hypervisor. Needs KVM_TEST_HOST/USER/SSH_KEY/BRIDGE/IMAGE_URL; see docs/testing.md.
|
test-integration-kvm: ## Run the KVM provider test against a real hypervisor. Needs KVM_TEST_HOST/USER/SSH_KEY/BRIDGE/IMAGE_URL; see docs/testing.md.
|
||||||
go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/
|
go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/
|
||||||
|
|
||||||
|
.PHONY: test-integration-kvm2
|
||||||
|
test-integration-kvm2: ## Run the kvm2 provider test against a real hypervisor. Needs KVM2_TEST_HOST/USER/SSH_KEY/BRIDGE/IMAGE_URL; see docs/testing.md.
|
||||||
|
go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM2 ./internal/provider/kvm2/
|
||||||
|
|
||||||
# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
|
# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
|
||||||
# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
|
# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
|
||||||
# kubectl kuberc is disabled by default for test isolation; enable with:
|
# kubectl kuberc is disabled by default for test isolation; enable with:
|
||||||
|
|||||||
@@ -149,9 +149,10 @@ spec:
|
|||||||
- name: providers-config
|
- name: providers-config
|
||||||
mountPath: /etc/proxy-operator
|
mountPath: /etc/proxy-operator
|
||||||
readOnly: true
|
readOnly: true
|
||||||
# SSH material for kvm providers (sshKeyFile / knownHostsFile in the
|
# SSH material for the kvm and kvm2 providers (sshKeyFile /
|
||||||
# providers config point here). Optional: the mount is empty when the
|
# knownHostsFile in the providers config point here). One Secret
|
||||||
# Secret does not exist, and only kvm entries need it. Create it with:
|
# serves both types. Optional: the mount is empty when the Secret
|
||||||
|
# does not exist, and only kvm and kvm2 entries need it. Create it with:
|
||||||
# kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
|
# kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
|
||||||
# --from-file=id_ed25519=<private key> \
|
# --from-file=id_ed25519=<private key> \
|
||||||
# --from-file=known_hosts=<known_hosts with every host's key>
|
# --from-file=known_hosts=<known_hosts with every host's key>
|
||||||
@@ -167,6 +168,7 @@ spec:
|
|||||||
- name: providers-config
|
- name: providers-config
|
||||||
configMap:
|
configMap:
|
||||||
name: providers-config
|
name: providers-config
|
||||||
|
# Shared by the kvm and kvm2 providers; see the mount above.
|
||||||
- name: kvm-ssh
|
- name: kvm-ssh
|
||||||
secret:
|
secret:
|
||||||
secretName: kvm-ssh
|
secretName: kvm-ssh
|
||||||
|
|||||||
@@ -51,4 +51,35 @@ data:
|
|||||||
bridge: br0 # to a LAN with DHCP; the VM's IP comes from there
|
bridge: br0 # to a LAN with DHCP; the VM's IP comes from there
|
||||||
# socket: /var/run/libvirt/libvirt-sock
|
# socket: /var/run/libvirt/libvirt-sock
|
||||||
# pool: default
|
# pool: default
|
||||||
|
# kvm2 takes the same block shape as kvm: an entry ports by changing
|
||||||
|
# type to kvm2 and the block name to kvm2. kvm2 does not adopt domains
|
||||||
|
# made by kvm, so delete those Proxies first. See docs/kvm2-provider.md.
|
||||||
|
# - name: kvm2-home
|
||||||
|
# type: kvm2
|
||||||
|
# kvm2:
|
||||||
|
# sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
|
||||||
|
# knownHostsFile: /etc/proxy-operator/kvm/known_hosts
|
||||||
|
# # insecureIgnoreHostKey: true
|
||||||
|
# # diskSizeGb: 10
|
||||||
|
# # installGuestAgent: false
|
||||||
|
# # vendorData: |
|
||||||
|
# # #cloud-config
|
||||||
|
# # ...
|
||||||
|
# images:
|
||||||
|
# debian-13:
|
||||||
|
# url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||||
|
# # sha256: "<64 hex>"
|
||||||
|
# # firmware: uefi
|
||||||
|
# ubuntu-24.04:
|
||||||
|
# url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
|
||||||
|
# machineTypes:
|
||||||
|
# small: {vcpus: 1, memoryMiB: 1024}
|
||||||
|
# medium: {vcpus: 2, memoryMiB: 2048}
|
||||||
|
# hosts:
|
||||||
|
# - name: beelink
|
||||||
|
# address: beelink.tail8c69c.ts.net:22
|
||||||
|
# user: novakj
|
||||||
|
# bridge: br0
|
||||||
|
# # socket: /var/run/libvirt/libvirt-sock
|
||||||
|
# # pool: default
|
||||||
|
|
||||||
|
|||||||
@@ -5,5 +5,6 @@ resources:
|
|||||||
- proxy_kubernetes.yaml
|
- proxy_kubernetes.yaml
|
||||||
- proxy_gcp.yaml
|
- proxy_gcp.yaml
|
||||||
- proxy_kvm.yaml
|
- proxy_kvm.yaml
|
||||||
|
- proxy_kvm2.yaml
|
||||||
- proxy_external.yaml
|
- proxy_external.yaml
|
||||||
# +kubebuilder:scaffold:manifestskustomizesamples
|
# +kubebuilder:scaffold:manifestskustomizesamples
|
||||||
|
|||||||
@@ -55,3 +55,34 @@ providers:
|
|||||||
bridge: br0 # to a LAN with DHCP; the VM's IP comes from there
|
bridge: br0 # to a LAN with DHCP; the VM's IP comes from there
|
||||||
# socket: /var/run/libvirt/libvirt-sock
|
# socket: /var/run/libvirt/libvirt-sock
|
||||||
# pool: default
|
# pool: default
|
||||||
|
# kvm2 takes the same block shape as kvm: an entry ports by changing type
|
||||||
|
# to kvm2 and the block name to kvm2. kvm2 does not adopt domains made by
|
||||||
|
# kvm, so delete those Proxies first. See docs/kvm2-provider.md.
|
||||||
|
# - name: kvm2-home
|
||||||
|
# type: kvm2
|
||||||
|
# kvm2:
|
||||||
|
# sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
|
||||||
|
# knownHostsFile: /etc/proxy-operator/kvm/known_hosts
|
||||||
|
# # insecureIgnoreHostKey: true
|
||||||
|
# # diskSizeGb: 10
|
||||||
|
# # installGuestAgent: false
|
||||||
|
# # vendorData: |
|
||||||
|
# # #cloud-config
|
||||||
|
# # ...
|
||||||
|
# images:
|
||||||
|
# debian-13:
|
||||||
|
# url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||||
|
# # sha256: "<64 hex>"
|
||||||
|
# # firmware: uefi
|
||||||
|
# ubuntu-24.04:
|
||||||
|
# url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
|
||||||
|
# machineTypes:
|
||||||
|
# small: {vcpus: 1, memoryMiB: 1024}
|
||||||
|
# medium: {vcpus: 2, memoryMiB: 2048}
|
||||||
|
# hosts:
|
||||||
|
# - name: beelink
|
||||||
|
# address: beelink:22
|
||||||
|
# user: novakj
|
||||||
|
# bridge: br0
|
||||||
|
# # socket: /var/run/libvirt/libvirt-sock
|
||||||
|
# # pool: default
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# A Managed proxy backed by a libvirt/KVM hypervisor through the kvm2
|
||||||
|
# provider: the operator boots a stock Debian cloud image as a VM on the
|
||||||
|
# named host, with a NIC on the host's bridge (IP from the LAN's DHCP), and
|
||||||
|
# installs Squid via cloud-init. Requires a providers-config entry named
|
||||||
|
# "kvm2-home" (see providers-config.yaml) and the kvm-ssh Secret, which the
|
||||||
|
# kvm and kvm2 providers share (docs/kvm2-provider.md).
|
||||||
|
#
|
||||||
|
# placement.zone is the host name from kvm2.hosts (optional when only one
|
||||||
|
# host is configured); machineType and image are keys in the provider's
|
||||||
|
# machineTypes and images maps. The first Proxy for an image on a host
|
||||||
|
# waits for the image download; later ones reuse the cached copy.
|
||||||
|
apiVersion: crawl.example.com/v1alpha1
|
||||||
|
kind: Proxy
|
||||||
|
metadata:
|
||||||
|
name: proxy-kvm2-sample
|
||||||
|
spec:
|
||||||
|
mode: Managed
|
||||||
|
provider: kvm2-home
|
||||||
|
placement:
|
||||||
|
zone: beelink
|
||||||
|
machineType: small
|
||||||
|
image: debian-13
|
||||||
|
port: 3128
|
||||||
|
cloudInit:
|
||||||
|
inline: |
|
||||||
|
#cloud-config
|
||||||
|
packages:
|
||||||
|
- squid
|
||||||
|
write_files:
|
||||||
|
- path: /etc/squid/conf.d/proxy-operator.conf
|
||||||
|
content: |
|
||||||
|
http_port 3128
|
||||||
|
http_access allow all
|
||||||
|
via off
|
||||||
|
forwarded_for off
|
||||||
|
runcmd:
|
||||||
|
- systemctl restart squid
|
||||||
|
# Optional: log in with `ssh debian@<ip>` (the Debian image's default
|
||||||
|
# user) to look around; `virsh console <name>` on the host works too.
|
||||||
|
# ssh_authorized_keys:
|
||||||
|
# - ssh-ed25519 AAAA... you@laptop
|
||||||
|
attributes:
|
||||||
|
geo: home
|
||||||
|
purpose: crawl
|
||||||
@@ -10,7 +10,7 @@ Plan: [docs/plans/2026-09-16-1333-kvm2-provider.md](../plans/2026-09-16-1333-kvm
|
|||||||
- [x] Step 5 — Image cache and pool-space precheck
|
- [x] Step 5 — Image cache and pool-space precheck
|
||||||
- [x] Step 6 — The four contract methods
|
- [x] Step 6 — The four contract methods
|
||||||
- [x] Step 7 — Checker and the CRD comment
|
- [x] Step 7 — Checker and the CRD comment
|
||||||
- [ ] Step 8 — Deployment files, Makefile, integration test
|
- [x] Step 8 — Deployment files, Makefile, integration test
|
||||||
- [ ] Step 9 — Documentation
|
- [ ] Step 9 — Documentation
|
||||||
- [ ] Step 10 — Milestone review
|
- [ ] Step 10 — Milestone review
|
||||||
- [ ] Step 11 — Real-host validation, changelog, MR
|
- [ ] Step 11 — Real-host validation, changelog, MR
|
||||||
@@ -281,3 +281,40 @@ so it built its own locked capture; the orchestrator fixed `captureContext`
|
|||||||
with a mutex in this commit and re-ran the package at `-race -count=3`.
|
with a mutex in this commit and re-ran the package at `-race -count=3`.
|
||||||
|
|
||||||
Attestation: "files read outside the allowed set: none".
|
Attestation: "files read outside the allowed set: none".
|
||||||
|
|
||||||
|
## Step 8 — Deployment files, Makefile, integration test
|
||||||
|
|
||||||
|
2026-09-16 19:39. Seventh subagent step. `kvm2_integration_test.go`
|
||||||
|
(`//go:build integration`, `KVM2_TEST_*` variables, the eleven ordered
|
||||||
|
assertions of spec entry 21 plus `TestKVM2Check_realHost`),
|
||||||
|
`config/samples/proxy_kvm2.yaml` (zone `beelink`, so the sample matches the
|
||||||
|
shipped config), a commented `kvm2-home` entry in both providers configs
|
||||||
|
with the porting note, comment-only edits on the shared `kvm-ssh` mount, the
|
||||||
|
`test-integration-kvm2` Makefile target and the kvm2 rows and section in
|
||||||
|
`docs/testing.md`. The integration test was not run; that is Step 11 with
|
||||||
|
the user's go-ahead.
|
||||||
|
|
||||||
|
The agent found that `New` builds the production dialer without tracing
|
||||||
|
options, so in production the dialer and decorator use the global tracer
|
||||||
|
provider (correct: `cmd/main.go` sets it up), while a test that wants every
|
||||||
|
span family in one recorder sets the dialer's option itself. The test does
|
||||||
|
so and asserts that `kvm2.DomainDefineXML` and `kvm2.dial` are children of
|
||||||
|
`provider.create`.
|
||||||
|
|
||||||
|
Decisions the agent made, accepted: the UID is generated from `crypto/rand`
|
||||||
|
rather than promoting `google/uuid` to a direct dependency; the sentinel
|
||||||
|
check looks for the full cloud-init string because the provider name also
|
||||||
|
appears in attributes; `-run TestKVM2` matches both tests, as the kvm
|
||||||
|
target's pattern does. Forward links to `docs/kvm2-provider.md` dangle until
|
||||||
|
Step 9.
|
||||||
|
|
||||||
|
Commands:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go vet ./... && go vet -tags=integration ./... && go vet -tags=e2e ./...
|
||||||
|
bin/kustomize build config/default > /dev/null && bin/kustomize build config/samples > /dev/null
|
||||||
|
make -n test-integration-kvm2
|
||||||
|
# go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM2 ./internal/provider/kvm2/
|
||||||
|
```
|
||||||
|
|
||||||
|
All clean. Attestation: "files read outside the allowed set: none".
|
||||||
|
|||||||
+100
-6
@@ -62,6 +62,11 @@ manual runs too.
|
|||||||
| | `trace_test.go`, `conn_test.go`, `errors_test.go` | every `libvirtAPI` method traced exactly once (guarded by reflection on the interface), results and errors pass through, libvirt error fields, detail keys only at V2, upload content never logged; pool connect/drop lines, no wrapping below V1 | `fakeAPI` + `funcr` capture |
|
| | `trace_test.go`, `conn_test.go`, `errors_test.go` | every `libvirtAPI` method traced exactly once (guarded by reflection on the interface), results and errors pass through, libvirt error fields, detail keys only at V2, upload content never logged; pool connect/drop lines, no wrapping below V1 | `fakeAPI` + `funcr` capture |
|
||||||
| | `check_test.go`, `conn_test.go` | health check: details, per-host verdicts in config order, hosts checked concurrently, a timeout that keeps the connection and refuses a second check while the first still waits, no waiting behind a reconcile's dial, the reason table; `probe` abandons without closing, but still drops on a transport error; waiting for a host's slot ends with the caller's deadline; an inactive or missing pool and daemon errors keep the connection; every dial stage tagged through the real SSH server, with classification unchanged | `fakeAPI`, fake dialers, in-process SSH server |
|
| | `check_test.go`, `conn_test.go` | health check: details, per-host verdicts in config order, hosts checked concurrently, a timeout that keeps the connection and refuses a second check while the first still waits, no waiting behind a reconcile's dial, the reason table; `probe` abandons without closing, but still drops on a transport error; waiting for a host's slot ends with the caller's deadline; an inactive or missing pool and daemon errors keep the connection; every dial stage tagged through the real SSH server, with classification unchanged | `fakeAPI`, fake dialers, in-process SSH server |
|
||||||
| | `kvm_integration_test.go` (`-tags=integration`) | one real VM: image fetch, Create, idempotent re-Create, Running with IP, ListByTag, Delete, pool state afterwards; `TestKVMCheck_realHost` runs the health check against the host and creates nothing | a **real hypervisor** — see below |
|
| | `kvm_integration_test.go` (`-tags=integration`) | one real VM: image fetch, Create, idempotent re-Create, Running with IP, ListByTag, Delete, pool state afterwards; `TestKVMCheck_realHost` runs the health check against the host and creates nothing | a **real hypervisor** — see below |
|
||||||
|
| `internal/provider/kvm2` | `xml_test.go`, `volume_test.go`, `seed_test.go`, `id_test.go`, `metadata_test.go`, `state_test.go` | pure builders: domain XML (file disks with resolved paths, UEFI with secure boot off, BIOS escape, the metadata element, the guest-agent channel), volume XML with and without a backing store, seed ISO reopened (files, label, user-data byte for byte, temp dir removed) and the vendor-data script vs. multipart rules, providerID round-trip and malformed forms, metadata in both namespace spellings, state mapping and the IPv4 filter | pure |
|
||||||
|
| | `errors_test.go`, `backend_test.go`, `hosts_test.go`, `ssh_test.go` | classification table (libvirt codes, SSH auth text, `knownhosts.KeyError`, missing files); the per-call budget adapter (deadline kills the connection and is Transient, cancel unblocks, killed by another caller, close idempotent) against a mocked libvirt socket; the host table (one dial per host under concurrency, redial after drop, `dialFresh` never cached, waiting ends with ctx); the real dialer end to end: socket forwarding, unknown and mismatched host keys, insecure mode, missing vs. unparsable key and known_hosts, wrong key, handshake timeout, cancel, every stage tagged, key material never in errors or logs | fake dialer; a **real in-process `x/crypto/ssh` server** forwarding a unix socket |
|
||||||
|
| | `image_test.go`, `create_test.go`, `get_test.go`, `delete_test.go`, `list_test.go`, `kvm2_test.go`, `provider_test.go` | image cache (fetch with the marker written last, cached, marker missing or without a digest, sha256 change or mismatch, 4xx/5xx, no Content-Length, short read, concurrent callers fetch once, cancel mid-stream, streamed without buffering, upload on a dedicated connection) and the pool-space quota precheck; Create resume after every partial step, returns only when running, adoption guard on UID, seed always rebuilt, cloud-init sentinel never logged; Get state and address mapping incl. agent-before-ARP; every Delete already-gone case, NVRAM flag, per-name lock; ListByTag across hosts, one down fails the whole call, unreadable domain skipped; `New` never dials and warns on missing files; every error wrapped with the configured name; logging tiers V0/V1/V2 | `httptest` image server + in-memory `fakeBackend` behind the `backend` seam |
|
||||||
|
| | `observe_test.go`, `check_test.go` | every seam method traced and logged exactly once (guarded by reflection), child spans under the parent, detail only at V2, upload bodies never logged, error fields; health check: one entry per host in config order, hosts checked concurrently, expired ctx returns promptly, never touches the pooled connection, the reason table, detail keys | `fakeBackend` + `funcr` capture + span recorder |
|
||||||
|
| | `kvm2_integration_test.go` (`-tags=integration`) | one real VM: `New` without a dial, `Check` Ready with details, Create, idempotent re-Create with an unchanged domain count, Running with IP and the metadata round-trip, ListByTag, Delete, NotFound, pool state afterwards, `kvm2.*` spans under `provider.create`; `TestKVM2Check_realHost` runs the health check against the host and creates nothing | a **real hypervisor** — see below |
|
||||||
| `internal/health` | `probe_test.go` | probes through a **real CONNECT-capable proxy stub** to a real TLS server: tunnel success, refused CONNECT, wrong status, dead proxy, plain-http forward | `httptest` + hijacked bidirectional tunnel |
|
| `internal/health` | `probe_test.go` | probes through a **real CONNECT-capable proxy stub** to a real TLS server: tunnel success, refused CONNECT, wrong status, dead proxy, plain-http forward | `httptest` + hijacked bidirectional tunnel |
|
||||||
| | `engine_test.go` | thresholds, first-verdict, latency suppression matrix, dropped-event retry, stale-probe UID guard, tick scheduling/seeding/pruning, `Start` end-to-end | fake `client.Reader` + injected `probeFn` |
|
| | `engine_test.go` | thresholds, first-verdict, latency suppression matrix, dropped-event retry, stale-probe UID guard, tick scheduling/seeding/pruning, `Start` end-to-end | fake `client.Reader` + injected `probeFn` |
|
||||||
| `internal/lease` | `store_test.go` | capacity, `MaxLeases=0`, all three selection tie-breaks, target-scoped vs global cooldowns, expiry via fake clock, report-on-retained-lease, idempotent release, **40 concurrent acquires never exceeding capacity** | injectable clock; the concurrency case is why `-race` matters |
|
| `internal/lease` | `store_test.go` | capacity, `MaxLeases=0`, all three selection tie-breaks, target-scoped vs global cooldowns, expiry via fake clock, report-on-retained-lease, idempotent release, **40 concurrent acquires never exceeding capacity** | injectable clock; the concurrency case is why `-race` matters |
|
||||||
@@ -93,12 +98,12 @@ envtest suite the scaffold generated.
|
|||||||
bug (below).
|
bug (below).
|
||||||
- **Live GCP**: no test talks to Google. The seam boundary
|
- **Live GCP**: no test talks to Google. The seam boundary
|
||||||
(`buildInsertRequest` + classification) is tested exhaustively instead.
|
(`buildInsertRequest` + classification) is tested exhaustively instead.
|
||||||
- **Live libvirt, by default**: the KVM unit tests run over an in-memory
|
- **Live libvirt, by default**: the KVM and kvm2 unit tests run over an
|
||||||
fake of the 20 libvirt calls the provider uses. Whether libvirt accepts
|
in-memory fake of the 20 libvirt calls each provider uses. Whether
|
||||||
the domain XML, whether a raw qcow2 upload plus pool refresh yields a
|
libvirt accepts the domain XML, whether a raw qcow2 upload plus pool
|
||||||
usable backing image, and whether the guest actually boots and reports
|
refresh yields a usable backing image, and whether the guest actually
|
||||||
an address are provable only on a real host — that is the opt-in
|
boots and reports an address are provable only on a real host — that is
|
||||||
integration test below.
|
the opt-in integration test below (one per provider).
|
||||||
|
|
||||||
## The KVM integration test (real hypervisor)
|
## The KVM integration test (real hypervisor)
|
||||||
|
|
||||||
@@ -162,6 +167,95 @@ go test -tags=integration -count=1 -v -run TestKVMCheck ./internal/provider/kvm/
|
|||||||
The test is not part of `make test` and not run in CI. Keep it compiling
|
The test is not part of `make test` and not run in CI. Keep it compiling
|
||||||
with `go vet -tags=integration ./...` when the provider's internals move.
|
with `go vet -tags=integration ./...` when the provider's internals move.
|
||||||
|
|
||||||
|
## The kvm2 integration test (real hypervisor)
|
||||||
|
|
||||||
|
`internal/provider/kvm2/kvm2_integration_test.go` compiles only under
|
||||||
|
`-tags=integration` and skips itself unless the environment names a
|
||||||
|
host. Like the kvm one it deviates from the "testcontainers for real
|
||||||
|
infra" convention on purpose: libvirt needs `/dev/kvm` and a bridge to a
|
||||||
|
DHCP network, and a container has neither. It targets whichever
|
||||||
|
hypervisor you point it at and leaves that host's pool with one more
|
||||||
|
cached base image (`base-test-image.qcow2` and its `.sha256` marker) than
|
||||||
|
it had. The kvm and kvm2 tests can share a host: kvm2 marks its domains
|
||||||
|
in its own metadata namespace and never lists or adopts the other's.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export KVM2_TEST_HOST=beelink:22 # SSH endpoint
|
||||||
|
export KVM2_TEST_USER=novakj # must be in the libvirt group
|
||||||
|
export KVM2_TEST_SSH_KEY=$HOME/.ssh/id_ed25519 # private key for that user
|
||||||
|
export KVM2_TEST_KNOWN_HOSTS=$HOME/.ssh/kvm_known_hosts # the hypervisors' host keys
|
||||||
|
export KVM2_TEST_BRIDGE=br0 # bridge to a network with DHCP
|
||||||
|
export KVM2_TEST_IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||||
|
make test-integration-kvm2
|
||||||
|
```
|
||||||
|
|
||||||
|
`KVM2_TEST_KNOWN_HOSTS` is technically optional, but leaving it out turns
|
||||||
|
host-key verification **off** for the run (the log says so); keep a
|
||||||
|
`kvm_known_hosts` holding just the hypervisors rather than pointing the
|
||||||
|
test at your everyday `known_hosts`, so a re-keyed hypervisor is a
|
||||||
|
one-line fix in a file nothing else uses.
|
||||||
|
|
||||||
|
Also optional: `KVM2_TEST_IMAGE_SHA256` (verified after download and
|
||||||
|
compared with the cached marker), `KVM2_TEST_POOL` (default `default`),
|
||||||
|
`KVM2_TEST_SOCKET` (default `/var/run/libvirt/libvirt-sock`),
|
||||||
|
`KVM2_TEST_TIMEOUT` (default `20m`, the budget for the whole lifecycle;
|
||||||
|
it must cover the first-time image download), `KVM2_TEST_KEEP` to leave
|
||||||
|
the VM behind for `virsh console` when something goes wrong (the log then
|
||||||
|
prints the `virsh undefine --nvram` and `virsh vol-delete` lines that
|
||||||
|
clean it up), and `KVM2_TEST_LOG_LEVEL` (default `1`) for how much of the
|
||||||
|
provider's logging the `-v` output shows: at `1` every libvirt RPC is one
|
||||||
|
compact `libvirt call` line (`HH:MM:SS.mmm kvm2 "level"=1 "msg"="libvirt
|
||||||
|
call" "rpc"="DomainLookupByName" "domain"=... "duration"=...`, plus the
|
||||||
|
per-operation outcome lines and connection events), interleaved with the
|
||||||
|
test's own `t.Log` lines; `2` adds the domain and volume XML, metadata
|
||||||
|
and addresses; `0` leaves only `t.Log`. The per-run constants (provider,
|
||||||
|
host, domain name) are stated once in the first line and left off every
|
||||||
|
provider line. See the Logging section of
|
||||||
|
[kvm2-provider.md](kvm2-provider.md).
|
||||||
|
|
||||||
|
What one run asserts, in order, each step logged with its duration:
|
||||||
|
|
||||||
|
1. `New` returns within 100 ms and without dialling (no `libvirt host
|
||||||
|
connected` line before the first call).
|
||||||
|
2. `Check` reports the one host `Ready` with reason `Reachable` and the
|
||||||
|
`hostname`, `libvirtVersion`, `poolState` (`running`) and
|
||||||
|
`poolAvailableGiB` details, which are logged.
|
||||||
|
3. Every later call goes through `provider.WithTracing` with an in-memory
|
||||||
|
span recorder that the package's own spans also land in.
|
||||||
|
4. `Create` returns `hosts/kvm2-test/domains/<name>` (downloading the
|
||||||
|
image on a cold pool — the log prints how long it took).
|
||||||
|
5. A second `Create` returns the same ID and the host's domain count,
|
||||||
|
read over a dedicated connection, is unchanged.
|
||||||
|
6. `Get`, polled every 5 s, reaches `Running` with an IPv4 within the
|
||||||
|
budget; its `UID` is the one Create wrote and `CreatedAt` is non-zero
|
||||||
|
and within a few minutes of now. The time from Create's return to the
|
||||||
|
first address is logged.
|
||||||
|
7. `ListByTag` contains the ID with that UID.
|
||||||
|
8. A TCP connect to port 22 on the guest, logged only — the machine
|
||||||
|
running the test may not route to the bridge network.
|
||||||
|
9. `Delete` returns nil, `Get` is `ErrNotFound`, a second `Delete` is a
|
||||||
|
no-op.
|
||||||
|
10. The pool, inspected over a dedicated connection, holds the base image
|
||||||
|
and its marker but no `<name>.qcow2` or `<name>-seed.iso`.
|
||||||
|
11. Among the recorded spans, a `provider.create` span has at least one
|
||||||
|
`kvm2.DomainDefineXML` and one `kvm2.dial` child, and no span
|
||||||
|
attribute carries the cloud-init text.
|
||||||
|
|
||||||
|
With `KVM2_TEST_KEEP` steps 9 and 10 are skipped; otherwise a `t.Cleanup`
|
||||||
|
attempts the Delete even when an earlier assertion failed.
|
||||||
|
|
||||||
|
`TestKVM2Check_realHost` runs the provider health check
|
||||||
|
([kvm2-provider.md](kvm2-provider.md)) against the same host. It must
|
||||||
|
find the host reachable and the pool running, and it creates nothing. It
|
||||||
|
needs the same variables; to run only this test:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test -tags=integration -count=1 -v -run TestKVM2Check ./internal/provider/kvm2/
|
||||||
|
```
|
||||||
|
|
||||||
|
The test is not part of `make test` and not run in CI. Keep it compiling
|
||||||
|
with `go vet -tags=integration ./...` when the provider's internals move.
|
||||||
|
|
||||||
## The scaffolded `make test-e2e` suite
|
## The scaffolded `make test-e2e` suite
|
||||||
|
|
||||||
`test/e2e/` is the kubebuilder-generated smoke suite (build image → kind
|
`test/e2e/` is the kubebuilder-generated smoke suite (build image → kind
|
||||||
|
|||||||
@@ -0,0 +1,590 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package kvm2
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/digitalocean/go-libvirt"
|
||||||
|
"github.com/go-logr/logr"
|
||||||
|
"github.com/go-logr/logr/funcr"
|
||||||
|
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||||
|
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||||
|
"k8s.io/apimachinery/pkg/types"
|
||||||
|
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||||
|
|
||||||
|
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||||
|
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The real-hypervisor test of spec entry 21. It runs only with
|
||||||
|
// -tags=integration and skips itself unless KVM2_TEST_* names a host; see
|
||||||
|
// docs/testing.md for the variables and what one run asserts.
|
||||||
|
const (
|
||||||
|
integrationProvider = "kvm2-integration"
|
||||||
|
integrationHost = "kvm2-test"
|
||||||
|
integrationImage = "test-image"
|
||||||
|
integrationMachine = "small"
|
||||||
|
integrationPort = 3128
|
||||||
|
|
||||||
|
// integrationCloudInit is the user-data the VM boots with. Nothing
|
||||||
|
// asserts on the guest running it; it exists so the sentinel check can
|
||||||
|
// prove that no span attribute carries the cloud-init text.
|
||||||
|
integrationCloudInit = "#cloud-config\nruncmd:\n - echo kvm2-integration\n"
|
||||||
|
|
||||||
|
defaultIntegrationTimeout = 20 * time.Minute
|
||||||
|
defaultIntegrationLogLvl = 1
|
||||||
|
getPollInterval = 5 * time.Second
|
||||||
|
cleanupBudget = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
type integrationEnv struct {
|
||||||
|
host, user, sshKey, knownHosts, bridge, imageURL string
|
||||||
|
imageSHA256, pool, socket string
|
||||||
|
timeout time.Duration
|
||||||
|
keep bool
|
||||||
|
logLevel int
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadIntegrationEnv reads the KVM2_TEST_* variables, skipping the test
|
||||||
|
// under -short or when a required one is missing.
|
||||||
|
func loadIntegrationEnv(t *testing.T) integrationEnv {
|
||||||
|
t.Helper()
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("real-hypervisor test skipped under -short")
|
||||||
|
}
|
||||||
|
env := integrationEnv{timeout: defaultIntegrationTimeout, logLevel: defaultIntegrationLogLvl}
|
||||||
|
required := []struct {
|
||||||
|
name string
|
||||||
|
dst *string
|
||||||
|
}{
|
||||||
|
{"KVM2_TEST_HOST", &env.host},
|
||||||
|
{"KVM2_TEST_USER", &env.user},
|
||||||
|
{"KVM2_TEST_SSH_KEY", &env.sshKey},
|
||||||
|
{"KVM2_TEST_BRIDGE", &env.bridge},
|
||||||
|
{"KVM2_TEST_IMAGE_URL", &env.imageURL},
|
||||||
|
}
|
||||||
|
for _, r := range required {
|
||||||
|
v := os.Getenv(r.name)
|
||||||
|
if v == "" {
|
||||||
|
t.Skipf("%s is not set; the kvm2 integration test needs a real hypervisor (see docs/testing.md)", r.name)
|
||||||
|
}
|
||||||
|
*r.dst = v
|
||||||
|
}
|
||||||
|
env.knownHosts = os.Getenv("KVM2_TEST_KNOWN_HOSTS")
|
||||||
|
env.imageSHA256 = os.Getenv("KVM2_TEST_IMAGE_SHA256")
|
||||||
|
env.pool = os.Getenv("KVM2_TEST_POOL")
|
||||||
|
env.socket = os.Getenv("KVM2_TEST_SOCKET")
|
||||||
|
env.keep = os.Getenv("KVM2_TEST_KEEP") != ""
|
||||||
|
if v := os.Getenv("KVM2_TEST_TIMEOUT"); v != "" {
|
||||||
|
d, err := time.ParseDuration(v)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("KVM2_TEST_TIMEOUT=%q: %v", v, err)
|
||||||
|
}
|
||||||
|
env.timeout = d
|
||||||
|
}
|
||||||
|
if v := os.Getenv("KVM2_TEST_LOG_LEVEL"); v != "" {
|
||||||
|
n, err := strconv.Atoi(v)
|
||||||
|
if err != nil || n < 0 {
|
||||||
|
t.Fatalf("KVM2_TEST_LOG_LEVEL=%q: want a non-negative integer", v)
|
||||||
|
}
|
||||||
|
env.logLevel = n
|
||||||
|
}
|
||||||
|
return env
|
||||||
|
}
|
||||||
|
|
||||||
|
// providerConfig is what the operator would load for this host: one host,
|
||||||
|
// one image, one machine type. Host keys are verified only when
|
||||||
|
// KVM2_TEST_KNOWN_HOSTS is set.
|
||||||
|
func (e integrationEnv) providerConfig() provider.ProviderConfig {
|
||||||
|
return provider.ProviderConfig{
|
||||||
|
Name: integrationProvider,
|
||||||
|
Type: "kvm2",
|
||||||
|
KVM2: &provider.KVMConfig{
|
||||||
|
SSHKeyFile: e.sshKey,
|
||||||
|
KnownHostsFile: e.knownHosts,
|
||||||
|
InsecureIgnoreHostKey: e.knownHosts == "",
|
||||||
|
Images: map[string]provider.KVMImage{
|
||||||
|
integrationImage: {URL: e.imageURL, SHA256: e.imageSHA256},
|
||||||
|
},
|
||||||
|
MachineTypes: map[string]provider.KVMMachineType{
|
||||||
|
integrationMachine: {VCPUs: 1, MemoryMiB: 1024},
|
||||||
|
},
|
||||||
|
Hosts: []provider.KVMHost{{
|
||||||
|
Name: integrationHost,
|
||||||
|
Address: e.host,
|
||||||
|
User: e.user,
|
||||||
|
Socket: e.socket,
|
||||||
|
Pool: e.pool,
|
||||||
|
Bridge: e.bridge,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// integrationLog routes the provider's logr output into t.Log through funcr
|
||||||
|
// at the chosen verbosity, keeps every line for assertions, and strips the
|
||||||
|
// key/value pairs that are constant for the run (stated once up front) so
|
||||||
|
// each line stays short. Lines arriving after the test ended are dropped:
|
||||||
|
// t.Log panics then.
|
||||||
|
type integrationLog struct {
|
||||||
|
t *testing.T
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
drop []string
|
||||||
|
lines []string
|
||||||
|
done bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newIntegrationLog(t *testing.T, level int, drop ...string) (*integrationLog, logr.Logger) {
|
||||||
|
t.Helper()
|
||||||
|
sink := &integrationLog{t: t, drop: drop}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
sink.mu.Lock()
|
||||||
|
defer sink.mu.Unlock()
|
||||||
|
sink.done = true
|
||||||
|
})
|
||||||
|
log := funcr.New(func(prefix, args string) { sink.write(prefix, args) }, funcr.Options{Verbosity: level})
|
||||||
|
return sink, log
|
||||||
|
}
|
||||||
|
|
||||||
|
// dropValue adds a rendered pair (`"key"="value"`) to strip from later lines.
|
||||||
|
func (l *integrationLog) dropValue(key, value string) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
l.drop = append(l.drop, fmt.Sprintf("%q=%q", key, value))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *integrationLog) write(prefix, args string) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
l.lines = append(l.lines, args)
|
||||||
|
if l.done {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, d := range l.drop {
|
||||||
|
args = strings.ReplaceAll(args, " "+d, "")
|
||||||
|
}
|
||||||
|
l.t.Logf("%s %s %s", time.Now().Format("15:04:05.000"), prefix, args)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *integrationLog) matching(substr string) []string {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
var out []string
|
||||||
|
for _, line := range l.lines {
|
||||||
|
if strings.Contains(line, substr) {
|
||||||
|
out = append(out, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// newIntegrationProvider builds the provider the way New does, plus the span
|
||||||
|
// recorder's tracer on both places the package starts spans: the Provider
|
||||||
|
// (kvm2.download) and the dialer (kvm2.dial and, through observe, every
|
||||||
|
// kvm2.<Method>). newLibvirtDialer has no option for it, so the field is set
|
||||||
|
// directly; that is what makes the kvm2.* spans land in the same recorder
|
||||||
|
// as provider.WithTracing's provider.* spans.
|
||||||
|
func newIntegrationProvider(pc provider.ProviderConfig, tp *sdktrace.TracerProvider) *Provider {
|
||||||
|
cfg := withDefaults(*pc.KVM2)
|
||||||
|
opt := tracing.WithTracerProvider(tp)
|
||||||
|
d := newLibvirtDialer(pc.Name, cfg)
|
||||||
|
d.tracing = []tracing.Option{opt}
|
||||||
|
return newProvider(pc.Name, cfg, d, withTracing(opt))
|
||||||
|
}
|
||||||
|
|
||||||
|
// randomUID is a UUID-shaped string, so the run's domain never collides with
|
||||||
|
// a previous run's leftovers or with a real Proxy's.
|
||||||
|
func randomUID(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
var b [16]byte
|
||||||
|
if _, err := rand.Read(b[:]); err != nil {
|
||||||
|
t.Fatalf("reading random bytes: %v", err)
|
||||||
|
}
|
||||||
|
h := hex.EncodeToString(b[:])
|
||||||
|
return h[0:8] + "-" + h[8:12] + "-" + h[12:16] + "-" + h[16:20] + "-" + h[20:32]
|
||||||
|
}
|
||||||
|
|
||||||
|
// stepDone logs one lifecycle step with its duration; the log is the record
|
||||||
|
// of how long a cold image download or a boot took on this host.
|
||||||
|
func stepDone(t *testing.T, step string, start time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
t.Logf("step %s: %s", step, time.Since(start).Round(time.Millisecond))
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitUntil polls cond every interval until it is true or ctx ends.
|
||||||
|
func waitUntil(ctx context.Context, t *testing.T, interval time.Duration, what string, cond func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
ticker := time.NewTicker(interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for !cond() {
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatalf("gave up waiting for %s: %v", what, ctx.Err())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// domainCount lists every domain on the host over a dedicated connection.
|
||||||
|
func domainCount(ctx context.Context, t *testing.T, p *Provider, host provider.KVMHost) int {
|
||||||
|
t.Helper()
|
||||||
|
be, err := p.hosts.dialFresh(ctx, host)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dialling %s to count domains: %v", host.Name, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = be.Close() }()
|
||||||
|
doms, _, err := be.ConnectListAllDomains(ctx, 1, libvirt.ConnectListDomainsActive|libvirt.ConnectListDomainsInactive)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("listing domains on %s: %v", host.Name, err)
|
||||||
|
}
|
||||||
|
return len(doms)
|
||||||
|
}
|
||||||
|
|
||||||
|
// poolVolumes reports, for each name, whether the host's pool holds a volume
|
||||||
|
// of that name, over a dedicated connection. Any error other than
|
||||||
|
// ErrNoStorageVol fails the test.
|
||||||
|
func poolVolumes(ctx context.Context, t *testing.T, p *Provider, host provider.KVMHost, names ...string) map[string]bool {
|
||||||
|
t.Helper()
|
||||||
|
be, err := p.hosts.dialFresh(ctx, host)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dialling %s to inspect the pool: %v", host.Name, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = be.Close() }()
|
||||||
|
pool, err := be.StoragePoolLookupByName(ctx, host.Pool)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("looking up pool %s: %v", host.Pool, err)
|
||||||
|
}
|
||||||
|
out := make(map[string]bool, len(names))
|
||||||
|
for _, name := range names {
|
||||||
|
_, err := be.StorageVolLookupByName(ctx, pool, name)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
out[name] = true
|
||||||
|
case hasCode(err, libvirt.ErrNoStorageVol):
|
||||||
|
out[name] = false
|
||||||
|
default:
|
||||||
|
t.Fatalf("looking up volume %s in pool %s: %v", name, host.Pool, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKVM2_lifecycle(t *testing.T) {
|
||||||
|
env := loadIntegrationEnv(t)
|
||||||
|
pc := env.providerConfig()
|
||||||
|
sink, log := newIntegrationLog(t, env.logLevel,
|
||||||
|
fmt.Sprintf("%q=%q", "provider", integrationProvider),
|
||||||
|
fmt.Sprintf("%q=%q", "host", integrationHost))
|
||||||
|
ctx, cancel := context.WithTimeout(logf.IntoContext(context.Background(), log), env.timeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
t.Logf("provider=%s host=%s address=%s user=%s bridge=%s image=%s budget=%s log level %d; provider, host and name are left off every provider log line",
|
||||||
|
integrationProvider, integrationHost, env.host, env.user, env.bridge, env.imageURL, env.timeout, env.logLevel)
|
||||||
|
if env.knownHosts == "" {
|
||||||
|
t.Log("KVM2_TEST_KNOWN_HOSTS is not set: host keys are NOT verified for this run")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. New returns without dialling.
|
||||||
|
start := time.Now()
|
||||||
|
built, err := New(ctx, pc)
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New() error = %v", err)
|
||||||
|
}
|
||||||
|
if elapsed > 100*time.Millisecond {
|
||||||
|
t.Errorf("New() took %s, want under 100ms: it must not dial", elapsed)
|
||||||
|
}
|
||||||
|
if lines := sink.matching("libvirt host connected"); len(lines) > 0 {
|
||||||
|
t.Errorf("New() dialled the host before any call: %q", lines)
|
||||||
|
}
|
||||||
|
stepDone(t, "1 New", start)
|
||||||
|
|
||||||
|
// 2. Check reports the host Ready with details.
|
||||||
|
checker, ok := built.(provider.Checker)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("New() returned %T, which is not a provider.Checker", built)
|
||||||
|
}
|
||||||
|
start = time.Now()
|
||||||
|
targets := checker.Check(ctx)
|
||||||
|
if len(targets) != 1 {
|
||||||
|
t.Fatalf("Check() returned %d targets, want 1: %+v", len(targets), targets)
|
||||||
|
}
|
||||||
|
ts := targets[0]
|
||||||
|
if !ts.Ready || ts.Reason != provider.ReasonReachable {
|
||||||
|
t.Fatalf("Check() = ready %v reason %q message %q, want Ready/%s (details %v)", ts.Ready, ts.Reason, ts.Message, provider.ReasonReachable, ts.Details)
|
||||||
|
}
|
||||||
|
for _, key := range []string{"hostname", "libvirtVersion", "poolState", "poolAvailableGiB"} {
|
||||||
|
if ts.Details[key] == "" {
|
||||||
|
t.Errorf("Check() details lack %q: %v", key, ts.Details)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ts.Details["poolState"] != "running" {
|
||||||
|
t.Errorf("Check() poolState = %q, want running", ts.Details["poolState"])
|
||||||
|
}
|
||||||
|
t.Logf("host %s: hostname=%s libvirt=%s pool=%s (%s) available=%s GiB capacity=%s GiB latency=%s",
|
||||||
|
ts.Name, ts.Details["hostname"], ts.Details["libvirtVersion"], ts.Details["pool"], ts.Details["poolState"],
|
||||||
|
ts.Details["poolAvailableGiB"], ts.Details["poolCapacityGiB"], ts.Latency.Round(time.Millisecond))
|
||||||
|
stepDone(t, "2 Check", start)
|
||||||
|
|
||||||
|
// 3. Every later call goes through the tracing decorator, with the
|
||||||
|
// package's own spans recorded alongside.
|
||||||
|
sr := tracetest.NewSpanRecorder()
|
||||||
|
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||||
|
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||||
|
p := newIntegrationProvider(pc, tp)
|
||||||
|
traced := provider.WithTracing(pc.Name, p, tracing.WithTracerProvider(tp))
|
||||||
|
host := p.cfg.Hosts[0]
|
||||||
|
|
||||||
|
// 4. Create.
|
||||||
|
uid := randomUID(t)
|
||||||
|
name := provider.NameFromUID(types.UID(uid))
|
||||||
|
sink.dropValue("name", name)
|
||||||
|
id := formatProviderID(integrationHost, name)
|
||||||
|
req := provider.CreateRequest{
|
||||||
|
Name: name,
|
||||||
|
UID: uid,
|
||||||
|
Namespace: "integration",
|
||||||
|
ProxyName: "kvm2-lifecycle",
|
||||||
|
Placement: provider.Placement{Zone: integrationHost, MachineType: integrationMachine, Image: integrationImage},
|
||||||
|
CloudInit: integrationCloudInit,
|
||||||
|
Port: integrationPort,
|
||||||
|
}
|
||||||
|
t.Logf("domain name %s, uid %s, id %s", name, uid, id)
|
||||||
|
|
||||||
|
deleted := false
|
||||||
|
if env.keep {
|
||||||
|
t.Logf("KVM2_TEST_KEEP is set: the VM stays behind. Clean up on the host with:\n"+
|
||||||
|
" virsh undefine --nvram %s\n virsh vol-delete --pool %s %s\n virsh vol-delete --pool %s %s",
|
||||||
|
name, host.Pool, overlayVolumeName(name), host.Pool, seedVolumeName(name))
|
||||||
|
} else {
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if deleted {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cctx, ccancel := context.WithTimeout(logf.IntoContext(context.Background(), log), cleanupBudget)
|
||||||
|
defer ccancel()
|
||||||
|
if err := p.Delete(cctx, id); err != nil {
|
||||||
|
t.Errorf("cleanup Delete(%s) error = %v; remove the domain and its volumes by hand", id, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
start = time.Now()
|
||||||
|
gotID, err := traced.Create(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Create() error = %v", err)
|
||||||
|
}
|
||||||
|
if gotID != id {
|
||||||
|
t.Fatalf("Create() = %q, want %q", gotID, id)
|
||||||
|
}
|
||||||
|
createReturned := time.Now()
|
||||||
|
stepDone(t, "4 Create (includes the image download on a cold pool)", start)
|
||||||
|
|
||||||
|
// 5. Create again: same ID, nothing new on the host.
|
||||||
|
start = time.Now()
|
||||||
|
before := domainCount(ctx, t, p, host)
|
||||||
|
again, err := traced.Create(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second Create() error = %v", err)
|
||||||
|
}
|
||||||
|
if again != id {
|
||||||
|
t.Errorf("second Create() = %q, want %q", again, id)
|
||||||
|
}
|
||||||
|
if after := domainCount(ctx, t, p, host); after != before {
|
||||||
|
t.Errorf("second Create() changed the host's domain count from %d to %d", before, after)
|
||||||
|
}
|
||||||
|
stepDone(t, "5 Create again", start)
|
||||||
|
|
||||||
|
// 6. Get reaches Running with an IPv4.
|
||||||
|
start = time.Now()
|
||||||
|
var inst *provider.Instance
|
||||||
|
waitUntil(ctx, t, getPollInterval, "the domain to be Running with an IPv4", func() bool {
|
||||||
|
var err error
|
||||||
|
inst, err = traced.Get(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Get(%s) error = %v", id, err)
|
||||||
|
}
|
||||||
|
return inst.State == provider.StateRunning && inst.IP != ""
|
||||||
|
})
|
||||||
|
if inst.UID != uid {
|
||||||
|
t.Errorf("Get() UID = %q, want %q", inst.UID, uid)
|
||||||
|
}
|
||||||
|
if inst.CreatedAt.IsZero() {
|
||||||
|
t.Error("Get() CreatedAt is zero; the metadata mark did not round-trip")
|
||||||
|
} else if skew := time.Since(inst.CreatedAt).Abs(); skew > 5*time.Minute {
|
||||||
|
t.Errorf("Get() CreatedAt = %s, %s from now; want within a few minutes", inst.CreatedAt, skew)
|
||||||
|
}
|
||||||
|
t.Logf("domain running at %s, %s after Create returned (createdAt %s)", inst.IP, time.Since(createReturned).Round(time.Millisecond), inst.CreatedAt.Format(time.RFC3339))
|
||||||
|
stepDone(t, "6 Get until Running", start)
|
||||||
|
|
||||||
|
// 7. ListByTag contains it.
|
||||||
|
start = time.Now()
|
||||||
|
list, err := traced.ListByTag(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListByTag() error = %v", err)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, li := range list {
|
||||||
|
if li.ID == id {
|
||||||
|
found = true
|
||||||
|
if li.UID != uid {
|
||||||
|
t.Errorf("ListByTag() entry %s has UID %q, want %q", id, li.UID, uid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Errorf("ListByTag() = %d instances, none with ID %s", len(list), id)
|
||||||
|
}
|
||||||
|
stepDone(t, "7 ListByTag", start)
|
||||||
|
|
||||||
|
// 8. Reachability, logged only: this machine may not route to the bridge.
|
||||||
|
start = time.Now()
|
||||||
|
addr := net.JoinHostPort(inst.IP, "22")
|
||||||
|
if conn, err := net.DialTimeout("tcp", addr, 3*time.Second); err != nil {
|
||||||
|
t.Logf("TCP connect to %s failed (informational; the test host may not route to the bridge): %v", addr, err)
|
||||||
|
} else {
|
||||||
|
_ = conn.Close()
|
||||||
|
t.Logf("TCP connect to %s succeeded", addr)
|
||||||
|
}
|
||||||
|
stepDone(t, "8 TCP connect", start)
|
||||||
|
|
||||||
|
if env.keep {
|
||||||
|
t.Log("KVM2_TEST_KEEP is set: skipping Delete and the pool inspection")
|
||||||
|
} else {
|
||||||
|
// 9. Delete, Get is NotFound, Delete again.
|
||||||
|
start = time.Now()
|
||||||
|
if err := traced.Delete(ctx, id); err != nil {
|
||||||
|
t.Fatalf("Delete() error = %v", err)
|
||||||
|
}
|
||||||
|
deleted = true
|
||||||
|
if _, err := traced.Get(ctx, id); !errors.Is(err, provider.ErrNotFound) {
|
||||||
|
t.Errorf("Get() after Delete error = %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
if err := traced.Delete(ctx, id); err != nil {
|
||||||
|
t.Errorf("second Delete() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
stepDone(t, "9 Delete, Get, Delete", start)
|
||||||
|
|
||||||
|
// 10. The pool keeps the shared base image and its marker, and holds
|
||||||
|
// no per-VM volume.
|
||||||
|
start = time.Now()
|
||||||
|
base := baseVolumeName(integrationImage, provider.KVMFormatQcow2)
|
||||||
|
marker := markerVolumeName(base)
|
||||||
|
overlay, seed := overlayVolumeName(name), seedVolumeName(name)
|
||||||
|
present := poolVolumes(ctx, t, p, host, base, marker, overlay, seed)
|
||||||
|
for _, want := range []string{base, marker} {
|
||||||
|
if !present[want] {
|
||||||
|
t.Errorf("pool %s lacks %s after Delete; the base image cache must stay", host.Pool, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, gone := range []string{overlay, seed} {
|
||||||
|
if present[gone] {
|
||||||
|
t.Errorf("pool %s still holds %s after Delete", host.Pool, gone)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stepDone(t, "10 pool inspection", start)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 11. Spans: the package's spans are children of the decorator's.
|
||||||
|
assertLifecycleSpans(t, sr.Ended())
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertLifecycleSpans checks that at least one provider.create span has a
|
||||||
|
// kvm2.DomainDefineXML and a kvm2.dial child (the first Create dials the
|
||||||
|
// pooled connection and defines the domain), and that no attribute of any
|
||||||
|
// span carries the cloud-init text.
|
||||||
|
func assertLifecycleSpans(t *testing.T, spans []sdktrace.ReadOnlySpan) {
|
||||||
|
t.Helper()
|
||||||
|
var creates []sdktrace.ReadOnlySpan
|
||||||
|
for _, s := range spans {
|
||||||
|
if s.Name() == "provider.create" {
|
||||||
|
creates = append(creates, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(creates) == 0 {
|
||||||
|
t.Fatalf("no provider.create span among %d spans: %v", len(spans), spanNames(spans))
|
||||||
|
}
|
||||||
|
satisfied := false
|
||||||
|
for _, create := range creates {
|
||||||
|
parent := create.SpanContext().SpanID()
|
||||||
|
var children, define, dial int
|
||||||
|
for _, s := range spans {
|
||||||
|
if s.Parent().SpanID() != parent || !strings.HasPrefix(s.Name(), "kvm2.") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
children++
|
||||||
|
switch s.Name() {
|
||||||
|
case "kvm2.DomainDefineXML":
|
||||||
|
define++
|
||||||
|
case "kvm2.dial":
|
||||||
|
dial++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Logf("provider.create span %s: %d kvm2.* children (%d DomainDefineXML, %d dial)", parent, children, define, dial)
|
||||||
|
if define > 0 && dial > 0 {
|
||||||
|
satisfied = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !satisfied {
|
||||||
|
t.Errorf("no provider.create span has both a kvm2.DomainDefineXML and a kvm2.dial child; spans: %v", spanNames(spans))
|
||||||
|
}
|
||||||
|
for _, s := range spans {
|
||||||
|
for _, a := range s.Attributes() {
|
||||||
|
v := a.Value.Emit()
|
||||||
|
if strings.Contains(v, integrationCloudInit) || strings.Contains(v, "#cloud-config") {
|
||||||
|
t.Errorf("span %s attribute %s carries the cloud-init text", s.Name(), a.Key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestKVM2Check_realHost runs the health check against the host and creates
|
||||||
|
// nothing.
|
||||||
|
func TestKVM2Check_realHost(t *testing.T) {
|
||||||
|
env := loadIntegrationEnv(t)
|
||||||
|
pc := env.providerConfig()
|
||||||
|
_, log := newIntegrationLog(t, env.logLevel,
|
||||||
|
fmt.Sprintf("%q=%q", "provider", integrationProvider),
|
||||||
|
fmt.Sprintf("%q=%q", "host", integrationHost))
|
||||||
|
ctx, cancel := context.WithTimeout(logf.IntoContext(context.Background(), log), env.timeout)
|
||||||
|
defer cancel()
|
||||||
|
if env.knownHosts == "" {
|
||||||
|
t.Log("KVM2_TEST_KNOWN_HOSTS is not set: host keys are NOT verified for this run")
|
||||||
|
}
|
||||||
|
|
||||||
|
built, err := New(ctx, pc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New() error = %v", err)
|
||||||
|
}
|
||||||
|
checker, ok := built.(provider.Checker)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("New() returned %T, which is not a provider.Checker", built)
|
||||||
|
}
|
||||||
|
targets := checker.Check(ctx)
|
||||||
|
if len(targets) != 1 {
|
||||||
|
t.Fatalf("Check() returned %d targets, want 1: %+v", len(targets), targets)
|
||||||
|
}
|
||||||
|
ts := targets[0]
|
||||||
|
if !ts.Ready || ts.Reason != provider.ReasonReachable {
|
||||||
|
t.Fatalf("Check() = ready %v reason %q message %q, want Ready/%s (details %v)", ts.Ready, ts.Reason, ts.Message, provider.ReasonReachable, ts.Details)
|
||||||
|
}
|
||||||
|
if ts.Details["poolState"] != "running" {
|
||||||
|
t.Errorf("Check() poolState = %q, want running", ts.Details["poolState"])
|
||||||
|
}
|
||||||
|
t.Logf("host %s (%s): hostname=%s libvirt=%s pool=%s (%s) available=%s GiB latency=%s",
|
||||||
|
ts.Name, ts.Address, ts.Details["hostname"], ts.Details["libvirtVersion"], ts.Details["pool"], ts.Details["poolState"],
|
||||||
|
ts.Details["poolAvailableGiB"], ts.Latency.Round(time.Millisecond))
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user