Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  kacerrandClaude Fable 5.1
|
a114201ad3 | ||
|
kacerrandClaude Fable 5.1
|
7d177b4817 | ||
|
kacerrandClaude Fable 5.1
|
d171c0426e | ||
|
kacerrandClaude Fable 5.1
|
7aea14fbb3 | ||
|
kacerrandClaude Fable 5.1
|
bd633c0288 | ||
|
kacerrandClaude Fable 5.1
|
1b5ddb8349 | ||
|
kacerr
|
20ffba604b |
@@ -84,16 +84,6 @@
|
||||
"Bash(kubectl -n egress-proxies-operator-system get deploy egress-proxies-operator-controller-manager -o jsonpath='{.spec.template.spec.containers[0].args}')",
|
||||
"Bash(kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager)",
|
||||
"Bash(python3 -c \"import json; d=json.load\\(open\\('docs/deploy/sa_key.json'\\)\\); print\\(d.get\\('type'\\), d.get\\('client_email'\\)\\)\")",
|
||||
"Bash(tea pr *)",
|
||||
"Bash(git worktree *)",
|
||||
"Bash(python3 -c \"import yaml; yaml.safe_load\\(open\\('.gitea/workflows/build.yaml'\\)\\); print\\('YAML OK'\\)\")",
|
||||
"Bash(ruby -ryaml -e \"YAML.load_file\\('.gitea/workflows/build.yaml'\\); puts 'YAML OK'\")",
|
||||
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator status --short --branch)",
|
||||
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator log --oneline -1)",
|
||||
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator tag 0.01)",
|
||||
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator push origin 0.01)",
|
||||
"Bash(chmod +x docs/demo/run-demo.sh)",
|
||||
"Bash(bash -n docs/demo/run-demo.sh)",
|
||||
"Bash(git add *)",
|
||||
"Bash(git commit -q -m 'Add plan: KVM/libvirt provider *)",
|
||||
"Bash(go get *)",
|
||||
|
||||
@@ -1,91 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-09-10 22:58 UTC — Code-review fixes on the KVM branch: deletion, validation, lazy SSH, guest agent
|
||||
|
||||
Five findings from a review of `feat/kvm-provider`, all fixed on the branch.
|
||||
|
||||
- **A Proxy can always finish deleting.** The delete path used to drop the finalizer
|
||||
only when `Get` answered not-found, so a permanent error there retried forever —
|
||||
a KVM host removed from the config was enough. It now calls `Delete` anyway, since
|
||||
that is idempotent and can finish where `Get` cannot, and
|
||||
`crawl.example.com/skip-cleanup: "true"` abandons the instance for whatever stays
|
||||
stuck. In the KVM provider, deleting over an unconfigured host is a no-op success;
|
||||
`Get` still fails permanently there so a typo cannot look like a missing VM.
|
||||
- **The guest agent ships as a vendor-data script**, superseding the cloud-config
|
||||
merge in the entry below. Merging could never be made safe: cloud-init drops
|
||||
vendor-data keys the user-data defines, and the strategy cannot be overridden from
|
||||
either document. The Proxy's cloud-init now reaches the seed byte for byte in any
|
||||
format — Jinja, MIME, gzip, scripts — where the round-trip previously re-typed
|
||||
unquoted scalars (`version: 1.10` arrived as `1.1`), dropped comments and reordered
|
||||
keys. New `installGuestAgent` setting, default on.
|
||||
- **A missing `kvm-ssh` Secret no longer crash-loops the manager.** The SSH material
|
||||
is read on first use and cached, so only KVM Proxies fail and they recover when the
|
||||
Secret appears. A missing file is transient, an unparseable key permanent, and
|
||||
startup still logs a warning.
|
||||
- **Host names and image keys are validated.** Either can end up in a providerID or a
|
||||
volume file name, where a slash made the VM unaddressable for good.
|
||||
- Verified end to end: unit suite, the integration test against a real hypervisor
|
||||
(22 s, agent reporting the IP), both in-cluster Proxies Ready in 2 min with 204
|
||||
through squid, the guest showing `qemu-guest-agent` active with its cloud-config
|
||||
comments intact, and the manager staying up with the Secret pointed away.
|
||||
- Key files: `internal/provider/kvm/{seed.go,conn.go,kvm.go}`,
|
||||
`internal/controller/proxy_controller.go`, `internal/provider/config_kvm.go`,
|
||||
`api/v1alpha1/proxy_types.go`.
|
||||
|
||||
## 2026-09-10 21:04 UTC — KVM provider passes in-cluster; guest agent merged into cloud-config
|
||||
|
||||
- Two KVM Proxies with real cloud-configs reach `Ready` from inside the kind cluster
|
||||
(2 min from apply, IP via the guest agent, 204 through squid from the LAN).
|
||||
- The `qemu-guest-agent` bootstrap is now merged into each Proxy's cloud-config
|
||||
(`packages` + `runcmd`) instead of shipped as vendor-data: cloud-init's default
|
||||
merger drops vendor-data keys the user-data also defines, so any Proxy with its own
|
||||
`packages:` never got the agent and stayed `Provisioning` forever. `vendorData` is
|
||||
now plain extra vendor-data with no default; non-cloud-config user-data (scripts,
|
||||
`#include`) still gets the bootstrap as vendor-data. A cloud-config that is not a
|
||||
YAML mapping fails the Proxy permanently instead of booting without its config.
|
||||
- The manager pod mounts a 64 MiB `emptyDir` on `/tmp`: the seed ISO writer stages
|
||||
files on disk and the root filesystem is read-only.
|
||||
- Errors already classified with a provider sentinel no longer drop the cached
|
||||
libvirt SSH connection; only libvirt and transport errors do.
|
||||
- Key files: `internal/provider/kvm/{seed.go,conn.go,errors.go}`,
|
||||
`config/manager/manager.yaml`, `docs/kvm-provider.md`.
|
||||
|
||||
## 2026-09-10 10:07 UTC — KVM provider validated on a real hypervisor; UEFI default; libvirt call tracing
|
||||
|
||||
- `make test-integration-kvm` passes end to end against a libvirt 11.6 / QEMU 10.1
|
||||
host (Debian 13 image, UEFI, guest agent reporting the IP after ~20 s). Four
|
||||
defects found only on the real host, all fixed on the branch:
|
||||
- domain disks are now `type="file"` with resolved paths: pool-volume references
|
||||
get no AppArmor rule from `virt-aa-helper`, so QEMU was denied the disks;
|
||||
- domain metadata is parsed by local name: libvirt returns the element without
|
||||
its namespace declaration;
|
||||
- guests boot **UEFI by default** (`images.<key>.firmware: uefi|bios`, OVMF now a
|
||||
host prerequisite): Debian 13's image hangs in GRUB under SeaBIOS on a headless
|
||||
VM with a serial console (Debian bug #1111240, open); bookworm is unaffected;
|
||||
- the image cache's raw-upload path is confirmed to produce a byte-identical file.
|
||||
- Every libvirt RPC is logged at `--zap-log-level=debug` as one `libvirt call` line
|
||||
(`rpc=virDomainCreate`, identifiers, duration, `libvirtError` on failure); level 2
|
||||
adds XML, metadata and addresses; cloud-init is never logged. The integration test
|
||||
shows the trace in `-v` output (`KVM_TEST_LOG_LEVEL`, default 1).
|
||||
- Key files: `internal/provider/kvm/{domainxml.go,metadata.go,trace.go,conn.go}`,
|
||||
`internal/provider/config_kvm.go`, `docs/kvm-provider.md`.
|
||||
|
||||
## 2026-09-08 11:00 UTC — KVM/libvirt provider
|
||||
|
||||
- New `kvm` provider type: Managed proxies as VMs on your own libvirt/KVM hosts. Each
|
||||
host is reached over SSH with the libvirt unix socket forwarded through the
|
||||
connection (pure Go, nothing to expose on the hypervisors beyond sshd); one provider
|
||||
instance can span several hosts, selected by `placement.zone`.
|
||||
- Base images (`images:` map or a direct URL in `placement.image`) are downloaded by the
|
||||
operator into each host's storage pool on first use, verified by sha256 when configured,
|
||||
and cached there. Per-VM disks are qcow2 overlays; cloud-init arrives on a NoCloud seed
|
||||
ISO built in-process. GC identity lives in the domain's `<metadata>` element.
|
||||
- Wiring: `"kvm": kvm.New` in `cmd/main.go`; optional `kvm-ssh` Secret mounted at
|
||||
`/etc/proxy-operator/kvm`; samples `config/samples/proxy_kvm.yaml` and a `kvm-home`
|
||||
block in `providers-config.yaml`; `docs/kvm-provider.md`; opt-in integration test
|
||||
(`make test-integration-kvm`) against a real hypervisor.
|
||||
- Not validated on a real host at the time of this entry; see 2026-09-10 above.
|
||||
|
||||
## 2026-08-10 09:34 CEST — kind e2e verified; fix Squid OOM in containers; quickstart goes in-cluster
|
||||
|
||||
- Full end-to-end pass on a throwaway kind cluster: Squid pod Ready with a real CONNECT
|
||||
|
||||
@@ -66,10 +66,6 @@ vet: ## Run go vet against code.
|
||||
test: manifests generate fmt vet setup-envtest ## Run tests.
|
||||
KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test -race $$(go list ./... | grep -v /e2e) -coverprofile cover.out
|
||||
|
||||
.PHONY: test-integration-kvm
|
||||
test-integration-kvm: ## Run the KVM provider test against a real hypervisor. Needs KVM_TEST_HOST/USER/SSH_KEY/BRIDGE/IMAGE_URL; see docs/testing.md.
|
||||
go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/
|
||||
|
||||
# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
|
||||
# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
|
||||
# kubectl kuberc is disabled by default for test isolation; enable with:
|
||||
@@ -94,8 +90,7 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist
|
||||
|
||||
.PHONY: test-e2e
|
||||
test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
|
||||
KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) TEMPO_URL=$(TEMPO_URL) OTLP_ENDPOINT=$(OTLP_ENDPOINT) \
|
||||
go test -tags=e2e ./test/e2e/ -v -ginkgo.v -timeout 30m
|
||||
KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v
|
||||
$(MAKE) cleanup-test-e2e
|
||||
|
||||
.PHONY: cleanup-test-e2e
|
||||
@@ -178,20 +173,6 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
|
||||
@out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \
|
||||
if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi
|
||||
|
||||
KVM_SSH_KEY ?= $(HOME)/.ssh/proxy-operator_ed25519
|
||||
KVM_KNOWN_HOSTS ?= $(HOME)/.ssh/kvm_known_hosts
|
||||
KVM_SSH_NAMESPACE ?= egress-proxies-operator-system
|
||||
|
||||
.PHONY: kvm-ssh-secret
|
||||
kvm-ssh-secret: ## Create or update the kvm-ssh Secret from KVM_SSH_KEY (private key) and KVM_KNOWN_HOSTS.
|
||||
@test -f "$(KVM_SSH_KEY)" || { echo "KVM_SSH_KEY=$(KVM_SSH_KEY) not found"; exit 1; }
|
||||
@test -f "$(KVM_KNOWN_HOSTS)" || { echo "KVM_KNOWN_HOSTS=$(KVM_KNOWN_HOSTS) not found"; exit 1; }
|
||||
@grep -q 'PRIVATE KEY' "$(KVM_SSH_KEY)" || { echo "KVM_SSH_KEY=$(KVM_SSH_KEY) is not a private key (did you pass the .pub?)"; exit 1; }
|
||||
"$(KUBECTL)" -n $(KVM_SSH_NAMESPACE) create secret generic kvm-ssh \
|
||||
--from-file=id_ed25519="$(KVM_SSH_KEY)" \
|
||||
--from-file=known_hosts="$(KVM_KNOWN_HOSTS)" \
|
||||
--dry-run=client -o yaml | "$(KUBECTL)" apply -f -
|
||||
|
||||
.PHONY: deploy
|
||||
deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
|
||||
cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG}
|
||||
|
||||
@@ -15,10 +15,9 @@ itself**, and hands out to crawler clients via an HTTP list/lease API.
|
||||
cattle**: any meaningful spec change (placement, cloud-init, port)
|
||||
deletes and recreates the VM — never in-place mutation.
|
||||
- **Providers** behind one minimal interface: `kubernetes` (a real Squid
|
||||
pod in this cluster — local dev/CI), `gcp` (Compute Engine VMs with
|
||||
ephemeral external IPs — the real egress fleet) and `kvm` (VMs on your
|
||||
own libvirt hypervisors, reached over SSH). Config is a YAML file
|
||||
(`--providers-config`) with named instances (`gcp-eu`, `kvm-home`, ...).
|
||||
pod in this cluster — local dev/CI) and `gcp` (Compute Engine VMs with
|
||||
ephemeral external IPs — the real egress fleet). Config is a YAML file
|
||||
(`--providers-config`) with named instances (`gcp-eu`, `gcp-us`, ...).
|
||||
- **Health engine** probes every proxy by fetching a URL *through* it (a
|
||||
real CONNECT tunnel — a proxy that accepts TCP but can't egress goes
|
||||
Unhealthy), with threshold logic and transition-only status writes.
|
||||
@@ -132,119 +131,11 @@ make deploy IMG=<registry>/egress-proxies-operator:dev
|
||||
which also installs Squid via cloud-init). A missing field fails the
|
||||
Proxy with a message naming it.
|
||||
|
||||
## KVM setup
|
||||
|
||||
Managed proxies as VMs on libvirt/KVM hosts you own. Full guide:
|
||||
[docs/kvm-provider.md](docs/kvm-provider.md).
|
||||
|
||||
1. Each host needs `libvirtd`, an SSH user in the `libvirt` group, a
|
||||
bridge to a network with DHCP, and a storage pool.
|
||||
2. Create the `kvm-ssh` Secret from the operator's **private** key and a
|
||||
`known_hosts` with each host's key; the manager mounts it at
|
||||
`/etc/proxy-operator/kvm`:
|
||||
|
||||
```sh
|
||||
ssh-keyscan -H <host> >> ~/.ssh/kvm_known_hosts
|
||||
kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
|
||||
--from-file=id_ed25519=$HOME/.ssh/proxy-operator_ed25519 \
|
||||
--from-file=known_hosts=$HOME/.ssh/kvm_known_hosts
|
||||
# or: make kvm-ssh-secret (same paths by default; KVM_SSH_KEY / KVM_KNOWN_HOSTS to override)
|
||||
```
|
||||
|
||||
3. Add a `kvm` entry to the providers config with `images`,
|
||||
`machineTypes` and `hosts` (see
|
||||
[config/samples/providers-config.yaml](config/samples/providers-config.yaml)).
|
||||
Base images are downloaded into each host's pool on first use.
|
||||
4. A Proxy names a host (`placement.zone`, optional with one host), a
|
||||
size (`placement.machineType`) and an image (`placement.image`) — see
|
||||
[config/samples/proxy_kvm.yaml](config/samples/proxy_kvm.yaml).
|
||||
|
||||
Cloud-init from a Secret: the Secret **must** carry the label
|
||||
`crawl.example.com/cloud-init: "true"` — the operator's cache only holds
|
||||
labelled Secrets, so an unlabelled one is invisible (the Proxy reports
|
||||
`CloudInitError`). Rotating the Secret's content triggers VM replacement.
|
||||
|
||||
Stuck deletion: a Proxy stays `Terminating` while the operator keeps failing
|
||||
to clean its instance up — credentials revoked, a provider or a KVM host
|
||||
dropped from the config. Annotate it with
|
||||
`crawl.example.com/skip-cleanup: "true"` to abandon the instance and release
|
||||
the object. Whatever is still running then has to be removed by hand.
|
||||
|
||||
## Providers
|
||||
|
||||
`Managed` proxies are provisioned by a provider — a small compute backend
|
||||
behind one minimal interface. Providers are configured in the
|
||||
`--providers-config` YAML file as **named instances**: `spec.provider` on a
|
||||
Proxy refers to an entry's `name`, not its `type`, so `gcp-eu` and `gcp-us`
|
||||
can be two differently-configured instances of the same `gcp` type (see
|
||||
[config/samples/providers-config.yaml](config/samples/providers-config.yaml)).
|
||||
|
||||
### Implemented providers
|
||||
|
||||
| Type | Creates | Per-instance config | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `kubernetes` | A real Squid pod (`ubuntu/squid:6.6-24.04_edge` by default) in the same cluster the operator runs in | `image` (optional) | Needs no cloud account — local dev, CI, and the kind quickstart. Pods share the cluster's egress IP, so it exercises the full lifecycle but not distinct egress paths. |
|
||||
| `gcp` | A Compute Engine VM with an ephemeral external IP | `project` (required), `network`, `networkTag`, `diskSizeGb` | The real egress fleet. Deliberately uses only four API calls (Insert / Get / Delete / AggregatedList), all fire-and-forget: `Create` returns as soon as the operation is submitted and the reconciler discovers progress by polling `Get`. Auth is Application Default Credentials — workload identity in-cluster, `gcloud` ADC locally; no key-file plumbing. |
|
||||
|
||||
### Adding a provider
|
||||
|
||||
A new backend (Hetzner, AWS, ...) is four pieces; the contract lives in
|
||||
[internal/provider/provider.go](internal/provider/provider.go):
|
||||
|
||||
1. **Implement the 4-method `Provider` interface** in a new
|
||||
`internal/provider/<type>/` package:
|
||||
- `Create` submits and returns — it never blocks until the VM runs, and
|
||||
must be idempotent keyed on `req.Name` (a deterministic name derived
|
||||
from the Proxy's UID), so a repeat call after a crash finds the
|
||||
existing instance instead of duplicating it.
|
||||
- `Get` returns `provider.ErrNotFound` as a *normal* outcome — the
|
||||
reconciler branches on it for replacement and adoption, so don't
|
||||
treat it as exceptional.
|
||||
- `Delete` is idempotent: deleting an already-gone instance is not an
|
||||
error.
|
||||
- `ListByTag` returns every instance the operator ever tagged, for
|
||||
orphan GC.
|
||||
2. **Tag every created resource** with `LabelManaged=true` and
|
||||
`LabelUID=<Proxy UID>`, and report `CreatedAt` — orphan GC relies on
|
||||
all three to find owned resources and skip in-flight creates.
|
||||
3. **Classify every returned error** with `provider.Wrap` into the
|
||||
four-sentinel taxonomy in
|
||||
[internal/provider/errors.go](internal/provider/errors.go)
|
||||
(`ErrNotFound` / `ErrQuotaExceeded` / `ErrTransient` / `ErrPermanent`)
|
||||
— the reconciler decides retry, slow backoff, or latching `Failed`
|
||||
purely from that classification, never from provider-specific types.
|
||||
4. **Wire it up**: add a type-specific config block in
|
||||
[internal/provider/config.go](internal/provider/config.go), and
|
||||
register the constructor in the builtins map in
|
||||
[cmd/main.go](cmd/main.go) (`"<type>": <pkg>.New`). The registry
|
||||
([internal/provider/registry](internal/provider/registry/registry.go))
|
||||
handles named instances, and the metrics wrapper is applied
|
||||
automatically.
|
||||
|
||||
Test against a fake API seam rather than the real cloud — see the
|
||||
`instancesAPI` seam in
|
||||
[internal/provider/gcp/gcp.go](internal/provider/gcp/gcp.go) for the
|
||||
pattern.
|
||||
|
||||
## Tracing
|
||||
|
||||
OpenTelemetry tracing is built in but **off by default** — the operator
|
||||
only exports spans when pointed at an OTLP receiver:
|
||||
|
||||
```bash
|
||||
# In config/manager/manager.yaml (commented-out block is already there):
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger-collector.observability:4318
|
||||
# Local dev: print spans to stdout instead
|
||||
OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml
|
||||
```
|
||||
|
||||
Every reconcile, discovery API request, and GC sweep becomes a trace, with
|
||||
child spans for provider and Kubernetes API calls; log lines inside a traced
|
||||
operation carry matching `traceID`/`spanID` fields (point Grafana/Loki
|
||||
derived fields at `traceID`). Sampling and endpoints follow the standard
|
||||
`OTEL_*` env vars; `--trace-health-probes` additionally traces each health
|
||||
probe (high volume). Details in `docs/architecture.md` §10.
|
||||
|
||||
## Caveats — read these two
|
||||
|
||||
**Changing a proxy changes its IP.** Proxies are immutable cattle: editing
|
||||
|
||||
@@ -62,15 +62,6 @@ const (
|
||||
// must be replaced.
|
||||
AnnotationSpecHash = "crawl.example.com/spec-hash"
|
||||
|
||||
// AnnotationSkipCleanup abandons the provider-side instance: set it to
|
||||
// "true" and deletion removes the finalizer without asking the provider
|
||||
// to clean anything up. It is the escape hatch for a cleanup that cannot
|
||||
// succeed — a provider dropped from the config, credentials revoked, a
|
||||
// backend that answers permanently — where the object would otherwise
|
||||
// stay in Terminating forever. The instance is left running and becomes
|
||||
// the administrator's to remove.
|
||||
AnnotationSkipCleanup = "crawl.example.com/skip-cleanup"
|
||||
|
||||
// LabelCloudInit must be set (to "true") on every Secret referenced by
|
||||
// spec.cloudInit.secretRef: the manager's cache only holds Secrets
|
||||
// carrying this label, so an unlabelled Secret is invisible to the
|
||||
|
||||
+5
-43
@@ -42,7 +42,6 @@ import (
|
||||
"sigs.k8s.io/controller-runtime/pkg/cache"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/healthz"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log/zap"
|
||||
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
|
||||
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
|
||||
@@ -58,9 +57,7 @@ import (
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kvm"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version"
|
||||
// +kubebuilder:scaffold:imports
|
||||
)
|
||||
@@ -96,7 +93,6 @@ func main() {
|
||||
var leaseCooldown, maxLeaseTTL time.Duration
|
||||
var showVersion bool
|
||||
var gcpWireFullPayloads bool
|
||||
var traceHealthProbes bool
|
||||
|
||||
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
|
||||
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
|
||||
@@ -137,9 +133,6 @@ func main() {
|
||||
"Print the commit the binary was built from and exit.")
|
||||
flag.BoolVar(&gcpWireFullPayloads, "gcp-wire-log-full-payloads", false,
|
||||
"Log GCP V(5) wire payloads verbatim instead of eliding fields larger than 1KiB.")
|
||||
flag.BoolVar(&traceHealthProbes, "trace-health-probes", false,
|
||||
"Emit one trace span per health probe. Off by default: probes run about "+
|
||||
"once per second per proxy and would dominate trace volume.")
|
||||
|
||||
opts := zap.Options{
|
||||
Development: true,
|
||||
@@ -155,17 +148,6 @@ func main() {
|
||||
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
|
||||
setupLog.Info("Starting egress-proxies-operator",
|
||||
"commit", version.Resolve(), "goVersion", goruntime.Version())
|
||||
|
||||
// Off (no-op spans, unchanged logs) unless OTEL_* env opts in; see
|
||||
// internal/tracing. Shutdown is called explicitly after mgr.Start
|
||||
// returns — the os.Exit paths below skip defers.
|
||||
tracingShutdown, err := tracing.Setup(logf.IntoContext(context.Background(), setupLog),
|
||||
"egress-proxies-operator", version.Resolve())
|
||||
if err != nil {
|
||||
setupLog.Error(err, "Failed to set up tracing")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx := ctrl.SetupSignalHandler()
|
||||
|
||||
// Providers load first and fail fast: a manager that comes up without
|
||||
@@ -180,15 +162,10 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
providers, err := registry.Build(ctx, cfg, map[string]registry.Constructor{
|
||||
"kubernetes": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
|
||||
// The kubernetes provider builds its own uncached client;
|
||||
// wrap its transport so its API calls join the caller's trace.
|
||||
return kubernetes.NewWithTransportWrapper(ctx, pc, tracing.RestConfigWrapper())
|
||||
},
|
||||
"kubernetes": kubernetes.New,
|
||||
"gcp": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
|
||||
return gcp.NewWithWireOptions(ctx, pc, gcp.WireLogOptions{FullPayloads: gcpWireFullPayloads})
|
||||
},
|
||||
"kvm": kvm.New,
|
||||
})
|
||||
if err != nil {
|
||||
setupLog.Error(err, "Failed to build providers")
|
||||
@@ -196,8 +173,7 @@ func main() {
|
||||
}
|
||||
m := metrics.New()
|
||||
for name, p := range providers {
|
||||
// Tracing outermost: the span covers the metrics recording too.
|
||||
providers[name] = provider.WithTracing(name, provider.WithMetrics(name, p, m))
|
||||
providers[name] = provider.WithMetrics(name, p, m)
|
||||
}
|
||||
|
||||
// if the enable-http2 flag is false (the default), http/2 should be disabled
|
||||
@@ -260,9 +236,7 @@ func main() {
|
||||
cacheOpts.DefaultNamespaces = map[string]cache.Config{proxyNamespace: {}}
|
||||
}
|
||||
|
||||
restCfg := ctrl.GetConfigOrDie()
|
||||
restCfg.Wrap(tracing.RestConfigWrapper())
|
||||
mgr, err := ctrl.NewManager(restCfg, ctrl.Options{
|
||||
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
|
||||
Scheme: scheme,
|
||||
Metrics: metricsServerOptions,
|
||||
HealthProbeBindAddress: probeAddr,
|
||||
@@ -284,7 +258,6 @@ func main() {
|
||||
engine := health.NewEngine(mgr.GetClient())
|
||||
engine.Workers = healthWorkers
|
||||
engine.Metrics = m
|
||||
engine.TraceProbes = traceHealthProbes
|
||||
if err := mgr.Add(engine); err != nil {
|
||||
setupLog.Error(err, "Failed to add health engine")
|
||||
os.Exit(1)
|
||||
@@ -350,19 +323,8 @@ func main() {
|
||||
}
|
||||
|
||||
setupLog.Info("Starting manager")
|
||||
startErr := mgr.Start(ctx)
|
||||
|
||||
// Flush pending spans on the way out, error path included. Fresh
|
||||
// context: the signal ctx is already cancelled by the time Start
|
||||
// returns.
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if err := tracingShutdown(shutdownCtx); err != nil {
|
||||
setupLog.Error(err, "Failed to flush traces on shutdown")
|
||||
}
|
||||
cancel()
|
||||
|
||||
if startErr != nil {
|
||||
setupLog.Error(startErr, "Failed to run manager")
|
||||
if err := mgr.Start(ctx); err != nil {
|
||||
setupLog.Error(err, "Failed to run manager")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,5 +5,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
images:
|
||||
- name: controller
|
||||
newName: egress-proxies-operator
|
||||
newTag: kvm-dev
|
||||
newName: example.com/egress-proxies-operator
|
||||
newTag: v0.0.1
|
||||
|
||||
@@ -64,8 +64,6 @@ spec:
|
||||
- --leader-elect
|
||||
- --health-probe-bind-address=:8081
|
||||
- --providers-config=/etc/proxy-operator/providers.yaml
|
||||
# Add --trace-health-probes to emit one span per health probe
|
||||
# (high volume: ~1/s per proxy).
|
||||
env:
|
||||
# Bearer token for the discovery API. Optional: without the
|
||||
# Secret the API serves unauthenticated (with a loud warning).
|
||||
@@ -78,33 +76,6 @@ spec:
|
||||
name: discovery-token
|
||||
key: token
|
||||
optional: true
|
||||
# --- OpenTelemetry tracing ---
|
||||
# Tracing is OFF until OTEL_EXPORTER_OTLP_ENDPOINT (or
|
||||
# OTEL_TRACES_EXPORTER) is set; without it the operator runs
|
||||
# exactly as before. The downward-API vars must stay listed
|
||||
# before OTEL_RESOURCE_ATTRIBUTES: $(VAR) expansion only sees
|
||||
# earlier-listed vars.
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: OTEL_SERVICE_NAME
|
||||
value: egress-proxies-operator
|
||||
- name: OTEL_RESOURCE_ATTRIBUTES
|
||||
value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)
|
||||
# Point at an OTLP receiver (Tempo, Jaeger, otel-collector) to
|
||||
# enable tracing; http/protobuf (4318) is the default protocol,
|
||||
# set OTEL_EXPORTER_OTLP_PROTOCOL=grpc for 4317.
|
||||
# - name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||
# value: http://jaeger-collector.observability:4318
|
||||
# - name: OTEL_TRACES_SAMPLER
|
||||
# value: parentbased_traceidratio
|
||||
# - name: OTEL_TRACES_SAMPLER_ARG
|
||||
# value: "0.1"
|
||||
image: controller:latest
|
||||
name: manager
|
||||
ports:
|
||||
@@ -145,30 +116,9 @@ spec:
|
||||
- name: providers-config
|
||||
mountPath: /etc/proxy-operator
|
||||
readOnly: true
|
||||
# SSH material for kvm providers (sshKeyFile / knownHostsFile in the
|
||||
# providers config point here). Optional: the mount is empty when the
|
||||
# Secret does not exist, and only kvm entries need it. Create it with:
|
||||
# kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
|
||||
# --from-file=id_ed25519=<private key> \
|
||||
# --from-file=known_hosts=<known_hosts with every host's key>
|
||||
- name: kvm-ssh
|
||||
mountPath: /etc/proxy-operator/kvm
|
||||
readOnly: true
|
||||
# Scratch space: the root filesystem is read-only, and the kvm
|
||||
# provider builds each VM's cloud-init seed ISO through a temp
|
||||
# directory (a few hundred KB, deleted right after upload).
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: providers-config
|
||||
configMap:
|
||||
name: providers-config
|
||||
- name: kvm-ssh
|
||||
secret:
|
||||
secretName: kvm-ssh
|
||||
optional: true
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
sizeLimit: 64Mi
|
||||
serviceAccountName: controller-manager
|
||||
terminationGracePeriodSeconds: 10
|
||||
|
||||
@@ -9,46 +9,9 @@ metadata:
|
||||
data:
|
||||
# Mounted at /etc/proxy-operator/providers.yaml (--providers-config).
|
||||
# The default ships only the kubernetes-pod provider so the operator runs
|
||||
# out of the box; add gcp entries (type: gcp, gcp.project: ...) or kvm
|
||||
# entries (type: kvm, hosts reached over SSH) for real egress fleets —
|
||||
# see config/samples/providers-config.yaml.
|
||||
# out of the box; add gcp entries (type: gcp, gcp.project: ...) for real
|
||||
# egress fleets — see config/samples/providers-config.yaml.
|
||||
providers.yaml: |
|
||||
providers:
|
||||
- name: kubernetes
|
||||
type: kubernetes
|
||||
- name: kvm-home
|
||||
type: kvm
|
||||
kvm:
|
||||
# SSH to each host with the libvirt unix socket forwarded through the
|
||||
# connection; nothing to open on the hypervisors beyond sshd. Both
|
||||
# files come from the optional kvm-ssh Secret mounted by
|
||||
# config/manager/manager.yaml — see docs/kvm-provider.md.
|
||||
sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
|
||||
knownHostsFile: /etc/proxy-operator/kvm/known_hosts
|
||||
# insecureIgnoreHostKey: true # skip host-key checks (not recommended)
|
||||
# diskSizeGb: 10 # per-VM overlay; cloud-init grows the root fs into it
|
||||
# vendorData: | # extra vendor-data; the qemu-guest-agent bootstrap is merged
|
||||
# #cloud-config # into each Proxy's cloud-config, not configured here
|
||||
# ...
|
||||
images:
|
||||
# placement.image names a key here (or is an https:// URL directly).
|
||||
# Fetched into each host's pool on first use and cached there as
|
||||
# base-<key>.qcow2; do not change an entry while Proxies use it.
|
||||
debian-13:
|
||||
url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||
# sha256: "<64 hex>" # optional; verified after download
|
||||
# firmware: uefi # default; "bios" only for images with no EFI partition
|
||||
# # (Debian 13 hangs in GRUB under BIOS headless: Debian bug #1111240)
|
||||
ubuntu-24.04:
|
||||
url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
|
||||
machineTypes: # placement.machineType names a key here
|
||||
small: {vcpus: 1, memoryMiB: 1024}
|
||||
medium: {vcpus: 2, memoryMiB: 2048}
|
||||
hosts: # placement.zone names a host; optional with one host
|
||||
- name: beelink
|
||||
address: beelink.tail8c69c.ts.net:22
|
||||
user: novakj # must be in the libvirt group
|
||||
bridge: br0 # to a LAN with DHCP; the VM's IP comes from there
|
||||
# socket: /var/run/libvirt/libvirt-sock
|
||||
# pool: default
|
||||
|
||||
|
||||
@@ -4,6 +4,5 @@
|
||||
resources:
|
||||
- proxy_kubernetes.yaml
|
||||
- proxy_gcp.yaml
|
||||
- proxy_kvm.yaml
|
||||
- proxy_external.yaml
|
||||
# +kubebuilder:scaffold:manifestskustomizesamples
|
||||
|
||||
@@ -19,39 +19,3 @@ providers:
|
||||
# diskSizeGb: 10
|
||||
# auth: Application Default Credentials (workload identity
|
||||
# in-cluster, gcloud ADC locally). No key-file plumbing.
|
||||
- name: kvm-home
|
||||
type: kvm
|
||||
kvm:
|
||||
# SSH to each host with the libvirt unix socket forwarded through the
|
||||
# connection; nothing to open on the hypervisors beyond sshd. Both
|
||||
# files come from the optional kvm-ssh Secret mounted by
|
||||
# config/manager/manager.yaml — see docs/kvm-provider.md.
|
||||
sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
|
||||
knownHostsFile: /etc/proxy-operator/kvm/known_hosts
|
||||
# insecureIgnoreHostKey: true # skip host-key checks (not recommended)
|
||||
# diskSizeGb: 10 # per-VM overlay; cloud-init grows the root fs into it
|
||||
# installGuestAgent: false # skip the vendor-data script that installs qemu-guest-agent
|
||||
# vendorData: | # extra vendor-data, carried beside the agent script as a
|
||||
# #cloud-config # MIME multipart. The Proxy's own cloud-init is never rewritten.
|
||||
# ...
|
||||
images:
|
||||
# placement.image names a key here (or is an https:// URL directly).
|
||||
# Fetched into each host's pool on first use and cached there as
|
||||
# base-<key>.qcow2; do not change an entry while Proxies use it.
|
||||
debian-13:
|
||||
url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||
# sha256: "<64 hex>" # optional; verified after download
|
||||
# firmware: uefi # default; "bios" only for images with no EFI partition
|
||||
# # (Debian 13 hangs in GRUB under BIOS headless: Debian bug #1111240)
|
||||
ubuntu-24.04:
|
||||
url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
|
||||
machineTypes: # placement.machineType names a key here
|
||||
small: {vcpus: 1, memoryMiB: 1024}
|
||||
medium: {vcpus: 2, memoryMiB: 2048}
|
||||
hosts: # placement.zone names a host; optional with one host
|
||||
- name: beelink
|
||||
address: beelink:22
|
||||
user: novakj # must be in the libvirt group
|
||||
bridge: br0 # to a LAN with DHCP; the VM's IP comes from there
|
||||
# socket: /var/run/libvirt/libvirt-sock
|
||||
# pool: default
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
# A Managed proxy backed by a libvirt/KVM hypervisor on the home network:
|
||||
# the operator boots a stock Debian cloud image as a VM on the named host,
|
||||
# with a NIC on the host's bridge (IP from the LAN's DHCP), and installs
|
||||
# Squid via cloud-init. Requires a providers-config entry named "kvm-home"
|
||||
# (see providers-config.yaml) and the kvm-ssh Secret (docs/kvm-provider.md).
|
||||
#
|
||||
# placement.zone is the host name from kvm.hosts (optional when only one
|
||||
# host is configured); machineType and image are keys in the provider's
|
||||
# machineTypes and images maps. The first Proxy for an image on a host
|
||||
# waits for the image download; later ones reuse the cached copy.
|
||||
apiVersion: crawl.example.com/v1alpha1
|
||||
kind: Proxy
|
||||
metadata:
|
||||
name: proxy-kvm-sample
|
||||
spec:
|
||||
mode: Managed
|
||||
provider: kvm-home
|
||||
placement:
|
||||
zone: nas1
|
||||
machineType: small
|
||||
image: debian-13
|
||||
port: 3128
|
||||
cloudInit:
|
||||
inline: |
|
||||
#cloud-config
|
||||
packages:
|
||||
- squid
|
||||
write_files:
|
||||
- path: /etc/squid/conf.d/proxy-operator.conf
|
||||
content: |
|
||||
http_port 3128
|
||||
http_access allow all
|
||||
via off
|
||||
forwarded_for off
|
||||
runcmd:
|
||||
- systemctl restart squid
|
||||
# Optional: log in with `ssh debian@<ip>` (the Debian image's default
|
||||
# user) to look around; `virsh console <name>` on the host works too.
|
||||
# ssh_authorized_keys:
|
||||
# - ssh-ed25519 AAAA... you@laptop
|
||||
attributes:
|
||||
geo: home
|
||||
purpose: crawl
|
||||
@@ -9,7 +9,6 @@
|
||||
| Provider contract | `internal/provider` | library | — | `Provider` interface, error taxonomy, deterministic naming, config, metrics decorator |
|
||||
| kubernetes provider | `internal/provider/kubernetes` | library | — | real Squid pods in this cluster (local dev/CI) |
|
||||
| gcp provider | `internal/provider/gcp` | library | — | Compute Engine VMs, four API calls, fire-and-forget ops |
|
||||
| kvm provider | `internal/provider/kvm` | library | — | VMs on libvirt hosts over SSH-forwarded sockets; image cache, overlay disks, NoCloud seeds |
|
||||
| Health engine | `internal/health` | Runnable | yes | through-the-proxy probes, thresholds, transition events |
|
||||
| Lease store | `internal/lease` | Runnable (expiry sweep) | no | in-memory leases + cooldowns, single mutex |
|
||||
| Discovery API | `internal/discovery` | Runnable | no | HTTP list/lease/release/report on `:8090` |
|
||||
@@ -148,25 +147,6 @@ prov.ListByTag ─► AggregatedList(label filter, ─┘ prov
|
||||
ReturnPartialSuccess: true)
|
||||
providerID = zones/<zone>/instances/<name> — zone-qualified, so Get/Delete
|
||||
stay correct even mid-replacement after a zone edit
|
||||
|
||||
kvm provider (internal/provider/kvm/) — libvirt RPC over an SSH-forwarded unix socket, per host
|
||||
prov.Create ──► DomainLookupByName: exists → start if needed, done ─┐ every step tolerates
|
||||
else: ensure base image in pool (download on first use) │ the previous attempt
|
||||
overlay volume (exists = reuse) │ having got that far;
|
||||
seed ISO (always rebuilt — a cut-off upload must │ the seed is the one
|
||||
never be trusted) │ thing redone
|
||||
DomainDefineXML + DomainCreate ─┘
|
||||
prov.Get ──► DomainGetState + DomainGetMetadata + DomainInterfaceAddresses
|
||||
(ARP, then guest agent; first routable IPv4)
|
||||
running without IP = Provisioning; nostate = Provisioning
|
||||
prov.Delete ──► destroy (not running = fine) → undefine → delete overlay + seed by name
|
||||
prov.ListByTag ─► ConnectListAllDomains on every host, keep domains whose
|
||||
<metadata> carries managed=true; one host down fails the sweep,
|
||||
one unreadable domain is skipped
|
||||
providerID = hosts/<host>/domains/<name>; GC identity lives in the domain's
|
||||
<metadata> element (libvirt has no labels). All RPCs run under a context-
|
||||
bounded call: go-libvirt never times out on its own, so a cancelled reconcile
|
||||
closes the SSH connection to unblock it.
|
||||
```
|
||||
|
||||
The reconciler never watches provider-side resources (Pods or GCP VMs).
|
||||
@@ -320,45 +300,6 @@ Each consuming package defines its own small recorder interface
|
||||
`metrics.Metrics` satisfies all of them structurally, so no package other
|
||||
than `cmd/main.go` imports the metrics package.
|
||||
|
||||
### 10. Tracing (`internal/tracing/`)
|
||||
|
||||
OpenTelemetry tracing, integrated with — not replacing — the logr/zap
|
||||
logging. Everything hangs off standard `OTEL_*` env vars: with no
|
||||
`OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` set, no SDK is
|
||||
installed, spans are no-ops, and logs are byte-for-byte what they were.
|
||||
`OTEL_SDK_DISABLED=true` and `OTEL_TRACES_EXPORTER=none` force it off;
|
||||
`console` prints spans to stdout for local dev; sampling follows
|
||||
`OTEL_TRACES_SAMPLER(_ARG)`.
|
||||
|
||||
Span topology — each unit of work is a new trace (watch events carry no
|
||||
incoming trace context):
|
||||
|
||||
- `Reconcile Proxy` (root, via a `reconcile.Reconciler` decorator) →
|
||||
`reconcile.managed` / `reconcile.replaceInstance` / `reconcile.delete`,
|
||||
plus `status.patch` from the deferred flush. `reconcileExternal` does no
|
||||
I/O and is unspanned.
|
||||
- `provider.create|get|delete|list` (client spans, `provider.WithTracing`
|
||||
decorator wired outermost around `WithMetrics`); the GCP SDK's own
|
||||
otelhttp transport contributes HTTP child spans automatically.
|
||||
- Kubernetes API calls become client child spans via a wrapped
|
||||
`rest.Config` transport — gated on an existing parent span, so informer
|
||||
list/watch long-polls and leader-election renewals never create root
|
||||
spans. The kube-apiserver ignores incoming `traceparent` by design;
|
||||
these spans are leaves.
|
||||
- Discovery API: one server span per request (named from the route
|
||||
pattern), incoming W3C `traceparent` honored so clients' traces continue
|
||||
into the operator; `/healthz` excluded.
|
||||
- `gc.sweep` per GC pass; `health.probe` per probe **only** with
|
||||
`--trace-health-probes` (default off — probes run ~1/s per proxy).
|
||||
|
||||
Log correlation: logr sinks never see a context, so trace IDs ride on the
|
||||
logger — `tracing.Start` re-derives the ctx logger from a captured base
|
||||
with `traceID`/`spanID` values (lowerCamel, matching `reconcileID`;
|
||||
Grafana/Loki derived-field regexes must match `traceID`, not `trace_id`).
|
||||
Re-deriving from the base rather than layering keeps zap from emitting
|
||||
duplicate keys on nested spans. GCP V(5) wire logs get the same keys from
|
||||
the slog handler's ctx.
|
||||
|
||||
## Decisions
|
||||
|
||||
Judgment calls the spec left open, and deliberate deviations — recorded so
|
||||
@@ -435,41 +376,6 @@ they read as choices, not accidents. Chronological by build step.
|
||||
- **Unknown GCP instance statuses map to `Stopped`:** the reconciler's
|
||||
answer to Stopped is delete-and-recreate, the always-safe move for
|
||||
cattle when the API grows a new state.
|
||||
- **KVM hosts are reached over SSH with the libvirt socket forwarded,**
|
||||
not libvirtd's TCP/TLS listener: nothing to configure or expose on the
|
||||
hypervisors beyond sshd, pure Go on the operator side (no cgo, no
|
||||
libvirt libraries in the image), and the hosts list can grow by a
|
||||
config edit because the providerID carries the host name.
|
||||
- **KVM base images are fetched by the operator into the host's pool**
|
||||
through libvirt's upload stream, rather than expected to be pre-placed:
|
||||
the operator then needs no filesystem access on the host and a new
|
||||
host is usable with zero manual staging. The cost is a synchronous
|
||||
first-time download inside Create.
|
||||
- **The KVM seed ISO is rebuilt on every Create attempt** before the
|
||||
domain exists, instead of "exists → skip": a reconcile cancelled mid
|
||||
upload leaves a truncated seed, and the compensating delete cannot run
|
||||
because the connection was already closed to unblock the RPC. Skipping
|
||||
it would boot a VM with no cloud-init and no retry would ever repair it.
|
||||
- **KVM `nostate` maps to `Provisioning`, not `Stopped`:** unlike GCP's
|
||||
unknown statuses, `nostate` means libvirt could not determine the
|
||||
state at all; answering that with delete-and-recreate is too drastic,
|
||||
so the next poll decides.
|
||||
- **KVM guests boot UEFI by default, secure boot off.** Found on the
|
||||
first real host: Debian 13's generic image hangs in GRUB under SeaBIOS
|
||||
on a headless VM (Debian bug #1111240: `terminal_output gfxterm
|
||||
serial` with a serial console and no video device) regardless of CPU
|
||||
model or seed, and boots under OVMF with the serial console working.
|
||||
The alternatives were a virtio console (loses the kernel's serial
|
||||
output) or a video device the VMs have no use for. Every
|
||||
current distro cloud image supports UEFI, so it is the default and
|
||||
`firmware: bios` is the per-image escape; libvirt's firmware
|
||||
autoselection (`<os firmware="efi">`) keeps the OVMF path out of the
|
||||
operator's config. Secure boot is off so unsigned images boot too.
|
||||
- **A KVM domain GC cannot read is skipped, a host it cannot reach fails
|
||||
the sweep.** GC never deletes what it does not see, so leaving one
|
||||
domain out is safe; one corrupt domain blocking GC for every other one
|
||||
is not. A whole host missing from the view would hide many, so that
|
||||
still aborts.
|
||||
- **Kubernetes 403s classify as `ErrPermanent`** even though quota
|
||||
exhaustion also surfaces as 403 (indistinguishable from RBAC denial in
|
||||
`apierrors`): not hammering an API server that may never allow the
|
||||
@@ -489,15 +395,6 @@ they read as choices, not accidents. Chronological by build step.
|
||||
cluster-wide Secret read RBAC — an unrestricted cache would hold every
|
||||
Secret in scope). Unlabelled referenced Secrets are invisible by
|
||||
construction, surfacing as `CloudInitError`.
|
||||
- **Deletion never gives up on its own, but can be released by hand:** a
|
||||
permanent cleanup failure keeps retrying with the finalizer on, because
|
||||
dropping it automatically would leak a running instance on any provider
|
||||
that answers "permission denied". Where `Get` fails permanently the
|
||||
reconciler still calls `Delete`, since it is idempotent and may finish
|
||||
where `Get` cannot converge. `crawl.example.com/skip-cleanup: "true"`
|
||||
abandons the instance and releases the object; it is checked before the
|
||||
provider lookup, so it also frees a Proxy whose provider is gone from the
|
||||
config.
|
||||
- **Events RBAC from the plan is omitted:** nothing wires an
|
||||
EventRecorder in the prototype, and granting verbs nothing uses would
|
||||
be RBAC lint noise. Add the marker together with the recorder if events
|
||||
@@ -507,24 +404,3 @@ they read as choices, not accidents. Chronological by build step.
|
||||
logger to everything running under the manager — fighting that would
|
||||
mean two logging systems in one process. Noted as a deviation rather
|
||||
than silently ignored.
|
||||
- **Tracing is env-gated, not flag-gated:** it activates only when
|
||||
`OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` is set (user
|
||||
decision). No collector configured means no SDK installed, no-op spans,
|
||||
unchanged logs — the safe default for every existing deployment. The
|
||||
one flag is `--trace-health-probes`, off by default, because probes at
|
||||
~1/s per proxy would dominate trace volume.
|
||||
- **Trace keys are `traceID`/`spanID`**, lowerCamel like `reconcileID` and
|
||||
`providerID`, deliberately not the `trace_id` many Grafana derived-field
|
||||
examples assume — configure the derived-field regex accordingly. logr
|
||||
sinks can't read ctx, so the IDs ride on the ctx logger, re-derived from
|
||||
a captured base per span so zap never emits duplicate keys.
|
||||
- **Probe→reconcile trace links are not attempted:** the health engine's
|
||||
`GenericEvent` carries only namespace/name (no ctx), and the workqueue
|
||||
coalesces events, so any link would be a guess. A health-triggered
|
||||
reconcile starts a fresh trace; the probe that caused it is findable via
|
||||
its own (opt-in) span and shared proxy attributes.
|
||||
- **No `traceparent` toward probe targets:** probe transports stay
|
||||
uninstrumented so trace headers can never leak through a proxy to
|
||||
external sites. The kube-apiserver ignores incoming `traceparent` by
|
||||
design (public endpoint), so k8s client spans are leaves — in-process
|
||||
traces, not cross-process ones.
|
||||
|
||||
@@ -1,318 +0,0 @@
|
||||
# The KVM provider
|
||||
|
||||
Runs Managed proxies as VMs on your own libvirt/KVM hypervisors. The
|
||||
operator reaches each host over SSH, forwards the libvirt unix socket
|
||||
through that connection, and talks libvirt's own RPC over it — so the
|
||||
hosts need nothing beyond `sshd` and `libvirtd`, and the operator image
|
||||
carries no libvirt libraries. VMs boot a stock distro cloud image that the
|
||||
operator downloads into the host's storage pool on first use, with the
|
||||
Proxy's cloud-init delivered on a NoCloud seed ISO built in-process.
|
||||
|
||||
## Hypervisor prerequisites
|
||||
|
||||
Per host, once:
|
||||
|
||||
1. **libvirt + KVM** (`libvirtd` running, `/dev/kvm` present).
|
||||
2. **An SSH user in the `libvirt` group**, so it may open
|
||||
`/var/run/libvirt/libvirt-sock`. The operator authenticates with a
|
||||
private key; nothing else on the host is touched.
|
||||
3. **A bridge to a network with DHCP** (e.g. `br0` bridged onto the LAN).
|
||||
Each VM gets one virtio NIC on it and takes its IP from that network's
|
||||
DHCP; the cluster must be able to reach those IPs for health probes
|
||||
and for crawlers to use the proxy.
|
||||
4. **A storage pool** (default `default`, usually `/var/lib/libvirt/images`)
|
||||
with room for one base image per distro (a few hundred MB) plus the
|
||||
per-VM overlays (`diskSizeGb` each, thin-provisioned).
|
||||
5. **UEFI firmware for guests**: the `ovmf` package on Debian/Ubuntu,
|
||||
`edk2-ovmf` on Fedora/RHEL. VMs boot UEFI by default (libvirt picks
|
||||
the OVMF binary itself); a domain start fails with "no firmware
|
||||
found" without it. Only `firmware: bios` images skip this.
|
||||
6. Optionally, **outbound internet from the VMs**: a vendor-data script
|
||||
installs `qemu-guest-agent` from the distro mirrors on first boot.
|
||||
Without it the VM still works — the operator falls back to the host's
|
||||
ARP table to find the IP, which fills in once the guest sends traffic —
|
||||
but the ARP table only sees traffic to the host itself, so on a plain
|
||||
bridge the agent is in practice required.
|
||||
|
||||
Check from the operator's point of view before wiring it in:
|
||||
|
||||
```sh
|
||||
ssh virt@nas1.home.lan virsh -c qemu:///system list --all # user can drive libvirt
|
||||
ssh virt@nas1.home.lan virsh -c qemu:///system pool-info default
|
||||
ssh virt@nas1.home.lan ip link show br0
|
||||
```
|
||||
|
||||
## Cluster-side setup
|
||||
|
||||
1. Create the SSH Secret from a key pair dedicated to the operator and a
|
||||
`known_hosts` file holding every host's key. The worked example below
|
||||
uses `~/.ssh/proxy-operator_ed25519` and `~/.ssh/kvm_known_hosts`,
|
||||
the same paths the integration test and `make kvm-ssh-secret` default
|
||||
to.
|
||||
|
||||
```sh
|
||||
# once: a key pair for the operator; the .pub goes on every host
|
||||
ssh-keygen -t ed25519 -f ~/.ssh/proxy-operator_ed25519 -N '' -C proxy-operator
|
||||
ssh-copy-id -i ~/.ssh/proxy-operator_ed25519.pub virt@nas1.home.lan
|
||||
|
||||
# once per host: its host key, under the address the *operator* will
|
||||
# dial (hosts[].address), which may differ from what your laptop uses
|
||||
ssh-keyscan -H nas1.home.lan >> ~/.ssh/kvm_known_hosts
|
||||
|
||||
# the Secret: the PRIVATE key (not the .pub) plus known_hosts
|
||||
kubectl -n egress-proxies-operator-system create secret generic kvm-ssh \
|
||||
--from-file=id_ed25519=$HOME/.ssh/proxy-operator_ed25519 \
|
||||
--from-file=known_hosts=$HOME/.ssh/kvm_known_hosts
|
||||
```
|
||||
|
||||
`make kvm-ssh-secret` runs that last command as an apply, so it also
|
||||
updates an existing Secret; override the paths with
|
||||
`KVM_SSH_KEY=... KVM_KNOWN_HOSTS=...`. Putting the `.pub` in the
|
||||
Secret is the classic mistake: the operator then fails at startup
|
||||
with a "not a private key" error. Without `known_hosts`, set
|
||||
`insecureIgnoreHostKey: true` and accept that a MITM could hand you a
|
||||
fake hypervisor.
|
||||
|
||||
`config/manager/manager.yaml` mounts the Secret (optional) at
|
||||
`/etc/proxy-operator/kvm` as two files, `id_ed25519` and
|
||||
`known_hosts`, which is what `sshKeyFile` and `knownHostsFile` in the
|
||||
provider config point at. The deployment starts fine without the
|
||||
Secret, even with a `kvm` provider configured: only that provider's
|
||||
proxies fail, and they recover once the Secret exists. Rotating the
|
||||
key: re-run the command (or the make target) and restart the manager;
|
||||
both files are read once, on the first connection, and cached.
|
||||
|
||||
2. Add a provider entry to the providers config
|
||||
(`config/manager/providers_config.yaml` in-cluster; full sample in
|
||||
`config/samples/providers-config.yaml`):
|
||||
|
||||
```yaml
|
||||
- name: kvm-home
|
||||
type: kvm
|
||||
kvm:
|
||||
sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
|
||||
knownHostsFile: /etc/proxy-operator/kvm/known_hosts
|
||||
images:
|
||||
debian-13:
|
||||
url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||
machineTypes:
|
||||
small: {vcpus: 1, memoryMiB: 1024}
|
||||
hosts:
|
||||
- name: nas1
|
||||
address: nas1.home.lan:22
|
||||
user: virt
|
||||
bridge: br0
|
||||
```
|
||||
|
||||
The operator checks both files at startup and logs a `WARNING:` line
|
||||
when either is unreadable, but it does not refuse to start: the
|
||||
provider set is built all-or-nothing, so exiting here would crash-loop
|
||||
the manager and take the `kubernetes` and `gcp` providers with it.
|
||||
The files are read for real on the first connection. A missing file is
|
||||
a transient failure, so proxies heal by themselves once the Secret
|
||||
lands; a key that does not parse is permanent and needs a fix.
|
||||
|
||||
3. Apply a Proxy (`config/samples/proxy_kvm.yaml`) and watch it:
|
||||
|
||||
```sh
|
||||
kubectl get px -w
|
||||
# NAME MODE PROVIDER PHASE IP HEALTHY
|
||||
# proxy-kvm-sample Managed kvm-home Provisioning
|
||||
# proxy-kvm-sample Managed kvm-home Running 192.168.1.73
|
||||
# proxy-kvm-sample Managed kvm-home Ready 192.168.1.73 True
|
||||
```
|
||||
|
||||
The first Proxy for a given image on a given host sits in
|
||||
`Provisioning` for the length of the image download; the operator log
|
||||
shows `fetching base image into host pool` and `base image cached`
|
||||
with the size and duration.
|
||||
|
||||
## Config reference
|
||||
|
||||
| Field | Required | Default | Meaning |
|
||||
|---|---|---|---|
|
||||
| `sshKeyFile` | yes | | private key used for every host |
|
||||
| `knownHostsFile` | unless `insecureIgnoreHostKey` | | OpenSSH `known_hosts` to verify host keys against |
|
||||
| `insecureIgnoreHostKey` | | `false` | skip host-key verification entirely |
|
||||
| `diskSizeGb` | | `10` | capacity of each VM's overlay disk; cloud-init's growpart expands the root fs into it |
|
||||
| `vendorData` | | | extra cloud-init vendor-data placed on every seed next to the Proxy's user-data. Beware that cloud-init drops any top-level key of a vendor-data **cloud-config** that the user-data also defines. Shipped as one part of a MIME multipart alongside the agent script when `installGuestAgent` is on |
|
||||
| `installGuestAgent` | | `true` | add the vendor-data script that installs and starts `qemu-guest-agent`. Turn off for images that ship it already or cannot reach a package mirror; IP discovery then depends on the host's ARP table |
|
||||
| `images.<key>` | | | the key names the image in `placement.image`. Letters, digits, dots, dashes and underscores only — it becomes the cached volume name `base-<key>.<format>` |
|
||||
| `images.<key>.url` | yes per entry | | http(s) URL of a cloud image (qcow2 or raw) |
|
||||
| `images.<key>.sha256` | | | expected digest; verified after download, a change triggers a re-download |
|
||||
| `images.<key>.format` | | `qcow2` | `qcow2` or `raw` |
|
||||
| `images.<key>.firmware` | | `uefi` | `uefi` (OVMF, secure boot off) or `bios` (SeaBIOS) for images without an EFI system partition; Debian 13 hangs in GRUB under BIOS on a headless VM, see *Known gaps* |
|
||||
| `machineTypes.<key>` | ≥ 1 | | `{vcpus, memoryMiB}` size profile |
|
||||
| `hosts[].name` | yes | | the `placement.zone` value that selects this host; unique. Letters, digits, dots, dashes and underscores only — it becomes part of the providerID `hosts/<name>/domains/<vm>` |
|
||||
| `hosts[].address` | yes | | SSH endpoint, `host:port` |
|
||||
| `hosts[].user` | yes | | SSH user, member of the `libvirt` group |
|
||||
| `hosts[].socket` | | `/var/run/libvirt/libvirt-sock` | libvirt socket path on the host |
|
||||
| `hosts[].pool` | | `default` | storage pool for base images, overlays and seeds |
|
||||
| `hosts[].bridge` | yes | | host bridge for the VM's NIC |
|
||||
|
||||
## Placement
|
||||
|
||||
| Proxy field | Meaning for kvm |
|
||||
|---|---|
|
||||
| `placement.zone` | host name from `hosts`. May be omitted when exactly one host is configured; otherwise required. Unknown → `Failed` |
|
||||
| `placement.machineType` | key in `machineTypes`. Unknown → `Failed` |
|
||||
| `placement.image` | key in `images`, or an `http(s)://` URL used directly. Anything else → `Failed` |
|
||||
| `placement.region` | ignored |
|
||||
|
||||
`Failed` here means the reconciler stops retrying and the message names
|
||||
the offending value; fix the spec (which replaces the VM) or the config.
|
||||
|
||||
## What lands on the host
|
||||
|
||||
For a Proxy whose deterministic name is `proxy-abc…`, on the chosen
|
||||
host's pool:
|
||||
|
||||
| Object | Name | Lifetime |
|
||||
|---|---|---|
|
||||
| base image | `base-<key>.qcow2` (or `base-<url-hash>.qcow2` for a URL) | cached; survives Proxy deletion |
|
||||
| image marker | `base-<key>.qcow2.sha256` | with the base image |
|
||||
| overlay disk | `proxy-abc….qcow2`, qcow2 backed by the base | with the Proxy |
|
||||
| seed ISO | `proxy-abc…-seed.iso` | with the Proxy |
|
||||
| domain | `proxy-abc…` | with the Proxy |
|
||||
|
||||
The domain is a q35 machine with host-passthrough CPU, the overlay on
|
||||
virtio, the seed on a SATA cdrom, one virtio NIC on the bridge, a serial
|
||||
console, the guest-agent channel, a virtio RNG and no graphics. Its
|
||||
`<metadata>` carries the operator's element (managed flag, owning UID,
|
||||
creation time) — that is how orphan GC recognises its own VMs, since
|
||||
libvirt has no labels.
|
||||
|
||||
The seed's `user-data` is the Proxy's `cloudInit`, **byte for byte**. Any
|
||||
format cloud-init accepts works, including a Jinja template, a MIME
|
||||
archive, a shell script or a gzipped payload; the operator never parses or
|
||||
re-renders it.
|
||||
|
||||
The guest agent rides along in `vendor-data`, as a **shell script** rather
|
||||
than a cloud-config. That is the only shape cloud-init cannot drop:
|
||||
vendor-data cloud-config is merged under the user's own document with
|
||||
mergers that keep the user-data's value for every key both define and never
|
||||
append lists, so a Proxy carrying its own `packages:` silently lost the
|
||||
agent and sat in `Provisioning` forever. The strategy cannot be overridden
|
||||
from either document — `merge_how` in user-data governs only the first
|
||||
source, and `merge_how` in vendor-data is consumed and discarded before the
|
||||
document is stored. A vendor-data script skips all of it: cloud-init gives
|
||||
vendor-data its own handler chain, writes scripts to a separate directory
|
||||
and runs them from `cc_scripts_vendor`, after the user's packages are
|
||||
installed and before the user's `runcmd`. The script is idempotent, picks
|
||||
whichever of `apt-get`, `dnf`, `yum`, `zypper` or `apk` the image has, and
|
||||
logs into `/var/log/cloud-init-output.log`. Set `installGuestAgent: false`
|
||||
to leave it out. When `vendorData` is also configured, the two travel as
|
||||
separate parts of a MIME multipart vendor-data.
|
||||
|
||||
## The image cache
|
||||
|
||||
The base image is downloaded **by the operator, through libvirt**, into
|
||||
the host's pool the first time a Proxy asks for it on that host — nothing
|
||||
is pre-placed on the hypervisor, and the operator needs no filesystem
|
||||
access there. The bytes stream from the URL through the operator into
|
||||
the pool with a running sha256; nothing is buffered. The marker volume is
|
||||
written last and holds the digest, so a crash mid-download leaves a base
|
||||
volume without a marker, which the next attempt deletes and redoes.
|
||||
|
||||
A re-download happens when: the marker is missing, or `sha256` is
|
||||
configured and differs from the marker's content. Otherwise the cached
|
||||
copy is used for as long as it exists, even if the URL's content changed
|
||||
upstream (a `latest` URL is pinned at the moment of first fetch, per
|
||||
host).
|
||||
|
||||
To evict manually:
|
||||
|
||||
```sh
|
||||
ssh virt@nas1 virsh -c qemu:///system vol-delete --pool default base-debian-13.qcow2.sha256
|
||||
ssh virt@nas1 virsh -c qemu:///system vol-delete --pool default base-debian-13.qcow2
|
||||
```
|
||||
|
||||
⚠️ **Do not change an image entry's `url` or `sha256`, and do not evict
|
||||
its base volume, while Proxies using that image exist on the host.** The
|
||||
overlays back onto the base file; deleting or replacing it corrupts every
|
||||
VM that uses it. Delete those Proxies first, or add the new image under a
|
||||
new key and move Proxies over by changing their `placement.image` (which
|
||||
replaces the VM).
|
||||
|
||||
## Logging into a VM
|
||||
|
||||
- Put your key in the Proxy's cloud-init (`ssh_authorized_keys`) and
|
||||
`ssh debian@<ip>` (Debian) / `ssh ubuntu@<ip>` (Ubuntu).
|
||||
- Or on the host: `virsh console proxy-abc…` attaches to the serial
|
||||
console; cloud-init's output scrolls by there on first boot.
|
||||
- `virsh domifaddr proxy-abc… --source arp` / `--source agent` shows what
|
||||
the operator sees when it looks for the IP.
|
||||
|
||||
## Logging
|
||||
|
||||
The provider logs through controller-runtime's logger, named `kvm`, with
|
||||
`provider`, `host` and `name` (the domain) on every line.
|
||||
|
||||
- `--zap-log-level=debug` (logr V(1)): one line per provider operation
|
||||
(`domain defined`, `KVM domain ensured running`, ...) **and one
|
||||
`libvirt call` line per libvirt RPC**, named `kvm.libvirt`, with
|
||||
`rpc` (the libvirt API function, e.g. `virDomainLookupByName`), the
|
||||
identifiers it was given (`domain`, `pool`, `volume`), curated results
|
||||
(`state`, `path`, `interfaces`), `duration`, and on failure `error`
|
||||
plus `libvirtError` (the code name, e.g. `ErrNoDomain`) and
|
||||
`libvirtCode`. Connection events (`libvirt host connected`, dial
|
||||
failures, `libvirt connection dropped` with a reason) log at the same
|
||||
level. A Create is about ten RPCs; a Get is four.
|
||||
- `--zap-log-level=2`: the same lines also carry the domain and volume
|
||||
`xml`, the `metadata` element, the interface `addresses` and the
|
||||
domain `names` from a listing.
|
||||
- Cloud-init is never logged at any level: the seed ISO upload reports
|
||||
only `length` and `duration`. There is no raw wire tier like the GCP
|
||||
provider's V(5); go-libvirt exposes no hook for it.
|
||||
|
||||
## Known gaps
|
||||
|
||||
- **No quota or capacity detection.** libvirt reports an out-of-space
|
||||
pool or an over-committed host as an ordinary error; the reconciler
|
||||
treats it as transient and retries with backoff rather than the slower
|
||||
quota backoff.
|
||||
- **Orphan volumes without a domain are invisible to GC.** A crash
|
||||
between creating the volumes and defining the domain leaves an overlay
|
||||
and seed that `ListByTag` cannot see (it lists domains). They are
|
||||
cleaned when the same Proxy reconciles again (Delete removes volumes by
|
||||
name even with no domain), but a Proxy CR that is gone for good leaves
|
||||
them for `virsh vol-list` to find.
|
||||
- **A host removed from the config takes its VMs with it.** `Delete` for a
|
||||
providerID naming an unconfigured host is a no-op success, so the Proxy
|
||||
finishes deleting instead of pinning its finalizer, but the domain and its
|
||||
volumes stay on the hypervisor and orphan GC cannot see them either (it
|
||||
only sweeps configured hosts). Delete the Proxies before retiring a host,
|
||||
or clean up afterwards with `virsh undefine --nvram` plus `virsh vol-delete`.
|
||||
`Get` still fails permanently for such a providerID on purpose: a typo in
|
||||
`hosts` must not look like a missing VM and trigger a rebuild elsewhere.
|
||||
- **The first image fetch is synchronous.** It holds a reconcile worker
|
||||
for the download (LAN speed, a few hundred MB, once per image per
|
||||
host). Other Proxies keep reconciling on the remaining workers.
|
||||
- **Replacing an image in use breaks running VMs** — see the warning
|
||||
under *The image cache*.
|
||||
- **IP discovery needs either guest traffic or the agent.** A VM on a
|
||||
bridge with no DHCP never gets an address and stays `Provisioning`
|
||||
forever; check `virsh console`. The same symptom with a booted, addressed
|
||||
VM means the guest agent never came up: check `systemctl status
|
||||
qemu-guest-agent`, `cloud-init status --long` and the vendor script's
|
||||
output in `/var/log/cloud-init-output.log` (a VM without outbound
|
||||
internet cannot install it). A user-data that disables vendor-data
|
||||
(`vendor_data: {enabled: false}`) also removes the agent, by design.
|
||||
- **Debian 13 cloud images hang in GRUB under BIOS on a headless VM**
|
||||
([Debian bug #1111240](https://bugs.debian.org/1111240), open, in
|
||||
`grub2`): trixie's image ships `terminal_output gfxterm serial` in its
|
||||
GRUB config, and with a serial console but no video device (exactly
|
||||
the domain this provider defines) GRUB prints "Booting" and never
|
||||
hands over to the kernel. Not host-specific; reproduced by the Debian
|
||||
cloud team. The same image boots under UEFI with the serial console
|
||||
intact, which is why `firmware` defaults to `uefi`. Debian 12
|
||||
(bookworm) images are unaffected and boot either way. Images that
|
||||
need `firmware: bios` are the exception, not the rule.
|
||||
|
||||
## Testing
|
||||
|
||||
Unit tests run over an in-memory fake of the libvirt API. The optional
|
||||
integration test against a real host is described in
|
||||
[testing.md](testing.md#the-kvm-integration-test-real-hypervisor); its
|
||||
`-v` output includes the `libvirt call` trace above, at the verbosity
|
||||
`KVM_TEST_LOG_LEVEL` selects (default 1).
|
||||
@@ -1,183 +0,0 @@
|
||||
# Execution log: OpenTelemetry tracing integrated with logr/zap logging
|
||||
|
||||
Plan: `docs/plans/2026-08-24-1025-otel-tracing.md`
|
||||
|
||||
- [x] Step 1 — Dependencies
|
||||
- [x] Step 2 — New package `internal/tracing`
|
||||
- [x] Step 3 — `provider.WithTracing` decorator
|
||||
- [x] Step 4 — `cmd/main.go` wiring
|
||||
- [x] Step 5 — Reconciler spans
|
||||
- [x] Step 6 — Discovery server
|
||||
- [x] Step 7 — GC + health
|
||||
- [x] Step 8 — GCP wire-log enrichment
|
||||
- [x] Step 9 — Manifests + docs
|
||||
|
||||
## Step 1 — Dependencies
|
||||
|
||||
Aligned the pre-existing indirect skew (otel core v1.44.0 vs otlptrace exporters
|
||||
v1.40.0) and added the new direct deps in one shot:
|
||||
|
||||
```bash
|
||||
go get go.opentelemetry.io/otel@v1.45.0 \
|
||||
go.opentelemetry.io/otel/sdk@v1.45.0 \
|
||||
go.opentelemetry.io/otel/trace@v1.45.0 \
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0 \
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp@v1.45.0 \
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace@v1.45.0 \
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
MVS side-effects worth recording: `logr v1.4.3→v1.4.4`, `httpsnoop
|
||||
v1.0.4→v1.1.0`, `grpc-gateway/v2 v2.27.7→v2.29.0`, `proto/otlp v1.9.0→v1.11.0`,
|
||||
plus a `genproto/googleapis/api` pseudo-version bump. Full `make test` passed
|
||||
against the bumped graph (`google.golang.org/api v0.292.0` and k8s v0.36
|
||||
tolerate otelhttp v0.70.0).
|
||||
|
||||
Worth noting: the first `go mod tidy` ran *before* any first-party code
|
||||
imported `otlptracehttp`/`stdouttrace`, so it silently dropped those two
|
||||
modules again; the Step 2 tidy re-added them. The plan's semconv question
|
||||
resolved to `semconv/v1.43.0` — that's what `sdk@v1.45.0/resource/builtin.go`
|
||||
imports, so first-party code uses the same version to avoid
|
||||
`ErrSchemaURLConflict` in the common path.
|
||||
|
||||
## Step 2 — `internal/tracing` package
|
||||
|
||||
Landed as planned: `tracing.go` (env-gated `Setup`, hand-rolled exporter
|
||||
selection), `logger.go` (`Start`/`StartSpan`/`ContextWithLogger` with the
|
||||
base-logger ctx key that prevents duplicate `traceID` zap fields on nested
|
||||
spans), `reconciler.go`, `transport.go`, `http.go`, `options.go`; tests for
|
||||
all of it (first use of `sdk/trace/tracetest` in the repo).
|
||||
|
||||
Two deviations from the plan's letter:
|
||||
|
||||
- **k8s transport gating is a custom RoundTripper, not `otelhttp.WithFilter`**
|
||||
(`parentGatedTransport`): requests without a parent span bypass the otel
|
||||
transport entirely, so the no-root-spans-from-informers guarantee doesn't
|
||||
depend on otelhttp filter semantics for transports.
|
||||
- **`HTTPMiddleware` must copy the route pattern back.** The logger-injecting
|
||||
inner handler wraps the request via `WithContext` (a shallow copy), so the
|
||||
mux records the matched pattern on the copy while otelhttp's post-routing
|
||||
span rename reads the original. Found by the middleware test (span named
|
||||
`"GET"` instead of `"GET /v1/things/{id}"`); fixed with `r.Pattern =
|
||||
r2.Pattern` after `next.ServeHTTP`.
|
||||
|
||||
Also: both the transport and the middleware pass explicit W3C propagators
|
||||
instead of relying on the global, so behavior is deterministic under tests
|
||||
and when tracing is disabled. `Setup` tests reset the global provider to a
|
||||
fresh noop per case — restoring otel's own default delegate triggers a
|
||||
"Setting tracer provider to its current value" warning from the SDK.
|
||||
|
||||
## Step 3 — `provider.WithTracing`
|
||||
|
||||
`internal/provider/tracing.go` mirrors `metrics.go` exactly (same wrap shape,
|
||||
same `resultLabel` classification reused as the `provider.result` span
|
||||
attribute); tests reuse `staticProvider` from `metrics_test.go`. `ErrNotFound`
|
||||
maps to span status Ok as planned — the reconciler polls `Get` to NotFound
|
||||
during replacement/deletion, so it's an answer, not a failure.
|
||||
|
||||
One small API addition to `internal/tracing` for this: exported
|
||||
`tracing.Tracer(opts...)`, since the decorator lives in package `provider`
|
||||
and couldn't reach the unexported option resolver. The `cmd/main.go` wiring
|
||||
(`WithTracing` outermost around `WithMetrics`) lands with Step 4's commit —
|
||||
same file, one commit.
|
||||
|
||||
## Step 4 — `cmd/main.go` wiring
|
||||
|
||||
As planned: `tracing.Setup` right after `ctrl.SetLogger`; manager rest.Config
|
||||
wrapped via `restCfg := ctrl.GetConfigOrDie(); restCfg.Wrap(...)` (the
|
||||
previous inline call left nowhere to wrap); providers decorated tracing-
|
||||
outermost; explicit trace flush after `mgr.Start` returns on both the error
|
||||
and clean paths, with a fresh 10s context since the signal ctx is already
|
||||
cancelled by then.
|
||||
|
||||
The kubernetes provider grew `NewWithTransportWrapper(ctx, cfg, wrap)` (its
|
||||
client is built from its own `ctrl.GetConfig()`, invisible to the manager's
|
||||
wrapped config); `New` now delegates with a nil wrapper, so `newWithClient`
|
||||
tests stayed untouched, and main registers a closure — the same pattern the
|
||||
gcp constructor already used for wire-log options.
|
||||
|
||||
Deviation: the plan put `--trace-health-probes` here, but the flag needs the
|
||||
`health.Engine.TraceProbes` field that Step 7 introduces — moved there to
|
||||
keep every commit compiling.
|
||||
|
||||
## Step 5 — Reconciler spans
|
||||
|
||||
`SetupWithManager` completes with `tracing.NewReconciler("Proxy", r)`;
|
||||
sub-spans `reconcile.managed` / `reconcile.replaceInstance` /
|
||||
`reconcile.delete` open at the top of each state machine, and `status.patch`
|
||||
opens inside the deferred flush closure so it stays within the root span
|
||||
while its error still folds into the recorded result. `reconcileExternal`
|
||||
left unspanned as planned (no I/O).
|
||||
|
||||
Two small judgment calls: the sub-spans carry no extra attributes — the
|
||||
provider decorator already records `provider.id`, and the root span carries
|
||||
the object identity, so duplicating them was noise; and `status.patch` is
|
||||
emitted every reconcile even when nothing changed (the no-op compare is the
|
||||
span's content — a real PATCH shows up as its k8s HTTP child). Tests that
|
||||
call `r.Reconcile` directly bypass the wrapper; with no global tracer set
|
||||
they see no-op spans, so the existing fake-client and envtest suites run
|
||||
unchanged.
|
||||
|
||||
## Step 6 — Discovery server
|
||||
|
||||
Middleware chain is now recover → tracing (server span + request logger) →
|
||||
request-log → body cap → auth; the request-log middleware and the three
|
||||
handler error sites log via `logf.FromContext(r.Context())` — which is
|
||||
`s.log` enriched with the request's traceID/spanID by the tracing
|
||||
middleware, not a different logger (logr sinks can't read ctx at log time,
|
||||
so per-request values must ride on the logger instance in the ctx; user
|
||||
asked, answered in-session).
|
||||
|
||||
Deviation from the plan's letter: `recoverMiddleware` keeps `s.log`. It sits
|
||||
*outside* the tracing layer, so its request ctx never has the enriched
|
||||
logger — switching it to `FromContext` would silently drop the `"discovery"`
|
||||
name and gain nothing.
|
||||
|
||||
## Step 7 — GC + health
|
||||
|
||||
`gc.sweep` runs in a root span with `gc.providers` / `gc.deleted` counters
|
||||
(a failed orphan delete no longer counts as deleted — the loop grew an
|
||||
explicit `continue`). The health engine got the deferred `--trace-health-probes`
|
||||
flag and `TraceProbes` field; the probe call moved into `runProbe`, which
|
||||
wraps it in a `health.probe` client span only when enabled, ending before
|
||||
`record` (which stays ctx-free by design). Probe transports remain
|
||||
uninstrumented so no traceparent can leak through a proxy to external
|
||||
targets. Probe→reconcile span links stay out of scope (GenericEvent carries
|
||||
no ctx; the workqueue coalesces events) — recorded in the architecture
|
||||
Decisions in Step 9.
|
||||
|
||||
## Step 8 — GCP wire-log enrichment
|
||||
|
||||
`wireFilterHandler.Handle` clones the record and appends `traceID`/`spanID`
|
||||
when the ctx carries a valid span — same keys as the logr enrichment, added
|
||||
before the elision pass (both values are well under the 1KiB threshold).
|
||||
The new `wirelog_test.go` covers the enrichment, that the JWT/token security
|
||||
filter still drops non-wire Debug records even with a span present, and that
|
||||
spanless records stay unchanged.
|
||||
|
||||
## Step 9 — Manifests, docs, lint
|
||||
|
||||
`config/manager/manager.yaml` gained the OTel env block (downward-API
|
||||
`POD_NAME`/`POD_NAMESPACE` deliberately listed *before*
|
||||
`OTEL_RESOURCE_ATTRIBUTES` — `$(VAR)` expansion only sees earlier vars),
|
||||
with the OTLP endpoint and sampler left as commented examples so deployments
|
||||
stay tracing-off by default. `docs/architecture.md` got "### 10. Tracing"
|
||||
plus five Decisions entries; README got a Tracing section.
|
||||
|
||||
`make lint` surfaced 16 new-on-branch issues (checked with
|
||||
`golangci-lint run --new-from-rev main`; the other ~43 pre-date this work):
|
||||
|
||||
- one gofmt slip, 13 goconst repeats — fixed with `env*`/`exporter*`
|
||||
constants in `internal/tracing/tracing.go` shared by the tests;
|
||||
- two logcheck hits ("function takes both a context and a logger"):
|
||||
`Setup` now reads its logger from ctx (`logf.FromContext`, caller seeds
|
||||
via `logf.IntoContext` — the linter's sanctioned pattern), and
|
||||
`ContextWithLogger` carries a justified `//nolint:logcheck`, since being
|
||||
the IntoContext-analogue is its purpose.
|
||||
|
||||
Worth noting: pre-existing lint debt (50 goconst etc. on `main`) was left
|
||||
untouched — `make lint` still fails overall; only branch-introduced issues
|
||||
were cleaned. The lint fixes changed `Setup`'s signature after Step 2/4 had
|
||||
landed, so those commits show the two-arg version; this commit is where it
|
||||
settles.
|
||||
@@ -1,68 +0,0 @@
|
||||
# Execution log: e2e test — OTel tracing against real Tempo
|
||||
|
||||
Plan: `docs/plans/2026-08-24-1224-tracing-e2e.md`
|
||||
|
||||
- [x] Step 1 — `test/e2e/tracing_test.go`
|
||||
- [x] Step 2 — Makefile
|
||||
- [x] Step 3 — Docs
|
||||
|
||||
## Steps 1–3 — spec, Makefile, docs (one commit)
|
||||
|
||||
The three steps landed together — the spec is one new file and the other
|
||||
two are its wiring. `Describe("OTel tracing", Ordered)` is fully
|
||||
self-contained (own ns create → `make install`/`make deploy` → teardown)
|
||||
because Ginkgo randomizes top-level container order, so it cannot share the
|
||||
Manager Describe's deployment. It reuses the package-level `namespace` /
|
||||
`managerImage` and the suite's idioms (`utils.Run`, curl-pod with the
|
||||
restricted-PSS overrides JSON, log-substring `Eventually`s).
|
||||
|
||||
Judgment calls beyond the plan's letter:
|
||||
|
||||
- The squid pre-pull (`docker pull` + `kind load`) is **best effort** — a
|
||||
missing docker binary logs a note and continues rather than failing the
|
||||
spec; the 5m Ready timeout still covers an in-cluster pull.
|
||||
- The OTLP preflight pod prints per-attempt HTTP codes and a final
|
||||
`OTLP_OK`/`OTLP_UNREACHABLE` marker; the assertion quotes the pod's
|
||||
output, so an unreachable endpoint names itself in the failure.
|
||||
- `kubectl set env` is passed the literal
|
||||
`OTEL_RESOURCE_ATTRIBUTES=...$(POD_NAME)...` string via `exec.Command` —
|
||||
no shell involved, kubectl stores `$()` verbatim, and the in-place update
|
||||
keeps the var after the downward-API vars it references.
|
||||
- Tempo helpers are stdlib-only; `/api/traces/<id>` is decoded as
|
||||
OTLP-JSON (`batches[].scopeSpans[].spans[].name`), which is Tempo's
|
||||
actual shape (not Jaeger's).
|
||||
|
||||
Verified so far: `go vet -tags=e2e ./...` clean.
|
||||
|
||||
## Live run against homelab Tempo
|
||||
|
||||
First attempt failed before the suite started — Docker Desktop wasn't
|
||||
running (`kind` could not create the cluster), and the failure was masked
|
||||
to exit 0 by a `| tail` pipe on the make invocation (no pipefail in that
|
||||
shell). Rerun with Docker started first and no pipe:
|
||||
|
||||
```bash
|
||||
TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e
|
||||
```
|
||||
|
||||
Result: **SUCCESS — 4/4 specs (Manager smoke + both tracing Its), 219 s**,
|
||||
kind cluster auto-deleted. Run id `e2e-1787567624494810000`; the traces are
|
||||
findable in Grafana with TraceQL
|
||||
`{resource.test.run.id="e2e-1787567624494810000"}`. Both proxies reached
|
||||
`Ready` on real squid pods; the create trace carried
|
||||
`Reconcile Proxy` / `reconcile.managed` / `provider.create` /
|
||||
`status.patch` and the expected resource attrs; the deletion trace carried
|
||||
`reconcile.delete` / `provider.delete`.
|
||||
|
||||
Worth noting: the OTLP preflight and both Tempo polls succeeded on the
|
||||
in-cluster → LAN path (kind on macOS reaches 192.168.0.30 through Docker
|
||||
Desktop's NAT), so no extra networking setup is needed on this machine.
|
||||
|
||||
The plan's negative verification also ran: with a deliberately wrong
|
||||
endpoint (`OTLP_ENDPOINT=http://192.168.0.30:9999`) the tracing spec
|
||||
failed in the BeforeAll preflight after ~52 s with
|
||||
`OTLP endpoint http://192.168.0.30:9999 is not reachable from inside the
|
||||
kind cluster; curl output: OTLP_UNREACHABLE` — a named, fast failure
|
||||
instead of a 2-minute opaque search timeout — and teardown still deleted
|
||||
the kind cluster (`make cleanup-test-e2e` run explicitly, since a failing
|
||||
`go test` skips the Makefile's cleanup step).
|
||||
@@ -8,9 +8,9 @@ Plan: [2026-09-04-1404-kvm-provider.md](../plans/2026-09-04-1404-kvm-provider.md
|
||||
- [x] Step 3 — Transport + errors (`conn.go`, `errors.go`)
|
||||
- [x] Step 3b — Image cache (`image.go`)
|
||||
- [x] Step 4 — Provider methods (`kvm.go`) over a fake API
|
||||
- [x] Step 5 — Wiring, manifests, samples, docs, CHANGELOG
|
||||
- [x] Step 6 — Integration test against a real hypervisor (written and compiled; first real run is part of Step 7)
|
||||
- [x] Step 7 — Manual validation + MR
|
||||
- [ ] Step 5 — Wiring, manifests, samples, docs, CHANGELOG
|
||||
- [ ] Step 6 — Integration test against a real hypervisor
|
||||
- [ ] Step 7 — Manual validation + MR
|
||||
|
||||
## Step 0 — Save plan + branch
|
||||
|
||||
@@ -69,9 +69,6 @@ Two design choices differ from the plan text:
|
||||
`<disk type="volume"><source pool=... volume=.../>`, so only the *base*
|
||||
image path is ever looked up (the overlay's `<backingStore>` needs a real
|
||||
path). Fewer RPCs and the builder's inputs are all names.
|
||||
**Reverted in Step 7**: volume-referenced disks cannot start on an
|
||||
AppArmor host, see there. The domain XML is back to `type="file"` with
|
||||
the two `StorageVolGetPath` calls the plan had.
|
||||
- **Capacities are always `unit="bytes"`.** libvirt's `G` means GiB but
|
||||
`GB` means 10^9; spelling the byte count out removes the ambiguity from
|
||||
both the overlay and the seed/base volumes.
|
||||
@@ -81,9 +78,7 @@ The metadata element is written with a default namespace
|
||||
marshal. libvirt may hand it back prefixed; `parseInstanceMetadata` is
|
||||
tested against both spellings since `encoding/xml` matches on the namespace
|
||||
URI. Whether libvirt accepts the default-namespace form on `DomainDefineXML`
|
||||
is confirmed in Step 6 against a real host, not here. (Step 7: it does,
|
||||
but `DomainGetMetadata` returns the element with no namespace at all; the
|
||||
parser now matches on local name only.)
|
||||
is confirmed in Step 6 against a real host, not here.
|
||||
|
||||
Checked before relying on `github.com/kdomanski/iso9660` for the seed: its
|
||||
filename mangling lowercases and keeps `-`, so `user-data`/`meta-data`/
|
||||
@@ -288,386 +283,3 @@ uncovered lines are the real SSH/libvirt dial path and log-only
|
||||
branches. `go vet` and `-race` (three runs) are clean. Wiring into
|
||||
`cmd/main.go` is Step 5, so `New` is not yet reachable from a running
|
||||
operator.
|
||||
|
||||
## Step 6 — Integration test
|
||||
|
||||
Done before Step 5, at the user's request, after a discussion of what the
|
||||
unit tests can and cannot prove: they run over an in-memory fake of the
|
||||
libvirt calls, so every assumption about libvirt's real behaviour (domain
|
||||
XML accepted, raw qcow2 upload + pool refresh usable as a backing image,
|
||||
guest boots and reports an address) needed a test against a real host.
|
||||
The wiring in Step 5 is not a prerequisite — the test drives the provider
|
||||
directly.
|
||||
|
||||
`internal/provider/kvm/kvm_integration_test.go` is behind
|
||||
`//go:build integration` and skips itself under `-short` or when any of
|
||||
`KVM_TEST_HOST`, `KVM_TEST_USER`, `KVM_TEST_SSH_KEY`, `KVM_TEST_BRIDGE`,
|
||||
`KVM_TEST_IMAGE_URL` is unset. Optional knobs beyond the plan's list:
|
||||
`KVM_TEST_KNOWN_HOSTS` (without it the run disables host-key checking
|
||||
and logs that), `KVM_TEST_IMAGE_SHA256`, `KVM_TEST_POOL`,
|
||||
`KVM_TEST_SOCKET`, `KVM_TEST_TIMEOUT` (default 20 m, must cover a cold
|
||||
image download) and `KVM_TEST_KEEP` to leave a failed run's VM behind for
|
||||
`virsh console`.
|
||||
|
||||
One sequential test, `TestKVM_lifecycle`, not subtests: each stage
|
||||
depends on hypervisor state the previous one left. Create (timed, so the
|
||||
log shows the image fetch), re-Create returns the same ID, poll Get to
|
||||
Running with an IPv4 (Stopped fails fast with a hint to use the console),
|
||||
UID and CreatedAt survive the metadata round-trip, ListByTag includes it,
|
||||
Delete, Get → NotFound, second Delete is a no-op, ListByTag excludes it,
|
||||
and the pool is inspected through the provider's own connection pool
|
||||
(`p.hosts.call`, reachable because the test is in package `kvm`): overlay
|
||||
and seed gone, base image and marker kept. A TCP connect to the guest's
|
||||
port 22 is attempted but only logged, since the test machine may not
|
||||
route to the bridge network. Cleanup deletes the VM on failure with its
|
||||
own two-minute budget, because the test context may already be spent.
|
||||
|
||||
Makefile target, so the invocation is not something to remember:
|
||||
|
||||
```bash
|
||||
make test-integration-kvm
|
||||
# = go test -race -tags=integration -count=1 -v -timeout 40m -run TestKVM ./internal/provider/kvm/
|
||||
```
|
||||
|
||||
`docs/testing.md` gained the kvm package rows in the suite table, a
|
||||
"live libvirt" bullet under the deliberately-not-covered list, and a
|
||||
section on running this test, with the deviation from the testcontainers
|
||||
convention called out as the plan asked.
|
||||
|
||||
Worth noting: **the test has not been executed yet.** This session had
|
||||
no hypervisor credentials; it was verified with
|
||||
`go vet -tags=integration ./...` and by running it without the
|
||||
environment (self-skip). Its first real run is the opening act of
|
||||
Step 7's manual validation, and it is the point at which the Step 2 and
|
||||
Step 3b "confirm on a real host" notes get answered. The test's config
|
||||
is built directly rather than through the provider package's validation
|
||||
(unexported), so a typo in an env var surfaces from `New` or from
|
||||
libvirt, not from a validation message.
|
||||
|
||||
## Step 5 — Wiring, manifests, samples, docs, CHANGELOG
|
||||
|
||||
Done after Step 6 (see that entry for why the order flipped). Everything
|
||||
the plan listed, plus a few things it did not:
|
||||
|
||||
- `cmd/main.go`: `"kvm": kvm.New` in the builtins map. No RBAC change —
|
||||
the provider talks to hypervisors, not the API server.
|
||||
- `config/manager/manager.yaml`: `kvm-ssh` Secret mounted read-only at
|
||||
`/etc/proxy-operator/kvm` with `optional: true`, so deployments without
|
||||
a kvm provider are unaffected. It nests under the providers-config
|
||||
mount at `/etc/proxy-operator`; kustomize renders it (`kubectl
|
||||
kustomize config/default` checked). The container runs as uid 65532
|
||||
and Secret files default to 0644, so the key is readable without an
|
||||
`fsGroup`; the `kubectl create secret` recipe is in a comment next to
|
||||
the mount, in `docs/kvm-provider.md` and in the README.
|
||||
- `config/samples/providers-config.yaml`: `kvm-home` block with Debian 13
|
||||
and Ubuntu 24.04 official image URLs, two machine types, one host, and
|
||||
every optional field shown commented out. Parsed through
|
||||
`provider.LoadConfigFile` in a throwaway test to make sure the sample
|
||||
validates. `config/samples/proxy_kvm.yaml` uses the GCP sample's squid
|
||||
cloud-init and shows `ssh_authorized_keys` commented out rather than
|
||||
set — a sample with a placeholder key would either fail cloud-init or
|
||||
invite pasting a real key into a committed file. Added to the samples
|
||||
kustomization; `config/manager/providers_config.yaml`'s comment now
|
||||
mentions kvm.
|
||||
- `docs/kvm-provider.md`: prerequisites with a three-line preflight
|
||||
(`virsh list`, `pool-info`, `ip link` over ssh), cluster setup, the full
|
||||
config reference table, placement table, a table of what lands on the
|
||||
host and its lifetime, how the image cache works and how to evict, the
|
||||
**warning not to change or evict an image while Proxies use it** (the
|
||||
reviewer's finding from Step 4, carried as a documented rule since the
|
||||
content-addressed-naming alternative was not chosen), how to log in,
|
||||
and the known gaps.
|
||||
- README: providers bullet and a "KVM setup" section mirroring the GCP
|
||||
one. `docs/architecture.md`: components row, a §5 block for the kvm
|
||||
call flow in the same style as the other two, and five decision-log
|
||||
entries (SSH transport, operator-fetched images, seed always rebuilt,
|
||||
`nostate` → Provisioning, skip-vs-fail in the GC sweep).
|
||||
- `CHANGELOG.md` entry at the top, stating explicitly that the provider
|
||||
is not yet validated on a real host.
|
||||
|
||||
Verification: `go build ./...`, `go vet ./...`, `go test -short` on
|
||||
`cmd` and `internal/provider/...`, and both kustomizations rendered.
|
||||
`hack/providers-dev.yaml` untouched, as planned.
|
||||
|
||||
Worth noting: the CRD has no rule requiring all three placement fields
|
||||
(that is GCP's own check), so the single-host "zone may be omitted"
|
||||
behaviour works without an API change. And with this step the branch is
|
||||
feature-complete but **unproven**: Step 7 runs the integration test and
|
||||
the end-to-end checklist against a real hypervisor before the MR.
|
||||
|
||||
## Step 7 — Manual validation + MR (in progress)
|
||||
|
||||
Host: `beelink` (Ubuntu, libvirt 11.6.0, QEMU 10.1.0, pool `default` at
|
||||
`/srv/vms/images` on local ext4, bridge `br0`), reached over Tailscale as
|
||||
`beelink.tail8c69c.ts.net`, SSH user `novakj`. Not the `nas1.home.lan`
|
||||
the plan and samples name; that host does not resolve from the dev box.
|
||||
|
||||
```bash
|
||||
export KVM_TEST_HOST=beelink.tail8c69c.ts.net:22
|
||||
export KVM_TEST_USER=novakj
|
||||
export KVM_TEST_SSH_KEY=$HOME/.ssh/proxy-operator_ed25519
|
||||
export KVM_TEST_KNOWN_HOSTS=$HOME/.ssh/kvm_known_hosts
|
||||
export KVM_TEST_BRIDGE=br0
|
||||
export KVM_TEST_IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||
make test-integration-kvm
|
||||
```
|
||||
|
||||
### Run 1 and 2: `Could not open '/srv/vms/images/base-test-image.qcow2': Permission denied`
|
||||
|
||||
Everything up to `DomainCreate` worked on the first try against a real
|
||||
host, which answers the open notes from Steps 2 and 3b: the SSH-forwarded
|
||||
libvirt handshake, the raw qcow2 upload plus pool refresh as a backing
|
||||
image, the default-namespace metadata element, and `DomainDefineXML` for
|
||||
the generated XML were all accepted by libvirt 11.6. QEMU then exited at
|
||||
start with `Permission denied` on the base image.
|
||||
|
||||
Ownership was ruled out the slow way: the file was `root:root 0600` (the
|
||||
dir-pool default; libvirt's DAC driver chowns the chain at start and
|
||||
restores it after a failed start, so that observation meant nothing),
|
||||
`qemu.conf` was all defaults (`libvirt-qemu`, `dynamic_ownership=1`), and
|
||||
after a manual `chown libvirt-qemu:kvm` the second run failed identically.
|
||||
|
||||
The actual cause is the Step 2 choice of `<disk type="volume">`.
|
||||
libvirt's AppArmor helper builds each domain's profile from the domain XML
|
||||
without consulting the storage driver, and its `add_file_path` only emits a
|
||||
rule when the source has a `path`; a volume-referenced disk has pool and
|
||||
volume names instead, so neither the overlay nor its backing image gets a
|
||||
rule and QEMU is denied on the first one it opens. Every stock Debian and
|
||||
Ubuntu hypervisor confines QEMU this way. The unit tests could not have
|
||||
caught it: the fake accepts either form.
|
||||
|
||||
Fix, in this step: `buildDomainXML` takes `DiskPath`/`SeedPath` and emits
|
||||
`type="file"` disks; `ensureOverlay` and `writeSeed` return the volume
|
||||
handles (the exists-already branch now does a `StorageVolLookupByName`),
|
||||
and Create resolves both paths with `StorageVolGetPath` before defining
|
||||
the domain. Two extra cheap RPCs per Create. `TestBuildDomainXML_fieldByField`
|
||||
now fails on any `type="volume"` or `pool=` attribute so the design cannot
|
||||
drift back, and `TestCreate_freshHost` asserts the disk sources equal the
|
||||
paths the fake reports.
|
||||
|
||||
Worth noting: the first two runs left the base image and its marker in
|
||||
the pool as designed, so later runs skip the 435 MB download.
|
||||
|
||||
### Run 3: `parsing domain metadata: expected element <instance> in name space … but have no name space`
|
||||
|
||||
With file-backed disks, Create returned in 1 s (image cached) and the
|
||||
domain started. Get then failed to parse the metadata: libvirt 11.6 finds
|
||||
the element by our URI but returns it as a bare `<instance>` with no
|
||||
namespace declaration at all. Step 2 had tested the default-namespace and
|
||||
prefixed spellings and left "does libvirt accept the default-namespace
|
||||
form" for the real host; it does, it just does not give it back that way.
|
||||
|
||||
Fix: `parseInstanceMetadata` matches on the local name only, since the
|
||||
lookup already selected the element by URI. The fake now strips the
|
||||
declaration when storing metadata, so the provider's Get and ListByTag
|
||||
tests parse the shape libvirt really returns, and a new
|
||||
`TestParseInstanceMetadata_libvirtBareForm` pins it.
|
||||
|
||||
### Run 4: guest hangs at GRUB's "Booting" line
|
||||
|
||||
Create and start passed; Get polled `Provisioning` with no IP while the
|
||||
VM sat at 100 % CPU after GRUB printed "Booting `Debian GNU/Linux`" and
|
||||
before the kernel wrote anything to the serial console. The domain XML
|
||||
matches a working VM on the same host (q35, BIOS, host-passthrough,
|
||||
virtio disk, SATA cdrom, br0), so the difference is the disk content:
|
||||
the cached image was downloaded without a checksum and never verified.
|
||||
Diagnosis was handed to the user (sha512 against Debian's published
|
||||
sums, `qemu-img check`, and a `virt-install --import` of the same image
|
||||
outside the operator). Because the provider's own logs were invisible in
|
||||
the test output, this run also produced a side plan:
|
||||
[2026-09-10-0805-kvm-libvirt-tracing](2026-09-10-0805-kvm-libvirt-tracing.md),
|
||||
which traces every libvirt RPC at V(1) and routes it into `-v` output.
|
||||
|
||||
### Run 5: the guest hang is Debian 13's BIOS boot path; UEFI fixes it
|
||||
|
||||
The cached image's sha512 matched Debian's published sum, and
|
||||
`qemu-img info` showed a sound 3 GiB qcow2, which also confirms that
|
||||
the raw upload over a libvirt-created volume plus pool refresh yields a
|
||||
byte-identical file (the Step 3b note is answered). Bisected by hand on
|
||||
beelink with `virt-install --import` on a fresh overlay of the same
|
||||
image:
|
||||
|
||||
| variant | seed | CPU | firmware | result |
|
||||
|---|---|---|---|---|
|
||||
| operator's domain | ours | host-passthrough | SeaBIOS | hangs after GRUB "Booting", 100 % CPU |
|
||||
| bare | none | host-passthrough | SeaBIOS | same hang |
|
||||
| bare | none | host-model | SeaBIOS | same hang |
|
||||
| bare | none | host-passthrough | **OVMF (`--boot uefi`)** | **boots to login** |
|
||||
|
||||
So neither the seed ISO, the domain XML nor the CPU model was the
|
||||
problem: the image's legacy GRUB → kernel handoff does not work on this
|
||||
host, while its EFI stub path does. Not diagnosed further; it is
|
||||
Debian's chain, not ours, and every current cloud image boots UEFI.
|
||||
|
||||
Change: `images.<key>.firmware` (`uefi` default, `bios` escape),
|
||||
threaded through `withDefaults` → `imageSource` → `domainParams`, and
|
||||
`buildDomainXML` emits `<os firmware="efi"><firmware><feature
|
||||
enabled="no" name="secure-boot"/></firmware>` so libvirt selects the
|
||||
OVMF binary and creates the NVRAM file; the `nvram` undefine flag Delete
|
||||
already passes removes it (seen on the host: `virsh undefine` without
|
||||
`--nvram` refuses such a domain). Direct-URL images default to UEFI
|
||||
too. Docs: OVMF is a new hypervisor prerequisite, config table row,
|
||||
sample comment, known-gaps entry, decision-log entry.
|
||||
|
||||
Worth noting: the first image download went through with no checksum
|
||||
configured, and the cache reuses a base image by key, so switching
|
||||
`KVM_TEST_IMAGE_URL` requires deleting `base-test-image.qcow2` and its
|
||||
marker by hand first. That is the reviewer's Step 4 finding showing up
|
||||
in practice; still documented rather than fixed.
|
||||
|
||||
Follow-up, same day: the cause is known upstream. [Debian bug
|
||||
#1111240](https://bugs.debian.org/1111240) (filed 2025-08-16 against
|
||||
the cloud image, reassigned to `grub2`, still open as of 2026-09-10;
|
||||
trixie ships grub2 2.12-9+deb13u2) reports exactly this with a domain
|
||||
XML almost identical to ours: BIOS, virtio disk, SATA cdrom seed, bridge
|
||||
NIC, `<console><target type='serial'/>`, no `<graphics>`/`<video>`.
|
||||
The cloud team reproduced it: trixie's grub.cfg has `terminal_output
|
||||
gfxterm serial`, and with a serial port present but no video adapter
|
||||
GRUB never hands over to the kernel. Their fix in the image
|
||||
(`terminal_output serial`) is not something an operator can apply;
|
||||
the reporter's workaround (console `target type='virtio'`) boots but
|
||||
takes the kernel's `console=ttyS0` output with it. UEFI sidesteps the
|
||||
gfxterm path and keeps the serial console, so the decision stands, but
|
||||
the docs now name the bug rather than blaming the host, and note that
|
||||
bookworm images are unaffected. `latest` on cloud.debian.org was the
|
||||
2026-08-31 build when this was hit, so every trixie point release to
|
||||
date carries it.
|
||||
|
||||
### Run 6: pass
|
||||
|
||||
With UEFI the full lifecycle passed in 23 s on a warm cache: Create 2 s
|
||||
(domain start 0.9 s), guest agent answering with `192.168.0.89` after
|
||||
20 s (ARP never did; the host had not talked to the guest), TCP 22
|
||||
reachable from the dev machine, ListByTag seeing it, Delete leaving only
|
||||
the base image and marker in the pool, second Delete a no-op. Trace
|
||||
excerpt worth keeping for the record:
|
||||
|
||||
```
|
||||
09:44:28.603 libvirt virDomainDefineXML xmlBytes=1509 domain=proxy-hk5oavoqk45n3twq duration=13.51ms
|
||||
09:44:28.603 kvm domain defined uid=9bbceb41-… firmware=uefi
|
||||
09:44:29.542 libvirt virDomainCreate domain=proxy-hk5oavoqk45n3twq duration=938.705ms
|
||||
09:44:49.573 libvirt virDomainInterfaceAddresses domain=proxy-hk5oavoqk45n3twq source=DomainInterfaceAddressesSrcAgent interfaces=2 duration=1.742ms
|
||||
09:44:49.573 kvm KVM domain fetched state=Running ip=192.168.0.89 uid=9bbceb41-…
|
||||
```
|
||||
|
||||
Worth noting: the sweep listed a second managed domain,
|
||||
`proxy-qxge3dbm2mnpc3ih`, shut off, left behind by an earlier failed
|
||||
run whose cleanup never ran. That is exactly the orphan GC exists for;
|
||||
on the host it needs `virsh undefine --nvram` plus its two volumes by
|
||||
hand. The `ListByTag` sweep costs one `virDomainGetMetadata` per
|
||||
domain on the host (21 here, ~0.4 ms each over the tunnel), which is
|
||||
fine at this scale but is the call to watch on a busy hypervisor.
|
||||
|
||||
### Run 7: in-cluster, two defects the integration test could not see
|
||||
|
||||
The e2e pass ran the operator inside the kind cluster
|
||||
(`egress-proxies-operator:kvm-dev`, built with `make docker-build`,
|
||||
loaded with `kind load docker-image`, deployed via the kustomize
|
||||
`config/manager` with the `kvm-home` provider pointing at beelink) and
|
||||
two Proxies with real cloud-configs: one plain squid, one with a debug
|
||||
login user. Both defects come from the difference between the test
|
||||
harness and the real thing, not from libvirt.
|
||||
|
||||
**Read-only root filesystem.** The manager runs with
|
||||
`readOnlyRootFilesystem: true`, and `iso9660.NewWriter` stages the seed
|
||||
in a temp directory, so the first Create failed with `creating iso
|
||||
writer: read-only file system`. Fix: a 64 MiB `emptyDir` on `/tmp` in
|
||||
`config/manager/manager.yaml`. Side finding: the error is classified
|
||||
`ErrPermanent` by `writeSeed`, yet `hostPool.call` treated any
|
||||
non-libvirt error as a transport failure and dropped the SSH connection,
|
||||
so every retry redialled. `call` now leaves the connection alone for
|
||||
errors already carrying a provider sentinel (`isClassified`), with a
|
||||
test.
|
||||
|
||||
**The guest agent never got installed.** Both VMs booted (confirmed on
|
||||
the console by hand) but sat in `Provisioning` for 25 minutes: the
|
||||
trace showed `virDomainInterfaceAddresses` from the agent source failing
|
||||
with `ErrAgentUnresponsive` on every poll and the ARP source empty. The
|
||||
integration test had passed with `CloudInit: "#cloud-config\n"`; the
|
||||
real Proxies have `packages: [squid]` and `runcmd:`. cloud-init merges
|
||||
vendor-data under user-data with its default mergers, which keep the
|
||||
user-data's value for any key both define and never append lists, so the
|
||||
vendor-data's `packages`/`runcmd` were dropped wholesale. Confirmed
|
||||
locally with the dev box's cloud-init 26.1 before changing anything:
|
||||
|
||||
```sh
|
||||
python3 - <<'EOF'
|
||||
from cloudinit import mergers
|
||||
user = {"packages": ["squid"], "runcmd": ["systemctl restart squid"]}
|
||||
vendor = {"packages": ["qemu-guest-agent"], "runcmd": [["systemctl", "enable", "--now", "qemu-guest-agent"]]}
|
||||
print(mergers.construct(mergers.default_mergers()).merge(dict(user), dict(vendor)))
|
||||
EOF
|
||||
# {'packages': ['squid'], 'runcmd': ['systemctl restart squid']}
|
||||
```
|
||||
|
||||
Fix: `withGuestAgent` in `seed.go` merges the bootstrap into the Proxy's
|
||||
cloud-config itself (append to `packages` and `runcmd`, skip if present,
|
||||
re-render with `sigs.k8s.io/yaml`), so vendor-data is no longer defaulted
|
||||
and `vendorData` is plain extra data. User-data that is not a cloud-config
|
||||
document (script, `#include`, MIME) still gets the bootstrap as
|
||||
vendor-data since there is nothing to collide with; a cloud-config that
|
||||
does not parse as a mapping fails the Proxy as `ErrPermanent`. The
|
||||
rendered document for the squid Proxy passed `cloud-init schema
|
||||
--config-file`. The logging test's fixture (`#cloud-config` + a bare
|
||||
sentinel line) is not a mapping and had to become `hostname: <sentinel>`.
|
||||
|
||||
Rebuilt, reloaded, restarted, deleted and recreated both Proxies: both
|
||||
`Ready`/`HEALTHY True` in 2 min 1 s from `kubectl apply`, IP via the
|
||||
agent 110 s after `virDomainCreate` (the guest installs squid and the
|
||||
agent from the mirrors first), and `curl -x http://<ip>:3128
|
||||
https://www.gstatic.com/generate_204` from the dev box returns 204
|
||||
through both.
|
||||
|
||||
Worth noting: the `#cloud-config` round-trip drops comments and reorders
|
||||
keys alphabetically; semantics are preserved (numbers, booleans and
|
||||
block scalars survive, checked in tests) but a user diffing the seed
|
||||
against their CR will see a different document. Vendor-data supplied via
|
||||
`vendorData` is subject to the same cloud-init merge rule and is now
|
||||
documented as such. The kind pod also restarted twice during the session
|
||||
from "Pod sandbox changed" (the node briefly lost the API server), which
|
||||
is unrelated to the operator. Still uncommitted on purpose: the
|
||||
`kustomization.yaml` image override (`egress-proxies-operator:kvm-dev`)
|
||||
and the `beelink` host entries in `config/manager/providers_config.yaml`
|
||||
and `config/samples/providers-config.yaml`, which are this machine's dev
|
||||
setup rather than the repo's defaults.
|
||||
|
||||
### Run 8: review fixes
|
||||
|
||||
A `/code-review high` pass over the branch found five defects, fixed in
|
||||
[plan 2026-09-10-2222](../plans/2026-09-10-2222-kvm-review-fixes.md) with
|
||||
its own execution log. Two of them land on Run 7's conclusions above and
|
||||
supersede them:
|
||||
|
||||
- The guest-agent **merge described in Run 7 is gone**. It could not be made
|
||||
safe for user-data that is not a plain cloud-config, and its YAML
|
||||
round-trip re-typed unquoted scalars. The bootstrap now ships as a
|
||||
vendor-data shell script, which cloud-init never merges, and the Proxy's
|
||||
cloud-init reaches the seed byte for byte.
|
||||
- The read-only-filesystem fix from Run 7 still stands, but the eager SSH
|
||||
key load it sat next to does not: a missing `kvm-ssh` Secret aborted the
|
||||
whole provider set and crash-looped the manager. The material is read on
|
||||
first use now.
|
||||
|
||||
Re-validated the same way: integration test against beelink, both Proxies
|
||||
recreated in kind, 204 through squid.
|
||||
|
||||
### MR opened
|
||||
|
||||
[PR #5](https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/pulls/5),
|
||||
`feat/kvm-provider` → `main`, 19 commits. `tea` is not installed on the
|
||||
dev box and the HTTPS remote has no credential helper, so the push used a
|
||||
one-off helper reading a Gitea access token from `~/.config/gitea-token`,
|
||||
and the MR was created with the API instead of `tea pr create`:
|
||||
|
||||
```sh
|
||||
git -c credential.helper='!f() { echo username=kacerr; echo "password=$(tr -d "[:space:]" < ~/.config/gitea-token)"; }; f' \
|
||||
push -u origin feat/kvm-provider
|
||||
curl -s -H "Authorization: token $(tr -d '[:space:]' < ~/.config/gitea-token)" \
|
||||
-H 'Content-Type: application/json' -d @pr.json \
|
||||
https://gitea.home.hrajfrisbee.cz/api/v1/repos/kacerr/egress-proxies-operator/pulls
|
||||
# pr.json: {"title": ..., "head": "feat/kvm-provider", "base": "main", "body": ...}
|
||||
```
|
||||
|
||||
The push printed a libsecret "store failed" warning from the system
|
||||
credential store; harmless, the token was not persisted anywhere but the
|
||||
file.
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
# Execution: Trace every libvirt call in the KVM provider
|
||||
|
||||
Plan: [2026-09-10-0805-kvm-libvirt-tracing.md](../plans/2026-09-10-0805-kvm-libvirt-tracing.md)
|
||||
|
||||
- [x] Step 0 — Save plan
|
||||
- [x] Step 1 — `errors.go`: shared error fields
|
||||
- [x] Step 2 — `trace.go`: the `tracingAPI` decorator
|
||||
- [x] Step 3 — `conn.go`: pool logging and wrapping
|
||||
- [x] Step 4 — Call sites
|
||||
- [x] Step 5 — Integration test output
|
||||
- [x] Step 6 — Unit tests
|
||||
- [x] Step 7 — Docs, execution log, CHANGELOG
|
||||
|
||||
## Steps 1–6 — Tracer, pool logging, tests, integration wiring
|
||||
|
||||
Done as one commit on `feat/kvm-provider`; the plan's assumption that
|
||||
this belongs on the KVM branch rather than its own feature branch was
|
||||
accepted at plan approval.
|
||||
|
||||
The code follows the plan closely. Things that came out differently
|
||||
or are worth knowing:
|
||||
|
||||
- **Logger names join with `/` in funcr, `.` in zap.** The trace lines
|
||||
are `kvm/libvirt` in the unit-test capture and in `testr` output, and
|
||||
`kvm.libvirt` under the operator's zap logger. The docs use the zap
|
||||
spelling; the unit test asserts the funcr one. First run of the tier
|
||||
test failed on exactly this.
|
||||
- **Cancel is not reported as a transport error.** The original `call`
|
||||
dropped the connection twice on cancellation (once from the abort
|
||||
closure, once from the "not a libvirt error" branch, harmlessly, since
|
||||
drop is idempotent). With a log line on each drop that would have
|
||||
printed both `context cancelled` and `transport error` for one event,
|
||||
so the transport branch now skips `ctx.Err()`. Covered by
|
||||
`TestHostPool_call_logsDropReason`.
|
||||
- **`ConnectListAllDomains` in the fake includes every defined domain**,
|
||||
so the trace test's fixture (one running managed domain plus one
|
||||
stopped domain for `DomainCreate` to start) lists two; the V(2)
|
||||
assertion checks for the `names` key rather than an exact list.
|
||||
- **`libvirtErrorFields` also feeds `logAPIError`**, which therefore
|
||||
gains `libvirtCodeName` (e.g. `ErrNoDomain`) next to the numeric code.
|
||||
- The seam-coverage guard (`reflect` count of `libvirtAPI` methods vs
|
||||
the test table) is the only thing standing between a future 18th RPC
|
||||
and a silently untraced call; keep it.
|
||||
|
||||
Verification run before committing:
|
||||
|
||||
```bash
|
||||
go build ./... && go vet ./... && go vet -tags=integration ./internal/provider/kvm/
|
||||
go test -short ./...
|
||||
go test -race -count=1 -cover ./internal/provider/kvm/ # 90.2 %, up from 88.2
|
||||
go test -tags=integration -count=1 -run TestKVM ./internal/provider/kvm/ # self-skips without env
|
||||
```
|
||||
|
||||
Worth noting: the live run (`make test-integration-kvm` against beelink)
|
||||
had not happened when this was written; it is what Step 7 of the KVM
|
||||
provider plan is waiting on, and its output is the acceptance test for
|
||||
this feature. Expected shape is in the plan's Verification section.
|
||||
|
||||
## Revision after the first live run — less noise, libvirt names
|
||||
|
||||
The first live run on beelink showed the trace working but hard to read:
|
||||
`testr` printed `"ts"`, `"level"`, `"msg"` and the three per-run
|
||||
constants on every line, the error text appeared twice (`error` and
|
||||
`libvirtMessage`), `host` appeared twice on image-cache lines (the
|
||||
logger already carried it), and `method=DomainLookupByName` was
|
||||
go-libvirt's name rather than something to look up in libvirt's docs.
|
||||
The user asked for "less verbose but more informative". Changes:
|
||||
|
||||
- The tracer's headline field is now `rpc=virDomainLookupByName`: the
|
||||
public libvirt API function, which go-libvirt's method names are
|
||||
exactly minus the `vir` prefix (they come from the same
|
||||
`remote_protocol.x`). Not gRPC, for the record: libvirt's own
|
||||
XDR-based RPC over the forwarded unix socket.
|
||||
- Errors are one `error` field plus `libvirtError=ErrNoDomain` and
|
||||
`libvirtCode=42`; `libvirtMessage` is gone from both the tracer and
|
||||
`logAPIError`, since it is the error text.
|
||||
- `image.go` no longer adds `host` to lines whose logger has it.
|
||||
- The integration test replaces `testr` with a small `compactSink`
|
||||
(`logr.LogSink` plus `GetCallStackHelper`, so `t.Log` still
|
||||
attributes lines to `trace.go`): `HH:MM:SS.mmm libvirt
|
||||
virDomainCreate domain=… duration=… error=… libvirtError=…`. The
|
||||
`rpc` value becomes the headline, `provider`/`host`/`name` are
|
||||
dropped (the run's first `t.Log` line states them), and values with
|
||||
spaces are quoted. Production zap output is unchanged apart from the
|
||||
field renames; structured logs are meant to repeat context.
|
||||
|
||||
Previewed by driving the sink through the fake provider with an
|
||||
injected `DomainCreate` failure (a throwaway integration-tagged test,
|
||||
not committed):
|
||||
|
||||
```
|
||||
trace.go:55: 09:04:33.165 libvirt virDomainLookupByName domain=proxy-abc123def456ghij duration=4µs error="Domain not found: …" libvirtError=ErrNoDomain libvirtCode=42
|
||||
trace.go:55: 09:04:33.165 libvirt virStorageVolUpload pool=default volume=proxy-abc123def456ghij-seed.iso offset=0 length=0 duration=71µs
|
||||
kvm.go:262: 09:04:33.165 kvm seed volume uploaded volume=proxy-abc123def456ghij-seed.iso bytes=45056
|
||||
trace.go:55: 09:04:33.165 libvirt virDomainCreate domain=proxy-abc123def456ghij duration=0s error="internal error: process exited…" libvirtError=ErrResourceBusy libvirtCode=87
|
||||
```
|
||||
|
||||
Worth noting: `length=0` on `virStorageVolUpload` is what the provider
|
||||
passes (libvirt reads to EOF); the following `seed volume uploaded`
|
||||
line has the byte count.
|
||||
|
||||
## Step 7 — Live run confirmed; CHANGELOG
|
||||
|
||||
The compact trace was accepted on the first passing live run
|
||||
(2026-09-10, beelink): one line per RPC with the libvirt function name,
|
||||
the provider's own lines between them, `Provisioning` polls readable at
|
||||
a glance (`SrcArp interfaces=0`, `SrcAgent … ErrAgentUnresponsive`
|
||||
until the agent came up at +20 s), and a full `ListByTag` sweep over 21
|
||||
domains in 15 ms. CHANGELOG entry added under the KVM validation entry.
|
||||
@@ -1,229 +0,0 @@
|
||||
# Execution: fix the five code-review findings on `feat/kvm-provider`
|
||||
Plan: [docs/plans/2026-09-10-2222-kvm-review-fixes.md](../plans/2026-09-10-2222-kvm-review-fixes.md)
|
||||
|
||||
- [x] Step 0 — Save plan into the repo
|
||||
- [x] Step 1 — Unwedge deletion (finding 1)
|
||||
- [x] Step 2 — Validate names that become identifiers (finding 2)
|
||||
- [x] Step 3 — Read the SSH material lazily (finding 3)
|
||||
- [x] Step 4 — Guest agent via vendor-data script (findings 4 and 5)
|
||||
- [x] Step 5 — Validate, record, push
|
||||
|
||||
## Step 0 — plan committed
|
||||
|
||||
Copied out of `~/.claude/plans/` and committed alone before touching code, as
|
||||
the repo's plan rules require.
|
||||
|
||||
## Step 1 — Unwedge deletion
|
||||
|
||||
Three changes, in the order the bug unfolds.
|
||||
|
||||
`reconcileDelete` calls `Get` first and only drops the finalizer when it
|
||||
answers not-found, so any permanent error there retried forever. It now
|
||||
switches on the class: a permanent `Get` still calls `Delete`, which is
|
||||
idempotent, and releases the object when that succeeds. Polling a `Get`
|
||||
that cannot converge was the wedge; `Delete` is the only call that can
|
||||
still make progress.
|
||||
|
||||
`crawl.example.com/skip-cleanup: "true"` abandons the instance and removes
|
||||
the finalizer. It is checked **before** the provider lookup, which also
|
||||
frees a Proxy whose provider was deleted from the config — a second wedge
|
||||
sitting three lines above the reported one, with the same shape. No
|
||||
`EventRecorder` exists on the reconciler and none was added, so the
|
||||
abandonment is an Info log; `docs/architecture.md` already records that
|
||||
decision for the whole controller.
|
||||
|
||||
In the KVM provider, `Delete` for a providerID naming an unconfigured host
|
||||
now logs and returns success. The precedent is two functions further down:
|
||||
`removeVolumes` treats a missing pool the same way, with a comment saying
|
||||
outright that refusing "would pin the Proxy's finalizer".
|
||||
|
||||
**`Get` deliberately still fails permanently for that case.** Making it
|
||||
answer not-found would have been a one-line fix for the wedge, but the live
|
||||
reconcile path reads not-found as "the VM is gone" and rebuilds it
|
||||
elsewhere. A typo in `hosts` would then quietly duplicate every Proxy on
|
||||
that host, with the originals still running where GC cannot see them, since
|
||||
`ListByTag` only sweeps configured hosts. The asymmetry is now pinned by a
|
||||
test that asserts both halves at once.
|
||||
|
||||
## Step 2 — Validate names that become identifiers
|
||||
|
||||
One regexp, `^[A-Za-z0-9][A-Za-z0-9_.-]*$`, applied to `hosts[].name` and to
|
||||
the `images` map keys. Dots stay legal because a host name like
|
||||
`nas1.home.lan` is ordinary and harmless; the character that actually
|
||||
breaks things is `/`.
|
||||
|
||||
The review only reported the host name. The image key has the same defect
|
||||
and is arguably worse: it is not merely embedded in an ID but pasted into a
|
||||
file name, `base-<key>.<format>`, so `../etc/x` would be a path traversal
|
||||
into the storage pool rather than a parse failure. Both are validated in the
|
||||
same pass. Machine-type keys are left alone deliberately — they are only
|
||||
ever compared against `placement.machineType`, never embedded anywhere.
|
||||
|
||||
Worth noting: this is validation of the operator's own config file, which
|
||||
only a cluster admin can write, so the traversal was never a privilege
|
||||
boundary. It is still cheaper to reject at load than to explain a stuck
|
||||
Proxy later. Both new table cases assert the message text, so the wording
|
||||
that names the consequence is now pinned by tests.
|
||||
|
||||
## Step 3 — Read the SSH material lazily
|
||||
|
||||
`kvm.New` no longer loads the key; `lazyAuth` does, on the first connection,
|
||||
caching the result once it works. `registry.Build` is all-or-nothing by
|
||||
design, so the old constructor error did not merely fail the KVM provider,
|
||||
it exited the manager before any controller started.
|
||||
|
||||
The plan said "lazy dialer", and the first attempt was exactly that — one
|
||||
closure wrapping both the loading and the dial. It could not be tested: the
|
||||
fixture's SSH server is real, but the socket behind it is an echo server, so
|
||||
every successful dial dies in the libvirt handshake and the assertion could
|
||||
not tell "the key worked" from "the key did not". Splitting the seam in two,
|
||||
`lazyAuth` for the material and `lazyDialer` for the wiring, made both
|
||||
testable: the healing path against `lazyAuth`, and a separate test that the
|
||||
dialer really authenticates, asserted by the error being past SSH and by the
|
||||
server having seen the forward request.
|
||||
|
||||
Error classes carry the intent. A missing file stays unclassified, so
|
||||
`provider.Class` defaults it to transient and a Proxy heals on its own when
|
||||
the Secret appears — the common case, since the Secret is usually created
|
||||
after the manager. A key that does not parse wraps `ErrPermanent`, because
|
||||
retrying cannot fix it. `newSSHAuth` distinguishes the two with
|
||||
`errors.Is(err, fs.ErrNotExist)`.
|
||||
|
||||
Worth noting: the startup check is kept, just non-fatally, logged with the
|
||||
`WARNING:` prefix the repo already uses where logr has no warn level. That
|
||||
preserves the diagnostic the eager version was written for. The caching test
|
||||
deliberately deletes the key file after a successful load and asserts the
|
||||
cached value is reused, which pins the rotation behaviour the docs promise:
|
||||
a rotated Secret needs a manager restart. `TestNew_badKeyPathFailsFast` was
|
||||
inverted rather than deleted, so the old contract stays visible in the diff
|
||||
next to the new one.
|
||||
|
||||
## Step 4 — Guest agent via vendor-data script
|
||||
|
||||
The review filed two findings against `withGuestAgent` — detection too
|
||||
narrow, and YAML re-typing on the round-trip. Reading cloud-init 26.1's
|
||||
sources while planning turned both into one finding: the operator should
|
||||
not be editing the user's document at all.
|
||||
|
||||
Vendor-data gets its own handler chain (`stages.py:_default_vendordata_handlers`),
|
||||
with separate script and cloud-config paths. A vendor-data **script** is
|
||||
therefore written to the vendor script directory and run by
|
||||
`cc_scripts_vendor`, never touching the config merger that drops colliding
|
||||
keys. On the final-stage module list it lands after
|
||||
`package_update_upgrade_install` and before the user's `runcmd`, so the
|
||||
agent now comes up *earlier* than the old `runcmd` append did.
|
||||
|
||||
That also closed the door the previous approach had left ajar. The merge
|
||||
cannot be rescued from inside either document: `merge_how` in user-data
|
||||
governs only the first source, and `merge_how` in vendor-data is popped by
|
||||
`dict_extract_mergers` and discarded before the document is written. There
|
||||
was no fix that kept the merge.
|
||||
|
||||
`withGuestAgent`, `isCloudConfig` and `appendUnique` are gone, along with
|
||||
seed.go's `sigs.k8s.io/yaml` dependency. `installGuestAgent` (default true)
|
||||
turns the script off. When `vendorData` is also configured the two ship as
|
||||
a MIME multipart, with the user's document declared `text/plain` — a type
|
||||
cloud-init re-sniffs from the content's own header line, so this code never
|
||||
classifies what the user wrote.
|
||||
|
||||
Before deleting the old path, its damage was captured rather than taken on
|
||||
trust. Round-tripping a plausible cloud-config through it:
|
||||
|
||||
```sh
|
||||
# temporary test in the package, deleted afterwards
|
||||
go test -run TestZZProbeRetyping -v ./internal/provider/kvm/
|
||||
# in: version: 1.10
|
||||
# out: version: 1.1
|
||||
```
|
||||
|
||||
`1.10` arriving as `1.1` is worse than the reviewer's `passwd` example,
|
||||
since nothing about it looks wrong in the seed. Comments were dropped and
|
||||
keys reordered on the same pass.
|
||||
|
||||
The generated vendor-data was then checked against the cloud-init actually
|
||||
installed on this machine, rather than against the docs:
|
||||
|
||||
```sh
|
||||
python3 -c 'from cloudinit import user_data, handlers; ...'
|
||||
# vendor-script: multipart=False sniffed=text/x-shellscript
|
||||
# vendor-multipart: multipart=True part 1 text/plain -> text/cloud-config
|
||||
# part 2 text/x-shellscript
|
||||
```
|
||||
|
||||
and the script itself through `sh -n`.
|
||||
|
||||
Worth noting: the script has to choose a package manager where cloud-init's
|
||||
`packages:` key was distro-agnostic. It covers apt, dnf, yum, zypper and
|
||||
apk, and both systemd and OpenRC for the enable step, but an image outside
|
||||
that set now needs `installGuestAgent: false` and the agent baked in. The
|
||||
seed's user-data is byte-identical to the Proxy's `cloudInit` from here on,
|
||||
which is what the new `TestBuildSeedISO_userDataIsNeverRewritten` pins
|
||||
across seven formats. One consequence worth remembering: a user-data that
|
||||
sets `vendor_data: {enabled: false}` disables the agent, and cloud-init
|
||||
documents that as intended behaviour.
|
||||
|
||||
## Step 5 — Validation
|
||||
|
||||
Unit suite, `go vet` and `gofmt` clean across the repo. Then the two real
|
||||
runs.
|
||||
|
||||
**Integration against beelink**, unchanged environment from the KVM plan's
|
||||
log, passed in 22 s on a warm cache, with
|
||||
`virDomainInterfaceAddresses source=DomainInterfaceAddressesSrcAgent
|
||||
interfaces=2` — the agent installed by the new vendor script answering, on
|
||||
a stock Debian 13 image.
|
||||
|
||||
**In-cluster**, both Proxies deleted and recreated against the rebuilt
|
||||
image: Ready with an address 2 min 3 s after `kubectl apply`, 204 through
|
||||
squid from the dev box for both. Inside the debug VM:
|
||||
|
||||
```sh
|
||||
ssh novakj@192.168.0.94 'systemctl is-active qemu-guest-agent; cloud-init status; \
|
||||
sudo head -4 /var/lib/cloud/instance/user-data.txt; sudo ls /var/lib/cloud/instance/scripts/vendor/'
|
||||
# active
|
||||
# status: done
|
||||
# #cloud-config
|
||||
# # Login for debugging: user "novakj", password "kvm-dev", passwordless sudo.
|
||||
# ...
|
||||
# part-001
|
||||
```
|
||||
|
||||
Two things worth having seen with your own eyes there: the user's **comment
|
||||
lines survived** into the guest, which the old merge stripped, and the
|
||||
script really did land in the vendor directory rather than being merged
|
||||
into the user's config.
|
||||
|
||||
**Negative check for the Secret**, done by pointing the deployment's volume
|
||||
at a name that does not exist rather than deleting the real Secret:
|
||||
|
||||
```sh
|
||||
kubectl -n egress-proxies-operator-system patch deploy egress-proxies-operator-controller-manager \
|
||||
--type=json -p '[{"op":"replace","path":"/spec/template/spec/volumes/1/secret/secretName","value":"kvm-ssh-absent"}]'
|
||||
```
|
||||
|
||||
The manager came up and stayed up, 0 restarts, logging
|
||||
`WARNING: kvm ssh material is not usable ... open /etc/proxy-operator/kvm/id_ed25519:
|
||||
no such file or directory`. The Kubernetes Proxies kept reconciling; the
|
||||
KVM ones failed per reconcile and retried. Before this branch the same
|
||||
config exited the process before any controller started. Patched back
|
||||
afterwards, everything Ready.
|
||||
|
||||
Worth noting: the KVM Proxies kept reporting `Ready` during that window
|
||||
rather than degrading visibly. That is existing behaviour, not a
|
||||
regression — a transient provider error rides the workqueue and does not
|
||||
clear a healthy status — but it does mean the only signal of a missing
|
||||
Secret is the log and the reconcile errors. Also, the ListByTag sweep on
|
||||
beelink still reports `managed=2` where one domain is an orphan from an
|
||||
older failed run; unrelated to this work, and the same leftover noted in
|
||||
the KVM plan's Run 6.
|
||||
|
||||
Worth noting: the review filed this as a KVM finding, but the fix is mostly
|
||||
in the controller, because KVM is the only provider that resolves a
|
||||
providerID segment against operator config at all. GCP and Kubernetes hand
|
||||
the lookup to the backend, which answers 404 for a retired zone or a deleted
|
||||
namespace, so they reach not-found and release on their own. Anything else
|
||||
that classifies permanently on the delete path — revoked RBAC on the
|
||||
Kubernetes provider is the realistic one — was equally stuck before this
|
||||
step and is what the annotation is really for. The KVM table case
|
||||
`"host not configured"` moved out of `TestDelete_errors` into its own named
|
||||
test, since it is no longer an error.
|
||||
@@ -1,107 +0,0 @@
|
||||
# Plan: OpenTelemetry tracing integrated with the existing logr/zap logging
|
||||
**Created:** 2026-08-24 10:25
|
||||
|
||||
## Context
|
||||
|
||||
The operator has structured logging (controller-runtime zap → logr, `logf.FromContext(ctx)` everywhere) and Prometheus metrics, but no tracing. A single reconcile fans out into k8s API calls, provider calls (GCP VMs / squid Pods), and status patches; the discovery API serves lease requests; GC and health engines run on tickers. Today correlating one operation across those hops means grepping for `reconcileID` or `providerID`. Goal: proper OTel traces — a new trace per unit of work (reconcile, HTTP request, GC sweep, optionally health probe), child spans for k8s/provider/HTTP calls — **and every log line inside a traced operation enriched with the trace context**, without disturbing the existing logr/zap system.
|
||||
|
||||
User decisions (confirmed):
|
||||
- **Enablement:** auto from standard `OTEL_*` env vars — no collector configured ⇒ tracing fully off, zero new flags for the common case.
|
||||
- **Health probes:** not traced by default; new `--trace-health-probes` flag enables one root span per probe.
|
||||
- **Backend:** none exists yet — verification uses the console exporter and a throwaway Jaeger all-in-one in kind; manifests ship commented-out OTLP env examples.
|
||||
|
||||
## Key design facts (verified, non-obvious)
|
||||
|
||||
- **logr can't read ctx:** `logr.LogSink` never sees `context.Context`, so trace IDs cannot be added inside the zap sink. They must be attached as `WithValues` at span-creation points via `logf.IntoContext`. zap does **not** dedupe repeated keys, so enrichment must happen exactly once per ctx chain (see `tracing.Start` below).
|
||||
- Log keys: **`traceID` / `spanID`** (lowerCamel matches `reconcileID`/`providerID` house style; note in docs that Grafana/Loki derived-field regexes must match `traceID`, not the `trace_id` default).
|
||||
- **GCP calls trace themselves:** `google.golang.org/api/transport/http` already wraps its transport in `otelhttp.NewTransport` — once a global TracerProvider is set, every Compute call emits an HTTP client span. Do not add HTTP-level spans in the gcp provider; the `provider.*` decorator span becomes their parent.
|
||||
- **kube-apiserver ignores incoming `traceparent`** by design; our k8s-API client spans are leaves. Don't promise cross-process traces into the apiserver.
|
||||
- The Go SDK does **not** implement `OTEL_SDK_DISABLED` — handle it ourselves.
|
||||
- `contrib/exporters/autoexport` drags in metric/log/prometheus exporters (binary + deps on a 128Mi pod) — hand-roll trace-exporter selection (~40 lines) instead.
|
||||
- Span events are deprecated (OTel spec, Mar 2026): use `span.SetStatus` + `error.type`-style attributes, not `RecordError`/`AddEvent`.
|
||||
- `rest.Config.Wrap` puts our transport innermost (after auth), so spans see the final request — correct; but informer list/watch + leader election run outside any span ⇒ the transport must **filter out requests with no parent span** or every watch becomes a long-lived root span.
|
||||
- Manager runnables receive no logger in ctx (only HTTP servers do) — `logf.FromContext` falls back to the global logger there, which is fine; the discovery server must keep seeding from `s.log`.
|
||||
|
||||
## Steps
|
||||
|
||||
### Step 1 — Dependencies
|
||||
|
||||
`go get` (aligning the existing indirect skew: otel core v1.44 / otlptrace v1.40):
|
||||
`go.opentelemetry.io/otel@v1.45.0`, `otel/sdk@v1.45.0`, `otel/trace@v1.45.0`,
|
||||
`otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0`, `.../otlptracehttp@v1.45.0`,
|
||||
`otel/exporters/stdout/stdouttrace@v1.45.0`,
|
||||
`go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0`, plus the
|
||||
`semconv` version matching the SDK's `resource.Default()` (treat `ErrSchemaURLConflict` from `resource.New` as non-fatal if versions drift).
|
||||
`go mod tidy`; full build + `make test` to confirm `google.golang.org/api v0.292.0` / k8s v0.36 tolerate the bump (MVS picks the higher otelhttp).
|
||||
|
||||
### Step 2 — New package `internal/tracing`
|
||||
|
||||
- **`tracing.go` — `Setup(ctx, log logr.Logger, service, version string) (shutdown func(context.Context) error, err error)`**
|
||||
- Disabled (returns no-op shutdown, installs nothing) when `OTEL_SDK_DISABLED=true`, or when neither `OTEL_TRACES_EXPORTER` nor `OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is set, or `OTEL_TRACES_EXPORTER=none`. One startup Info line either way stating the effective state/exporter.
|
||||
- Exporter selection (hand-rolled, no autoexport): `OTEL_TRACES_EXPORTER` ∈ `otlp` (default) | `console`; for otlp, `OTEL_EXPORTER_OTLP_(TRACES_)PROTOCOL` ∈ `http/protobuf` (default) | `grpc` → `otlptracehttp` / `otlptracegrpc` (endpoint/TLS/headers via their native env handling).
|
||||
- `sdktrace.NewTracerProvider` with batch processor (`WithMaxQueueSize(512)` — bounded on the 128Mi pod; sampler stays the SDK default so `OTEL_TRACES_SAMPLER(_ARG)` works), resource = `resource.New(WithAttributes(service.name, service.version=commit), WithFromEnv())` with env last so it wins.
|
||||
- Globals: `otel.SetTracerProvider`, `otel.SetTextMapPropagator(TraceContext+Baggage)`, `otel.SetLogger(log.WithName("otel"))`, `otel.SetErrorHandler` → logr at `V(1)` (an unreachable collector errors every few seconds; Error level would spam).
|
||||
- **`logger.go` — the one logger-enrichment mechanism** (used by everything; prevents duplicate `traceID` keys):
|
||||
- private ctx key holding the *base* (pre-enrichment) logr.Logger.
|
||||
- `Start(ctx, spanName, opts...) (context.Context, trace.Span)`: starts a span from the global tracer; if the resulting span context is valid, capture base = stored base or `logf.FromContext(ctx)` on first call, then `logf.IntoContext(ctx, base.WithValues("traceID", ..., "spanID", ...))`. Nested `Start` re-derives from base — never stacks. Documented trade-off: `WithValues` pushed via `logf.IntoContext` *between* two `Start` calls is dropped (nothing first-party does that; controller-runtime's `reconcileID` logger is the base and survives).
|
||||
- `ContextWithLogger(ctx, base)` seeds the base explicitly (discovery middleware).
|
||||
- **`reconciler.go` — `NewReconciler(kind string, inner reconcile.Reconciler, opts...) reconcile.Reconciler`**: root span `Reconcile <kind>` (SpanKind Internal) via `Start`; attrs `k8s.namespace.name`, `k8s.object.name`, `reconcile.id` (`controller.ReconcileIDFromContext`); on return records `reconcile.requeue_after`, sets Error status from err. `WithTracerProvider` option for tests.
|
||||
- **`transport.go` — `RestConfigWrapper() transport.WrapperFunc`**: `otelhttp.NewTransport` with `WithFilter` requiring `trace.SpanContextFromContext(r.Context()).IsValid()` — k8s API spans only under an existing trace (excludes informers, leader election, metrics authn).
|
||||
- **`http.go` — `HTTPMiddleware(operation string, base logr.Logger) func(http.Handler) http.Handler`**: `otelhttp.NewHandler` (filter `/healthz`; otelhttp renames the span from `r.Pattern` after routing — no custom naming needed) + inner layer calling `ContextWithLogger`-then-enrich so handler logs carry `traceID`. Accepts incoming `traceparent` (clients continue their traces).
|
||||
|
||||
Tests (all with `sdktrace` + `tracetest.NewSpanRecorder` via the `WithTracerProvider` options, parallel; `Setup` env tests use `t.Setenv`, not parallel):
|
||||
- `Start`: nested spans → child span recorded; **funcr-captured log line contains `traceID` exactly once** (pattern: `internal/provider/gcp/gcp_test.go:277`); disabled tracer → no `traceID` in logs.
|
||||
- `Setup`: table over env combinations (unset→disabled, `none`→disabled, endpoint set→otlp, `console`, `OTEL_SDK_DISABLED`).
|
||||
- Reconciler decorator: fake inner returns result/err → span name/attrs/status.
|
||||
- Transport filter: request without parent span produces no span.
|
||||
- Middleware: `httptest` request → server span named from route pattern; `/healthz` unspanned; handler logger carries `traceID`.
|
||||
|
||||
### Step 3 — `provider.WithTracing` decorator
|
||||
|
||||
`internal/provider/tracing.go`, mirroring `metrics.go`: `WithTracing(name string, p Provider, opts...) Provider` — one Client span per call (`provider.create|get|delete|list`), attrs `provider.name`, `provider.id` (where present), result class attr from `resultLabel(err)`; status Error only for quota/permanent/transient — `ErrNotFound` stays Ok (expected during deletion polling). Returns the inner error **unmodified**. Uses `tracing.Start` so provider-internal logs (gcp `p.logger(ctx)`) inherit `traceID`. Tests in `internal/provider/tracing_test.go` reusing `staticProvider` from `metrics_test.go:19`.
|
||||
|
||||
Wiring in `cmd/main.go:174-177` — tracing outermost: `provider.WithTracing(name, provider.WithMetrics(name, p, m))`.
|
||||
|
||||
### Step 4 — `cmd/main.go` wiring
|
||||
|
||||
- `tracing.Setup(...)` immediately after `ctrl.SetLogger` (cmd/main.go:148); on error: log + `os.Exit(1)`.
|
||||
- Explicit `shutdown(context.WithTimeout(context.Background(), 10*time.Second))` (fresh ctx — signal ctx is already cancelled) after `mgr.Start` returns, on **both** the error path before `os.Exit(1)` and the clean path. Earlier `os.Exit` sites have nothing to flush.
|
||||
- `cfg := ctrl.GetConfigOrDie(); cfg.Wrap(tracing.RestConfigWrapper())` before `ctrl.NewManager` (cmd/main.go:239).
|
||||
- kubernetes provider: add `kubernetes.NewWithTransportWrapper(ctx, cfg, wrap transport.WrapperFunc)` beside `New` (internal/provider/kubernetes/kubernetes.go:42) and register a closure in main's constructor map (same pattern as gcp at cmd/main.go:166) passing `tracing.RestConfigWrapper()` — keeps `internal/tracing` out of provider packages and `newWithClient` tests untouched.
|
||||
- New flag `--trace-health-probes` (default false) → `engine.TraceProbes`.
|
||||
|
||||
### Step 5 — Reconciler spans
|
||||
|
||||
- `SetupWithManager` (proxy_controller.go:412): `b.Complete(tracing.NewReconciler("Proxy", r))`.
|
||||
- Sub-spans via `tracing.Start`: `reconcile.managed` (:116), `reconcile.replaceInstance` (:226), `reconcile.delete` (:254), and `status.patch` opened **inside the deferred flush closure** (:98-104, runs within the root span; its error already folds into `err`). `reconcileExternal` stays unspanned — no I/O, pure noise.
|
||||
- Useful attrs where cheap: phase transitions, `providerID`.
|
||||
|
||||
### Step 6 — Discovery server
|
||||
|
||||
- `handler()` (internal/discovery/server.go:151-156): chain becomes recover → `tracing.HTTPMiddleware("discovery", s.log)` → log → maxBytes → auth (otelhttp must stay outside `maxBytesMiddleware`).
|
||||
- Handlers and `logMiddleware`/`recoverMiddleware` switch `s.log` → `logf.FromContext(r.Context())` (handlers.go:85, :141, :234; server.go:163, :191) so request logs carry `traceID`. `s.log` remains for startup lines.
|
||||
- k8s `List` calls in handlers already use `r.Context()` → child spans appear via the wrapped rest transport.
|
||||
|
||||
### Step 7 — GC + health
|
||||
|
||||
- `gc.Sweeper.sweep` (internal/gc/gc.go:83): wrap in root span `gc.sweep` via `tracing.Start` (before the `.WithName("orphan-gc")` derivation so it inherits), attrs: providers swept, orphans deleted. Provider/k8s child spans come free from Steps 3/4.
|
||||
- `health.Engine`: new field `TraceProbes bool`. In the worker loop (engine.go:158) when enabled: root span `health.probe` (SpanKind Client) around `probeFn`, attrs proxy key/ok/latency/error class, ended before `record` (which stays ctx-free). No transport wrap and no propagation — never inject `traceparent` toward external targets through the proxies. Probe→reconcile span links are **out of scope**: `GenericEvent` (engine.go:330) carries no ctx and the workqueue coalesces events; documented in architecture Decisions.
|
||||
- Lease store sweep: not traced (in-memory only).
|
||||
|
||||
### Step 8 — GCP wire-log enrichment
|
||||
|
||||
`wireFilterHandler.Handle` (internal/provider/gcp/wirelog.go:46) receives the real request ctx from the SDK's slog calls: append `traceID`/`spanID` attrs when `trace.SpanContextFromContext(ctx)` is valid — V(5) wire logs become correlatable with the `provider.*` span. Add `wirelog_test.go` case.
|
||||
|
||||
### Step 9 — Manifests + docs
|
||||
|
||||
- `config/manager/manager.yaml` env (after `DISCOVERY_TOKEN`): `POD_NAME`/`POD_NAMESPACE` via downward API **first**, then `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)` ($(VAR) expansion only sees earlier vars); commented-out `OTEL_EXPORTER_OTLP_ENDPOINT` + `OTEL_TRACES_SAMPLER` examples with a "tracing is off until an endpoint is set" comment. Note `--trace-health-probes` beside the args list.
|
||||
- `docs/architecture.md`: new "### 10. Tracing" section after Metrics (:281) — span topology, enablement, `traceID` log-key choice; Decisions entries (:303): env-gated enablement, probes off by default, no probe→reconcile links (why), `traceID` naming vs Grafana defaults, no traceparent into apiserver.
|
||||
- README ops note; CHANGELOG entry (dated via `date`) once the user confirms it works.
|
||||
- Run `make lint` (CI has no lint step; `lll`/`revive` will hit new files).
|
||||
|
||||
## Verification
|
||||
|
||||
1. `make test` (unit, includes all new `tracetest` assertions) and `make lint`.
|
||||
2. Console smoke: `OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml ...` against kind — create a Proxy, confirm (a) `Reconcile Proxy` span JSON on stdout with nested `provider.create` + k8s PATCH spans, (b) the reconcile log lines carry the same `traceID` as the span, (c) no spans from informer watches or leader election.
|
||||
3. End-to-end in kind: deploy Jaeger all-in-one (`jaegertracing/all-in-one`, OTLP 4318), set `OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318` on the manager Deployment; exercise a reconcile + a `POST /v1/leases` (with and without a client `traceparent`) + wait one GC interval; in the Jaeger UI: reconcile trace shows provider+k8s children; lease trace shows route-named server span with k8s child; grep manager logs for a `traceID` and resolve it in Jaeger.
|
||||
4. Negative: run with no `OTEL_*` env — startup line says tracing disabled, no export-error spam, logs unchanged (no `traceID` keys).
|
||||
@@ -1,150 +0,0 @@
|
||||
# Plan: e2e test — OTel tracing against real Tempo
|
||||
**Created:** 2026-08-24 12:24
|
||||
|
||||
## Context
|
||||
|
||||
The tracing feature (MR #4, branch `feat/otel-tracing`) is covered by unit
|
||||
tests with an in-memory span recorder, but nothing proves the full pipeline:
|
||||
operator in a real cluster → OTLP export → Tempo ingest → queryable traces
|
||||
with the documented span topology. The homelab Tempo lives at
|
||||
`http://192.168.0.30:3200` (query API — verified live from this host:
|
||||
`/api/echo`, TraceQL `GET /api/search?q=…`, `GET /api/traces/<id>`), with
|
||||
OTLP ingest on `:4318` (HTTP, verified 200) and `:4317` (gRPC, open). User
|
||||
decisions: **Go Ginkgo e2e test** (not a shell script), **structural
|
||||
assertions** (span tree, not just trace-exists), flow per the user's sketch:
|
||||
deploy operator in kind with tracing on → create kubernetes-provider proxies
|
||||
→ delete them → shut down — with traces tagged for easy discovery per test
|
||||
run. This work continues on `feat/otel-tracing`; MR #4 stays the vehicle.
|
||||
|
||||
## Design decisions (from review, with evidence)
|
||||
|
||||
- **Proxy CRs go in the `default` namespace, not the operator namespace.**
|
||||
The squid pods the kubernetes provider creates carry no securityContext
|
||||
(internal/provider/kubernetes/pod.go:33-49) and are created in the CR's
|
||||
namespace; the operator ns is labeled `pod-security enforce=restricted`
|
||||
(test/e2e/e2e_test.go:61-62) and would reject them. Pod RBAC is
|
||||
cluster-scoped (config/rbac/role.yaml:10), so `default` works. All
|
||||
kubectl calls use explicit `-n`.
|
||||
- **Anchor assertions on `provider.create`, not the root-span search.** A
|
||||
`name="Reconcile Proxy"` hit may be the finalizer-add or a drift
|
||||
reconcile (no provider call inside). TraceQL matches span names anywhere
|
||||
in a trace, so search `{resource.test.run.id="<runID>" && name="provider.create"}`,
|
||||
fetch *that* traceID, then assert companion span names.
|
||||
- **Run identification with zero code changes:** OTel resource attribute
|
||||
`test.run.id=<runID>` via `OTEL_RESOURCE_ATTRIBUTES` (env wins —
|
||||
`resource.WithFromEnv()` merges last, internal/tracing/tracing.go). Tempo
|
||||
vParquet searches arbitrary resource attrs without config; fallback query
|
||||
documented: `{resource.service.name="egress-proxies-operator" && name="provider.create"}`
|
||||
+ start/end window (Unix **seconds**).
|
||||
- **OTLP preflight from inside the cluster:** host-reachability of
|
||||
192.168.0.30 doesn't prove pod-reachability from kind on macOS, and
|
||||
export failures log only at V(1) (invisible) — without a preflight, a
|
||||
broken path is a 2-minute opaque timeout. Reuse the suite's curl-pod
|
||||
pattern (e2e_test.go:218-248, incl. the restricted-PSS overrides JSON)
|
||||
to POST to `<OTLP_ENDPOINT>/v1/traces` and fail fast.
|
||||
- **No homelab defaults baked into the Makefile.** The spec Skips unless
|
||||
`TEMPO_URL`/`OTLP_ENDPOINT` are set (otherwise everyone without that LAN
|
||||
host inherits a 2-minute failure). The copy-pasteable invocation lives in
|
||||
docs/testing.md.
|
||||
- **Ginkgo ordering:** top-level container order is randomized, so the
|
||||
tracing Describe is fully self-contained (own ns create → make install →
|
||||
make deploy → teardown), same shape as the Manager Describe. `kubectl
|
||||
delete ns` blocks until termination, so no create/delete race between
|
||||
the two Describes.
|
||||
|
||||
## Steps
|
||||
|
||||
### Step 1 — `test/e2e/tracing_test.go` (new; `//go:build e2e`, package e2e)
|
||||
|
||||
`Describe("OTel tracing", Ordered)`:
|
||||
|
||||
- **BeforeAll:**
|
||||
1. Read `TEMPO_URL` + `OTLP_ENDPOINT`; `Skip("TEMPO_URL/OTLP_ENDPOINT not set")` when empty.
|
||||
2. `runID := "e2e-" + strconv.FormatInt(time.Now().UnixNano(), 10)`;
|
||||
print it to `GinkgoWriter` so the run is findable in Grafana by hand.
|
||||
3. Create operator ns + restricted-PSS label; `make install`;
|
||||
`make deploy IMG=<managerImage>` (same commands as e2e_test.go:54-75,
|
||||
via `utils.Run`).
|
||||
4. Pre-pull the proxy image to kill the biggest flake source:
|
||||
`docker pull ubuntu/squid:6.6-24.04_edge` + `kind load docker-image`
|
||||
(via `utils.Run`, honoring `KIND`/`KIND_CLUSTER` env like
|
||||
utils.LoadImageToKindClusterWithName).
|
||||
5. OTLP preflight: curl pod POSTing `{}` to `<OTLP_ENDPOINT>/v1/traces`,
|
||||
assert HTTP 200 in its logs (fail message names the endpoint).
|
||||
6. `kubectl set env deployment/egress-proxies-operator-controller-manager
|
||||
-n <ns> OTEL_EXPORTER_OTLP_ENDPOINT=<OTLP_ENDPOINT>
|
||||
"OTEL_RESOURCE_ATTRIBUTES=k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE),test.run.id=<runID>"`
|
||||
— in-place update keeps the var after the downward-API vars ($(VAR)
|
||||
expansion needs that ordering); `exec.Command` means no shell mangling
|
||||
of `$()`. Then `kubectl rollout status`.
|
||||
7. `Eventually`: newest controller pod (sorted by creationTimestamp — a
|
||||
terminating old pod may otherwise be picked) logs contain
|
||||
`"tracing enabled"`.
|
||||
- **It "creates proxies and reports reconcile traces":**
|
||||
- Write 2 Proxy manifests (names `proxy-tracing-e2e-1/2`,
|
||||
`provider: kubernetes`, modeled on config/samples/proxy_kubernetes.yaml)
|
||||
to `GinkgoT().TempDir()` (absolute paths — `utils.Run` chdirs the
|
||||
process); `kubectl apply -n default`.
|
||||
- `Eventually` (5m — first squid start) `.status.phase == "Ready"`
|
||||
(confirmed field, api/v1alpha1/proxy_types.go:41).
|
||||
- `Eventually`: Tempo search
|
||||
`{resource.test.run.id="<runID>" && name="provider.create"}` ≥1 hit
|
||||
(BSP flushes every ~5s; 2m default timeout is ample).
|
||||
- Fetch that traceID via `/api/traces/<id>`; assert span names include
|
||||
`Reconcile Proxy`, `reconcile.managed`, `provider.create`,
|
||||
`status.patch`, and resource attr `service.name=egress-proxies-operator`.
|
||||
- **It "traces deletion":**
|
||||
- `kubectl delete proxy -n default …`; `Eventually` CRs gone (finalizer
|
||||
→ `provider.delete`, DeletionPoll 10s).
|
||||
- `Eventually`: Tempo search finds `provider.delete` for the runID; fetch
|
||||
and assert `reconcile.delete` in the same trace.
|
||||
- **AfterAll** (= the sketch's "shut down operator"): delete leftover
|
||||
proxies in `default`; `make undeploy` (SIGTERM → `tracingShutdown`
|
||||
flushes); `make uninstall`; delete operator ns — errors discarded
|
||||
(`_, _ = utils.Run(...)`), mirroring e2e_test.go:79-95.
|
||||
- **Tempo client helpers** (same file, stdlib only — the test process runs
|
||||
on the host, which reaches Tempo directly):
|
||||
- `tempoSearch(traceql string)` → `GET {TEMPO_URL}/api/search?q=…&start=…&end=…`
|
||||
(Unix seconds, window = suite start − 5m → now); minimal struct
|
||||
`{Traces []struct{ TraceID string }}`.
|
||||
- `tempoTrace(id string)` → `/api/traces/<id>`, decoded as **OTLP-JSON**
|
||||
(`batches[].scopeSpans[].spans[].name`, resource attrs as
|
||||
`{key, value:{stringValue}}`) — not Jaeger's shape. Helper flattens to
|
||||
a span-name set + resource attr map.
|
||||
|
||||
### Step 2 — Makefile
|
||||
|
||||
- `test-e2e`: pass `TEMPO_URL`/`OTLP_ENDPOINT` through to `go test` env
|
||||
(no defaults) and add `-timeout 30m` — the suite already runs
|
||||
docker-build + kind-load in BeforeSuite plus 3–5m Eventuallys, and this
|
||||
adds a second full deploy cycle + image pulls; the 10m default will be
|
||||
exceeded on cold caches.
|
||||
|
||||
### Step 3 — Docs
|
||||
|
||||
- `docs/testing.md` (§ e2e, currently "scaffold… only asserts manager runs
|
||||
and serves metrics"): document the tracing spec, its Skip gate, and the
|
||||
invocation:
|
||||
`TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`.
|
||||
- Execution-log entries per step in `docs/plans-executions/2026-08-24-1224-tracing-e2e.md`.
|
||||
- CHANGELOG entry only after the user confirms a green run (house rule).
|
||||
|
||||
## Not doing (and why)
|
||||
|
||||
- No dedicated flush assertion on shutdown — teardown exercises the flush
|
||||
path, but attributing a specific late span to it is guesswork.
|
||||
- No gRPC (4317) variant — http/protobuf is the operator default and the
|
||||
verified path; a variant run is a one-env-var change if ever wanted.
|
||||
- No changes to `internal/` — the whole test works through public surface
|
||||
(env vars, kubectl, Tempo API), which is the point of an e2e test.
|
||||
|
||||
## Verification
|
||||
|
||||
1. `TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`
|
||||
→ tracing spec passes (both Its); without the env vars → spec reports
|
||||
Skipped, remainder of suite unaffected.
|
||||
2. In Grafana: TraceQL `{resource.test.run.id="<runID printed in test log>"}`
|
||||
shows the run's traces with the expected tree.
|
||||
3. `go vet -tags=e2e ./...` clean (documented routine).
|
||||
4. A deliberately wrong `OTLP_ENDPOINT` (e.g. port 9) fails fast in the
|
||||
preflight step with a clear message, not a 2-minute search timeout.
|
||||
@@ -1,186 +0,0 @@
|
||||
# Plan: Trace every libvirt call in the KVM provider
|
||||
**Created:** 2026-09-10 08:05
|
||||
|
||||
## Context
|
||||
|
||||
Step 7 of the KVM provider plan (live validation on `beelink`) has gone through
|
||||
three fixes found only on the real host, and the current failure (guest hangs at
|
||||
GRUB's "Booting" line) is being diagnosed blind: the provider logs at V(1)/V(2)
|
||||
but nothing shows them. The integration test never installs a logger, so
|
||||
controller-runtime prints "log.SetLogger(...) was never called" and swallows
|
||||
everything; and even with a logger, the provider only logs per *operation*
|
||||
(Create, Get…), not per libvirt RPC, so the sequence and timing of what was
|
||||
actually sent to the hypervisor is invisible.
|
||||
|
||||
Goal: one log line per libvirt call at debug level (`--zap-log-level=debug`, logr
|
||||
V(1) in this repo's scheme: V(1) = outcome per operation, V(2) = curated detail,
|
||||
V(5) = raw wire, per the GCP verbose-logging plans), with method, host, domain or
|
||||
volume, duration and the libvirt error code on failure; and those lines visible in
|
||||
`make test-integration-kvm` output.
|
||||
|
||||
Assumption: this lands on the current `feat/kvm-provider` branch, since it exists
|
||||
to finish that branch's validation, and the plan file plus execution log follow
|
||||
CLAUDE.md conventions (plan committed alone first, execution summary alongside
|
||||
each step). Say so if you want a separate branch instead.
|
||||
|
||||
## Design
|
||||
|
||||
- **Decorator, not call-site logging.** New `tracingAPI` implements the 17-method
|
||||
`libvirtAPI` seam (`internal/provider/kvm/conn.go:26-44`), wrapping the real
|
||||
client. Every method logs one V(1) line after the inner call returns. Applied in
|
||||
`hostPool.call`, so every provider method, `image.go`'s cache and the
|
||||
integration test's own `p.hosts.call` are traced with no per-site changes.
|
||||
- **Logger passed explicitly.** `call(ctx, log, host, fn)` and `get(ctx, log, host)`
|
||||
gain a `logr.Logger` parameter, matching how this package already threads `log`
|
||||
into every helper. The five call sites (`kvm.go:169, 322, 458, 530`,
|
||||
`kvm_integration_test.go:220`) already hold the right logger with
|
||||
provider/host/name values. `call` does `log = log.WithName("libvirt")`, so trace
|
||||
lines are named `kvm.libvirt`.
|
||||
- **Wrap only when `log.V(1).Enabled()`.** Zero cost at info level; the
|
||||
controller-runtime unfulfilled root sink reports disabled, so unit tests and
|
||||
`conn_test.go`'s `hostClient{api: nil}` cases are unaffected.
|
||||
- **One line per RPC at every level.** At V(2) the same line gains extra keys
|
||||
(`xml`, `metadata`, `addresses`, `names`) instead of a second line. Upload and
|
||||
download log offset/length/duration only, never bytes (the seed ISO carries
|
||||
cloud-init). Domain and volume XML contain no secrets, so V(2) is fine for them.
|
||||
- **Connection events traced too** in the pool: dial with duration, dial failure,
|
||||
dead cached client redial, and both drop reasons (context cancelled, transport
|
||||
error). `dialSSH`/`connectLibvirt`/`keepalive` stay untouched: changing
|
||||
`hostDialer`'s signature would ripple into every test dialer, and the pool
|
||||
already observes every event.
|
||||
- **Rendering.** Handles by name only (`domain`, `pool`, `volume`), never UUID/Key.
|
||||
`libvirt.DomainState` and `DomainInterfaceAddressesSource` have stringers, pass
|
||||
them typed. Bitmask flags don't stringify, log as integers only when non-zero.
|
||||
`libvirt.OptString` → first element or `""` under `uri`.
|
||||
- **awaitOrCancel.** `done` runs inside fn's goroutine; on the normal path it logs
|
||||
before `call` returns. On cancel, the abandoned RPC logs one late line when the
|
||||
closed connection makes it fail: desirable in production, and in the integration
|
||||
test only reachable in an already-failing timeout run (testr panics on a log after
|
||||
the test ends; note that in the header, don't engineer around it).
|
||||
|
||||
## Steps
|
||||
|
||||
### Step 0 — Save plan
|
||||
Copy this file to `docs/plans/2026-09-10-0805-kvm-libvirt-tracing.md`, commit alone.
|
||||
|
||||
### Step 1 — `errors.go`: shared error fields
|
||||
Factor `libvirtErrorFields(err error) []any` out of `logAPIError` (`errors.go:62-72`):
|
||||
returns `libvirtCode` (uint32), `libvirtCodeName` (`libvirt.ErrorNumber(code)`,
|
||||
e.g. `ErrNoDomain`), `libvirtMessage`; nil for non-libvirt errors; works through
|
||||
`%w` wrapping via `errors.As`. `logAPIError` keeps its message and gains
|
||||
`libvirtCodeName`. Test `TestLibvirtErrorFields` (table: libvirt.Error, wrapped,
|
||||
plain error) in `errors_test.go`.
|
||||
|
||||
### Step 2 — new `trace.go`
|
||||
```go
|
||||
type tracingAPI struct { inner libvirtAPI; log logr.Logger }
|
||||
type tracedCall struct { t *tracingAPI; method string; start time.Time; kv []any }
|
||||
func (t *tracingAPI) begin(method string, kv ...any) *tracedCall
|
||||
func (c *tracedCall) detail(kv ...any) *tracedCall // appended only if V(2) enabled
|
||||
func (c *tracedCall) done(err error, kv ...any) // one V(1) line
|
||||
```
|
||||
`done` emits `log.V(1).Info("libvirt call", ...)` with keys in order: `method`,
|
||||
begin args, result kv, `duration` (rounded to µs), then on error `error` plus
|
||||
`libvirtErrorFields`. Per-method keys:
|
||||
|
||||
| method | V(1) args | V(1) results | V(2) extra |
|
||||
|---|---|---|---|
|
||||
| DomainLookupByName | domain | | |
|
||||
| DomainDefineXML | xmlBytes | domain | xml |
|
||||
| DomainCreate, DomainDestroy | domain | | |
|
||||
| DomainUndefineFlags | domain, flags | | |
|
||||
| DomainGetState | domain | state, reason | |
|
||||
| DomainGetMetadata | domain, type, uri | metadataBytes | metadata |
|
||||
| DomainInterfaceAddresses | domain, source | interfaces (count) | addresses |
|
||||
| ConnectListAllDomains | | count | names |
|
||||
| StoragePoolLookupByName, StoragePoolRefresh | pool | | |
|
||||
| StorageVolLookupByName | pool, volume | | |
|
||||
| StorageVolGetPath | pool, volume | path | |
|
||||
| StorageVolCreateXML | pool, xmlBytes | volume | xml |
|
||||
| StorageVolUpload, StorageVolDownload | pool, volume, offset, length | | (never content) |
|
||||
| StorageVolDelete | pool, volume | | |
|
||||
|
||||
Return values pass through untouched.
|
||||
|
||||
### Step 3 — `conn.go`: pool logging and wrapping
|
||||
- `get(ctx, log, host)`: dead cached client → `"libvirt connection lost, redialing"`;
|
||||
time the dial; `"libvirt host connected"` with `address`, `duration`; failure →
|
||||
`"libvirt host dial failed"` with `address`, `duration`, `error`.
|
||||
- `call(ctx, log, host, fn)`: `log = log.WithName("libvirt")`; wrap when
|
||||
`log.V(1).Enabled()`; abort closure logs `"libvirt connection dropped"`
|
||||
`reason="context cancelled"` before `drop`; transport-error branch logs the same
|
||||
message with `reason="transport error"`. Update `call`'s doc comment.
|
||||
|
||||
### Step 4 — call sites
|
||||
`kvm.go` Create/Get/Delete pass `log`, ListByTag passes `hlog`;
|
||||
`kvm_integration_test.go:220` passes `p.logger(ctx).WithValues("host", host.Name)`;
|
||||
`conn_test.go` (12 `p.call(` sites) gets `var discard = logr.Discard()`.
|
||||
|
||||
### Step 5 — integration test output
|
||||
- `integrationLogger(t)`: `KVM_TEST_LOG_LEVEL` (default `1`, `strconv.Atoi`,
|
||||
`t.Fatalf` on garbage) → `testr.NewWithOptions(t, testr.Options{LogTimestamp:
|
||||
true, Verbosity: n})` (`github.com/go-logr/logr/testr`, present in logr v1.4.3,
|
||||
no go.mod change); `logf.SetLogger(log)` once so nothing falls back to the
|
||||
unfulfilled root.
|
||||
- `TestKVM_lifecycle`: `ctx = logr.NewContext(ctx, log)` after `WithTimeout`; the
|
||||
cleanup ctx gets the same logger.
|
||||
- Header comment: document `KVM_TEST_LOG_LEVEL` (1 = one line per RPC, 2 adds
|
||||
XML/metadata/addresses, 0 = only `t.Log`) and the late-line-after-timeout caveat.
|
||||
|
||||
### Step 6 — unit tests
|
||||
- `kvm_test.go`: `captureContext(verbosity)` like `gcp_test.go:277-286` but with a
|
||||
mutex around the slice; `TestLogging_verbosityTiers` (mirror `gcp_test.go:322-399`):
|
||||
V0 → no lines; V1 → `"msg"="libvirt call"`, `"method"="DomainDefineXML"`,
|
||||
`"method"="StorageVolUpload"`, `"msg"="libvirt host connected"`, provider/host/name
|
||||
values, `"libvirtCodeName"="ErrNoDomain"` from the first lookup, no `"xml"=`;
|
||||
V2 → `"xml"="<domain` and `"metadata"=`; a cloud-init sentinel absent at every tier.
|
||||
- `trace_test.go`: `TestTracingAPI_logsEveryMethod` (table of 17 calls over a
|
||||
`readyHost()` fake, exactly one line each, plus
|
||||
`reflect.TypeOf((*libvirtAPI)(nil)).Elem().NumMethod() == len(cases)` so a new
|
||||
interface method without tracing fails); `TestTracingAPI_passesResultsThrough`;
|
||||
`TestTracingAPI_errorFields` (libvirt error → code/name/message; `io.EOF` →
|
||||
`"error"="EOF"`, no code); `TestTracingAPI_detailOnlyAtV2`;
|
||||
`TestTracingAPI_uploadLogsSizeNotContent`.
|
||||
- `conn_test.go`: `fakeDialer` gains an `api` field; `TestHostPool_call_tracesAtV1`
|
||||
(V1 → connected + method lines; V0 → no lines and fn receives the bare
|
||||
`*fakeAPI`); `TestHostPool_call_logsDropReason` (transport error, cancel).
|
||||
|
||||
### Step 7 — docs, execution log, CHANGELOG
|
||||
- `docs/kvm-provider.md`: `## Logging` section before "Known gaps":
|
||||
`--zap-log-level=debug` = one `libvirt call` line per RPC plus connect/drop lines;
|
||||
`--zap-log-level=2` adds XML, metadata, addresses; cloud-init never logged; no
|
||||
V(5) wire tier because go-libvirt exposes no hook. Mention `KVM_TEST_LOG_LEVEL`
|
||||
in its Testing paragraph.
|
||||
- `docs/testing.md`: `KVM_TEST_LOG_LEVEL` in the optional vars; `-v` now shows every
|
||||
RPC; extend the `kvm_test.go` table row.
|
||||
- `docs/plans-executions/2026-09-10-0805-kvm-libvirt-tracing.md` with the checklist
|
||||
and per-step entries; also a short cross-reference line in the KVM provider
|
||||
execution log's Step 7 section.
|
||||
- `CHANGELOG.md` entry once the user confirms the live run shows the trace.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
go build ./... && go vet ./... && go vet -tags=integration ./internal/provider/kvm/
|
||||
go test -race -count=1 ./internal/provider/kvm/
|
||||
go test -short ./...
|
||||
```
|
||||
Live, with the usual env: `make test-integration-kvm`. Expect, interleaved with
|
||||
`t.Log` lines and attributed to `trace.go:<line>`, lines like
|
||||
|
||||
```
|
||||
kvm.libvirt: "level"=1 "msg"="libvirt call" "provider"="kvm-integration" "host"="kvm-test" "name"="proxy-…" "method"="DomainLookupByName" "domain"="proxy-…" "duration"="2.1ms" "error"="…" "libvirtCode"=42 "libvirtCodeName"="ErrNoDomain" "libvirtMessage"="…"
|
||||
```
|
||||
|
||||
plus one `libvirt host connected` line with `duration`, a `StorageVolUpload` line
|
||||
whose duration is the seed upload, and no "log.SetLogger(...) was never called"
|
||||
text. `KVM_TEST_LOG_LEVEL=2` shows `"xml"="<domain` and `"addresses"=`;
|
||||
`KVM_TEST_LOG_LEVEL=0` shows only `t.Log` output. No line at any level contains
|
||||
`#cloud-config`.
|
||||
|
||||
## Critical files
|
||||
- `internal/provider/kvm/conn.go`, `trace.go` (new), `trace_test.go` (new)
|
||||
- `internal/provider/kvm/errors.go`, `errors_test.go`
|
||||
- `internal/provider/kvm/kvm.go`, `kvm_test.go`, `conn_test.go`
|
||||
- `internal/provider/kvm/kvm_integration_test.go`
|
||||
- `docs/kvm-provider.md`, `docs/testing.md`
|
||||
@@ -1,195 +0,0 @@
|
||||
# Plan: Fix the five code-review findings on `feat/kvm-provider`
|
||||
**Created:** 2026-09-10 22:22 UTC
|
||||
|
||||
## Context
|
||||
|
||||
PR #5 (`feat/kvm-provider` → `main`, 19 commits) is open and awaiting merge. A
|
||||
`/code-review high` pass over the branch returned five confirmed findings: two that can
|
||||
pin a Proxy's finalizer forever, one startup crash-loop, and two in the guest-agent merge
|
||||
committed earlier today. All five are fixed on the same branch so the PR updates in place,
|
||||
then re-validated against the real hypervisor before merging.
|
||||
|
||||
Four decisions, made by the user during planning:
|
||||
|
||||
1. **Wedged deletes** — attempt `Delete` even when `Get` fails permanently, make an
|
||||
unconfigured KVM host a no-op delete, and add an annotation that abandons cleanup for
|
||||
whatever still cannot finish.
|
||||
2. **Missing `kvm-ssh` Secret** — read the SSH material lazily, so only KVM Proxies fail.
|
||||
3. **Guest agent** — stop rewriting the user's cloud-config. Ship the bootstrap as a
|
||||
vendor-data shell script instead.
|
||||
4. Validation is unit tests, the integration test against beelink, and the kind e2e. The
|
||||
uncommitted dev-only files stay exactly as they are and out of the PR.
|
||||
|
||||
## Why the guest-agent fix is a rewrite, not a patch
|
||||
|
||||
Both cloud-init findings come from the same root cause: this operator edits the user's
|
||||
document. Reading cloud-init 26.1's sources settles that it never has to.
|
||||
|
||||
Vendor-data is consumed by its **own** handler chain with separate script and config paths
|
||||
(`stages.py:_default_vendordata_handlers`). A vendor-data **shell script** is therefore
|
||||
written to the vendor script directory and run by `cc_scripts_vendor`, never passing
|
||||
through the config merger that drops colliding keys. On the final-stage module list it
|
||||
runs after `package_update_upgrade_install` and before the user's own `runcmd`, so the
|
||||
agent comes up early regardless of what the user's config does.
|
||||
|
||||
That also closes the door the previous approach left open. The merge cannot be rescued
|
||||
from inside the documents: `merge_how` in the user-data governs only the first source, and
|
||||
`merge_how` in vendor-data is popped and discarded by `dict_extract_mergers` before the
|
||||
document is ever written. There is no supported way to make vendor-data cloud-config keys
|
||||
survive a user-data that defines the same key.
|
||||
|
||||
Consequences: user-data reaches the seed byte for byte in any format, so Jinja templates,
|
||||
MIME archives, gzip payloads and shell scripts stop being special cases (finding 4), and
|
||||
nothing re-serialises YAML, so unquoted scalars keep their type (finding 5). The cost is
|
||||
that the script picks the package manager itself where cloud-init's `packages` key was
|
||||
distro-agnostic.
|
||||
|
||||
## Why the delete fix is shaped this way
|
||||
|
||||
KVM is the only provider that resolves a providerID segment against operator config. GCP
|
||||
and Kubernetes let the backend answer "not found", so retiring a location yields
|
||||
`ErrNotFound` and the finalizer drops. `Provider.host()`
|
||||
([kvm.go:118-139](internal/provider/kvm/kvm.go#L118-L139)) returns `ErrPermanent`, and
|
||||
`reconcileDelete` ([proxy_controller.go:281-291](internal/controller/proxy_controller.go#L281-L291))
|
||||
drops the finalizer only on `ErrNotFound`. `deletionFailure`
|
||||
([:327-336](internal/controller/proxy_controller.go#L327-L336)) says in its own comment
|
||||
that it retries "until an operator intervenes", and no intervention mechanism exists. The
|
||||
annotation is that mechanism, and it also rescues the unconfigured-provider wedge at
|
||||
[:276-279](internal/controller/proxy_controller.go#L276-L279).
|
||||
|
||||
`Get` keeps returning `ErrPermanent` for an unknown host deliberately. Turning it into
|
||||
`ErrNotFound` would make a typo in the host list look like a missing VM on the live path,
|
||||
and the reconciler would recreate every affected Proxy elsewhere while the originals keep
|
||||
running where garbage collection cannot see them.
|
||||
|
||||
## Steps
|
||||
|
||||
One commit per step on `feat/kvm-provider`, each ending with
|
||||
`Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>`.
|
||||
|
||||
### Step 0 — Save this plan into the repo
|
||||
Copy to `docs/plans/2026-09-10-2222-kvm-review-fixes.md` with the same header timestamp and
|
||||
commit it alone, before any code change. Start
|
||||
`docs/plans-executions/2026-09-10-2222-kvm-review-fixes.md` with the status checklist when
|
||||
Step 1 lands.
|
||||
|
||||
### Step 1 — Unwedge deletion (finding 1)
|
||||
- `api/v1alpha1/proxy_types.go`: add `AnnotationSkipCleanup = "crawl.example.com/skip-cleanup"`
|
||||
beside `AnnotationSpecHash` ([:59-63](api/v1alpha1/proxy_types.go#L59-L63)), documented as
|
||||
abandoning the backend instance.
|
||||
- `internal/controller/proxy_controller.go`, `reconcileDelete`: honour the annotation after
|
||||
the empty-providerID branch and **before** the provider lookup, so it also covers a
|
||||
provider that is no longer configured. Log the abandoned providerID at Info.
|
||||
`ProxyReconciler` has no `EventRecorder`; do not add one for this.
|
||||
- Same function: replace the two `if`s after `Get` with a switch. On
|
||||
`provider.Class(err) == provider.ErrPermanent`, call `Delete` anyway, since it is
|
||||
idempotent, and remove the finalizer when it succeeds rather than requeueing into a `Get`
|
||||
that can never converge. On failure, `deletionFailure(errors.Join(getErr, delErr))`.
|
||||
Guard on `err != nil`, since `Class(nil)` is nil.
|
||||
- `internal/provider/kvm/kvm.go`, `Delete`: an unconfigured host segment means there is
|
||||
nothing this operator can clean, so log it and return success, mirroring `removeVolumes`
|
||||
([kvm.go:506-514](internal/provider/kvm/kvm.go#L506-L514)) for a missing pool. `Get` is
|
||||
unchanged.
|
||||
- Tests: extend the deletion table in `internal/controller/reconcile_test.go`
|
||||
([:357-393](internal/controller/reconcile_test.go#L357-L393)) with permanent-Get plus
|
||||
succeeding Delete, permanent-Get plus failing Delete, the annotation with the provider
|
||||
never called, and the annotation on an unconfigured provider. In `kvm_test.go`, the
|
||||
`"host not configured"` case at [:737](internal/provider/kvm/kvm_test.go#L737) flips from
|
||||
`ErrPermanent` to success; add a named test beside `TestDelete_missingPoolIsNotAnError`
|
||||
([:715](internal/provider/kvm/kvm_test.go#L715)).
|
||||
- Docs: the annotation in `README.md` (~line 163) and `docs/architecture.md` (~line 487);
|
||||
a *Known gaps* entry in `docs/kvm-provider.md` saying a domain on a host removed from
|
||||
config can no longer be cleaned up by the operator, the Proxy still finishes deleting,
|
||||
and the VM must be removed by hand because garbage collection only sweeps configured
|
||||
hosts.
|
||||
|
||||
### Step 2 — Validate names that become identifiers (finding 2)
|
||||
- `internal/provider/config_kvm.go`: add a name regexp beside `sha256Hex`
|
||||
([:6](internal/provider/config_kvm.go#L6)) allowing letters, digits, dot, dash and
|
||||
underscore. Apply it to `hosts[].name` after the existing empty check
|
||||
([:129-131](internal/provider/config_kvm.go#L129-L131)) and to each `images` map key,
|
||||
which becomes the cached volume name `base-<key>.<format>`
|
||||
([image.go:49-55](internal/provider/kvm/image.go#L49-L55)) and half of the image-cache
|
||||
lock key ([image.go:96](internal/provider/kvm/image.go#L96)). `machineTypes` keys are only
|
||||
compared, never embedded, so leave them.
|
||||
- Keep the file's first-error-wins style and `fmt.Sprintf` field paths. The host message
|
||||
should name the consequence: the value becomes part of `hosts/<name>/domains/<vm>`, which
|
||||
`parseProviderID` splits on `/` ([id.go:11-21](internal/provider/kvm/id.go#L11-L21)).
|
||||
- Tests: two entries in the `TestLoadConfig_kvmInvalid` table
|
||||
([config_kvm_test.go:76](internal/provider/config_kvm_test.go#L76)), existing
|
||||
name/yaml/wantErrSub shape.
|
||||
|
||||
### Step 3 — Read the SSH material lazily (finding 3)
|
||||
- `internal/provider/kvm/conn.go`: keep `newSSHAuth`, and add a lazy dialer that builds it
|
||||
on first dial and caches it once it succeeds, so a `kvm-ssh` Secret created after the
|
||||
manager starts heals on the next reconcile with no restart.
|
||||
- `internal/provider/kvm/kvm.go`, `New`: stop calling `newSSHAuth`; pass the lazy dialer.
|
||||
Use the context parameter it currently ignores to log a warning when either file is
|
||||
unreadable at startup, keeping the diagnostic without the exit.
|
||||
- Error classes: a missing or unreadable file stays unclassified, so `Class` defaults it to
|
||||
transient and the Proxy retries until the Secret appears; an unparseable key or malformed
|
||||
known_hosts wraps `ErrPermanent`. Distinguish with `errors.Is(err, fs.ErrNotExist)`.
|
||||
- Fix the two doc comments promising startup failure
|
||||
([kvm.go:79-81](internal/provider/kvm/kvm.go#L79-L81),
|
||||
[conn.go:182-185](internal/provider/kvm/conn.go#L182-L185)) and the matching claim in
|
||||
`docs/kvm-provider.md` (~lines 107-109). `DISCOVERY_TOKEN` already sets the house
|
||||
convention for an optional mount: degrade with a loud warning.
|
||||
- Tests in `conn_test.go`, which already writes key files: `New` succeeds with no key
|
||||
present, the first dial fails transiently, a dial after the file appears succeeds, and a
|
||||
corrupt key is permanent.
|
||||
|
||||
### Step 4 — Guest agent via vendor-data script (findings 4 and 5)
|
||||
- `internal/provider/kvm/seed.go`: delete `withGuestAgent`, `isCloudConfig`, `appendUnique`
|
||||
and the `sigs.k8s.io/yaml` import. User-data goes onto the seed exactly as the Proxy
|
||||
supplied it.
|
||||
- Add the bootstrap as a `#!/bin/sh` vendor-data script: skip when the agent is already
|
||||
present, otherwise install `qemu-guest-agent` through whichever of `apt-get`, `dnf`,
|
||||
`yum`, `zypper` or `apk` exists, then `systemctl enable --now`. Run `apt-get update`
|
||||
before installing, since nothing else guarantees a fresh index when the user's config has
|
||||
no `packages` key. Keep it idempotent and let it log to stdout, which lands in
|
||||
`cloud-init-output.log`.
|
||||
- When the provider config also sets `vendorData`, wrap both into a MIME multipart
|
||||
vendor-data: the user's document as one part, the script as another. Cloud-init detects
|
||||
multipart by finding `mime-version:` in the first 4096 bytes, so the header must lead.
|
||||
- `internal/provider/config_kvm.go`: add `installGuestAgent`, defaulting to on. Off means
|
||||
the seed carries only the user's own vendor-data and IP discovery falls back to the host
|
||||
ARP table. Nothing fails permanently any more, so the error path in `writeSeed`
|
||||
([kvm.go:252-255](internal/provider/kvm/kvm.go#L252-L255)) narrows to MIME assembly.
|
||||
- Tests in `seed_test.go`: user-data byte-identical for cloud-config, Jinja, MIME, gzip and
|
||||
script inputs; the script present in vendor-data; the opt-out; the multipart case
|
||||
including the `mime-version:` position; keep the existing ISO tests.
|
||||
- Docs: rewrite the *What lands on the host* passage added earlier today, the `vendorData`
|
||||
row and a new `installGuestAgent` row in the config table, and the ARP note in
|
||||
*Known gaps*. Cite the cloud-init behaviour that forced the change.
|
||||
|
||||
### Step 5 — Validate, record, push
|
||||
- `gofmt -l internal/`, `go vet ./...`, `go test -short ./...`.
|
||||
- `make test-integration-kvm` against beelink, environment as recorded in the KVM execution
|
||||
log.
|
||||
- Rebuild the image, `kind load`, restart the manager, delete and recreate both Proxies,
|
||||
curl a test URL through each.
|
||||
- `CHANGELOG.md` entry at the top with a real `date` timestamp, noting it supersedes the
|
||||
merge approach described in today's earlier entry. Add a Run 8 section to
|
||||
`docs/plans-executions/2026-09-04-1404-kvm-provider.md` plus this plan's own execution
|
||||
file. Push with the token helper; PR #5 updates in place.
|
||||
|
||||
## Verification
|
||||
- Unit: the controller cases prove a permanent `Get` no longer pins the finalizer and that
|
||||
the annotation short-circuits before the provider is touched. `TestDelete_*` proves an
|
||||
unconfigured host is a clean delete while `TestGet_*` stays permanent, which is what keeps
|
||||
the live path from recreating VMs.
|
||||
- Seed: assert the uploaded user-data equals the Proxy's `cloudInit` byte for byte, which is
|
||||
the property the old merge could not offer.
|
||||
- Integration on beelink: Create, IP via the guest agent, ListByTag, Delete. In the guest,
|
||||
`cloud-init status --long` clean and `systemctl is-active qemu-guest-agent` active.
|
||||
- In-cluster: both Proxies reach Ready with an IP and return 204 through squid. Their
|
||||
cloud-config defines its own `packages`, which is the exact case that failed in Run 7.
|
||||
- Negative check in kind: rename the `kvm-ssh` Secret away and restart. The pod must stay
|
||||
up, the Kubernetes Proxies must stay Ready, and only the KVM Proxies degrade. Restore the
|
||||
Secret and confirm they recover without a restart.
|
||||
|
||||
## Out of scope
|
||||
- The two items the review noted but did not report: deleting a cached base image while
|
||||
running VMs back onto it, and an ad-hoc image URL always being treated as qcow2. Both are
|
||||
documented known gaps and neither is a regression from this branch.
|
||||
- The uncommitted dev-only files, by the user's decision.
|
||||
+2
-90
@@ -54,11 +54,6 @@ manual runs too.
|
||||
| | `metrics_test.go` | `WithMetrics` decorator: classified result labels, error passthrough | fake recorder + static provider |
|
||||
| `internal/provider/kubernetes` | `kubernetes_test.go`, `pod_test.go` | Create/Get/Delete/ListByTag over real Pod objects; pure `buildPod`/`squidConf` incl. the `max_filedescriptors` OOM-regression assertion | `client/fake` (real `corev1.Pod`s, no kubelet) |
|
||||
| `internal/provider/gcp` | `insert_test.go`, `errors_test.go`, `gcp_test.go` | field-by-field `buildInsertRequest`; HTTP-code classification table; zone-qualified IDs, 409-is-success, 9-row state mapping, ListByTag filter + `ReturnPartialSuccess` | pure builder needs **no fake**; the rest uses the flattened `instancesAPI` seam |
|
||||
| `internal/provider/kvm` | `domainxml_test.go`, `volume_test.go`, `seed_test.go`, `id_test.go`, `metadata_test.go` | pure builders: domain/volume XML unmarshalled back and checked field by field, seed ISO reopened with the same library, providerID round-trip, metadata in both namespace spellings | pure |
|
||||
| | `errors_test.go`, `conn_test.go` | libvirt-code classification table; connection pool (reuse, redial, drop on transport error, cancel releases a blocked RPC, one dial per host under concurrency); SSH dial, host-key policy, keepalive | fake dialer; a **real in-process `x/crypto/ssh` server** forwarding a unix socket |
|
||||
| | `image_test.go`, `kvm_test.go` | image cache (fresh fetch, ready, marker missing, checksum change/mismatch, 4xx/5xx, concurrent callers); Create resume after every partial step, leftover truncated seed, state/IP mapping, every Delete already-gone case, ListByTag across hosts / one down / unreadable domain; logging tiers (V0 silent, V1 one line per RPC, V2 XML) with cloud-init never logged | `httptest` image server + in-memory `fakeAPI` behind the `libvirtAPI` seam |
|
||||
| | `trace_test.go`, `conn_test.go`, `errors_test.go` | every `libvirtAPI` method traced exactly once (guarded by reflection on the interface), results and errors pass through, libvirt error fields, detail keys only at V2, upload content never logged; pool connect/drop lines, no wrapping below V1 | `fakeAPI` + `funcr` capture |
|
||||
| | `kvm_integration_test.go` (`-tags=integration`) | one real VM: image fetch, Create, idempotent re-Create, Running with IP, ListByTag, Delete, pool state afterwards | a **real hypervisor** — see below |
|
||||
| `internal/health` | `probe_test.go` | probes through a **real CONNECT-capable proxy stub** to a real TLS server: tunnel success, refused CONNECT, wrong status, dead proxy, plain-http forward | `httptest` + hijacked bidirectional tunnel |
|
||||
| | `engine_test.go` | thresholds, first-verdict, latency suppression matrix, dropped-event retry, stale-probe UID guard, tick scheduling/seeding/pruning, `Start` end-to-end | fake `client.Reader` + injected `probeFn` |
|
||||
| `internal/lease` | `store_test.go` | capacity, `MaxLeases=0`, all three selection tie-breaks, target-scoped vs global cooldowns, expiry via fake clock, report-on-retained-lease, idempotent release, **40 concurrent acquires never exceeding capacity** | injectable clock; the concurrency case is why `-race` matters |
|
||||
@@ -88,59 +83,6 @@ envtest suite the scaffold generated.
|
||||
bug (below).
|
||||
- **Live GCP**: no test talks to Google. The seam boundary
|
||||
(`buildInsertRequest` + classification) is tested exhaustively instead.
|
||||
- **Live libvirt, by default**: the KVM unit tests run over an in-memory
|
||||
fake of the 17 libvirt calls the provider uses. Whether libvirt accepts
|
||||
the domain XML, whether a raw qcow2 upload plus pool refresh yields a
|
||||
usable backing image, and whether the guest actually boots and reports
|
||||
an address are provable only on a real host — that is the opt-in
|
||||
integration test below.
|
||||
|
||||
## The KVM integration test (real hypervisor)
|
||||
|
||||
`internal/provider/kvm/kvm_integration_test.go` compiles only under
|
||||
`-tags=integration` and skips itself unless the environment names a
|
||||
host. It deviates from the "testcontainers for real infra" convention on
|
||||
purpose: libvirt needs `/dev/kvm` and a bridge to a DHCP network, and a
|
||||
container has neither. So it targets whichever hypervisor you point it
|
||||
at and leaves that host's pool with one more cached base image than it
|
||||
had.
|
||||
|
||||
```sh
|
||||
export KVM_TEST_HOST=nas1.home.lan:22 # SSH endpoint
|
||||
export KVM_TEST_USER=virt # must be in the libvirt group
|
||||
export KVM_TEST_SSH_KEY=$HOME/.ssh/id_ed25519 # private key for that user
|
||||
export KVM_TEST_BRIDGE=br0 # bridge to a network with DHCP
|
||||
export KVM_TEST_IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||
make test-integration-kvm
|
||||
```
|
||||
|
||||
Optional: `KVM_TEST_KNOWN_HOSTS` (without it host keys are **not**
|
||||
verified — the log says so), `KVM_TEST_IMAGE_SHA256`, `KVM_TEST_POOL`
|
||||
(default `default`), `KVM_TEST_SOCKET`, `KVM_TEST_TIMEOUT` (default
|
||||
`20m`, must cover the first-time image download), `KVM_TEST_KEEP`
|
||||
to leave the VM behind for `virsh console` when something goes wrong,
|
||||
and `KVM_TEST_LOG_LEVEL` (default `1`) for how much of the provider's
|
||||
logging the `-v` output shows: at `1` every libvirt RPC is one compact
|
||||
line (`HH:MM:SS.mmm libvirt virDomainLookupByName domain=... duration=...
|
||||
error=... libvirtError=ErrNoDomain`), interleaved with the test's own
|
||||
`t.Log` lines; `2` adds the domain and volume XML, metadata and
|
||||
addresses; `0` leaves only `t.Log`. The per-run constants (provider,
|
||||
host, domain name) are left off each line. See the Logging section of
|
||||
[kvm-provider.md](kvm-provider.md#logging).
|
||||
|
||||
What one run asserts, in order: `New` dials the host; `Create` returns a
|
||||
`hosts/<host>/domains/<name>` ID (downloading the image on a cold host —
|
||||
the log prints how long it took); a second `Create` returns the same ID;
|
||||
`Get` reaches `Running` with an IPv4 within the budget, and the UID and
|
||||
`CreatedAt` survive the metadata round-trip through libvirt; `ListByTag`
|
||||
lists it; `Delete` removes it, `Get` says `ErrNotFound`, a second
|
||||
`Delete` is a no-op, and the pool holds the base image and its marker
|
||||
but no per-VM overlay or seed. It also tries a TCP connect to port 22 on
|
||||
the guest, but only logs the outcome — the machine running the test may
|
||||
not route to the bridge network.
|
||||
|
||||
The test is not part of `make test` and not run in CI. Keep it compiling
|
||||
with `go vet -tags=integration ./...` when the provider's internals move.
|
||||
|
||||
## The scaffolded `make test-e2e` suite
|
||||
|
||||
@@ -150,38 +92,8 @@ compiles only under `-tags=e2e`, manages its own kind cluster
|
||||
(`make test-e2e` / `make cleanup-test-e2e`), and has been kept compiling
|
||||
(`go vet -tags=e2e ./...` is part of the routine) but is **not part of
|
||||
`make test` and was not used for the release verification** — the manual
|
||||
kind run below covers strictly more.
|
||||
|
||||
### The OTel tracing spec (Tempo-gated)
|
||||
|
||||
`test/e2e/tracing_test.go` proves the full tracing pipeline against a
|
||||
**real Tempo**: deploy in kind with tracing enabled, create two
|
||||
kubernetes-provider proxies, delete them, and assert in Tempo that the
|
||||
traces exist with the documented span topology (`Reconcile Proxy` →
|
||||
`reconcile.managed` / `provider.create` / `status.patch`, and
|
||||
`reconcile.delete` / `provider.delete` on the way out).
|
||||
|
||||
It **skips unless both env vars are set** (so the rest of the suite runs
|
||||
anywhere). Homelab invocation:
|
||||
|
||||
```bash
|
||||
TEMPO_URL=http://192.168.0.30:3200 \
|
||||
OTLP_ENDPOINT=http://192.168.0.30:4318 \
|
||||
make test-e2e
|
||||
```
|
||||
|
||||
Worth knowing:
|
||||
|
||||
- Every span of a run carries the resource attribute
|
||||
`test.run.id=e2e-<nanos>` (injected via `OTEL_RESOURCE_ATTRIBUTES`, no
|
||||
code involved); the run ID is printed in the test log, and
|
||||
`{resource.test.run.id="<id>"}` in Grafana shows exactly that run.
|
||||
- The spec preflights the OTLP endpoint **from inside the cluster** with a
|
||||
curl pod and fails fast with a clear message if it's unreachable —
|
||||
export failures are otherwise only visible at `-zap-log-level=1`.
|
||||
- Proxy CRs are created in `default`, not the operator namespace: the
|
||||
squid pods carry no securityContext and the operator namespace enforces
|
||||
restricted PSS.
|
||||
kind run below covers strictly more. Treat it as scaffold to grow into if
|
||||
CI wants an automated cluster smoke test.
|
||||
|
||||
## The kind verification run (the real end-to-end)
|
||||
|
||||
|
||||
@@ -5,19 +5,12 @@ go 1.26.0
|
||||
require (
|
||||
cloud.google.com/go/compute v1.65.0
|
||||
github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858
|
||||
github.com/go-logr/logr v1.4.4
|
||||
github.com/go-logr/logr v1.4.3
|
||||
github.com/kdomanski/iso9660 v0.4.0
|
||||
github.com/onsi/ginkgo/v2 v2.27.4
|
||||
github.com/onsi/gomega v1.39.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
|
||||
go.opentelemetry.io/otel v1.45.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0
|
||||
go.opentelemetry.io/otel/sdk v1.45.0
|
||||
go.opentelemetry.io/otel/trace v1.45.0
|
||||
golang.org/x/crypto v0.54.0
|
||||
golang.org/x/crypto v0.56.0
|
||||
google.golang.org/api v0.292.0
|
||||
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
|
||||
k8s.io/api v0.36.0
|
||||
@@ -41,7 +34,7 @@ require (
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/emicklei/go-restful/v3 v3.13.0 // indirect
|
||||
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
@@ -58,7 +51,7 @@ require (
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.23.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
@@ -76,26 +69,31 @@ require (
|
||||
github.com/stoewer/go-strcase v1.3.0 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
|
||||
golang.org/x/mod v0.37.0 // indirect
|
||||
golang.org/x/mod v0.38.0 // indirect
|
||||
golang.org/x/net v0.57.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/term v0.45.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/tools v0.47.0 // indirect
|
||||
golang.org/x/tools v0.48.0 // indirect
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
|
||||
google.golang.org/grpc v1.83.0 // indirect
|
||||
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
|
||||
|
||||
@@ -36,8 +36,8 @@ github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8
|
||||
github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
|
||||
github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
|
||||
github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
|
||||
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
|
||||
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
|
||||
@@ -49,8 +49,8 @@ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6O
|
||||
github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
|
||||
github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
|
||||
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
|
||||
@@ -88,8 +88,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.19 h1:mMOE7DN2+p76/EdIrm
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k=
|
||||
github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE=
|
||||
github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
@@ -174,28 +174,24 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
|
||||
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
|
||||
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=
|
||||
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
|
||||
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
|
||||
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
|
||||
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
|
||||
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
|
||||
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
@@ -206,12 +202,12 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
|
||||
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
|
||||
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
||||
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
||||
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
|
||||
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
@@ -222,12 +218,12 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
|
||||
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
|
||||
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
|
||||
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
@@ -236,8 +232,8 @@ google.golang.org/api v0.292.0 h1:Ewiwo/GTtiaPZSNAZQUcWLh8AYDEoPmIXyJfeoTSMHU=
|
||||
google.golang.org/api v0.292.0/go.mod h1:07kjmMnFGm2RQuCza2EZM/5N68G/fVvFb1xKjWqoFA0=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
|
||||
|
||||
@@ -41,7 +41,6 @@ import (
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
// HealthSnapshotter provides the current probe verdict for a proxy. The
|
||||
@@ -97,14 +96,9 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
|
||||
}
|
||||
base := p.DeepCopy()
|
||||
defer func() {
|
||||
// Runs inside the root reconcile span (this defer fires before the
|
||||
// tracing.NewReconciler wrapper sees the return), and its error is
|
||||
// folded into err, which that wrapper records.
|
||||
pctx, span := tracing.Start(ctx, "status.patch")
|
||||
defer span.End()
|
||||
// NotFound is expected when this reconcile just removed the last
|
||||
// finalizer and the object is already gone.
|
||||
if perr := r.patchStatusIfChanged(pctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
|
||||
if perr := r.patchStatusIfChanged(ctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
|
||||
err = errors.Join(err, perr)
|
||||
}
|
||||
}()
|
||||
@@ -120,8 +114,6 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
|
||||
}
|
||||
|
||||
func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
|
||||
ctx, span := tracing.Start(ctx, "reconcile.managed")
|
||||
defer span.End()
|
||||
log := logf.FromContext(ctx)
|
||||
|
||||
if controllerutil.AddFinalizer(p, crawlv1alpha1.FinalizerName) {
|
||||
@@ -232,8 +224,6 @@ func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1
|
||||
// exists" — hence: delete, poll to NotFound, only then advance the hash and
|
||||
// let the create branch run.
|
||||
func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.Proxy, prov provider.Provider, hash string) (ctrl.Result, error) {
|
||||
ctx, span := tracing.Start(ctx, "reconcile.replaceInstance")
|
||||
defer span.End()
|
||||
_, err := prov.Get(ctx, p.Status.ProviderID)
|
||||
if provider.Class(err) == provider.ErrNotFound {
|
||||
// Old instance is gone. The Update inside setSpecHash refreshes p
|
||||
@@ -262,22 +252,14 @@ func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.
|
||||
}
|
||||
|
||||
func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
|
||||
ctx, span := tracing.Start(ctx, "reconcile.delete")
|
||||
defer span.End()
|
||||
if !controllerutil.ContainsFinalizer(p, crawlv1alpha1.FinalizerName) {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
if p.Status.ProviderID == "" {
|
||||
// Nothing was ever recorded as created; orphan GC reaps any stray
|
||||
// instance a crashed create might have left behind.
|
||||
return r.releaseFinalizer(ctx, p)
|
||||
}
|
||||
if p.Annotations[crawlv1alpha1.AnnotationSkipCleanup] == "true" {
|
||||
// Checked before the provider lookup so it also releases a Proxy whose
|
||||
// provider is gone from the config. The instance is left running.
|
||||
logf.FromContext(ctx).Info("skip-cleanup annotation set; abandoning the instance",
|
||||
"providerID", p.Status.ProviderID, "provider", p.Spec.Provider)
|
||||
return r.releaseFinalizer(ctx, p)
|
||||
controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
|
||||
return ctrl.Result{}, r.Update(ctx, p)
|
||||
}
|
||||
prov, ok := r.Providers[p.Spec.Provider]
|
||||
if !ok {
|
||||
@@ -285,22 +267,11 @@ func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.
|
||||
"provider %q is not configured; cannot clean up instance %s", p.Spec.Provider, p.Status.ProviderID)
|
||||
}
|
||||
_, err := prov.Get(ctx, p.Status.ProviderID)
|
||||
switch {
|
||||
case err == nil:
|
||||
case provider.Class(err) == provider.ErrNotFound:
|
||||
return r.releaseFinalizer(ctx, p)
|
||||
case provider.Class(err) == provider.ErrPermanent:
|
||||
// Get will never answer for this instance — a host retired from the
|
||||
// config, a providerID the provider can no longer resolve. Delete is
|
||||
// idempotent and may still finish, so try it instead of polling a Get
|
||||
// that cannot converge, and release the object when it does.
|
||||
if delErr := prov.Delete(ctx, p.Status.ProviderID); delErr != nil {
|
||||
return r.deletionFailure(errors.Join(err, delErr))
|
||||
}
|
||||
logf.FromContext(ctx).Info("instance unreachable but cleanup succeeded; removing the finalizer",
|
||||
"providerID", p.Status.ProviderID, "error", err.Error())
|
||||
return r.releaseFinalizer(ctx, p)
|
||||
default:
|
||||
if provider.Class(err) == provider.ErrNotFound {
|
||||
controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
|
||||
return ctrl.Result{}, r.Update(ctx, p)
|
||||
}
|
||||
if err != nil {
|
||||
return r.deletionFailure(err)
|
||||
}
|
||||
if err := prov.Delete(ctx, p.Status.ProviderID); err != nil {
|
||||
@@ -311,11 +282,6 @@ func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.
|
||||
return ctrl.Result{RequeueAfter: r.DeletionPoll}, nil
|
||||
}
|
||||
|
||||
func (r *ProxyReconciler) releaseFinalizer(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
|
||||
controllerutil.RemoveFinalizer(p, crawlv1alpha1.FinalizerName)
|
||||
return ctrl.Result{}, r.Update(ctx, p)
|
||||
}
|
||||
|
||||
func (r *ProxyReconciler) reconcileExternal(_ context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
|
||||
if p.Spec.Endpoint == nil {
|
||||
// CEL guarantees an endpoint on any object that went through the API
|
||||
@@ -443,10 +409,7 @@ func (r *ProxyReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
if r.HealthEvents != nil {
|
||||
b = b.WatchesRawSource(source.Channel(r.HealthEvents, &handler.EnqueueRequestForObject{}))
|
||||
}
|
||||
// Root span per reconcile; sub-reconcilers and the status flush hang
|
||||
// their spans off it. Tests calling r.Reconcile directly bypass the
|
||||
// wrapper and see no-op spans — the global tracer is never set there.
|
||||
return b.Complete(tracing.NewReconciler("Proxy", r))
|
||||
return b.Complete(r)
|
||||
}
|
||||
|
||||
func (r *ProxyReconciler) applyDefaults() {
|
||||
|
||||
@@ -2,7 +2,6 @@ package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -79,10 +78,6 @@ func notFoundErr() error {
|
||||
return provider.Wrap(provider.ErrNotFound, "get", "stub", "some-id", nil)
|
||||
}
|
||||
|
||||
func permanentErr() error {
|
||||
return provider.Wrap(provider.ErrPermanent, "get", "stub", "some-id", nil)
|
||||
}
|
||||
|
||||
func testScheme(t *testing.T) *runtime.Scheme {
|
||||
t.Helper()
|
||||
s := runtime.NewScheme()
|
||||
@@ -130,15 +125,6 @@ func withSpecHashAnnotation(hash string) func(*crawlv1alpha1.Proxy) {
|
||||
}
|
||||
}
|
||||
|
||||
func withSkipCleanup() func(*crawlv1alpha1.Proxy) {
|
||||
return func(p *crawlv1alpha1.Proxy) {
|
||||
if p.Annotations == nil {
|
||||
p.Annotations = map[string]string{}
|
||||
}
|
||||
p.Annotations[crawlv1alpha1.AnnotationSkipCleanup] = "true"
|
||||
}
|
||||
}
|
||||
|
||||
func deleting() func(*crawlv1alpha1.Proxy) {
|
||||
return func(p *crawlv1alpha1.Proxy) {
|
||||
now := metav1.Now()
|
||||
@@ -389,59 +375,6 @@ func TestReconcile_actionTable(t *testing.T) {
|
||||
assertProxyGone(t, r)
|
||||
},
|
||||
},
|
||||
{
|
||||
// A permanent Get (a retired KVM host, a providerID the provider
|
||||
// cannot resolve) used to retry forever with the finalizer on.
|
||||
name: "deletion with an unreachable instance still deletes and releases",
|
||||
proxy: managedProxy(deleting(), withProviderID("stub-id-1")),
|
||||
stub: &stubProvider{getErr: permanentErr()},
|
||||
wantResult: ctrl.Result{},
|
||||
verify: func(t *testing.T, r *ProxyReconciler, stub *stubProvider) {
|
||||
assertProxyGone(t, r)
|
||||
if stub.deleteCalls != 1 {
|
||||
t.Errorf("deleteCalls = %d, want 1 — Delete must be tried when Get cannot answer", stub.deleteCalls)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "deletion keeps the finalizer when cleanup itself fails",
|
||||
proxy: managedProxy(deleting(), withProviderID("stub-id-1")),
|
||||
stub: &stubProvider{
|
||||
getErr: permanentErr(),
|
||||
deleteErr: provider.Wrap(provider.ErrPermanent, "delete", "stub", "stub-id-1", nil),
|
||||
},
|
||||
wantResult: ctrl.Result{},
|
||||
wantErr: true,
|
||||
verify: func(t *testing.T, r *ProxyReconciler, _ *stubProvider) {
|
||||
// getProxy fails the test if the object is gone, which is what
|
||||
// finalizer removal would have caused.
|
||||
if p := getProxy(t, r); !slices.Contains(p.Finalizers, crawlv1alpha1.FinalizerName) {
|
||||
t.Error("finalizer removed although cleanup failed; the instance would leak silently")
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "skip-cleanup annotation abandons the instance",
|
||||
proxy: managedProxy(deleting(), withProviderID("stub-id-1"), withSkipCleanup()),
|
||||
stub: &stubProvider{getInst: &provider.Instance{ID: "stub-id-1", State: provider.StateRunning}},
|
||||
wantResult: ctrl.Result{},
|
||||
verify: func(t *testing.T, r *ProxyReconciler, stub *stubProvider) {
|
||||
assertProxyGone(t, r)
|
||||
if stub.getCalls != 0 || stub.deleteCalls != 0 {
|
||||
t.Errorf("getCalls/deleteCalls = %d/%d, want 0/0 — the provider must not be asked", stub.getCalls, stub.deleteCalls)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "skip-cleanup annotation releases a proxy whose provider is gone",
|
||||
proxy: managedProxy(deleting(), withProviderID("stub-id-1"), withSkipCleanup(),
|
||||
func(p *crawlv1alpha1.Proxy) { p.Spec.Provider = "retired" }),
|
||||
stub: &stubProvider{},
|
||||
wantResult: ctrl.Result{},
|
||||
verify: func(t *testing.T, r *ProxyReconciler, _ *stubProvider) {
|
||||
assertProxyGone(t, r)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "deletion deletes the instance and polls",
|
||||
proxy: managedProxy(deleting(), withProviderID("stub-id-1")),
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
|
||||
@@ -83,10 +82,7 @@ func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
var list crawlv1alpha1.ProxyList
|
||||
if err := s.Reader.List(r.Context(), &list); err != nil {
|
||||
// The ctx logger is s.log enriched with this request's
|
||||
// traceID/spanID by the tracing middleware (logr sinks can't read
|
||||
// ctx at log time, so per-request values ride on the logger).
|
||||
logf.FromContext(r.Context()).Error(err, "listing proxies")
|
||||
s.log.Error(err, "listing proxies")
|
||||
writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
|
||||
return
|
||||
}
|
||||
@@ -142,7 +138,7 @@ func (s *Server) handleAcquireLease(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
var list crawlv1alpha1.ProxyList
|
||||
if err := s.Reader.List(r.Context(), &list); err != nil {
|
||||
logf.FromContext(r.Context()).Error(err, "listing proxies for lease")
|
||||
s.log.Error(err, "listing proxies for lease")
|
||||
writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
|
||||
return
|
||||
}
|
||||
@@ -235,7 +231,7 @@ func (s *Server) handleReportLease(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusNotFound, "unknown_lease", "no such lease")
|
||||
return
|
||||
}
|
||||
logf.FromContext(r.Context()).Error(err, "reporting lease", "leaseID", r.PathValue("id"))
|
||||
s.log.Error(err, "reporting lease", "leaseID", r.PathValue("id"))
|
||||
writeError(w, http.StatusInternalServerError, "internal", "report failed")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
// LeaseStore is what the handlers need from a lease backend. Defined here,
|
||||
@@ -137,9 +136,7 @@ func (s *Server) Start(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// handler assembles the mux and the middleware chain, outermost first:
|
||||
// recover → tracing (server span + request logger) → request-log →
|
||||
// body-size cap → bearer auth. Everything inside the tracing layer logs via
|
||||
// logf.FromContext(r.Context()) and so carries traceID/spanID.
|
||||
// recover → request-log → body-size cap → bearer auth.
|
||||
func (s *Server) handler() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
@@ -155,7 +152,6 @@ func (s *Server) handler() http.Handler {
|
||||
h = s.authMiddleware(h)
|
||||
h = maxBytesMiddleware(h)
|
||||
h = s.logMiddleware(h)
|
||||
h = tracing.HTTPMiddleware("discovery", s.log)(h)
|
||||
h = s.recoverMiddleware(h)
|
||||
return h
|
||||
}
|
||||
@@ -192,7 +188,7 @@ func (s *Server) logMiddleware(next http.Handler) http.Handler {
|
||||
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
|
||||
start := time.Now()
|
||||
next.ServeHTTP(rec, r)
|
||||
logf.FromContext(r.Context()).Info("request",
|
||||
s.log.Info("request",
|
||||
"method", r.Method, "path", r.URL.Path,
|
||||
"status", rec.status, "duration", time.Since(start).String())
|
||||
})
|
||||
|
||||
@@ -12,11 +12,8 @@ import (
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
// Sweeper is the manager Runnable running the sweep loop.
|
||||
@@ -84,8 +81,6 @@ func (s *Sweeper) Start(ctx context.Context) error {
|
||||
// that deletion, and GC racing it would double-delete. A UID becomes
|
||||
// orphan-eligible only once the object is fully gone.
|
||||
func (s *Sweeper) sweep(ctx context.Context) {
|
||||
ctx, span := tracing.Start(ctx, "gc.sweep")
|
||||
defer span.End()
|
||||
log := logf.FromContext(ctx).WithName("orphan-gc")
|
||||
|
||||
var list crawlv1alpha1.ProxyList
|
||||
@@ -100,7 +95,6 @@ func (s *Sweeper) sweep(ctx context.Context) {
|
||||
live[string(list.Items[i].UID)] = true
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
for name, prov := range s.Providers {
|
||||
instances, err := prov.ListByTag(ctx)
|
||||
if err != nil {
|
||||
@@ -125,13 +119,7 @@ func (s *Sweeper) sweep(ctx context.Context) {
|
||||
if err := prov.Delete(ctx, inst.ID); err != nil {
|
||||
log.Error(err, "deleting orphaned instance",
|
||||
"provider", name, "providerID", inst.ID, "uid", inst.UID)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
}
|
||||
span.SetAttributes(
|
||||
attribute.Int("gc.providers", len(s.Providers)),
|
||||
attribute.Int("gc.deleted", deleted),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -10,9 +10,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
@@ -21,7 +18,6 @@ import (
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
// Snapshot is the engine's current verdict for one proxy, read by the
|
||||
@@ -103,10 +99,6 @@ type Engine struct {
|
||||
// transition-only by design, so metrics are where high-frequency
|
||||
// signal (true probe recency, every latency sample) lives.
|
||||
Metrics ProbeMetrics
|
||||
// TraceProbes emits one root span per probe (--trace-health-probes).
|
||||
// Off by default: probes run about once per second per proxy and would
|
||||
// dominate trace volume.
|
||||
TraceProbes bool
|
||||
|
||||
probeFn func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult
|
||||
|
||||
@@ -163,7 +155,8 @@ func (e *Engine) Start(ctx context.Context) error {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case job := <-jobs:
|
||||
e.record(job, e.runProbe(ctx, job), time.Now())
|
||||
res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
|
||||
e.record(job, res, time.Now())
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -271,29 +264,6 @@ func newState(p *crawlv1alpha1.Proxy, now time.Time, interval time.Duration) *st
|
||||
return st
|
||||
}
|
||||
|
||||
// runProbe executes one probe, inside its own root span when TraceProbes is
|
||||
// on. The span ends before record folds the result (record stays ctx-free),
|
||||
// and the probe transport is deliberately uninstrumented — no traceparent
|
||||
// must ever leak through a proxy toward external targets.
|
||||
func (e *Engine) runProbe(ctx context.Context, job probeJob) probeResult {
|
||||
if !e.TraceProbes {
|
||||
return e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
|
||||
}
|
||||
ctx, span := tracing.Start(ctx, "health.probe",
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("proxy", job.key.String())))
|
||||
defer span.End()
|
||||
res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
|
||||
span.SetAttributes(
|
||||
attribute.Bool("probe.ok", res.ok),
|
||||
attribute.Int64("probe.latency_ms", res.latency.Milliseconds()),
|
||||
)
|
||||
if !res.ok {
|
||||
span.SetStatus(codes.Error, res.err.Error())
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// record folds one probe result into the proxy's threshold state and emits
|
||||
// an event when the result is status-affecting: a first-ever verdict, a
|
||||
// threshold-crossing flip, or a material latency change (beyond
|
||||
|
||||
@@ -22,18 +22,10 @@ type KVMConfig struct {
|
||||
// DiskSizeGB is the capacity of each VM's overlay disk. Default 10. The
|
||||
// guest's cloud-init growpart expands the root filesystem into it.
|
||||
DiskSizeGB int64 `json:"diskSizeGb,omitempty"`
|
||||
// VendorData is optional cloud-init vendor-data placed on every seed next
|
||||
// to the Proxy's own user-data. Note that cloud-init drops any top-level
|
||||
// key of a vendor-data cloud-config that the user-data also defines.
|
||||
// When InstallGuestAgent is on, this document and the agent script are
|
||||
// carried as separate parts of a MIME multipart vendor-data.
|
||||
// VendorData is cloud-init vendor-data placed on every seed next to the
|
||||
// Proxy's own user-data. Default: installs and starts qemu-guest-agent so
|
||||
// the provider can read the VM's IP even before it shows up in ARP.
|
||||
VendorData string `json:"vendorData,omitempty"`
|
||||
// InstallGuestAgent adds a vendor-data script that installs and starts
|
||||
// qemu-guest-agent, which is how the provider learns a VM's address
|
||||
// before the host's ARP table has seen traffic from it. Default true.
|
||||
// Turn it off for images that already ship the agent, or that cannot
|
||||
// reach a package mirror; discovery then depends on ARP alone.
|
||||
InstallGuestAgent *bool `json:"installGuestAgent,omitempty"`
|
||||
// Images are named base images; placement.image names a key here (or is
|
||||
// an http(s) URL directly). The provider downloads each image into a
|
||||
// host's storage pool on first use and caches it there.
|
||||
@@ -55,11 +47,6 @@ type KVMImage struct {
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
// Format is the image's disk format, "qcow2" (default) or "raw".
|
||||
Format string `json:"format,omitempty"`
|
||||
// Firmware is how VMs from this image boot: "uefi" (default; OVMF with
|
||||
// secure boot off) or "bios" (SeaBIOS) for images with no EFI system
|
||||
// partition. Current Debian, Ubuntu, Fedora and Rocky cloud images all
|
||||
// boot UEFI, and Debian 13's BIOS path was found not to boot at all.
|
||||
Firmware string `json:"firmware,omitempty"`
|
||||
}
|
||||
|
||||
// KVMMachineType is a VM size profile.
|
||||
@@ -88,31 +75,8 @@ type KVMHost struct {
|
||||
Bridge string `json:"bridge"`
|
||||
}
|
||||
|
||||
// Accepted KVMImage.Format and KVMImage.Firmware values.
|
||||
const (
|
||||
KVMFormatQcow2 = "qcow2"
|
||||
KVMFormatRaw = "raw"
|
||||
KVMFirmwareUEFI = "uefi"
|
||||
KVMFirmwareBIOS = "bios"
|
||||
)
|
||||
|
||||
// IsHTTPURL reports whether s is an absolute http(s) URL with a host, the
|
||||
// only kind of image source the kvm provider can fetch.
|
||||
func IsHTTPURL(s string) bool {
|
||||
u, err := url.Parse(s)
|
||||
return err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != ""
|
||||
}
|
||||
|
||||
var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
|
||||
|
||||
// kvmIdentifier constrains the config names that are pasted into
|
||||
// identifiers elsewhere: a host name goes into the providerID
|
||||
// hosts/<name>/domains/<vm>, which is split on "/" to parse, and an image
|
||||
// key becomes the file name of the cached base volume. A slash in either
|
||||
// produces an ID or a path that never resolves again, so the VM leaks with
|
||||
// nothing able to address it.
|
||||
var kvmIdentifier = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`)
|
||||
|
||||
func validateKVM(prefix string, c *KVMConfig) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("%s: kvm block is required", prefix)
|
||||
@@ -128,21 +92,15 @@ func validateKVM(prefix string, c *KVMConfig) error {
|
||||
}
|
||||
for name, img := range c.Images {
|
||||
p := fmt.Sprintf("%s: kvm.images[%q]", prefix, name)
|
||||
if !kvmIdentifier.MatchString(name) {
|
||||
return fmt.Errorf("%s: key must be letters, digits, dots, dashes or underscores"+
|
||||
" — it becomes the cached volume name base-<key>.<format>", p)
|
||||
}
|
||||
if !IsHTTPURL(img.URL) {
|
||||
u, err := url.Parse(img.URL)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return fmt.Errorf("%s.url must be an http(s) URL (got %q)", p, img.URL)
|
||||
}
|
||||
if img.SHA256 != "" && !sha256Hex.MatchString(img.SHA256) {
|
||||
return fmt.Errorf("%s.sha256 must be 64 hex characters", p)
|
||||
}
|
||||
if img.Format != "" && img.Format != KVMFormatQcow2 && img.Format != KVMFormatRaw {
|
||||
return fmt.Errorf("%s.format must be %s or %s (got %q)", p, KVMFormatQcow2, KVMFormatRaw, img.Format)
|
||||
}
|
||||
if img.Firmware != "" && img.Firmware != KVMFirmwareUEFI && img.Firmware != KVMFirmwareBIOS {
|
||||
return fmt.Errorf("%s.firmware must be %s or %s (got %q)", p, KVMFirmwareUEFI, KVMFirmwareBIOS, img.Firmware)
|
||||
if img.Format != "" && img.Format != "qcow2" && img.Format != "raw" {
|
||||
return fmt.Errorf("%s.format must be qcow2 or raw (got %q)", p, img.Format)
|
||||
}
|
||||
}
|
||||
if len(c.MachineTypes) == 0 {
|
||||
@@ -162,10 +120,6 @@ func validateKVM(prefix string, c *KVMConfig) error {
|
||||
if h.Name == "" {
|
||||
return fmt.Errorf("%s: name is required", p)
|
||||
}
|
||||
if !kvmIdentifier.MatchString(h.Name) {
|
||||
return fmt.Errorf("%s: name %q must be letters, digits, dots, dashes or underscores"+
|
||||
" — it becomes part of the providerID hosts/<name>/domains/<vm>", p, h.Name)
|
||||
}
|
||||
if seen[h.Name] {
|
||||
return fmt.Errorf("%s: duplicate host name %q", p, h.Name)
|
||||
}
|
||||
|
||||
@@ -129,19 +129,6 @@ hosts:
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n - {name: nas1, address: a:22, user: virt, bridge: br0}\n - {name: nas1, address: b:22, user: virt, bridge: br0}\n"),
|
||||
wantErrSub: `duplicate host name "nas1"`,
|
||||
},
|
||||
{
|
||||
// A slash would produce hosts/dc1/rack2/domains/<vm>, which
|
||||
// parseProviderID can never split back, so the VM would be
|
||||
// unreachable for Get, Delete and GC alike.
|
||||
name: "host name with a slash",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n - {name: dc1/rack2, address: a:22, user: virt, bridge: br0}\n"),
|
||||
wantErrSub: `name "dc1/rack2" must be letters, digits, dots, dashes or underscores`,
|
||||
},
|
||||
{
|
||||
name: "image key with a slash",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n ../etc/d: {url: https://x/y.qcow2}\n" + hostsAndTypes),
|
||||
wantErrSub: `images["../etc/d"]: key must be letters, digits, dots, dashes or underscores`,
|
||||
},
|
||||
{
|
||||
name: "host missing address",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n - {name: nas1, user: virt, bridge: br0}\n"),
|
||||
@@ -167,11 +154,6 @@ hosts:
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n d: {url: https://x/y.qcow2, sha256: abc}\n" + hostsAndTypes),
|
||||
wantErrSub: `images["d"].sha256 must be 64 hex characters`,
|
||||
},
|
||||
{
|
||||
name: "image with unknown firmware",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n d: {url: https://x/y.qcow2, firmware: coreboot}\n" + hostsAndTypes),
|
||||
wantErrSub: `images["d"].firmware must be uefi or bios`,
|
||||
},
|
||||
{
|
||||
name: "image with unknown format",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n d: {url: https://x/y.vmdk, format: vmdk}\n" + hostsAndTypes),
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// wireLogMaxFieldBytes is the elision threshold for string fields in wire
|
||||
@@ -48,15 +47,6 @@ func (h *wireFilterHandler) Handle(ctx context.Context, rec slog.Record) error {
|
||||
if rec.Level <= slog.LevelDebug && rec.Message != "api request" && rec.Message != "api response" {
|
||||
return nil
|
||||
}
|
||||
// The SDK logs with the request ctx, so wire records can carry the
|
||||
// surrounding provider span — the one place slog's ctx-aware handlers
|
||||
// beat logr, and the same keys the logr enrichment uses.
|
||||
if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
|
||||
rec = rec.Clone()
|
||||
rec.AddAttrs(
|
||||
slog.String("traceID", sc.TraceID().String()),
|
||||
slog.String("spanID", sc.SpanID().String()))
|
||||
}
|
||||
if h.fullPayloads {
|
||||
return h.inner.Handle(ctx, rec)
|
||||
}
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
package gcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-logr/logr/funcr"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
)
|
||||
|
||||
func TestWireFilterHandler_addsTraceContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var lines []string
|
||||
base := funcr.New(func(prefix, args string) {
|
||||
lines = append(lines, prefix+" "+args)
|
||||
}, funcr.Options{Verbosity: 5})
|
||||
log := wireLogger(base, WireLogOptions{})
|
||||
|
||||
tp := sdktrace.NewTracerProvider()
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
ctx, span := tp.Tracer("test").Start(context.Background(), "provider.create")
|
||||
defer span.End()
|
||||
|
||||
log.DebugContext(ctx, "api request", "url", "https://compute.googleapis.com/x")
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("got %d lines, want 1: %v", len(lines), lines)
|
||||
}
|
||||
traceID := span.SpanContext().TraceID().String()
|
||||
if !strings.Contains(lines[0], "traceID") || !strings.Contains(lines[0], traceID) {
|
||||
t.Errorf("wire record missing trace context %s: %s", traceID, lines[0])
|
||||
}
|
||||
|
||||
// The security filter must still win: non-wire Debug records are
|
||||
// dropped even when a span is present.
|
||||
log.DebugContext(ctx, "token exchange", "assertion", "secret-jwt")
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("filtered record leaked: %v", lines[1:])
|
||||
}
|
||||
|
||||
// No span in ctx: record passes through without trace keys.
|
||||
log.DebugContext(context.Background(), "api response", "status", 200)
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("got %d lines, want 2", len(lines))
|
||||
}
|
||||
if strings.Contains(lines[1], "traceID") {
|
||||
t.Errorf("spanless record must not carry traceID: %s", lines[1])
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/transport"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
@@ -40,22 +39,11 @@ type Provider struct {
|
||||
|
||||
// New builds a kubernetes Provider from its config block. Satisfies
|
||||
// registry.Constructor.
|
||||
func New(ctx context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
|
||||
return NewWithTransportWrapper(ctx, cfg, nil)
|
||||
}
|
||||
|
||||
// NewWithTransportWrapper is New with an optional transport wrapper applied
|
||||
// to the provider's own rest.Config (this client is built independently of
|
||||
// the manager's, so the composition root must wrap it separately for
|
||||
// tracing). A nil wrapper means a plain client.
|
||||
func NewWithTransportWrapper(_ context.Context, cfg provider.ProviderConfig, wrap transport.WrapperFunc) (provider.Provider, error) {
|
||||
func New(_ context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
|
||||
restCfg, err := ctrl.GetConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kubernetes provider %q: loading kubeconfig: %w", cfg.Name, err)
|
||||
}
|
||||
if wrap != nil {
|
||||
restCfg.Wrap(wrap)
|
||||
}
|
||||
scheme := runtime.NewScheme()
|
||||
if err := corev1.AddToScheme(scheme); err != nil {
|
||||
return nil, fmt.Errorf("kubernetes provider %q: %w", cfg.Name, err)
|
||||
|
||||
@@ -2,17 +2,14 @@ package kvm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
"github.com/go-logr/logr"
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
|
||||
@@ -98,7 +95,7 @@ func (p *hostPool) entry(name string) *hostEntry {
|
||||
return e
|
||||
}
|
||||
|
||||
func (p *hostPool) get(ctx context.Context, log logr.Logger, host provider.KVMHost) (*hostClient, error) {
|
||||
func (p *hostPool) get(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
|
||||
e := p.entry(host.Name)
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
@@ -106,17 +103,13 @@ func (p *hostPool) get(ctx context.Context, log logr.Logger, host provider.KVMHo
|
||||
if e.client.alive() {
|
||||
return e.client, nil
|
||||
}
|
||||
log.V(1).Info("libvirt connection lost, redialing", "address", host.Address)
|
||||
e.client.close()
|
||||
e.client = nil
|
||||
}
|
||||
start := time.Now()
|
||||
c, err := p.dial(ctx, host)
|
||||
if err != nil {
|
||||
log.V(1).Info("libvirt host dial failed", "address", host.Address, "duration", time.Since(start).Round(time.Millisecond), "error", err.Error())
|
||||
return nil, fmt.Errorf("connecting to host %s (%s): %w", host.Name, host.Address, err)
|
||||
}
|
||||
log.V(1).Info("libvirt host connected", "address", host.Address, "duration", time.Since(start).Round(time.Millisecond))
|
||||
e.client = c
|
||||
return c, nil
|
||||
}
|
||||
@@ -136,29 +129,16 @@ func (p *hostPool) drop(host string, c *hostClient) {
|
||||
// call runs fn against host's connection. go-libvirt RPCs take no context
|
||||
// and block until the daemon answers, so when ctx ends first the connection
|
||||
// is closed — which unblocks fn with an error — and ctx.Err() is returned.
|
||||
// A transport error also drops the cached connection so the next call
|
||||
// redials instead of reusing a dead link. "Transport error" means anything
|
||||
// that is neither a libvirt.Error nor an error fn already classified with a
|
||||
// provider sentinel (a failed seed build, a bad placement): those are local
|
||||
// verdicts, not evidence about the link. When log has V(1) enabled, every
|
||||
// RPC fn makes is traced through it (see tracingAPI), as are connection
|
||||
// events.
|
||||
func (p *hostPool) call(ctx context.Context, log logr.Logger, host provider.KVMHost, fn func(api libvirtAPI) error) error {
|
||||
log = log.WithName("libvirt")
|
||||
c, err := p.get(ctx, log, host)
|
||||
// A transport error (anything that is not a libvirt.Error) also drops the
|
||||
// cached connection so the next call redials instead of reusing a dead
|
||||
// link.
|
||||
func (p *hostPool) call(ctx context.Context, host provider.KVMHost, fn func(api libvirtAPI) error) error {
|
||||
c, err := p.get(ctx, host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
api := c.api
|
||||
if log.V(1).Enabled() {
|
||||
api = &tracingAPI{inner: c.api, log: log}
|
||||
}
|
||||
err = awaitOrCancel(ctx, func() error { return fn(api) }, func() {
|
||||
log.V(1).Info("libvirt connection dropped", "reason", "context cancelled", "error", ctx.Err().Error())
|
||||
p.drop(host.Name, c)
|
||||
})
|
||||
if err != nil && !isLibvirtError(err) && !isClassified(err) && !errors.Is(err, ctx.Err()) {
|
||||
log.V(1).Info("libvirt connection dropped", "reason", "transport error", "error", err.Error())
|
||||
err = awaitOrCancel(ctx, func() error { return fn(c.api) }, func() { p.drop(host.Name, c) })
|
||||
if err != nil && !isLibvirtError(err) {
|
||||
p.drop(host.Name, c)
|
||||
}
|
||||
return err
|
||||
@@ -181,17 +161,14 @@ func awaitOrCancel(ctx context.Context, fn func() error, abort func()) error {
|
||||
}
|
||||
|
||||
// sshAuth holds the pieces of the SSH client config shared by every host:
|
||||
// the private key and the host-key policy. Both are loaded on the first
|
||||
// connection rather than at startup — see lazySSH.
|
||||
// the private key and the host-key policy, both loaded once at startup so a
|
||||
// missing or unreadable file fails the operator fast rather than on the
|
||||
// first Create.
|
||||
type sshAuth struct {
|
||||
signer ssh.Signer
|
||||
hostKey ssh.HostKeyCallback
|
||||
}
|
||||
|
||||
// newSSHAuth loads the key material. A file that is not there yet stays
|
||||
// unclassified, so the provider treats it as transient and heals when the
|
||||
// Secret appears; content that will never parse is permanent, since
|
||||
// retrying cannot fix it.
|
||||
func newSSHAuth(cfg provider.KVMConfig) (*sshAuth, error) {
|
||||
key, err := os.ReadFile(cfg.SSHKeyFile)
|
||||
if err != nil {
|
||||
@@ -199,79 +176,34 @@ func newSSHAuth(cfg provider.KVMConfig) (*sshAuth, error) {
|
||||
}
|
||||
signer, err := ssh.ParsePrivateKey(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: parsing kvm.sshKeyFile: %v", provider.ErrPermanent, err)
|
||||
return nil, fmt.Errorf("parsing kvm.sshKeyFile: %w", err)
|
||||
}
|
||||
hostKey := ssh.InsecureIgnoreHostKey() //nolint:gosec // opt-in via config, documented as insecure
|
||||
if !cfg.InsecureIgnoreHostKey {
|
||||
hostKey, err = knownhosts.New(cfg.KnownHostsFile)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, fmt.Errorf("loading kvm.knownHostsFile: %w", err)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: loading kvm.knownHostsFile: %v", provider.ErrPermanent, err)
|
||||
return nil, fmt.Errorf("loading kvm.knownHostsFile: %w", err)
|
||||
}
|
||||
}
|
||||
return &sshAuth{signer: signer, hostKey: hostKey}, nil
|
||||
}
|
||||
|
||||
// lazySSH defers loading the SSH material until it is first needed and
|
||||
// caches it once it works. The kvm-ssh Secret is an optional mount: the
|
||||
// manager has to start without it and keep every other provider running,
|
||||
// and a Secret created afterwards has to heal the KVM proxies on the next
|
||||
// reconcile rather than needing a restart.
|
||||
type lazySSH struct {
|
||||
cfg provider.KVMConfig
|
||||
|
||||
mu sync.Mutex
|
||||
auth *sshAuth
|
||||
}
|
||||
|
||||
// newLazySSH keeps only the SSH fields of cfg, since the loader lives as
|
||||
// long as the Provider.
|
||||
func newLazySSH(cfg provider.KVMConfig) *lazySSH {
|
||||
return &lazySSH{cfg: provider.KVMConfig{
|
||||
SSHKeyFile: cfg.SSHKeyFile,
|
||||
KnownHostsFile: cfg.KnownHostsFile,
|
||||
InsecureIgnoreHostKey: cfg.InsecureIgnoreHostKey,
|
||||
}}
|
||||
}
|
||||
|
||||
func (l *lazySSH) load() (*sshAuth, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.auth != nil {
|
||||
return l.auth, nil
|
||||
// dialer returns the production hostDialer: SSH to the host, forward its
|
||||
// libvirt unix socket through the connection, and open a libvirt session
|
||||
// over that.
|
||||
func (a *sshAuth) dialer() hostDialer {
|
||||
return func(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
|
||||
client, err := a.dialSSH(ctx, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hc, err := connectLibvirt(ctx, client, host.Socket)
|
||||
if err != nil {
|
||||
client.Close()
|
||||
return nil, err
|
||||
}
|
||||
return hc, nil
|
||||
}
|
||||
auth, err := newSSHAuth(l.cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l.auth = auth
|
||||
return auth, nil
|
||||
}
|
||||
|
||||
// dial is the production hostDialer.
|
||||
func (l *lazySSH) dial(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
|
||||
a, err := l.load()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.dial(ctx, host)
|
||||
}
|
||||
|
||||
// dial SSHes to the host, forwards its libvirt unix socket through the
|
||||
// connection, and opens a libvirt session over that.
|
||||
func (a *sshAuth) dial(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
|
||||
client, err := a.dialSSH(ctx, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hc, err := connectLibvirt(ctx, client, host.Socket)
|
||||
if err != nil {
|
||||
client.Close()
|
||||
return nil, err
|
||||
}
|
||||
return hc, nil
|
||||
}
|
||||
|
||||
// dialSSH establishes the SSH connection. The handshake is bounded by ctx's
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
@@ -19,7 +18,6 @@ import (
|
||||
"time"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
"github.com/go-logr/logr"
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
|
||||
@@ -45,8 +43,6 @@ type fakeDialer struct {
|
||||
err error
|
||||
delay time.Duration
|
||||
clients []*fakeClient
|
||||
// api is what the dialled client exposes; nil for pool-only tests.
|
||||
api libvirtAPI
|
||||
}
|
||||
|
||||
func (d *fakeDialer) dial(ctx context.Context, _ provider.KVMHost) (*hostClient, error) {
|
||||
@@ -65,7 +61,7 @@ func (d *fakeDialer) dial(ctx context.Context, _ provider.KVMHost) (*hostClient,
|
||||
}
|
||||
fc := &fakeClient{done: make(chan struct{})}
|
||||
d.clients = append(d.clients, fc)
|
||||
return &hostClient{api: d.api, done: fc.done, close: fc.close}, nil
|
||||
return &hostClient{api: nil, done: fc.done, close: fc.close}, nil
|
||||
}
|
||||
|
||||
func (d *fakeDialer) dialCount() int {
|
||||
@@ -76,8 +72,6 @@ func (d *fakeDialer) dialCount() int {
|
||||
|
||||
var testHost = provider.KVMHost{Name: "nas1", Address: "nas1:22", User: "virt", Socket: defaultSocket, Pool: "default", Bridge: "br0"}
|
||||
|
||||
var discard = logr.Discard()
|
||||
|
||||
func noop(libvirtAPI) error { return nil }
|
||||
|
||||
func TestHostPool_call_reusesConnection(t *testing.T) {
|
||||
@@ -85,7 +79,7 @@ func TestHostPool_call_reusesConnection(t *testing.T) {
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
for range 3 {
|
||||
if err := p.call(context.Background(), discard, testHost, noop); err != nil {
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("call() error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -98,11 +92,11 @@ func TestHostPool_call_redialsAfterDisconnect(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
if err := p.call(context.Background(), discard, testHost, noop); err != nil {
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("call() error = %v", err)
|
||||
}
|
||||
d.clients[0].close()
|
||||
if err := p.call(context.Background(), discard, testHost, noop); err != nil {
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("second call() error = %v", err)
|
||||
}
|
||||
if d.dialCount() != 2 {
|
||||
@@ -110,48 +104,41 @@ func TestHostPool_call_redialsAfterDisconnect(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Only a transport error is evidence about the link. A daemon-reported
|
||||
// libvirt error, or a verdict fn reached locally, must not cost the
|
||||
// connection.
|
||||
func TestHostPool_call_dropsConnectionOnlyOnTransportError(t *testing.T) {
|
||||
func TestHostPool_call_transportErrorDropsConnection(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
wantDropped bool
|
||||
}{
|
||||
{"transport error", io.ErrUnexpectedEOF, true},
|
||||
{"libvirt error", lvErr(libvirt.ErrNoDomain), false},
|
||||
{"locally classified error", fmt.Errorf("%w: creating iso writer: read-only file system", provider.ErrPermanent), false},
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
err := p.call(context.Background(), testHost, func(libvirtAPI) error { return io.ErrUnexpectedEOF })
|
||||
if !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
t.Fatalf("call() error = %v, want the transport error passed through", err)
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
log, snapshot := captureLogger(1)
|
||||
err := p.call(context.Background(), log, testHost, func(libvirtAPI) error { return tc.err })
|
||||
if !errors.Is(err, tc.err) {
|
||||
t.Fatalf("call() error = %v, want %v passed through", err, tc.err)
|
||||
}
|
||||
if got := d.clients[0].closed.Load(); got != tc.wantDropped {
|
||||
t.Errorf("connection closed = %v, want %v", got, tc.wantDropped)
|
||||
}
|
||||
joined := strings.Join(snapshot(), "\n")
|
||||
if logged := strings.Contains(joined, `"msg"="libvirt connection dropped"`) && strings.Contains(joined, `"reason"="transport error"`); logged != tc.wantDropped {
|
||||
t.Errorf("transport drop logged = %v, want %v:\n%s", logged, tc.wantDropped, joined)
|
||||
}
|
||||
if err := p.call(context.Background(), discard, testHost, noop); err != nil {
|
||||
t.Fatalf("next call() error = %v", err)
|
||||
}
|
||||
wantDials := 1
|
||||
if tc.wantDropped {
|
||||
wantDials = 2
|
||||
}
|
||||
if d.dialCount() != wantDials {
|
||||
t.Errorf("dials = %d, want %d", d.dialCount(), wantDials)
|
||||
}
|
||||
})
|
||||
if !d.clients[0].closed.Load() {
|
||||
t.Error("connection not closed after a transport error")
|
||||
}
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("call() after transport error = %v", err)
|
||||
}
|
||||
if d.dialCount() != 2 {
|
||||
t.Errorf("dials = %d, want 2", d.dialCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPool_call_libvirtErrorKeepsConnection(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
err := p.call(context.Background(), testHost, func(libvirtAPI) error { return lvErr(libvirt.ErrNoDomain) })
|
||||
if !hasCode(err, libvirt.ErrNoDomain) {
|
||||
t.Fatalf("call() error = %v, want the libvirt error passed through", err)
|
||||
}
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("call() error = %v", err)
|
||||
}
|
||||
if d.dialCount() != 1 {
|
||||
t.Errorf("dials = %d, want 1: a daemon-reported error must not cost the connection", d.dialCount())
|
||||
}
|
||||
if d.clients[0].closed.Load() {
|
||||
t.Error("connection closed after a libvirt error")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,7 +146,7 @@ func TestHostPool_call_dialErrorPropagates(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{err: errors.New("ssh: handshake failed")}
|
||||
p := newHostPool(d.dial)
|
||||
err := p.call(context.Background(), discard, testHost, noop)
|
||||
err := p.call(context.Background(), testHost, noop)
|
||||
if err == nil || !strings.Contains(err.Error(), "nas1") || !strings.Contains(err.Error(), "handshake failed") {
|
||||
t.Errorf("call() error = %v, want the dial error naming the host", err)
|
||||
}
|
||||
@@ -175,7 +162,7 @@ func TestHostPool_call_cancelAbandonsBlockedCall(t *testing.T) {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
err := p.call(ctx, discard, testHost, func(libvirtAPI) error {
|
||||
err := p.call(ctx, testHost, func(libvirtAPI) error {
|
||||
// Simulates an RPC stuck on a hung hypervisor: it only returns once
|
||||
// the pool closes the connection out from under it.
|
||||
<-d.clients[0].done
|
||||
@@ -190,7 +177,7 @@ func TestHostPool_call_cancelAbandonsBlockedCall(t *testing.T) {
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("blocked call was not released by closing the connection")
|
||||
}
|
||||
if err := p.call(context.Background(), discard, testHost, noop); err != nil {
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("call() after cancel = %v", err)
|
||||
}
|
||||
if d.dialCount() != 2 {
|
||||
@@ -205,7 +192,7 @@ func TestHostPool_call_concurrentCallersShareOneDial(t *testing.T) {
|
||||
var wg sync.WaitGroup
|
||||
for range 8 {
|
||||
wg.Go(func() {
|
||||
if err := p.call(context.Background(), discard, testHost, noop); err != nil {
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Errorf("call() error = %v", err)
|
||||
}
|
||||
})
|
||||
@@ -223,7 +210,7 @@ func TestHostPool_call_hostsAreIndependent(t *testing.T) {
|
||||
other := testHost
|
||||
other.Name = "lab2"
|
||||
for _, h := range []provider.KVMHost{testHost, other, testHost, other} {
|
||||
if err := p.call(context.Background(), discard, h, noop); err != nil {
|
||||
if err := p.call(context.Background(), h, noop); err != nil {
|
||||
t.Fatalf("call(%s) error = %v", h.Name, err)
|
||||
}
|
||||
}
|
||||
@@ -395,19 +382,13 @@ func newSSHFixture(t *testing.T, answerGlobalRequests bool) *sshFixture {
|
||||
}
|
||||
}
|
||||
|
||||
func mustSSHAuth(t *testing.T, cfg provider.KVMConfig) *sshAuth {
|
||||
t.Helper()
|
||||
auth, err := newSSHAuth(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
return auth
|
||||
}
|
||||
|
||||
func TestSSHAuth_dialSSH_forwardsUnixSocket(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
auth := mustSSHAuth(t, f.cfg)
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
client, err := auth.dialSSH(ctx, f.host)
|
||||
@@ -441,7 +422,10 @@ func TestSSHAuth_dialSSH_rejectsUnknownHostKey(t *testing.T) {
|
||||
if err := os.WriteFile(f.cfg.KnownHostsFile, []byte(line+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
auth := mustSSHAuth(t, f.cfg)
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
client, err := auth.dialSSH(context.Background(), f.host)
|
||||
if err == nil {
|
||||
client.Close()
|
||||
@@ -458,7 +442,11 @@ func TestSSHAuth_dialSSH_insecureIgnoreHostKey(t *testing.T) {
|
||||
f := newSSHFixture(t, true)
|
||||
f.cfg.KnownHostsFile = ""
|
||||
f.cfg.InsecureIgnoreHostKey = true
|
||||
client, err := mustSSHAuth(t, f.cfg).dialSSH(context.Background(), f.host)
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
client, err := auth.dialSSH(context.Background(), f.host)
|
||||
if err != nil {
|
||||
t.Fatalf("dialSSH() error = %v", err)
|
||||
}
|
||||
@@ -472,7 +460,11 @@ func TestSSHAuth_dialSSH_rejectsWrongClientKey(t *testing.T) {
|
||||
if err := os.WriteFile(f.keyFile, otherPEM, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := mustSSHAuth(t, f.cfg).dialSSH(context.Background(), f.host)
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
client, err := auth.dialSSH(context.Background(), f.host)
|
||||
if err == nil {
|
||||
client.Close()
|
||||
t.Fatal("dialSSH() succeeded with a key the server does not know")
|
||||
@@ -485,9 +477,13 @@ func TestSSHAuth_dialSSH_rejectsWrongClientKey(t *testing.T) {
|
||||
func TestSSHAuth_dialSSH_cancelledContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if client, err := mustSSHAuth(t, f.cfg).dialSSH(ctx, f.host); err == nil {
|
||||
if client, err := auth.dialSSH(ctx, f.host); err == nil {
|
||||
client.Close()
|
||||
t.Fatal("dialSSH() succeeded with an already-cancelled context")
|
||||
}
|
||||
@@ -523,101 +519,14 @@ func TestNewSSHAuth_missingFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The kvm-ssh Secret is an optional mount, so the provider has to be
|
||||
// constructible without it and has to start working once it appears — the
|
||||
// alternative was a constructor error that crash-looped the whole manager.
|
||||
func TestLazySSH_load_healsWhenTheKeyAppears(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
keyPEM, err := os.ReadFile(f.keyFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(f.keyFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
auth := newLazySSH(f.cfg).load
|
||||
|
||||
if _, err := auth(); err == nil {
|
||||
t.Fatal("loading with no key file succeeded, want an error")
|
||||
} else if class := classify(err); class != provider.ErrTransient {
|
||||
t.Errorf("class = %v, want ErrTransient so the proxy retries once the Secret lands", class)
|
||||
}
|
||||
if err := os.WriteFile(f.keyFile, keyPEM, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := auth()
|
||||
if err != nil {
|
||||
t.Fatalf("loading after the key appeared: %v", err)
|
||||
}
|
||||
// Cached from here on: the file may vanish again (a Secret rotated out)
|
||||
// without breaking connections that already work.
|
||||
if err := os.Remove(f.keyFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, err := auth(); err != nil || again != loaded {
|
||||
t.Errorf("auth() = %v, %v; want the cached value reused", again, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLazySSH_load_unparseableKeyIsPermanent(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
if err := os.WriteFile(f.keyFile, []byte("not a key"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := newLazySSH(f.cfg).load()
|
||||
if classify(err) != provider.ErrPermanent {
|
||||
t.Errorf("class = %v, want ErrPermanent: retrying cannot fix a key that does not parse", classify(err))
|
||||
}
|
||||
}
|
||||
|
||||
// The dialer must authenticate with the material the lazy loader produced,
|
||||
// not a stale or empty one. The echo socket is not a libvirt server, so the
|
||||
// call gets as far as the libvirt handshake and no further; what matters is
|
||||
// that it is past SSH by then.
|
||||
func TestLazySSH_dial_authenticatesOverSSH(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := newLazySSH(f.cfg).dial(ctx, f.host)
|
||||
if err == nil {
|
||||
t.Fatal("dial to an echo socket succeeded, want a libvirt handshake error")
|
||||
}
|
||||
if strings.Contains(err.Error(), "sshKeyFile") || strings.Contains(err.Error(), "ssh handshake") {
|
||||
t.Errorf("dial failed before libvirt: %v", err)
|
||||
}
|
||||
if paths := f.server.requestedPaths(); len(paths) != 1 || paths[0] != f.host.Socket {
|
||||
t.Errorf("server saw forward requests %v, want [%s]", paths, f.host.Socket)
|
||||
}
|
||||
}
|
||||
|
||||
// New must not fail on missing key material, or one misconfigured provider
|
||||
// aborts registry.Build and the manager exits before any controller starts.
|
||||
func TestNew_missingSSHMaterialStillBuilds(t *testing.T) {
|
||||
t.Parallel()
|
||||
p, err := New(context.Background(), provider.ProviderConfig{
|
||||
Name: "kvm-home", Type: "kvm",
|
||||
KVM: &provider.KVMConfig{
|
||||
SSHKeyFile: filepath.Join(t.TempDir(), "absent"),
|
||||
KnownHostsFile: filepath.Join(t.TempDir(), "absent"),
|
||||
Hosts: []provider.KVMHost{{Name: "nas1", Address: "nas1:22", User: "virt", Bridge: "br0"}},
|
||||
},
|
||||
})
|
||||
if err != nil || p == nil {
|
||||
t.Fatalf("New() = %v, %v; want a usable provider", p, err)
|
||||
}
|
||||
if _, err := p.Get(context.Background(), "hosts/nas1/domains/proxy-x"); provider.Class(err) != provider.ErrTransient {
|
||||
t.Errorf("Get class = %v, want ErrTransient", provider.Class(err))
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeepalive_closesClientWhenServerStalls(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, false)
|
||||
client, err := mustSSHAuth(t, f.cfg).dialSSH(context.Background(), f.host)
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
client, err := auth.dialSSH(context.Background(), f.host)
|
||||
if err != nil {
|
||||
t.Fatalf("dialSSH() error = %v", err)
|
||||
}
|
||||
@@ -638,7 +547,11 @@ func TestKeepalive_closesClientWhenServerStalls(t *testing.T) {
|
||||
func TestKeepalive_keepsHealthyClientOpen(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
client, err := mustSSHAuth(t, f.cfg).dialSSH(context.Background(), f.host)
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
client, err := auth.dialSSH(context.Background(), f.host)
|
||||
if err != nil {
|
||||
t.Fatalf("dialSSH() error = %v", err)
|
||||
}
|
||||
@@ -654,69 +567,3 @@ func TestKeepalive_keepsHealthyClientOpen(t *testing.T) {
|
||||
}
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
func TestHostPool_call_tracesAtV1(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{api: readyHost()}
|
||||
p := newHostPool(d.dial)
|
||||
log, snapshot := captureLogger(1)
|
||||
err := p.call(context.Background(), log, testHost, func(api libvirtAPI) error {
|
||||
_, err := api.StoragePoolLookupByName("default")
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("call() error = %v", err)
|
||||
}
|
||||
joined := strings.Join(snapshot(), "\n")
|
||||
for _, want := range []string{`"msg"="libvirt host connected"`, `"address"="nas1:22"`, `"rpc"="virStoragePoolLookupByName"`, `"pool"="default"`, "libvirt "} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("output missing %s:\n%s", want, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPool_call_noTracingBelowV1(t *testing.T) {
|
||||
t.Parallel()
|
||||
inner := readyHost()
|
||||
d := &fakeDialer{api: inner}
|
||||
p := newHostPool(d.dial)
|
||||
log, snapshot := captureLogger(0)
|
||||
err := p.call(context.Background(), log, testHost, func(api libvirtAPI) error {
|
||||
if api != libvirtAPI(inner) {
|
||||
t.Errorf("fn received %T, want the bare fake when V(1) is off", api)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("call() error = %v", err)
|
||||
}
|
||||
if lines := snapshot(); len(lines) != 0 {
|
||||
t.Errorf("verbosity 0 logged %d lines:\n%s", len(lines), strings.Join(lines, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPool_call_logsCancelDropReason(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
log, snapshot := captureLogger(1)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
released := make(chan struct{})
|
||||
err := p.call(ctx, log, testHost, func(libvirtAPI) error {
|
||||
<-d.clients[0].done
|
||||
close(released)
|
||||
return io.EOF
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("call() error = %v, want context.Canceled", err)
|
||||
}
|
||||
<-released
|
||||
joined := strings.Join(snapshot(), "\n")
|
||||
if !strings.Contains(joined, `"reason"="context cancelled"`) {
|
||||
t.Errorf("output missing the cancel drop line:\n%s", joined)
|
||||
}
|
||||
if strings.Contains(joined, `"reason"="transport error"`) {
|
||||
t.Errorf("cancel was also reported as a transport error:\n%s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,54 +9,35 @@ import (
|
||||
const guestAgentChannel = "org.qemu.guest_agent.0"
|
||||
|
||||
// domainParams is everything buildDomainXML needs, already resolved from the
|
||||
// config and placement so the builder stays a pure function. DiskPath and
|
||||
// SeedPath are the volumes' host paths as reported by libvirt; Firmware is
|
||||
// firmwareUEFI or firmwareBIOS.
|
||||
// config and placement so the builder stays a pure function.
|
||||
type domainParams struct {
|
||||
Name string
|
||||
UID string
|
||||
CreatedAt time.Time
|
||||
VCPUs int32
|
||||
MemoryMiB int64
|
||||
Firmware string
|
||||
Pool string
|
||||
Bridge string
|
||||
DiskPath string
|
||||
SeedPath string
|
||||
DiskVol string
|
||||
SeedVol string
|
||||
}
|
||||
|
||||
// buildDomainXML renders a q35 KVM guest: host-passthrough CPU, a virtio
|
||||
// overlay disk and a SATA cdrom for the seed, one virtio NIC on the host
|
||||
// bridge, a serial console, the guest-agent channel, a virtio RNG, no
|
||||
// graphics, and the operator's metadata element.
|
||||
//
|
||||
// Disks are type="file" with host paths, not type="volume" with pool/volume
|
||||
// names: libvirt's AppArmor helper (virt-aa-helper) only grants QEMU access
|
||||
// to sources that carry a path, so a volume-referenced disk fails to start
|
||||
// with "Permission denied" on any AppArmor-confined host (stock Debian and
|
||||
// Ubuntu). The paths cost one StorageVolGetPath round-trip each per Create.
|
||||
//
|
||||
// UEFI is expressed as <os firmware="efi"> with secure boot off, and libvirt
|
||||
// picks the OVMF binary and creates the NVRAM file itself (removed again by
|
||||
// the nvram undefine flag). Under SeaBIOS, Debian 13's GRUB hangs on a
|
||||
// headless domain with a serial console (Debian bug #1111240:
|
||||
// "terminal_output gfxterm serial" with no video device); the same image
|
||||
// boots under UEFI with the serial console intact.
|
||||
// overlay disk and a SATA cdrom for the seed (both referenced as pool
|
||||
// volumes so no host paths are needed), one virtio NIC on the host bridge,
|
||||
// a serial console, the guest-agent channel, a virtio RNG, no graphics, and
|
||||
// the operator's metadata element.
|
||||
func buildDomainXML(p domainParams) (string, error) {
|
||||
os := osXML{
|
||||
Type: osTypeXML{Arch: "x86_64", Machine: "q35", Value: "hvm"},
|
||||
Boot: bootXML{Dev: "hd"},
|
||||
}
|
||||
if p.Firmware == firmwareUEFI {
|
||||
os.Firmware = "efi"
|
||||
os.FirmwareFeatures = &firmwareXML{Features: []firmwareFeatureXML{{Enabled: "no", Name: "secure-boot"}}}
|
||||
}
|
||||
d := domainXML{
|
||||
Type: "kvm",
|
||||
Name: p.Name,
|
||||
Metadata: domainMetadataXML{Instance: newInstanceMetadata(p.UID, p.CreatedAt)},
|
||||
Memory: sizeXML{Unit: "MiB", Value: p.MemoryMiB},
|
||||
VCPU: p.VCPUs,
|
||||
OS: os,
|
||||
Type: "kvm",
|
||||
Name: p.Name,
|
||||
Metadata: domainMetadataXML{Instance: newInstanceMetadata(p.UID, p.CreatedAt)},
|
||||
Memory: sizeXML{Unit: "MiB", Value: p.MemoryMiB},
|
||||
VCPU: p.VCPUs,
|
||||
OS: osXML{
|
||||
Type: osTypeXML{Arch: "x86_64", Machine: "q35", Value: "hvm"},
|
||||
Boot: bootXML{Dev: "hd"},
|
||||
},
|
||||
Features: featuresXML{ACPI: &struct{}{}, APIC: &struct{}{}},
|
||||
CPU: cpuXML{Mode: "host-passthrough"},
|
||||
Clock: clockXML{Offset: "utc"},
|
||||
@@ -66,17 +47,17 @@ func buildDomainXML(p domainParams) (string, error) {
|
||||
Devices: devicesXML{
|
||||
Disks: []diskXML{
|
||||
{
|
||||
Type: "file",
|
||||
Type: "volume",
|
||||
Device: "disk",
|
||||
Driver: driverXML{Name: "qemu", Type: formatQcow2},
|
||||
Source: diskSourceXML{File: p.DiskPath},
|
||||
Source: diskSourceXML{Pool: p.Pool, Volume: p.DiskVol},
|
||||
Target: targetXML{Dev: "vda", Bus: "virtio"},
|
||||
},
|
||||
{
|
||||
Type: "file",
|
||||
Type: "volume",
|
||||
Device: "cdrom",
|
||||
Driver: driverXML{Name: "qemu", Type: formatRaw},
|
||||
Source: diskSourceXML{File: p.SeedPath},
|
||||
Source: diskSourceXML{Pool: p.Pool, Volume: p.SeedVol},
|
||||
Target: targetXML{Dev: "sda", Bus: "sata"},
|
||||
ReadOnly: &struct{}{},
|
||||
},
|
||||
@@ -129,19 +110,8 @@ type sizeXML struct {
|
||||
}
|
||||
|
||||
type osXML struct {
|
||||
Firmware string `xml:"firmware,attr,omitempty"`
|
||||
Type osTypeXML `xml:"type"`
|
||||
FirmwareFeatures *firmwareXML `xml:"firmware"`
|
||||
Boot bootXML `xml:"boot"`
|
||||
}
|
||||
|
||||
type firmwareXML struct {
|
||||
Features []firmwareFeatureXML `xml:"feature"`
|
||||
}
|
||||
|
||||
type firmwareFeatureXML struct {
|
||||
Enabled string `xml:"enabled,attr"`
|
||||
Name string `xml:"name,attr"`
|
||||
Type osTypeXML `xml:"type"`
|
||||
Boot bootXML `xml:"boot"`
|
||||
}
|
||||
|
||||
type osTypeXML struct {
|
||||
@@ -191,7 +161,8 @@ type driverXML struct {
|
||||
}
|
||||
|
||||
type diskSourceXML struct {
|
||||
File string `xml:"file,attr"`
|
||||
Pool string `xml:"pool,attr"`
|
||||
Volume string `xml:"volume,attr"`
|
||||
}
|
||||
|
||||
type targetXML struct {
|
||||
|
||||
@@ -14,10 +14,10 @@ func testDomainParams() domainParams {
|
||||
CreatedAt: time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC),
|
||||
VCPUs: 2,
|
||||
MemoryMiB: 2048,
|
||||
Firmware: firmwareUEFI,
|
||||
Pool: "default",
|
||||
Bridge: "br0",
|
||||
DiskPath: "/var/lib/libvirt/images/proxy-abc123def456ghij.qcow2",
|
||||
SeedPath: "/var/lib/libvirt/images/proxy-abc123def456ghij-seed.iso",
|
||||
DiskVol: "proxy-abc123def456ghij.qcow2",
|
||||
SeedVol: "proxy-abc123def456ghij-seed.iso",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,10 +41,6 @@ func TestBuildDomainXML_fieldByField(t *testing.T) {
|
||||
if d.OS.Type.Value != "hvm" || d.OS.Type.Arch != "x86_64" || d.OS.Type.Machine != "q35" || d.OS.Boot.Dev != "hd" {
|
||||
t.Errorf("os = %+v", d.OS)
|
||||
}
|
||||
if d.OS.Firmware != "efi" || d.OS.FirmwareFeatures == nil || len(d.OS.FirmwareFeatures.Features) != 1 ||
|
||||
d.OS.FirmwareFeatures.Features[0] != (firmwareFeatureXML{Enabled: "no", Name: "secure-boot"}) {
|
||||
t.Errorf("os firmware = %q %+v, want efi with secure-boot disabled", d.OS.Firmware, d.OS.FirmwareFeatures)
|
||||
}
|
||||
if d.Features.ACPI == nil || d.Features.APIC == nil {
|
||||
t.Errorf("features = %+v, want acpi and apic", d.Features)
|
||||
}
|
||||
@@ -67,19 +63,16 @@ func TestBuildDomainXML_fieldByField(t *testing.T) {
|
||||
t.Fatalf("disks = %d, want 2", len(d.Devices.Disks))
|
||||
}
|
||||
disk, seed := d.Devices.Disks[0], d.Devices.Disks[1]
|
||||
if disk.Type != "file" || disk.Device != "disk" || disk.Driver.Type != "qcow2" ||
|
||||
disk.Source.File != "/var/lib/libvirt/images/proxy-abc123def456ghij.qcow2" ||
|
||||
if disk.Type != "volume" || disk.Device != "disk" || disk.Driver.Type != "qcow2" ||
|
||||
disk.Source.Pool != "default" || disk.Source.Volume != "proxy-abc123def456ghij.qcow2" ||
|
||||
disk.Target.Dev != "vda" || disk.Target.Bus != "virtio" || disk.ReadOnly != nil {
|
||||
t.Errorf("overlay disk = %+v", disk)
|
||||
}
|
||||
if seed.Type != "file" || seed.Device != "cdrom" || seed.Driver.Type != "raw" ||
|
||||
seed.Source.File != "/var/lib/libvirt/images/proxy-abc123def456ghij-seed.iso" ||
|
||||
if seed.Type != "volume" || seed.Device != "cdrom" || seed.Driver.Type != "raw" ||
|
||||
seed.Source.Pool != "default" || seed.Source.Volume != "proxy-abc123def456ghij-seed.iso" ||
|
||||
seed.Target.Bus != "sata" || seed.ReadOnly == nil {
|
||||
t.Errorf("seed cdrom = %+v", seed)
|
||||
}
|
||||
if strings.Contains(raw, `type="volume"`) || strings.Contains(raw, " pool=") {
|
||||
t.Errorf("disks reference pool volumes; virt-aa-helper cannot grant QEMU access to those:\n%s", raw)
|
||||
}
|
||||
|
||||
if len(d.Devices.Interfaces) != 1 {
|
||||
t.Fatalf("interfaces = %d, want 1", len(d.Devices.Interfaces))
|
||||
@@ -118,20 +111,3 @@ func TestBuildDomainXML_escapesUserValues(t *testing.T) {
|
||||
t.Errorf("bridge = %q, want the raw value back", d.Devices.Interfaces[0].Source.Bridge)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDomainXML_biosFirmware(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := testDomainParams()
|
||||
p.Firmware = firmwareBIOS
|
||||
raw, err := buildDomainXML(p)
|
||||
if err != nil {
|
||||
t.Fatalf("buildDomainXML() error = %v", err)
|
||||
}
|
||||
var d domainXML
|
||||
if err := xml.Unmarshal([]byte(raw), &d); err != nil {
|
||||
t.Fatalf("output is not well-formed XML: %v", err)
|
||||
}
|
||||
if d.OS.Firmware != "" || d.OS.FirmwareFeatures != nil || strings.Contains(raw, "firmware") {
|
||||
t.Errorf("bios domain still carries firmware settings:\n%s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,8 +19,10 @@ import (
|
||||
// function to recognise. libvirt exposes no quota signal, so nothing maps
|
||||
// to QuotaExceeded.
|
||||
func classify(err error) error {
|
||||
if isClassified(err) {
|
||||
return provider.Class(err)
|
||||
for _, class := range []error{provider.ErrNotFound, provider.ErrQuotaExceeded, provider.ErrPermanent, provider.ErrTransient} {
|
||||
if errors.Is(err, class) {
|
||||
return class
|
||||
}
|
||||
}
|
||||
var lerr libvirt.Error
|
||||
if !errors.As(err, &lerr) {
|
||||
@@ -49,18 +51,6 @@ func isLibvirtError(err error) bool {
|
||||
return errors.As(err, &lerr)
|
||||
}
|
||||
|
||||
// isClassified reports whether err already carries one of the provider's
|
||||
// sentinels, i.e. the package reached a verdict itself rather than hitting
|
||||
// a transport failure.
|
||||
func isClassified(err error) bool {
|
||||
for _, class := range []error{provider.ErrNotFound, provider.ErrQuotaExceeded, provider.ErrPermanent, provider.ErrTransient} {
|
||||
if errors.Is(err, class) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasCode(err error, code libvirt.ErrorNumber) bool {
|
||||
var lerr libvirt.Error
|
||||
return errors.As(err, &lerr) && libvirt.ErrorNumber(lerr.Code) == code
|
||||
@@ -73,18 +63,10 @@ func logAPIError(log logr.Logger, op string, err error) {
|
||||
if !log.V(1).Enabled() {
|
||||
return
|
||||
}
|
||||
kv := append([]any{"op", op, "error", err.Error()}, libvirtErrorFields(err)...)
|
||||
kv := []any{"op", op, "error", err.Error()}
|
||||
var lerr libvirt.Error
|
||||
if errors.As(err, &lerr) {
|
||||
kv = append(kv, "libvirtCode", lerr.Code, "libvirtMessage", lerr.Message)
|
||||
}
|
||||
log.V(1).Info("libvirt call failed", kv...)
|
||||
}
|
||||
|
||||
// libvirtErrorFields renders a daemon-reported error as log key/values:
|
||||
// its constant name (ErrNoDomain, ...) and numeric code. The daemon's
|
||||
// message is already the error text. Nil for transport and other
|
||||
// non-libvirt errors.
|
||||
func libvirtErrorFields(err error) []any {
|
||||
var lerr libvirt.Error
|
||||
if !errors.As(err, &lerr) {
|
||||
return nil
|
||||
}
|
||||
return []any{"libvirtError", libvirt.ErrorNumber(lerr.Code).String(), "libvirtCode", lerr.Code}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
@@ -68,26 +67,3 @@ func TestHasCode(t *testing.T) {
|
||||
t.Error("io.EOF classified as a libvirt error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibvirtErrorFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want []any
|
||||
}{
|
||||
{"libvirt error", lvErr(libvirt.ErrNoDomain), []any{"libvirtError", "ErrNoDomain", "libvirtCode", uint32(42)}},
|
||||
{"wrapped libvirt error", fmt.Errorf("looking up: %w", lvErr(libvirt.ErrNoDomain)), []any{"libvirtError", "ErrNoDomain", "libvirtCode", uint32(42)}},
|
||||
{"transport error", io.EOF, nil},
|
||||
{"cancelled", context.Canceled, nil},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := libvirtErrorFields(tc.err)
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("libvirtErrorFields() = %#v, want %#v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
@@ -31,6 +30,7 @@ type fakeAPI struct {
|
||||
}
|
||||
|
||||
type fakeVol struct {
|
||||
name string
|
||||
format string
|
||||
capacity int64
|
||||
backing string
|
||||
@@ -93,7 +93,7 @@ func (f *fakeAPI) vol(pool, name string) *fakeVol {
|
||||
func (f *fakeAPI) putVol(pool, name, format string, data []byte) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.pools[pool][name] = &fakeVol{format: format, capacity: int64(len(data)), data: data}
|
||||
f.pools[pool][name] = &fakeVol{name: name, format: format, capacity: int64(len(data)), data: data}
|
||||
}
|
||||
|
||||
func (f *fakeAPI) volNames(pool string) []string {
|
||||
@@ -151,11 +151,8 @@ func (f *fakeAPI) DomainDefineXML(raw string) (libvirt.Domain, error) {
|
||||
if err != nil {
|
||||
return libvirt.Domain{}, lvError(libvirt.ErrXMLError, "%v", err)
|
||||
}
|
||||
// libvirt hands the element back without its namespace declaration
|
||||
// (seen on 11.6); mimic that so Get/ListByTag parse the real shape.
|
||||
stored := strings.Replace(string(meta), ` xmlns="`+metadataURI+`"`, "", 1)
|
||||
existing := f.domains[d.Name]
|
||||
nd := &fakeDomain{name: d.Name, xml: raw, metadata: stored}
|
||||
nd := &fakeDomain{name: d.Name, xml: raw, metadata: string(meta)}
|
||||
if existing != nil {
|
||||
nd.running = existing.running
|
||||
nd.addrs = existing.addrs
|
||||
@@ -347,7 +344,7 @@ func (f *fakeAPI) StorageVolCreateXML(pool libvirt.StoragePool, raw string, _ li
|
||||
if _, exists := vols[v.Name]; exists {
|
||||
return libvirt.StorageVol{}, lvError(libvirt.ErrStorageVolExist, "storage volume '%s' exists already", v.Name)
|
||||
}
|
||||
fv := &fakeVol{format: v.Target.Format.Type, capacity: v.Capacity.Value}
|
||||
fv := &fakeVol{name: v.Name, format: v.Target.Format.Type, capacity: v.Capacity.Value}
|
||||
if v.BackingStore != nil {
|
||||
fv.backing = v.BackingStore.Path
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -21,8 +22,10 @@ import (
|
||||
// imageSource is a resolved placement.image: either an entry from the
|
||||
// provider's images map (Key set) or a URL given directly.
|
||||
type imageSource struct {
|
||||
Key string
|
||||
provider.KVMImage
|
||||
Key string
|
||||
URL string
|
||||
SHA256 string
|
||||
Format string
|
||||
}
|
||||
|
||||
// resolveImage maps placement.image to a source. Anything that is neither a
|
||||
@@ -30,10 +33,11 @@ type imageSource struct {
|
||||
// surface, not retry.
|
||||
func resolveImage(cfg provider.KVMConfig, image string) (imageSource, error) {
|
||||
if img, ok := cfg.Images[image]; ok {
|
||||
return imageSource{Key: image, KVMImage: img}, nil
|
||||
return imageSource{Key: image, URL: img.URL, SHA256: img.SHA256, Format: img.Format}, nil
|
||||
}
|
||||
if provider.IsHTTPURL(image) {
|
||||
return imageSource{KVMImage: imageDefaults(provider.KVMImage{URL: image})}, nil
|
||||
u, err := url.Parse(image)
|
||||
if err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != "" {
|
||||
return imageSource{URL: image, Format: formatQcow2}, nil
|
||||
}
|
||||
return imageSource{}, fmt.Errorf("%w: placement.image %q is neither a key in kvm.images nor an http(s) URL",
|
||||
provider.ErrPermanent, image)
|
||||
@@ -97,7 +101,7 @@ func (c *imageCache) ensure(ctx context.Context, log logr.Logger, api libvirtAPI
|
||||
return "", err
|
||||
}
|
||||
if ready {
|
||||
log.V(1).Info("base image already cached", "volume", src.volumeName())
|
||||
log.V(1).Info("base image already cached", "host", hostName, "volume", src.volumeName())
|
||||
return api.StorageVolGetPath(base)
|
||||
}
|
||||
if err := deleteIfExists(api, pool, src.markerName()); err != nil {
|
||||
@@ -166,7 +170,7 @@ func (c *imageCache) fetch(ctx context.Context, log logr.Logger, api libvirtAPI,
|
||||
}
|
||||
|
||||
log.Info("fetching base image into host pool",
|
||||
"pool", pool.Name, "volume", src.volumeName(),
|
||||
"host", hostName, "pool", pool.Name, "volume", src.volumeName(),
|
||||
"url", src.URL, "bytes", resp.ContentLength)
|
||||
start := time.Now()
|
||||
|
||||
@@ -208,7 +212,7 @@ func (c *imageCache) fetch(ctx context.Context, log logr.Logger, api libvirtAPI,
|
||||
return libvirt.StorageVol{}, fmt.Errorf("writing image marker: %w", err)
|
||||
}
|
||||
log.Info("base image cached",
|
||||
"volume", src.volumeName(), "sha256", digest,
|
||||
"host", hostName, "volume", src.volumeName(), "sha256", digest,
|
||||
"bytes", resp.ContentLength, "duration", time.Since(start).Round(time.Second))
|
||||
return base, nil
|
||||
}
|
||||
|
||||
@@ -37,11 +37,12 @@ type imageServer struct {
|
||||
status int
|
||||
noLength bool
|
||||
delay time.Duration
|
||||
body []byte
|
||||
}
|
||||
|
||||
func newImageServer(t *testing.T) *imageServer {
|
||||
t.Helper()
|
||||
s := &imageServer{status: http.StatusOK}
|
||||
s := &imageServer{status: http.StatusOK, body: testImage}
|
||||
s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
s.hits.Add(1)
|
||||
time.Sleep(s.delay)
|
||||
@@ -54,16 +55,16 @@ func newImageServer(t *testing.T) *imageServer {
|
||||
if s.noLength {
|
||||
w.(http.Flusher).Flush()
|
||||
} else {
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(testImage)))
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(s.body)))
|
||||
}
|
||||
_, _ = w.Write(testImage)
|
||||
_, _ = w.Write(s.body)
|
||||
}))
|
||||
t.Cleanup(s.Close)
|
||||
return s
|
||||
}
|
||||
|
||||
func testSource(srv *imageServer, sha string) imageSource {
|
||||
return imageSource{Key: "debian-13", KVMImage: provider.KVMImage{URL: srv.URL + "/debian.qcow2", SHA256: sha, Format: formatQcow2}}
|
||||
return imageSource{Key: "debian-13", URL: srv.URL + "/debian.qcow2", SHA256: sha, Format: formatQcow2}
|
||||
}
|
||||
|
||||
func ensureTest(t *testing.T, api *fakeAPI, src imageSource) (string, error) {
|
||||
@@ -87,9 +88,9 @@ func TestResolveImage(t *testing.T) {
|
||||
want imageSource
|
||||
wantErr bool
|
||||
}{
|
||||
{"configured key", "debian-13", imageSource{Key: "debian-13", KVMImage: provider.KVMImage{URL: "https://cloud.debian.org/d.qcow2", SHA256: "abc", Format: "qcow2", Firmware: "uefi"}}, false},
|
||||
{"direct https url", "https://x.example/img.qcow2", imageSource{KVMImage: provider.KVMImage{URL: "https://x.example/img.qcow2", Format: "qcow2", Firmware: "uefi"}}, false},
|
||||
{"direct http url", "http://mirror/img.qcow2", imageSource{KVMImage: provider.KVMImage{URL: "http://mirror/img.qcow2", Format: "qcow2", Firmware: "uefi"}}, false},
|
||||
{"configured key", "debian-13", imageSource{Key: "debian-13", URL: "https://cloud.debian.org/d.qcow2", SHA256: "abc", Format: "qcow2"}, false},
|
||||
{"direct https url", "https://x.example/img.qcow2", imageSource{URL: "https://x.example/img.qcow2", Format: "qcow2"}, false},
|
||||
{"direct http url", "http://mirror/img.qcow2", imageSource{URL: "http://mirror/img.qcow2", Format: "qcow2"}, false},
|
||||
{"unknown key", "ubuntu", imageSource{}, true},
|
||||
{"file url", "file:///tmp/x.qcow2", imageSource{}, true},
|
||||
{"empty", "", imageSource{}, true},
|
||||
@@ -99,7 +100,9 @@ func TestResolveImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := resolveImage(cfg, tc.image)
|
||||
if tc.wantErr {
|
||||
wantClass(t, err, provider.ErrPermanent)
|
||||
if !errors.Is(err, provider.ErrPermanent) {
|
||||
t.Fatalf("resolveImage(%q) error = %v, want ErrPermanent", tc.image, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
@@ -114,16 +117,16 @@ func TestResolveImage(t *testing.T) {
|
||||
|
||||
func TestImageSource_volumeNames(t *testing.T) {
|
||||
t.Parallel()
|
||||
keyed := imageSource{Key: "debian-13", KVMImage: provider.KVMImage{URL: "https://a/b.qcow2", Format: "qcow2"}}
|
||||
keyed := imageSource{Key: "debian-13", URL: "https://a/b.qcow2", Format: "qcow2"}
|
||||
if keyed.volumeName() != "base-debian-13.qcow2" || keyed.markerName() != "base-debian-13.qcow2.sha256" {
|
||||
t.Errorf("keyed names = %s / %s", keyed.volumeName(), keyed.markerName())
|
||||
}
|
||||
direct := imageSource{KVMImage: provider.KVMImage{URL: "https://a/b.qcow2", Format: "qcow2"}}
|
||||
direct := imageSource{URL: "https://a/b.qcow2", Format: "qcow2"}
|
||||
name := direct.volumeName()
|
||||
if !strings.HasPrefix(name, "base-") || !strings.HasSuffix(name, ".qcow2") || len(name) != len("base-")+16+len(".qcow2") {
|
||||
t.Errorf("direct-url name = %q, want base-<16 hex>.qcow2", name)
|
||||
}
|
||||
other := imageSource{KVMImage: provider.KVMImage{URL: "https://a/c.qcow2", Format: "qcow2"}}
|
||||
other := imageSource{URL: "https://a/c.qcow2", Format: "qcow2"}
|
||||
if other.volumeName() == name {
|
||||
t.Error("different URLs produced the same volume name")
|
||||
}
|
||||
@@ -178,47 +181,57 @@ func TestImageCache_ensure_noChecksumConfigured(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_existingVolumes(t *testing.T) {
|
||||
func TestImageCache_ensure_alreadyCached(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
base []byte
|
||||
marker []byte
|
||||
wantHits int32
|
||||
}{
|
||||
{"already cached", testImage, []byte(testImageSHA()), 0},
|
||||
{"marker missing is a partial upload", []byte("half an image"), nil, 1},
|
||||
{"configured checksum changed", []byte("an older release"), []byte(strings.Repeat("0", 64)), 1},
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
api.putVol("default", "base-debian-13.qcow2", "qcow2", testImage)
|
||||
api.putVol("default", "base-debian-13.qcow2.sha256", "raw", []byte(testImageSHA()))
|
||||
path, err := ensureTest(t, api, testSource(srv, testImageSHA()))
|
||||
if err != nil {
|
||||
t.Fatalf("ensure() error = %v", err)
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
api.putVol("default", "base-debian-13.qcow2", "qcow2", tc.base)
|
||||
if tc.marker != nil {
|
||||
api.putVol("default", "base-debian-13.qcow2.sha256", "raw", tc.marker)
|
||||
}
|
||||
path, err := ensureTest(t, api, testSource(srv, testImageSHA()))
|
||||
if err != nil {
|
||||
t.Fatalf("ensure() error = %v", err)
|
||||
}
|
||||
if path != "/var/lib/libvirt/images/base-debian-13.qcow2" {
|
||||
t.Errorf("path = %q", path)
|
||||
}
|
||||
if srv.hits.Load() != tc.wantHits {
|
||||
t.Errorf("http hits = %d, want %d", srv.hits.Load(), tc.wantHits)
|
||||
}
|
||||
if touched := api.hasCall("StorageVolCreateXML") || api.hasCall("StorageVolDelete"); touched != (tc.wantHits > 0) {
|
||||
t.Errorf("volumes touched = %v, want %v: calls = %v", touched, tc.wantHits > 0, api.calls)
|
||||
}
|
||||
if base := api.vol("default", "base-debian-13.qcow2"); base == nil || !bytes.Equal(base.data, testImage) {
|
||||
t.Error("base volume does not hold the verified image")
|
||||
}
|
||||
if marker := api.vol("default", "base-debian-13.qcow2.sha256"); marker == nil || string(marker.data) != testImageSHA() {
|
||||
t.Errorf("marker = %+v, want the verified digest", marker)
|
||||
}
|
||||
})
|
||||
if path != "/var/lib/libvirt/images/base-debian-13.qcow2" {
|
||||
t.Errorf("path = %q", path)
|
||||
}
|
||||
if srv.hits.Load() != 0 {
|
||||
t.Errorf("http hits = %d, want 0 for a cached image", srv.hits.Load())
|
||||
}
|
||||
if api.hasCall("StorageVolCreateXML") || api.hasCall("StorageVolDelete") {
|
||||
t.Errorf("cached image was touched: calls = %v", api.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_markerMissingRedownloads(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
api.putVol("default", "base-debian-13.qcow2", "qcow2", []byte("half an image"))
|
||||
if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
|
||||
t.Fatalf("ensure() error = %v", err)
|
||||
}
|
||||
if srv.hits.Load() != 1 {
|
||||
t.Errorf("http hits = %d, want 1: a base without its marker is a partial upload", srv.hits.Load())
|
||||
}
|
||||
if base := api.vol("default", "base-debian-13.qcow2"); base == nil || !bytes.Equal(base.data, testImage) {
|
||||
t.Error("partial base volume was not replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_checksumChangedRedownloads(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
api.putVol("default", "base-debian-13.qcow2", "qcow2", []byte("an older release"))
|
||||
api.putVol("default", "base-debian-13.qcow2.sha256", "raw", []byte(strings.Repeat("0", 64)))
|
||||
if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
|
||||
t.Fatalf("ensure() error = %v", err)
|
||||
}
|
||||
if srv.hits.Load() != 1 {
|
||||
t.Errorf("http hits = %d, want 1 after the configured checksum changed", srv.hits.Load())
|
||||
}
|
||||
if marker := api.vol("default", "base-debian-13.qcow2.sha256"); marker == nil || string(marker.data) != testImageSHA() {
|
||||
t.Errorf("marker not updated: %+v", marker)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,7 +240,9 @@ func TestImageCache_ensure_checksumMismatchIsPermanent(t *testing.T) {
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
_, err := ensureTest(t, api, testSource(srv, strings.Repeat("a", 64)))
|
||||
wantClass(t, err, provider.ErrPermanent)
|
||||
if !errors.Is(err, provider.ErrPermanent) {
|
||||
t.Fatalf("ensure() error = %v, want ErrPermanent", err)
|
||||
}
|
||||
if names := api.volNames("default"); len(names) != 0 {
|
||||
t.Errorf("volumes left behind after a checksum mismatch: %v", names)
|
||||
}
|
||||
@@ -268,9 +283,8 @@ func TestImageCache_ensure_noContentLengthIsPermanent(t *testing.T) {
|
||||
srv.noLength = true
|
||||
api := newFakeAPI("default")
|
||||
_, err := ensureTest(t, api, testSource(srv, ""))
|
||||
wantClass(t, err, provider.ErrPermanent)
|
||||
if !strings.Contains(err.Error(), "Content-Length") {
|
||||
t.Errorf("ensure() error = %v, want one about Content-Length", err)
|
||||
if !errors.Is(err, provider.ErrPermanent) || !strings.Contains(err.Error(), "Content-Length") {
|
||||
t.Fatalf("ensure() error = %v, want ErrPermanent about Content-Length", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -26,10 +26,8 @@ const (
|
||||
defaultDiskSizeGB = 10
|
||||
defaultSocket = "/var/run/libvirt/libvirt-sock"
|
||||
defaultPool = "default"
|
||||
formatQcow2 = provider.KVMFormatQcow2
|
||||
formatRaw = provider.KVMFormatRaw
|
||||
firmwareUEFI = provider.KVMFirmwareUEFI
|
||||
firmwareBIOS = provider.KVMFirmwareBIOS
|
||||
formatQcow2 = "qcow2"
|
||||
formatRaw = "raw"
|
||||
|
||||
// undefineFlags removes everything libvirt may keep alongside a domain
|
||||
// definition, so an undefine never fails on leftover state.
|
||||
@@ -42,9 +40,15 @@ func withDefaults(cfg provider.KVMConfig) provider.KVMConfig {
|
||||
if cfg.DiskSizeGB == 0 {
|
||||
cfg.DiskSizeGB = defaultDiskSizeGB
|
||||
}
|
||||
if cfg.VendorData == "" {
|
||||
cfg.VendorData = defaultVendorData
|
||||
}
|
||||
images := make(map[string]provider.KVMImage, len(cfg.Images))
|
||||
for k, img := range cfg.Images {
|
||||
images[k] = imageDefaults(img)
|
||||
if img.Format == "" {
|
||||
img.Format = formatQcow2
|
||||
}
|
||||
images[k] = img
|
||||
}
|
||||
cfg.Images = images
|
||||
hosts := make([]provider.KVMHost, len(cfg.Hosts))
|
||||
@@ -70,41 +74,30 @@ type Provider struct {
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func imageDefaults(img provider.KVMImage) provider.KVMImage {
|
||||
if img.Format == "" {
|
||||
img.Format = formatQcow2
|
||||
}
|
||||
if img.Firmware == "" {
|
||||
img.Firmware = firmwareUEFI
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
// New builds a Provider that reaches its hosts over SSH. The key and
|
||||
// known_hosts files are tried once here and kept when usable, but a failure
|
||||
// is only a warning, retried on the first connection: they come from an
|
||||
// optional Secret mount, and a constructor that failed without it would
|
||||
// abort the whole provider set and crash-loop the manager, taking the
|
||||
// kubernetes and gcp providers down with it. Satisfies registry.Constructor.
|
||||
func New(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
|
||||
// known_hosts files are read here so a bad path fails at startup rather
|
||||
// than on the first Create. Satisfies registry.Constructor.
|
||||
func New(_ context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
|
||||
cfg := provider.KVMConfig{}
|
||||
if pc.KVM != nil {
|
||||
cfg = *pc.KVM
|
||||
}
|
||||
loader := newLazySSH(cfg)
|
||||
if _, err := loader.load(); err != nil {
|
||||
logf.FromContext(ctx).WithName("kvm").WithValues("provider", pc.Name).Info(
|
||||
"WARNING: kvm ssh material is not usable; proxies on this provider will fail until it is",
|
||||
"error", err.Error())
|
||||
auth, err := newSSHAuth(cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kvm provider %q: %w", pc.Name, err)
|
||||
}
|
||||
return newWithDialer(pc.Name, cfg, loader.dial, nil), nil
|
||||
return newWithDialer(pc, auth.dialer(), nil), nil
|
||||
}
|
||||
|
||||
// newWithDialer is the test seam: the dialer decides what libvirtAPI each
|
||||
// host resolves to, and the HTTP client is what image downloads go through.
|
||||
func newWithDialer(name string, cfg provider.KVMConfig, dial hostDialer, httpClient *http.Client) *Provider {
|
||||
func newWithDialer(pc provider.ProviderConfig, dial hostDialer, httpClient *http.Client) *Provider {
|
||||
cfg := provider.KVMConfig{}
|
||||
if pc.KVM != nil {
|
||||
cfg = *pc.KVM
|
||||
}
|
||||
return &Provider{
|
||||
name: name,
|
||||
name: pc.Name,
|
||||
cfg: withDefaults(cfg),
|
||||
hosts: newHostPool(dial),
|
||||
images: newImageCache(httpClient),
|
||||
@@ -173,7 +166,7 @@ func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (stri
|
||||
"vcpus", mt.VCPUs, "memoryMiB", mt.MemoryMiB, "diskSizeGb", p.cfg.DiskSizeGB,
|
||||
"image", src.volumeName(), "cloudInitBytes", len(req.CloudInit))
|
||||
|
||||
err = p.hosts.call(ctx, log, host, func(api libvirtAPI) error {
|
||||
err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
|
||||
dom, err := api.DomainLookupByName(req.Name)
|
||||
if err == nil {
|
||||
log.V(1).Info("domain already defined")
|
||||
@@ -190,23 +183,13 @@ func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (stri
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
overlay, err := p.ensureOverlay(log, api, pool, req.Name, basePath, src.Format)
|
||||
if err != nil {
|
||||
if err := p.ensureOverlay(log, api, pool, req.Name, basePath, src.Format); err != nil {
|
||||
return err
|
||||
}
|
||||
seed, err := p.writeSeed(log, api, pool, req.Name, req.CloudInit)
|
||||
if err != nil {
|
||||
if err := p.writeSeed(log, api, pool, req.Name, req.CloudInit); err != nil {
|
||||
return err
|
||||
}
|
||||
diskPath, err := api.StorageVolGetPath(overlay)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolving overlay path: %w", err)
|
||||
}
|
||||
seedPath, err := api.StorageVolGetPath(seed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolving seed path: %w", err)
|
||||
}
|
||||
dom, err = p.defineDomain(log, api, host, req, mt, src.Firmware, diskPath, seedPath)
|
||||
dom, err = p.defineDomain(log, api, host, req, mt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -220,71 +203,60 @@ func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (stri
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// ensureOverlay creates the per-VM overlay volume, or returns the existing
|
||||
// one: creation is atomic, so an overlay that exists is complete.
|
||||
func (p *Provider) ensureOverlay(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, basePath, baseFormat string) (libvirt.StorageVol, error) {
|
||||
func (p *Provider) ensureOverlay(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, basePath, baseFormat string) error {
|
||||
xml, err := buildVolumeXML(overlayVolume(name, p.cfg.DiskSizeGB, basePath, baseFormat))
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, err
|
||||
return err
|
||||
}
|
||||
vol, err := api.StorageVolCreateXML(pool, xml, 0)
|
||||
_, err = api.StorageVolCreateXML(pool, xml, 0)
|
||||
switch {
|
||||
case err == nil:
|
||||
log.V(1).Info("overlay volume created", "volume", overlayVolumeName(name), "base", basePath)
|
||||
return vol, nil
|
||||
case hasCode(err, libvirt.ErrStorageVolExist):
|
||||
log.V(1).Info("overlay volume already exists", "volume", overlayVolumeName(name))
|
||||
vol, err := api.StorageVolLookupByName(pool, overlayVolumeName(name))
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, fmt.Errorf("looking up existing overlay volume: %w", err)
|
||||
}
|
||||
return vol, nil
|
||||
default:
|
||||
return libvirt.StorageVol{}, fmt.Errorf("creating overlay volume: %w", err)
|
||||
return fmt.Errorf("creating overlay volume: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeSeed replaces the seed volume with a freshly built NoCloud ISO.
|
||||
// Only called while no domain references it.
|
||||
func (p *Provider) writeSeed(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, userData string) (libvirt.StorageVol, error) {
|
||||
func (p *Provider) writeSeed(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, userData string) error {
|
||||
seedName := seedVolumeName(name)
|
||||
if err := deleteIfExists(api, pool, seedName); err != nil {
|
||||
return libvirt.StorageVol{}, err
|
||||
return err
|
||||
}
|
||||
vendorData, err := buildVendorData(p.cfg.VendorData, p.cfg.InstallGuestAgent == nil || *p.cfg.InstallGuestAgent)
|
||||
iso, err := buildSeedISO(name, userData, p.cfg.VendorData)
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
|
||||
}
|
||||
iso, err := buildSeedISO(name, userData, vendorData)
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
|
||||
return fmt.Errorf("%w: %v", provider.ErrPermanent, err)
|
||||
}
|
||||
xml, err := buildVolumeXML(rawVolume(seedName, int64(len(iso))))
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, err
|
||||
return err
|
||||
}
|
||||
vol, err := api.StorageVolCreateXML(pool, xml, 0)
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, fmt.Errorf("creating seed volume: %w", err)
|
||||
return fmt.Errorf("creating seed volume: %w", err)
|
||||
}
|
||||
if err := api.StorageVolUpload(vol, bytes.NewReader(iso), 0, 0, 0); err != nil {
|
||||
return libvirt.StorageVol{}, fmt.Errorf("uploading seed volume: %w", err)
|
||||
return fmt.Errorf("uploading seed volume: %w", err)
|
||||
}
|
||||
log.V(1).Info("seed volume uploaded", "volume", seedName, "bytes", len(iso))
|
||||
return vol, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Provider) defineDomain(log logr.Logger, api libvirtAPI, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType, firmware, diskPath, seedPath string) (libvirt.Domain, error) {
|
||||
func (p *Provider) defineDomain(log logr.Logger, api libvirtAPI, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType) (libvirt.Domain, error) {
|
||||
xml, err := buildDomainXML(domainParams{
|
||||
Name: req.Name,
|
||||
UID: req.UID,
|
||||
CreatedAt: p.now(),
|
||||
VCPUs: mt.VCPUs,
|
||||
MemoryMiB: mt.MemoryMiB,
|
||||
Firmware: firmware,
|
||||
Pool: host.Pool,
|
||||
Bridge: host.Bridge,
|
||||
DiskPath: diskPath,
|
||||
SeedPath: seedPath,
|
||||
DiskVol: overlayVolumeName(req.Name),
|
||||
SeedVol: seedVolumeName(req.Name),
|
||||
})
|
||||
if err != nil {
|
||||
return libvirt.Domain{}, err
|
||||
@@ -293,7 +265,7 @@ func (p *Provider) defineDomain(log logr.Logger, api libvirtAPI, host provider.K
|
||||
if err != nil {
|
||||
return libvirt.Domain{}, fmt.Errorf("defining domain: %w", err)
|
||||
}
|
||||
log.V(1).Info("domain defined", "uid", req.UID, "firmware", firmware)
|
||||
log.V(1).Info("domain defined", "uid", req.UID)
|
||||
return dom, nil
|
||||
}
|
||||
|
||||
@@ -331,7 +303,7 @@ func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instan
|
||||
}
|
||||
log := p.logger(ctx).WithValues("host", host.Name, "name", name)
|
||||
var inst *provider.Instance
|
||||
err = p.hosts.call(ctx, log, host, func(api libvirtAPI) error {
|
||||
err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
|
||||
dom, err := api.DomainLookupByName(name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("looking up domain: %w", err)
|
||||
@@ -456,8 +428,7 @@ func mapState(s libvirt.DomainState, ip string) provider.InstanceState {
|
||||
// domain is not destroyed again, a missing domain is not undefined, and
|
||||
// the per-VM volumes are removed by name even when the domain is already
|
||||
// gone, so a crash between volume creation and define is cleaned up on the
|
||||
// next reconcile. A providerID naming a host that is no longer configured
|
||||
// is a no-op success rather than an error.
|
||||
// next reconcile.
|
||||
func (p *Provider) Delete(ctx context.Context, providerID string) error {
|
||||
hostName, name, err := parseProviderID(providerID)
|
||||
if err != nil {
|
||||
@@ -465,17 +436,10 @@ func (p *Provider) Delete(ctx context.Context, providerID string) error {
|
||||
}
|
||||
host, err := p.host(hostName)
|
||||
if err != nil {
|
||||
// The host is no longer configured, so this operator has no way to
|
||||
// reach the domain and nothing left to clean. Same reasoning as
|
||||
// removeVolumes over a missing pool: refusing would pin the Proxy's
|
||||
// finalizer. Whatever is still running there is the admin's to remove,
|
||||
// and orphan GC cannot see it either — see docs/kvm-provider.md.
|
||||
p.logger(ctx).Info("host is not configured; delete has nothing to clean up",
|
||||
"host", hostName, "providerID", providerID)
|
||||
return nil
|
||||
return p.wrapErr("delete", providerID, err)
|
||||
}
|
||||
log := p.logger(ctx).WithValues("host", host.Name, "name", name)
|
||||
err = p.hosts.call(ctx, log, host, func(api libvirtAPI) error {
|
||||
err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
|
||||
if err := p.removeDomain(log, api, name); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -547,7 +511,7 @@ func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
|
||||
var out []provider.Instance
|
||||
for _, host := range p.cfg.Hosts {
|
||||
hlog := log.WithValues("host", host.Name)
|
||||
err := p.hosts.call(ctx, hlog, host, func(api libvirtAPI) error {
|
||||
err := p.hosts.call(ctx, host, func(api libvirtAPI) error {
|
||||
doms, _, err := api.ConnectListAllDomains(1, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing domains: %w", err)
|
||||
|
||||
@@ -1,396 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package kvm
|
||||
|
||||
// Integration test against a real libvirt/KVM hypervisor. It deviates from
|
||||
// the repo's testcontainers convention on purpose: libvirt needs /dev/kvm
|
||||
// and a bridge, neither of which a container provides, so this runs
|
||||
// against whatever host the KVM_TEST_* variables name. It creates one VM
|
||||
// from a stock cloud image, waits for it to report an address, deletes it,
|
||||
// and checks the pool afterwards. The base image stays cached on the host.
|
||||
//
|
||||
// Required:
|
||||
//
|
||||
// KVM_TEST_HOST SSH endpoint, host:port
|
||||
// KVM_TEST_USER SSH user (member of the libvirt group)
|
||||
// KVM_TEST_SSH_KEY path to the private key
|
||||
// KVM_TEST_BRIDGE host bridge for the VM's NIC
|
||||
// KVM_TEST_IMAGE_URL cloud image to fetch (http/https)
|
||||
//
|
||||
// Optional:
|
||||
//
|
||||
// KVM_TEST_KNOWN_HOSTS known_hosts file; without it host keys are NOT verified
|
||||
// KVM_TEST_IMAGE_SHA256 expected digest of the image
|
||||
// KVM_TEST_POOL storage pool (default "default")
|
||||
// KVM_TEST_SOCKET libvirt socket path on the host
|
||||
// KVM_TEST_TIMEOUT overall budget incl. the image download (default 20m)
|
||||
// KVM_TEST_KEEP set to anything to leave the VM behind for inspection
|
||||
// KVM_TEST_LOG_LEVEL provider log verbosity shown in -v output (default 1:
|
||||
// one line per libvirt call; 2 adds XML, metadata and
|
||||
// addresses; 0 leaves only the t.Log lines)
|
||||
//
|
||||
// Provider logs are printed through t.Log in a compact form: time, logger
|
||||
// name, then the libvirt API function (or the message) and its fields.
|
||||
// The provider/host/name values that are the same on every line of a run
|
||||
// are dropped; the first t.Log line of the run states them. A run that
|
||||
// hits its timeout may log one more line from the RPC the pool abandoned,
|
||||
// after the test has ended; that is a symptom of the timeout, not a bug
|
||||
// in the tracer.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
"github.com/go-logr/logr"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
const (
|
||||
integrationHostName = "kvm-test"
|
||||
integrationImageKey = "test-image"
|
||||
integrationPoll = 5 * time.Second
|
||||
)
|
||||
|
||||
type integrationEnv struct {
|
||||
cfg provider.ProviderConfig
|
||||
timeout time.Duration
|
||||
keep bool
|
||||
logLevel int
|
||||
}
|
||||
|
||||
func loadIntegrationEnv(t *testing.T) integrationEnv {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("integration test skipped under -short")
|
||||
}
|
||||
required := map[string]string{}
|
||||
for _, key := range []string{"KVM_TEST_HOST", "KVM_TEST_USER", "KVM_TEST_SSH_KEY", "KVM_TEST_BRIDGE", "KVM_TEST_IMAGE_URL"} {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
t.Skipf("%s not set; skipping KVM integration test", key)
|
||||
}
|
||||
required[key] = v
|
||||
}
|
||||
cfg := &provider.KVMConfig{
|
||||
SSHKeyFile: required["KVM_TEST_SSH_KEY"],
|
||||
KnownHostsFile: os.Getenv("KVM_TEST_KNOWN_HOSTS"),
|
||||
Images: map[string]provider.KVMImage{
|
||||
integrationImageKey: {URL: required["KVM_TEST_IMAGE_URL"], SHA256: os.Getenv("KVM_TEST_IMAGE_SHA256")},
|
||||
},
|
||||
MachineTypes: map[string]provider.KVMMachineType{"small": {VCPUs: 1, MemoryMiB: 1024}},
|
||||
Hosts: []provider.KVMHost{{
|
||||
Name: integrationHostName,
|
||||
Address: required["KVM_TEST_HOST"],
|
||||
User: required["KVM_TEST_USER"],
|
||||
Socket: os.Getenv("KVM_TEST_SOCKET"),
|
||||
Pool: os.Getenv("KVM_TEST_POOL"),
|
||||
Bridge: required["KVM_TEST_BRIDGE"],
|
||||
}},
|
||||
}
|
||||
if cfg.KnownHostsFile == "" {
|
||||
cfg.InsecureIgnoreHostKey = true
|
||||
t.Log("KVM_TEST_KNOWN_HOSTS not set: host key verification disabled for this run")
|
||||
}
|
||||
pc := provider.ProviderConfig{Name: "kvm-integration", Type: "kvm", KVM: cfg}
|
||||
timeout := 20 * time.Minute
|
||||
if v := os.Getenv("KVM_TEST_TIMEOUT"); v != "" {
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil {
|
||||
t.Fatalf("KVM_TEST_TIMEOUT: %v", err)
|
||||
}
|
||||
timeout = d
|
||||
}
|
||||
logLevel := 1
|
||||
if v := os.Getenv("KVM_TEST_LOG_LEVEL"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
t.Fatalf("KVM_TEST_LOG_LEVEL: %v", err)
|
||||
}
|
||||
logLevel = n
|
||||
}
|
||||
return integrationEnv{cfg: pc, timeout: timeout, keep: os.Getenv("KVM_TEST_KEEP") != "", logLevel: logLevel}
|
||||
}
|
||||
|
||||
// integrationLogger routes provider logs into the test output at the
|
||||
// requested verbosity, and installs it as controller-runtime's root logger
|
||||
// so nothing falls back to the unfulfilled delegating sink (which would
|
||||
// print its "SetLogger was never called" warning and swallow the lines).
|
||||
func integrationLogger(t *testing.T, verbosity int) logr.Logger {
|
||||
t.Helper()
|
||||
log := logr.New(&compactSink{t: t, verbosity: verbosity})
|
||||
logf.SetLogger(log)
|
||||
return log
|
||||
}
|
||||
|
||||
// compactSink is a logr sink for test output: one line per record as
|
||||
// "HH:MM:SS.mmm name headline k=v ...", where the headline is the rpc
|
||||
// field when present (the libvirt API function) and the message
|
||||
// otherwise. Keys that are constant for the whole run are omitted.
|
||||
type compactSink struct {
|
||||
t *testing.T
|
||||
verbosity int
|
||||
name string
|
||||
values []any
|
||||
}
|
||||
|
||||
var compactOmit = map[string]bool{"provider": true, "host": true, "name": true}
|
||||
|
||||
func (s *compactSink) Init(logr.RuntimeInfo) {}
|
||||
|
||||
func (s *compactSink) Enabled(level int) bool { return level <= s.verbosity }
|
||||
|
||||
// GetCallStackHelper lets logr mark its frames as helpers, so t.Log
|
||||
// attributes each line to the provider code that logged it.
|
||||
func (s *compactSink) GetCallStackHelper() func() { return s.t.Helper }
|
||||
|
||||
func (s *compactSink) Info(_ int, msg string, kv ...any) {
|
||||
s.t.Helper()
|
||||
s.t.Log(s.format(msg, kv))
|
||||
}
|
||||
|
||||
func (s *compactSink) Error(err error, msg string, kv ...any) {
|
||||
s.t.Helper()
|
||||
s.t.Log(s.format(msg, append(kv, "error", err)))
|
||||
}
|
||||
|
||||
func (s *compactSink) WithValues(kv ...any) logr.LogSink {
|
||||
c := *s
|
||||
c.values = append(append([]any(nil), s.values...), kv...)
|
||||
return &c
|
||||
}
|
||||
|
||||
func (s *compactSink) WithName(name string) logr.LogSink {
|
||||
c := *s
|
||||
if c.name != "" {
|
||||
name = c.name + "/" + name
|
||||
}
|
||||
c.name = name
|
||||
return &c
|
||||
}
|
||||
|
||||
func (s *compactSink) format(msg string, kv []any) string {
|
||||
all := append(append([]any(nil), s.values...), kv...)
|
||||
headline := msg
|
||||
var b strings.Builder
|
||||
for i := 0; i+1 < len(all); i += 2 {
|
||||
key := fmt.Sprint(all[i])
|
||||
if key == "rpc" {
|
||||
headline = fmt.Sprint(all[i+1])
|
||||
continue
|
||||
}
|
||||
if compactOmit[key] {
|
||||
continue
|
||||
}
|
||||
val := fmt.Sprint(all[i+1])
|
||||
if strings.ContainsAny(val, " \t\"=") {
|
||||
val = strconv.Quote(val)
|
||||
}
|
||||
b.WriteString(" " + key + "=" + val)
|
||||
}
|
||||
name := s.name[strings.LastIndex(s.name, "/")+1:]
|
||||
return fmt.Sprintf("%s %s %s%s", time.Now().Format("15:04:05.000"), name, headline, b.String())
|
||||
}
|
||||
|
||||
func randomUID(t *testing.T) string {
|
||||
t.Helper()
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := hex.EncodeToString(b[:])
|
||||
return fmt.Sprintf("%s-%s-%s-%s-%s", h[0:8], h[8:12], h[12:16], h[16:20], h[20:32])
|
||||
}
|
||||
|
||||
// TestKVM_lifecycle is one sequential story, not parallel subtests: each
|
||||
// stage depends on the state the previous one left on the hypervisor.
|
||||
func TestKVM_lifecycle(t *testing.T) {
|
||||
env := loadIntegrationEnv(t)
|
||||
log := integrationLogger(t, env.logLevel)
|
||||
ctx, cancel := context.WithTimeout(logr.NewContext(context.Background(), log), env.timeout)
|
||||
defer cancel()
|
||||
|
||||
prov, err := New(ctx, env.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
p := prov.(*Provider)
|
||||
host := p.cfg.Hosts[0]
|
||||
|
||||
uid := randomUID(t)
|
||||
req := provider.CreateRequest{
|
||||
Name: provider.NameFromUID(types.UID(uid)),
|
||||
UID: uid,
|
||||
Namespace: "integration",
|
||||
ProxyName: "kvm-integration",
|
||||
Placement: provider.Placement{Zone: integrationHostName, MachineType: "small", Image: integrationImageKey},
|
||||
CloudInit: "#cloud-config\n",
|
||||
Port: 3128,
|
||||
}
|
||||
t.Logf("host %s (%s), pool %q, bridge %s, domain %s", host.Name, host.Address, host.Pool, host.Bridge, req.Name)
|
||||
|
||||
// Cleanup runs even when a stage fails, with its own budget: the test
|
||||
// ctx may already be spent by then.
|
||||
t.Cleanup(func() {
|
||||
if env.keep {
|
||||
t.Logf("KVM_TEST_KEEP set: leaving domain %s and its volumes on %s", req.Name, host.Address)
|
||||
return
|
||||
}
|
||||
cctx, ccancel := context.WithTimeout(logr.NewContext(context.Background(), log), 2*time.Minute)
|
||||
defer ccancel()
|
||||
if err := p.Delete(cctx, formatProviderID(host.Name, req.Name)); err != nil {
|
||||
t.Errorf("cleanup Delete: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// --- create (includes the image fetch on a cold host) ---
|
||||
start := time.Now()
|
||||
id, err := p.Create(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
t.Logf("Create returned %s after %s", id, time.Since(start).Round(time.Second))
|
||||
if want := formatProviderID(host.Name, req.Name); id != want {
|
||||
t.Fatalf("providerID = %s, want %s", id, want)
|
||||
}
|
||||
|
||||
// --- create again: must find what it made, not duplicate it ---
|
||||
id2, err := p.Create(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("second Create: %v", err)
|
||||
}
|
||||
if id2 != id {
|
||||
t.Fatalf("second Create returned %s, want %s", id2, id)
|
||||
}
|
||||
|
||||
// --- wait for the guest to come up with an address ---
|
||||
inst := waitForRunning(ctx, t, p, id)
|
||||
t.Logf("domain Running with IP %s (state reported after %s)", inst.IP, time.Since(start).Round(time.Second))
|
||||
if inst.UID != uid {
|
||||
t.Errorf("UID read back = %q, want %q (metadata round-trip through libvirt)", inst.UID, uid)
|
||||
}
|
||||
if age := time.Since(inst.CreatedAt); inst.CreatedAt.IsZero() || age < 0 || age > time.Hour {
|
||||
t.Errorf("CreatedAt = %s (age %s), want a recent timestamp", inst.CreatedAt, age)
|
||||
}
|
||||
if conn, err := net.DialTimeout("tcp", net.JoinHostPort(inst.IP, "22"), 5*time.Second); err != nil {
|
||||
t.Logf("note: tcp %s:22 not reachable from this machine (%v); fine if the bridge network is not routed here", inst.IP, err)
|
||||
} else {
|
||||
conn.Close()
|
||||
t.Logf("tcp %s:22 reachable from this machine", inst.IP)
|
||||
}
|
||||
|
||||
// --- the GC view must include it ---
|
||||
listed, err := p.ListByTag(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListByTag: %v", err)
|
||||
}
|
||||
if !containsID(listed, id) {
|
||||
t.Errorf("ListByTag = %+v, want it to include %s", listed, id)
|
||||
}
|
||||
|
||||
// --- delete, then verify nothing of the VM remains but the base image does ---
|
||||
if err := p.Delete(ctx, id); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
if _, err := p.Get(ctx, id); !errors.Is(err, provider.ErrNotFound) {
|
||||
t.Errorf("Get after Delete = %v, want ErrNotFound", err)
|
||||
}
|
||||
if err := p.Delete(ctx, id); err != nil {
|
||||
t.Errorf("second Delete: %v, want nil (idempotent)", err)
|
||||
}
|
||||
listed, err = p.ListByTag(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListByTag after Delete: %v", err)
|
||||
}
|
||||
if containsID(listed, id) {
|
||||
t.Errorf("ListByTag still includes %s after Delete", id)
|
||||
}
|
||||
|
||||
src, err := resolveImage(p.cfg, integrationImageKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = p.hosts.call(ctx, p.logger(ctx).WithValues("host", host.Name), host, func(api libvirtAPI) error {
|
||||
pool, err := api.StoragePoolLookupByName(host.Pool)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, gone := range []string{overlayVolumeName(req.Name), seedVolumeName(req.Name)} {
|
||||
if _, err := api.StorageVolLookupByName(pool, gone); !hasCode(err, libvirt.ErrNoStorageVol) {
|
||||
t.Errorf("volume %s after Delete: lookup err = %v, want ErrNoStorageVol", gone, err)
|
||||
}
|
||||
}
|
||||
for _, kept := range []string{src.volumeName(), src.markerName()} {
|
||||
if _, err := api.StorageVolLookupByName(pool, kept); err != nil {
|
||||
t.Errorf("cached image volume %s: %v, want it kept after Delete", kept, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("inspecting pool after Delete: %v", err)
|
||||
}
|
||||
// KVM_TEST_KEEP is for inspecting a failed run; a passed run has
|
||||
// nothing left to keep, so let cleanup's Delete confirm that.
|
||||
env.keep = false
|
||||
}
|
||||
|
||||
// waitForRunning polls Get until the instance is Running with an IP or ctx
|
||||
// ends. Transient errors are retried; anything else fails the test.
|
||||
func waitForRunning(ctx context.Context, t *testing.T, p *Provider, id string) *provider.Instance {
|
||||
t.Helper()
|
||||
var last provider.InstanceState
|
||||
ticker := time.NewTicker(integrationPoll)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
inst, err := p.Get(ctx, id)
|
||||
switch {
|
||||
case err == nil:
|
||||
if inst.State != last {
|
||||
t.Logf("state %s ip %q", inst.State, inst.IP)
|
||||
last = inst.State
|
||||
}
|
||||
if inst.State == provider.StateRunning {
|
||||
return inst
|
||||
}
|
||||
if inst.State == provider.StateStopped {
|
||||
t.Fatalf("domain reported %s while waiting for Running; check `virsh console %s` on the host", inst.State, idName(id))
|
||||
}
|
||||
case errors.Is(err, provider.ErrTransient):
|
||||
t.Logf("Get: transient error, retrying: %v", err)
|
||||
default:
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("timed out waiting for Running (last state %s): %v", last, ctx.Err())
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func idName(id string) string {
|
||||
_, name, _ := parseProviderID(id)
|
||||
return name
|
||||
}
|
||||
|
||||
func containsID(list []provider.Instance, id string) bool {
|
||||
for _, inst := range list {
|
||||
if inst.ID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -20,7 +20,7 @@ func TestWithDefaults(t *testing.T) {
|
||||
cfg := withDefaults(provider.KVMConfig{
|
||||
Images: map[string]provider.KVMImage{
|
||||
"debian": {URL: "https://x/d.qcow2"},
|
||||
"raw": {URL: "https://x/r.img", Format: "raw", Firmware: "bios"},
|
||||
"raw": {URL: "https://x/r.img", Format: "raw"},
|
||||
},
|
||||
Hosts: []provider.KVMHost{
|
||||
{Name: "a", Address: "a:22", User: "virt", Bridge: "br0"},
|
||||
@@ -30,15 +30,12 @@ func TestWithDefaults(t *testing.T) {
|
||||
if cfg.DiskSizeGB != defaultDiskSizeGB {
|
||||
t.Errorf("diskSizeGb = %d, want %d", cfg.DiskSizeGB, defaultDiskSizeGB)
|
||||
}
|
||||
if cfg.VendorData != "" {
|
||||
t.Errorf("vendorData = %q, want none: the guest-agent bootstrap is merged into the user-data, not defaulted here", cfg.VendorData)
|
||||
if cfg.VendorData != defaultVendorData {
|
||||
t.Errorf("vendorData = %q, want the default", cfg.VendorData)
|
||||
}
|
||||
if cfg.Images["debian"].Format != "qcow2" || cfg.Images["raw"].Format != "raw" {
|
||||
t.Errorf("image formats = %q/%q, want qcow2 default and raw kept", cfg.Images["debian"].Format, cfg.Images["raw"].Format)
|
||||
}
|
||||
if cfg.Images["debian"].Firmware != firmwareUEFI || cfg.Images["raw"].Firmware != firmwareBIOS {
|
||||
t.Errorf("image firmware = %q/%q, want uefi default and bios kept", cfg.Images["debian"].Firmware, cfg.Images["raw"].Firmware)
|
||||
}
|
||||
if a := cfg.Hosts[0]; a.Socket != defaultSocket || a.Pool != defaultPool {
|
||||
t.Errorf("host a = %+v, want default socket and pool", a)
|
||||
}
|
||||
@@ -95,8 +92,7 @@ func testConfig(hosts ...string) provider.ProviderConfig {
|
||||
}
|
||||
|
||||
func newTestProvider(fakes hostFakes, hosts ...string) *Provider {
|
||||
pc := testConfig(hosts...)
|
||||
p := newWithDialer(pc.Name, *pc.KVM, fakes.dial, nil)
|
||||
p := newWithDialer(testConfig(hosts...), fakes.dial, nil)
|
||||
p.now = func() time.Time { return testNow }
|
||||
return p
|
||||
}
|
||||
@@ -104,7 +100,7 @@ func newTestProvider(fakes hostFakes, hosts ...string) *Provider {
|
||||
// cacheImage pre-populates the pool with the configured image and its
|
||||
// marker so Create never touches the network.
|
||||
func cacheImage(api *fakeAPI) {
|
||||
src := imageSource{Key: "debian-13", KVMImage: provider.KVMImage{URL: testImageURL, Format: formatQcow2}}
|
||||
src := imageSource{Key: "debian-13", URL: testImageURL, Format: formatQcow2}
|
||||
api.putVol("default", src.volumeName(), formatQcow2, []byte("qcow2"))
|
||||
api.putVol("default", src.markerName(), formatRaw, []byte(testImageSHA()))
|
||||
}
|
||||
@@ -188,13 +184,6 @@ func assertCreated(t *testing.T, api *fakeAPI) {
|
||||
if seed.format != formatRaw || len(seed.data) == 0 || int64(len(seed.data)) != seed.capacity {
|
||||
t.Errorf("seed = format %s, %d bytes, capacity %d; want raw with capacity == uploaded size", seed.format, len(seed.data), seed.capacity)
|
||||
}
|
||||
_, files := readSeed(t, seed.data)
|
||||
if files["user-data"] != testCloudInit {
|
||||
t.Errorf("seed user-data = %q, want the Proxy's cloud-init verbatim", files["user-data"])
|
||||
}
|
||||
if !strings.Contains(files["vendor-data"], "qemu-guest-agent") {
|
||||
t.Errorf("seed vendor-data = %q, want the guest-agent script", files["vendor-data"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_freshHost(t *testing.T) {
|
||||
@@ -221,13 +210,6 @@ func TestCreate_freshHost(t *testing.T) {
|
||||
if d.Devices.Interfaces[0].Source.Bridge != "br0" {
|
||||
t.Errorf("bridge = %q, want br0", d.Devices.Interfaces[0].Source.Bridge)
|
||||
}
|
||||
wantDisk, wantSeed := api.poolPath+"/"+overlayVolumeName(testName), api.poolPath+"/"+seedVolumeName(testName)
|
||||
if got := d.Devices.Disks[0].Source.File; got != wantDisk {
|
||||
t.Errorf("overlay disk source = %q, want the path libvirt reported, %q", got, wantDisk)
|
||||
}
|
||||
if got := d.Devices.Disks[1].Source.File; got != wantSeed {
|
||||
t.Errorf("seed cdrom source = %q, want the path libvirt reported, %q", got, wantSeed)
|
||||
}
|
||||
if m := d.Metadata.Instance; m.UID != testUID || !m.managed() || !m.createdAt().Equal(testNow) {
|
||||
t.Errorf("metadata = %+v, want managed, uid %s, createdAt %s", m, testUID, testNow)
|
||||
}
|
||||
@@ -242,7 +224,7 @@ func TestCreate_fetchesImageOnFirstUse(t *testing.T) {
|
||||
api := newFakeAPI("default")
|
||||
cfg := testConfig("nas1")
|
||||
cfg.KVM.Images["debian-13"] = provider.KVMImage{URL: srv.URL + "/debian.qcow2", SHA256: testImageSHA()}
|
||||
p := newWithDialer(cfg.Name, *cfg.KVM, hostFakes{"nas1": api}.dial, nil)
|
||||
p := newWithDialer(cfg, hostFakes{"nas1": api}.dial, nil)
|
||||
|
||||
if _, err := p.Create(context.Background(), testRequest()); err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
@@ -480,25 +462,28 @@ func TestCreate_hostFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
setup func() hostFakes
|
||||
setup func() (hostFakes, string)
|
||||
class error
|
||||
}{
|
||||
{
|
||||
name: "host unreachable",
|
||||
setup: func() hostFakes { return hostFakes{} },
|
||||
setup: func() (hostFakes, string) { return hostFakes{}, "nas1" },
|
||||
class: provider.ErrTransient,
|
||||
},
|
||||
{
|
||||
name: "pool missing",
|
||||
setup: func() hostFakes { return hostFakes{"nas1": newFakeAPI("other")} },
|
||||
name: "pool missing",
|
||||
setup: func() (hostFakes, string) {
|
||||
api := newFakeAPI("other")
|
||||
return hostFakes{"nas1": api}, "nas1"
|
||||
},
|
||||
class: provider.ErrPermanent,
|
||||
},
|
||||
{
|
||||
name: "domain xml rejected",
|
||||
setup: func() hostFakes {
|
||||
setup: func() (hostFakes, string) {
|
||||
api := readyHost()
|
||||
api.fail["DomainDefineXML"] = lvError(libvirt.ErrConfigUnsupported, "no q35 here")
|
||||
return hostFakes{"nas1": api}
|
||||
return hostFakes{"nas1": api}, "nas1"
|
||||
},
|
||||
class: provider.ErrPermanent,
|
||||
},
|
||||
@@ -506,7 +491,8 @@ func TestCreate_hostFailures(t *testing.T) {
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := newTestProvider(tc.setup(), "nas1")
|
||||
fakes, host := tc.setup()
|
||||
p := newTestProvider(fakes, host)
|
||||
_, err := p.Create(context.Background(), testRequest())
|
||||
wantClass(t, err, tc.class)
|
||||
})
|
||||
@@ -726,28 +712,6 @@ func TestDelete_missingPoolIsNotAnError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A host dropped from the config leaves the operator no way to reach the
|
||||
// domain, so Delete has nothing left to clean. Failing instead would pin
|
||||
// the Proxy's finalizer forever, since the reconciler's delete path cannot
|
||||
// make progress without a Delete that can succeed.
|
||||
func TestDelete_unconfiguredHostIsNotAnError(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
managedDomain(t, api, testName, testUID, true, nil)
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
|
||||
if err := p.Delete(context.Background(), "hosts/nas9/domains/"+testName); err != nil {
|
||||
t.Fatalf("Delete over an unconfigured host: %v", err)
|
||||
}
|
||||
if api.domain(testName) == nil {
|
||||
t.Error("domain on the configured host was touched; only the named host's state may change")
|
||||
}
|
||||
// Get still refuses, so the live path cannot mistake a config typo for a
|
||||
// missing VM and recreate it somewhere else.
|
||||
_, err := p.Get(context.Background(), "hosts/nas9/domains/"+testName)
|
||||
wantClass(t, err, provider.ErrPermanent)
|
||||
}
|
||||
|
||||
func TestDelete_errors(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
@@ -757,6 +721,7 @@ func TestDelete_errors(t *testing.T) {
|
||||
class error
|
||||
}{
|
||||
{"malformed id", "nope", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
|
||||
{"host not configured", "hosts/nas9/domains/x", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
|
||||
{"host unreachable", "hosts/nas1/domains/x", func(*testing.T) hostFakes { return hostFakes{} }, provider.ErrTransient},
|
||||
{
|
||||
"undefine fails", "hosts/nas1/domains/" + testName,
|
||||
@@ -905,95 +870,11 @@ func TestMapState(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// runAllOps drives one full lifecycle over a ready fake host so logging
|
||||
// tests see every provider method and every libvirt call.
|
||||
func runAllOps(t *testing.T, ctx context.Context, req provider.CreateRequest) {
|
||||
t.Helper()
|
||||
p := newTestProvider(hostFakes{"nas1": readyHost()}, "nas1")
|
||||
id, err := p.Create(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
if _, err := p.Get(ctx, id); err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if _, err := p.ListByTag(ctx); err != nil {
|
||||
t.Fatalf("ListByTag: %v", err)
|
||||
}
|
||||
if err := p.Delete(ctx, id); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogging_verbosityTiers(t *testing.T) {
|
||||
func TestNew_badKeyPathFailsFast(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
verbosity int
|
||||
wantLines []string
|
||||
absentLines []string
|
||||
}{
|
||||
{
|
||||
name: "v0 stays silent",
|
||||
verbosity: 0,
|
||||
absentLines: []string{"libvirt call", "KVM domain ensured running"},
|
||||
},
|
||||
{
|
||||
name: "v1 logs one line per libvirt call plus the operation outcomes",
|
||||
verbosity: 1,
|
||||
wantLines: []string{
|
||||
`kvm/libvirt "level"=1 "msg"="libvirt host connected"`,
|
||||
`"msg"="libvirt call"`,
|
||||
`"rpc"="virDomainLookupByName"`,
|
||||
`"libvirtError"="ErrNoDomain"`,
|
||||
`"rpc"="virStorageVolUpload"`,
|
||||
`"rpc"="virDomainDefineXML"`,
|
||||
`"rpc"="virDomainCreate"`,
|
||||
`"rpc"="virDomainInterfaceAddresses"`,
|
||||
`"rpc"="virConnectListAllDomains"`,
|
||||
`"rpc"="virDomainUndefineFlags"`,
|
||||
`"rpc"="virStorageVolDelete"`,
|
||||
`"provider"="kvm-home"`,
|
||||
`"host"="nas1"`,
|
||||
`"name"="` + testName + `"`,
|
||||
`"msg"="KVM domain ensured running"`,
|
||||
},
|
||||
absentLines: []string{`"xml"=`, `"metadata"=`},
|
||||
},
|
||||
{
|
||||
name: "v2 adds xml, metadata and addresses on the same lines",
|
||||
verbosity: 2,
|
||||
wantLines: []string{`"xml"="<domain`, `"xml"="<volume`, `"metadata"="<instance`, `"names"=[`},
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, snapshot := captureContext(tc.verbosity)
|
||||
req := testRequest()
|
||||
const sentinel = "SENTINEL-cloud-init-must-never-be-logged"
|
||||
req.CloudInit = "#cloud-config\nhostname: " + sentinel + "\n"
|
||||
|
||||
runAllOps(t, ctx, req)
|
||||
|
||||
lines := snapshot()
|
||||
joined := strings.Join(lines, "\n")
|
||||
if tc.verbosity == 0 && len(lines) != 0 {
|
||||
t.Errorf("verbosity 0 logged %d lines:\n%s", len(lines), joined)
|
||||
}
|
||||
for _, want := range tc.wantLines {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("output missing %s:\n%s", want, joined)
|
||||
}
|
||||
}
|
||||
for _, absent := range tc.absentLines {
|
||||
if strings.Contains(joined, absent) {
|
||||
t.Errorf("output unexpectedly contains %s:\n%s", absent, joined)
|
||||
}
|
||||
}
|
||||
if strings.Contains(joined, sentinel) {
|
||||
t.Errorf("cloud-init content leaked into logs:\n%s", joined)
|
||||
}
|
||||
})
|
||||
cfg := testConfig("nas1")
|
||||
cfg.KVM.SSHKeyFile = t.TempDir() + "/missing"
|
||||
if _, err := New(context.Background(), cfg); err == nil {
|
||||
t.Fatal("New succeeded with a missing key file")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,19 +31,15 @@ func newInstanceMetadata(uid string, createdAt time.Time) instanceMetadata {
|
||||
}
|
||||
|
||||
// parseInstanceMetadata decodes the element as DomainGetMetadata returns it.
|
||||
// libvirt selects the element by namespace URI but serialises it without the
|
||||
// declaration (observed on libvirt 11.6: a bare <instance>), and other
|
||||
// versions may keep a default or prefixed form. The URI is already checked
|
||||
// by the lookup, so only the local name is matched here.
|
||||
// libvirt may re-serialise it with a namespace prefix rather than the default
|
||||
// namespace it was defined with; encoding/xml matches on namespace URI, so
|
||||
// both spellings decode.
|
||||
func parseInstanceMetadata(s string) (instanceMetadata, error) {
|
||||
var raw struct {
|
||||
XMLName xml.Name `xml:"instance"`
|
||||
instanceMetadata
|
||||
}
|
||||
if err := xml.Unmarshal([]byte(s), &raw); err != nil {
|
||||
var m instanceMetadata
|
||||
if err := xml.Unmarshal([]byte(s), &m); err != nil {
|
||||
return instanceMetadata{}, fmt.Errorf("parsing domain metadata: %w", err)
|
||||
}
|
||||
return raw.instanceMetadata, nil
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m instanceMetadata) managed() bool {
|
||||
|
||||
@@ -29,37 +29,32 @@ func TestInstanceMetadata_roundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseInstanceMetadata(t *testing.T) {
|
||||
func TestParseInstanceMetadata_libvirtPrefixedForm(t *testing.T) {
|
||||
t.Parallel()
|
||||
// libvirt re-serialises the stored element with whatever prefix it
|
||||
// chooses; only the namespace URI is contractual.
|
||||
in := `<op:instance xmlns:op="` + metadataURI + `">` +
|
||||
`<op:managed>true</op:managed><op:uid>uid-2</op:uid>` +
|
||||
`<op:createdAt>2026-09-04T10:00:00Z</op:createdAt></op:instance>`
|
||||
m, err := parseInstanceMetadata(in)
|
||||
if err != nil {
|
||||
t.Fatalf("parseInstanceMetadata() error = %v", err)
|
||||
}
|
||||
if !m.managed() || m.UID != "uid-2" || m.createdAt().Hour() != 10 {
|
||||
t.Errorf("parsed = %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseInstanceMetadata_notManaged(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
wantErr bool
|
||||
wantManaged bool
|
||||
wantUID string
|
||||
wantCreated time.Time
|
||||
name string
|
||||
in string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
// libvirt re-serialises the stored element with whatever prefix it
|
||||
// chooses; only the namespace URI is contractual.
|
||||
name: "libvirt prefixed form",
|
||||
in: `<op:instance xmlns:op="` + metadataURI + `">` +
|
||||
`<op:managed>true</op:managed><op:uid>uid-2</op:uid>` +
|
||||
`<op:createdAt>2026-09-04T10:00:00Z</op:createdAt></op:instance>`,
|
||||
wantManaged: true, wantUID: "uid-2", wantCreated: time.Date(2026, 9, 4, 10, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
// What libvirt 11.6 actually returns from DomainGetMetadata: the
|
||||
// element selected by our URI, serialised without any namespace
|
||||
// declaration.
|
||||
name: "libvirt bare form",
|
||||
in: `<instance><managed>true</managed><uid>uid-3</uid>` +
|
||||
`<createdAt>2026-09-10T07:30:00Z</createdAt></instance>`,
|
||||
wantManaged: true, wantUID: "uid-3", wantCreated: time.Date(2026, 9, 10, 7, 30, 0, 0, time.UTC),
|
||||
},
|
||||
{name: "managed false", in: `<instance xmlns="` + metadataURI + `"><managed>false</managed><uid>u</uid></instance>`, wantUID: "u"},
|
||||
{name: "foreign element", in: `<foo xmlns="urn:other"><managed>true</managed></foo>`, wantErr: true},
|
||||
{name: "garbage", in: `not xml`, wantErr: true},
|
||||
{"managed false", `<instance xmlns="` + metadataURI + `"><managed>false</managed><uid>u</uid></instance>`, false},
|
||||
{"foreign element", `<foo xmlns="urn:other"><managed>true</managed></foo>`, true},
|
||||
{"garbage", `not xml`, true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -68,8 +63,8 @@ func TestParseInstanceMetadata(t *testing.T) {
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("error = %v, wantErr %v", err, tc.wantErr)
|
||||
}
|
||||
if m.managed() != tc.wantManaged || m.UID != tc.wantUID || !m.createdAt().Equal(tc.wantCreated) {
|
||||
t.Errorf("parsed = %+v, want managed=%v uid=%q createdAt=%v", m, tc.wantManaged, tc.wantUID, tc.wantCreated)
|
||||
if m.managed() {
|
||||
t.Errorf("managed() = true for %q", tc.in)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3,9 +3,6 @@ package kvm
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/textproto"
|
||||
"strings"
|
||||
|
||||
"github.com/kdomanski/iso9660"
|
||||
@@ -14,97 +11,20 @@ import (
|
||||
// seedLabel is the volume label cloud-init's NoCloud datasource looks for.
|
||||
const seedLabel = "cidata"
|
||||
|
||||
// guestAgentScript installs and starts the QEMU guest agent, which is how
|
||||
// the provider learns a VM's address before the host's ARP table has seen
|
||||
// any traffic from it.
|
||||
//
|
||||
// It ships as vendor-data, and as a *script* rather than a cloud-config,
|
||||
// because that is the only shape cloud-init cannot drop. Vendor-data
|
||||
// cloud-config is merged under the Proxy's own user-data with mergers that
|
||||
// keep the user-data's value for every key both documents define and never
|
||||
// append lists, so a Proxy carrying its own packages: silently lost the
|
||||
// agent. The strategy cannot be overridden from either document: merge_how
|
||||
// in user-data governs only the first source, and merge_how in vendor-data
|
||||
// is consumed and discarded before the document is stored. A vendor-data
|
||||
// script bypasses all of it — cloud-init gives vendor-data its own handler
|
||||
// chain, writes scripts to a separate directory and runs them from
|
||||
// cc_scripts_vendor, after the user's packages are installed and before the
|
||||
// user's runcmd.
|
||||
const guestAgentScript = `#!/bin/sh
|
||||
# Added by egress-proxies-operator so the hypervisor can report this VM's
|
||||
# address. Set installGuestAgent: false on the provider to leave it out.
|
||||
set -e
|
||||
if ! command -v qemu-ga >/dev/null 2>&1 && [ ! -x /usr/sbin/qemu-ga ]; then
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y qemu-guest-agent
|
||||
elif command -v dnf >/dev/null 2>&1; then
|
||||
dnf install -y qemu-guest-agent
|
||||
elif command -v yum >/dev/null 2>&1; then
|
||||
yum install -y qemu-guest-agent
|
||||
elif command -v zypper >/dev/null 2>&1; then
|
||||
zypper --non-interactive install qemu-guest-agent
|
||||
elif command -v apk >/dev/null 2>&1; then
|
||||
apk add qemu-guest-agent
|
||||
else
|
||||
echo "no known package manager; install qemu-guest-agent in the image" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if command -v systemctl >/dev/null 2>&1; then
|
||||
systemctl enable --now qemu-guest-agent
|
||||
elif command -v rc-update >/dev/null 2>&1; then
|
||||
rc-update add qemu-guest-agent default
|
||||
rc-service qemu-guest-agent start
|
||||
fi
|
||||
// defaultVendorData is the cloud-init vendor-data shipped on every seed
|
||||
// unless the config overrides it. cloud-init merges it under the Proxy's own
|
||||
// user-data, so stock images grow a guest agent (which lets the provider
|
||||
// read the VM's IP before ARP sees traffic) without the user's cloud-init
|
||||
// having to know about it.
|
||||
const defaultVendorData = `#cloud-config
|
||||
packages:
|
||||
- qemu-guest-agent
|
||||
runcmd:
|
||||
- [systemctl, enable, --now, qemu-guest-agent]
|
||||
`
|
||||
|
||||
// buildVendorData renders the seed's vendor-data. With nothing configured
|
||||
// it is the guest-agent script alone; with both, a MIME multipart carrying
|
||||
// the configured document and the script as separate parts, since
|
||||
// vendor-data is a single file and the two must not be concatenated.
|
||||
func buildVendorData(configured string, installAgent bool) (string, error) {
|
||||
switch {
|
||||
case !installAgent:
|
||||
return configured, nil
|
||||
case strings.TrimSpace(configured) == "":
|
||||
return guestAgentScript, nil
|
||||
default:
|
||||
return multipartVendorData(configured, guestAgentScript)
|
||||
}
|
||||
}
|
||||
|
||||
// multipartVendorData packs several documents into one vendor-data blob.
|
||||
// cloud-init decides a payload is multipart by finding "mime-version:"
|
||||
// within its first 4096 bytes, so those headers lead. The configured
|
||||
// document goes in as text/plain on purpose: cloud-init re-sniffs that type
|
||||
// from the content's own header line, which keeps this code out of the
|
||||
// business of classifying what the user wrote.
|
||||
func multipartVendorData(configured, script string) (string, error) {
|
||||
var body bytes.Buffer
|
||||
w := multipart.NewWriter(&body)
|
||||
parts := []struct{ contentType, content string }{
|
||||
{"text/plain", configured},
|
||||
{"text/x-shellscript", script},
|
||||
}
|
||||
for _, p := range parts {
|
||||
part, err := w.CreatePart(textproto.MIMEHeader{"Content-Type": {p.contentType}})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("building vendor-data part: %w", err)
|
||||
}
|
||||
if _, err := io.WriteString(part, p.content); err != nil {
|
||||
return "", fmt.Errorf("writing vendor-data part: %w", err)
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return "", fmt.Errorf("closing vendor-data: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=%q\r\n\r\n%s",
|
||||
w.Boundary(), body.String()), nil
|
||||
}
|
||||
|
||||
// buildSeedISO renders a NoCloud seed: user-data and vendor-data verbatim,
|
||||
// and a meta-data whose instance-id is the deterministic VM name, so
|
||||
// buildSeedISO renders a NoCloud seed: user-data verbatim, vendor-data as
|
||||
// given, and a meta-data whose instance-id is the deterministic VM name, so
|
||||
// cloud-init never mistakes a reboot for a new instance. The ISO's volume
|
||||
// descriptor carries the build time, so two builds differ in a few
|
||||
// timestamp bytes; the files and label are identical.
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"bytes"
|
||||
"io"
|
||||
"maps"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -44,7 +42,7 @@ func readSeed(t *testing.T, iso []byte) (label string, files map[string]string)
|
||||
func TestBuildSeedISO_contents(t *testing.T) {
|
||||
t.Parallel()
|
||||
userData := "#cloud-config\npackages: [squid]\n"
|
||||
iso, err := buildSeedISO("proxy-abc123def456ghij", userData, guestAgentScript)
|
||||
iso, err := buildSeedISO("proxy-abc123def456ghij", userData, defaultVendorData)
|
||||
if err != nil {
|
||||
t.Fatalf("buildSeedISO() error = %v", err)
|
||||
}
|
||||
@@ -58,7 +56,7 @@ func TestBuildSeedISO_contents(t *testing.T) {
|
||||
if files["user-data"] != userData {
|
||||
t.Errorf("user-data = %q, want verbatim cloud-init", files["user-data"])
|
||||
}
|
||||
if files["vendor-data"] != guestAgentScript {
|
||||
if files["vendor-data"] != defaultVendorData {
|
||||
t.Errorf("vendor-data = %q", files["vendor-data"])
|
||||
}
|
||||
if got, want := files["meta-data"], "instance-id: proxy-abc123def456ghij\nlocal-hostname: proxy-abc123def456ghij\n"; got != want {
|
||||
@@ -91,7 +89,7 @@ func TestBuildSeedISO_deterministicContent(t *testing.T) {
|
||||
|
||||
func TestBuildSeedISO_emptyUserData(t *testing.T) {
|
||||
t.Parallel()
|
||||
iso, err := buildSeedISO("proxy-x", "", guestAgentScript)
|
||||
iso, err := buildSeedISO("proxy-x", "", defaultVendorData)
|
||||
if err != nil {
|
||||
t.Fatalf("buildSeedISO() error = %v", err)
|
||||
}
|
||||
@@ -100,134 +98,3 @@ func TestBuildSeedISO_emptyUserData(t *testing.T) {
|
||||
t.Errorf("user-data = %q (present=%v), want an empty file to still exist", body, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// The whole point of shipping the agent as vendor-data: whatever the Proxy
|
||||
// wrote reaches the guest untouched, in any of cloud-init's formats. The
|
||||
// previous implementation parsed and re-rendered cloud-config user-data,
|
||||
// which dropped comments, reordered keys and re-typed unquoted scalars
|
||||
// (version: 1.10 arrived as 1.1).
|
||||
func TestBuildSeedISO_userDataIsNeverRewritten(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
userData string
|
||||
}{
|
||||
{"cloud-config with its own packages", "#cloud-config\n# keep me\npackages:\n - squid\nversion: 1.10\nchpasswd: {list: 'u:0123456789'}\n"},
|
||||
{"jinja template", "## template: jinja\n#cloud-config\nhostname: {{ v1.local_hostname }}\n"},
|
||||
{"shell script", "#!/bin/sh\napt-get install -y squid\n"},
|
||||
{"include", "#include\nhttps://example.invalid/x\n"},
|
||||
{"mime multipart", "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"x\"\r\n\r\n--x\r\nContent-Type: text/cloud-config\r\n\r\n#cloud-config\npackages: [squid]\n\r\n--x--\r\n"},
|
||||
{"gzip payload", "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03binary\x00bytes"},
|
||||
{"leading blank line", "\n\n#cloud-config\npackages: [squid]\n"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
vendor, err := buildVendorData("", true)
|
||||
if err != nil {
|
||||
t.Fatalf("buildVendorData() error = %v", err)
|
||||
}
|
||||
iso, err := buildSeedISO("proxy-x", tc.userData, vendor)
|
||||
if err != nil {
|
||||
t.Fatalf("buildSeedISO() error = %v", err)
|
||||
}
|
||||
_, files := readSeed(t, iso)
|
||||
if files["user-data"] != tc.userData {
|
||||
t.Errorf("user-data was rewritten:\n got %q\nwant %q", files["user-data"], tc.userData)
|
||||
}
|
||||
if !strings.Contains(files["vendor-data"], "qemu-guest-agent") {
|
||||
t.Errorf("vendor-data = %q, want the agent script", files["vendor-data"])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildVendorData_agentScriptAlone(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := buildVendorData("", true)
|
||||
if err != nil {
|
||||
t.Fatalf("buildVendorData() error = %v", err)
|
||||
}
|
||||
if got != guestAgentScript {
|
||||
t.Errorf("vendor-data = %q, want the bare script", got)
|
||||
}
|
||||
if !strings.HasPrefix(got, "#!") {
|
||||
t.Error("vendor-data must start with a shebang or cloud-init treats it as something other than a script")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildVendorData_optOutKeepsConfiguredOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
configured := "#cloud-config\nntp: {enabled: true}\n"
|
||||
tests := []struct {
|
||||
name string
|
||||
configured string
|
||||
want string
|
||||
}{
|
||||
{"nothing configured", "", ""},
|
||||
{"configured document passes through", configured, configured},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := buildVendorData(tc.configured, false)
|
||||
if err != nil {
|
||||
t.Fatalf("buildVendorData() error = %v", err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Errorf("vendor-data = %q, want %q", got, tc.want)
|
||||
}
|
||||
if strings.Contains(got, "qemu-guest-agent") {
|
||||
t.Error("agent script present although installGuestAgent is off")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Both documents have to survive in one file, and cloud-init only looks for
|
||||
// the multipart marker in the first 4096 bytes.
|
||||
func TestBuildVendorData_multipartCarriesBoth(t *testing.T) {
|
||||
t.Parallel()
|
||||
configured := "#cloud-config\nntp:\n enabled: true\n"
|
||||
got, err := buildVendorData(configured, true)
|
||||
if err != nil {
|
||||
t.Fatalf("buildVendorData() error = %v", err)
|
||||
}
|
||||
head := got
|
||||
if len(head) > 4096 {
|
||||
head = head[:4096]
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(head), "mime-version:") {
|
||||
t.Fatalf("no mime-version: in the first 4096 bytes; cloud-init would treat this as a single document:\n%s", head)
|
||||
}
|
||||
_, params, err := mime.ParseMediaType(strings.SplitN(got, "\r\n\r\n", 2)[0][strings.Index(got, "Content-Type:")+len("Content-Type: "):])
|
||||
if err != nil {
|
||||
t.Fatalf("parsing the multipart content type: %v", err)
|
||||
}
|
||||
body := strings.SplitN(got, "\r\n\r\n", 2)[1]
|
||||
r := multipart.NewReader(strings.NewReader(body), params["boundary"])
|
||||
var bodies, types []string
|
||||
for {
|
||||
part, err := r.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("reading part: %v", err)
|
||||
}
|
||||
content, err := io.ReadAll(part)
|
||||
if err != nil {
|
||||
t.Fatalf("reading part body: %v", err)
|
||||
}
|
||||
bodies = append(bodies, string(content))
|
||||
types = append(types, part.Header.Get("Content-Type"))
|
||||
}
|
||||
if len(bodies) != 2 || bodies[0] != configured || bodies[1] != guestAgentScript {
|
||||
t.Fatalf("parts = %q, want the configured document then the script", bodies)
|
||||
}
|
||||
// text/plain makes cloud-init re-sniff the user's document from its own
|
||||
// header line, so this code never has to classify it.
|
||||
if types[0] != "text/plain" || types[1] != "text/x-shellscript" {
|
||||
t.Errorf("part types = %v, want [text/plain text/x-shellscript]", types)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,211 +0,0 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
"github.com/go-logr/logr"
|
||||
)
|
||||
|
||||
// tracingAPI wraps a libvirtAPI and logs one V(1) line per call after it
|
||||
// returns: the libvirt API function (rpc=virDomainLookupByName, ...), the
|
||||
// identifiers it was given, curated results, duration, and on failure the
|
||||
// error with libvirt's code name. At V(2) the same line also carries XML
|
||||
// bodies, metadata and address lists. Upload and download bodies are never
|
||||
// logged: the seed ISO carries cloud-init.
|
||||
type tracingAPI struct {
|
||||
inner libvirtAPI
|
||||
log logr.Logger
|
||||
}
|
||||
|
||||
type tracedCall struct {
|
||||
t *tracingAPI
|
||||
method string
|
||||
start time.Time
|
||||
kv []any
|
||||
}
|
||||
|
||||
// begin names the call by its libvirt API function: go-libvirt's method
|
||||
// names are the remote procedure names, which are the public vir* API
|
||||
// names without the prefix.
|
||||
func (t *tracingAPI) begin(method string, kv ...any) *tracedCall {
|
||||
return &tracedCall{t: t, method: "vir" + method, start: time.Now(), kv: kv}
|
||||
}
|
||||
|
||||
// detail adds key/values that are only worth the bytes at V(2).
|
||||
func (c *tracedCall) detail(kv ...any) *tracedCall {
|
||||
if c.t.log.V(2).Enabled() {
|
||||
c.kv = append(c.kv, kv...)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *tracedCall) done(err error, kv ...any) {
|
||||
fields := make([]any, 0, 2+len(c.kv)+len(kv)+10)
|
||||
fields = append(fields, "rpc", c.method)
|
||||
fields = append(fields, c.kv...)
|
||||
fields = append(fields, kv...)
|
||||
fields = append(fields, "duration", time.Since(c.start).Round(time.Microsecond))
|
||||
if err != nil {
|
||||
fields = append(fields, "error", err.Error())
|
||||
fields = append(fields, libvirtErrorFields(err)...)
|
||||
}
|
||||
c.t.log.V(1).Info("libvirt call", fields...)
|
||||
}
|
||||
|
||||
// flagKV renders a flags value only when something is set; go-libvirt's
|
||||
// generated stringers do not know bitmask combinations.
|
||||
func flagKV(v int64) []any {
|
||||
if v == 0 {
|
||||
return nil
|
||||
}
|
||||
return []any{"flags", v}
|
||||
}
|
||||
|
||||
func optString(s libvirt.OptString) string {
|
||||
if len(s) == 0 {
|
||||
return ""
|
||||
}
|
||||
return s[0]
|
||||
}
|
||||
|
||||
func (t *tracingAPI) DomainLookupByName(name string) (libvirt.Domain, error) {
|
||||
c := t.begin("DomainLookupByName", "domain", name)
|
||||
d, err := t.inner.DomainLookupByName(name)
|
||||
c.done(err)
|
||||
return d, err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) DomainDefineXML(xml string) (libvirt.Domain, error) {
|
||||
c := t.begin("DomainDefineXML", "xmlBytes", len(xml)).detail("xml", xml)
|
||||
d, err := t.inner.DomainDefineXML(xml)
|
||||
c.done(err, "domain", d.Name)
|
||||
return d, err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) DomainCreate(d libvirt.Domain) error {
|
||||
c := t.begin("DomainCreate", "domain", d.Name)
|
||||
err := t.inner.DomainCreate(d)
|
||||
c.done(err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) DomainDestroy(d libvirt.Domain) error {
|
||||
c := t.begin("DomainDestroy", "domain", d.Name)
|
||||
err := t.inner.DomainDestroy(d)
|
||||
c.done(err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) DomainUndefineFlags(d libvirt.Domain, flags libvirt.DomainUndefineFlagsValues) error {
|
||||
c := t.begin("DomainUndefineFlags", append([]any{"domain", d.Name}, flagKV(int64(flags))...)...)
|
||||
err := t.inner.DomainUndefineFlags(d, flags)
|
||||
c.done(err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) DomainGetState(d libvirt.Domain, flags uint32) (int32, int32, error) {
|
||||
c := t.begin("DomainGetState", "domain", d.Name)
|
||||
state, reason, err := t.inner.DomainGetState(d, flags)
|
||||
c.done(err, "state", libvirt.DomainState(state), "reason", reason)
|
||||
return state, reason, err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, flags libvirt.DomainModificationImpact) (string, error) {
|
||||
c := t.begin("DomainGetMetadata", "domain", d.Name, "type", typ, "uri", optString(uri))
|
||||
s, err := t.inner.DomainGetMetadata(d, typ, uri, flags)
|
||||
c.detail("metadata", s).done(err, "metadataBytes", len(s))
|
||||
return s, err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) DomainInterfaceAddresses(d libvirt.Domain, source uint32, flags uint32) ([]libvirt.DomainInterface, error) {
|
||||
c := t.begin("DomainInterfaceAddresses", "domain", d.Name, "source", libvirt.DomainInterfaceAddressesSource(source))
|
||||
ifaces, err := t.inner.DomainInterfaceAddresses(d, source, flags)
|
||||
if t.log.V(2).Enabled() {
|
||||
c.detail("addresses", formatAddresses(ifaces))
|
||||
}
|
||||
c.done(err, "interfaces", len(ifaces))
|
||||
return ifaces, err
|
||||
}
|
||||
|
||||
func formatAddresses(ifaces []libvirt.DomainInterface) []string {
|
||||
var out []string
|
||||
for _, iface := range ifaces {
|
||||
for _, a := range iface.Addrs {
|
||||
out = append(out, fmt.Sprintf("%s=%s/%d", iface.Name, a.Addr, a.Prefix))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (t *tracingAPI) ConnectListAllDomains(needResults int32, flags libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error) {
|
||||
c := t.begin("ConnectListAllDomains", flagKV(int64(flags))...)
|
||||
doms, n, err := t.inner.ConnectListAllDomains(needResults, flags)
|
||||
if t.log.V(2).Enabled() {
|
||||
names := make([]string, 0, len(doms))
|
||||
for _, d := range doms {
|
||||
names = append(names, d.Name)
|
||||
}
|
||||
c.detail("names", names)
|
||||
}
|
||||
c.done(err, "count", len(doms))
|
||||
return doms, n, err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) StoragePoolLookupByName(name string) (libvirt.StoragePool, error) {
|
||||
c := t.begin("StoragePoolLookupByName", "pool", name)
|
||||
pool, err := t.inner.StoragePoolLookupByName(name)
|
||||
c.done(err)
|
||||
return pool, err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) StoragePoolRefresh(pool libvirt.StoragePool, flags uint32) error {
|
||||
c := t.begin("StoragePoolRefresh", "pool", pool.Name)
|
||||
err := t.inner.StoragePoolRefresh(pool, flags)
|
||||
c.done(err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) StorageVolLookupByName(pool libvirt.StoragePool, name string) (libvirt.StorageVol, error) {
|
||||
c := t.begin("StorageVolLookupByName", "pool", pool.Name, "volume", name)
|
||||
vol, err := t.inner.StorageVolLookupByName(pool, name)
|
||||
c.done(err)
|
||||
return vol, err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) StorageVolGetPath(vol libvirt.StorageVol) (string, error) {
|
||||
c := t.begin("StorageVolGetPath", "pool", vol.Pool, "volume", vol.Name)
|
||||
path, err := t.inner.StorageVolGetPath(vol)
|
||||
c.done(err, "path", path)
|
||||
return path, err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) StorageVolCreateXML(pool libvirt.StoragePool, xml string, flags libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error) {
|
||||
c := t.begin("StorageVolCreateXML", append([]any{"pool", pool.Name, "xmlBytes", len(xml)}, flagKV(int64(flags))...)...).detail("xml", xml)
|
||||
vol, err := t.inner.StorageVolCreateXML(pool, xml, flags)
|
||||
c.done(err, "volume", vol.Name)
|
||||
return vol, err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) StorageVolUpload(vol libvirt.StorageVol, r io.Reader, offset, length uint64, flags libvirt.StorageVolUploadFlags) error {
|
||||
c := t.begin("StorageVolUpload", append([]any{"pool", vol.Pool, "volume", vol.Name, "offset", offset, "length", length}, flagKV(int64(flags))...)...)
|
||||
err := t.inner.StorageVolUpload(vol, r, offset, length, flags)
|
||||
c.done(err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) StorageVolDownload(vol libvirt.StorageVol, w io.Writer, offset, length uint64, flags libvirt.StorageVolDownloadFlags) error {
|
||||
c := t.begin("StorageVolDownload", append([]any{"pool", vol.Pool, "volume", vol.Name, "offset", offset, "length", length}, flagKV(int64(flags))...)...)
|
||||
err := t.inner.StorageVolDownload(vol, w, offset, length, flags)
|
||||
c.done(err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *tracingAPI) StorageVolDelete(vol libvirt.StorageVol, flags libvirt.StorageVolDeleteFlags) error {
|
||||
c := t.begin("StorageVolDelete", append([]any{"pool", vol.Pool, "volume", vol.Name}, flagKV(int64(flags))...)...)
|
||||
err := t.inner.StorageVolDelete(vol, flags)
|
||||
c.done(err)
|
||||
return err
|
||||
}
|
||||
@@ -1,300 +0,0 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/go-logr/logr/funcr"
|
||||
)
|
||||
|
||||
// captureLogger returns a funcr logger capped at verbosity (the test
|
||||
// stand-in for --zap-log-level=<verbosity>) and a snapshot function for the
|
||||
// lines it recorded. The sink is mutex-guarded: a cancelled call logs from
|
||||
// the abandoned goroutine.
|
||||
func captureLogger(verbosity int) (logr.Logger, func() []string) {
|
||||
var mu sync.Mutex
|
||||
var lines []string
|
||||
log := funcr.New(func(prefix, args string) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
lines = append(lines, prefix+" "+args)
|
||||
}, funcr.Options{Verbosity: verbosity})
|
||||
return log, func() []string {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return append([]string(nil), lines...)
|
||||
}
|
||||
}
|
||||
|
||||
func captureContext(verbosity int) (context.Context, func() []string) {
|
||||
log, snapshot := captureLogger(verbosity)
|
||||
return logr.NewContext(context.Background(), log), snapshot
|
||||
}
|
||||
|
||||
const stoppedName = "proxy-stopped"
|
||||
|
||||
// tracedHost is a fake with one managed running domain and its volumes, so
|
||||
// every method of the seam has something to act on.
|
||||
func tracedHost(t *testing.T) *fakeAPI {
|
||||
t.Helper()
|
||||
api := readyHost()
|
||||
managedDomain(t, api, testName, testUID, true, map[uint32][]libvirt.DomainInterface{
|
||||
uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Name: "vnet0", Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.50")}}},
|
||||
})
|
||||
api.putDomain(&fakeDomain{name: stoppedName})
|
||||
return api
|
||||
}
|
||||
|
||||
func TestTracingAPI_logsEveryMethod(t *testing.T) {
|
||||
t.Parallel()
|
||||
pool := libvirt.StoragePool{Name: "default"}
|
||||
dom := libvirt.Domain{Name: testName}
|
||||
overlay := libvirt.StorageVol{Pool: "default", Name: overlayVolumeName(testName)}
|
||||
cases := []struct {
|
||||
method string
|
||||
call func(api libvirtAPI) error
|
||||
}{
|
||||
{"DomainLookupByName", func(api libvirtAPI) error { _, err := api.DomainLookupByName(testName); return err }},
|
||||
{"DomainDefineXML", func(api libvirtAPI) error {
|
||||
xml, err := buildDomainXML(testDomainParams())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = api.DomainDefineXML(xml)
|
||||
return err
|
||||
}},
|
||||
{"DomainCreate", func(api libvirtAPI) error { return api.DomainCreate(libvirt.Domain{Name: stoppedName}) }},
|
||||
{"DomainDestroy", func(api libvirtAPI) error { return api.DomainDestroy(dom) }},
|
||||
{"DomainUndefineFlags", func(api libvirtAPI) error { return api.DomainUndefineFlags(dom, undefineFlags) }},
|
||||
{"DomainGetState", func(api libvirtAPI) error { _, _, err := api.DomainGetState(dom, 0); return err }},
|
||||
{"DomainGetMetadata", func(api libvirtAPI) error {
|
||||
_, err := api.DomainGetMetadata(dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, 0)
|
||||
return err
|
||||
}},
|
||||
{"DomainInterfaceAddresses", func(api libvirtAPI) error {
|
||||
_, err := api.DomainInterfaceAddresses(dom, uint32(libvirt.DomainInterfaceAddressesSrcArp), 0)
|
||||
return err
|
||||
}},
|
||||
{"ConnectListAllDomains", func(api libvirtAPI) error { _, _, err := api.ConnectListAllDomains(1, 0); return err }},
|
||||
{"StoragePoolLookupByName", func(api libvirtAPI) error { _, err := api.StoragePoolLookupByName("default"); return err }},
|
||||
{"StoragePoolRefresh", func(api libvirtAPI) error { return api.StoragePoolRefresh(pool, 0) }},
|
||||
{"StorageVolLookupByName", func(api libvirtAPI) error { _, err := api.StorageVolLookupByName(pool, overlay.Name); return err }},
|
||||
{"StorageVolGetPath", func(api libvirtAPI) error { _, err := api.StorageVolGetPath(overlay); return err }},
|
||||
{"StorageVolCreateXML", func(api libvirtAPI) error {
|
||||
xml, err := buildVolumeXML(rawVolume("new.iso", 3))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = api.StorageVolCreateXML(pool, xml, 0)
|
||||
return err
|
||||
}},
|
||||
{"StorageVolUpload", func(api libvirtAPI) error {
|
||||
return api.StorageVolUpload(overlay, strings.NewReader("abc"), 0, 3, 0)
|
||||
}},
|
||||
{"StorageVolDownload", func(api libvirtAPI) error {
|
||||
return api.StorageVolDownload(overlay, io.Discard, 0, 0, 0)
|
||||
}},
|
||||
{"StorageVolDelete", func(api libvirtAPI) error { return api.StorageVolDelete(overlay, 0) }},
|
||||
}
|
||||
if n := reflect.TypeOf((*libvirtAPI)(nil)).Elem().NumMethod(); n != len(cases) {
|
||||
t.Fatalf("libvirtAPI has %d methods, this test covers %d; trace the new one", n, len(cases))
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.method, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
log, snapshot := captureLogger(1)
|
||||
api := &tracingAPI{inner: tracedHost(t), log: log}
|
||||
if err := tc.call(api); err != nil {
|
||||
t.Fatalf("%s: %v", tc.method, err)
|
||||
}
|
||||
lines := snapshot()
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("logged %d lines, want exactly 1:\n%s", len(lines), strings.Join(lines, "\n"))
|
||||
}
|
||||
for _, want := range []string{`"msg"="libvirt call"`, `"rpc"="vir` + tc.method + `"`, `"duration"=`} {
|
||||
if !strings.Contains(lines[0], want) {
|
||||
t.Errorf("line missing %s:\n%s", want, lines[0])
|
||||
}
|
||||
}
|
||||
if strings.Contains(lines[0], `"error"=`) {
|
||||
t.Errorf("successful call logged an error:\n%s", lines[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTracingAPI_passesResultsThrough(t *testing.T) {
|
||||
t.Parallel()
|
||||
inner := tracedHost(t)
|
||||
log, _ := captureLogger(1)
|
||||
traced := &tracingAPI{inner: inner, log: log}
|
||||
dom := libvirt.Domain{Name: testName}
|
||||
|
||||
wantState, wantReason, err := inner.DomainGetState(dom, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, reason, err := traced.DomainGetState(dom, 0)
|
||||
if err != nil || state != wantState || reason != wantReason {
|
||||
t.Errorf("DomainGetState = %d/%d/%v, want %d/%d/nil", state, reason, err, wantState, wantReason)
|
||||
}
|
||||
|
||||
overlay := libvirt.StorageVol{Pool: "default", Name: overlayVolumeName(testName)}
|
||||
wantPath, _ := inner.StorageVolGetPath(overlay)
|
||||
if path, err := traced.StorageVolGetPath(overlay); err != nil || path != wantPath {
|
||||
t.Errorf("StorageVolGetPath = %q/%v, want %q", path, err, wantPath)
|
||||
}
|
||||
|
||||
wantDoms, _, _ := inner.ConnectListAllDomains(1, 0)
|
||||
doms, _, err := traced.ConnectListAllDomains(1, 0)
|
||||
if err != nil || !reflect.DeepEqual(doms, wantDoms) {
|
||||
t.Errorf("ConnectListAllDomains = %v/%v, want %v", doms, err, wantDoms)
|
||||
}
|
||||
|
||||
inner.fail["DomainCreate"] = lvError(libvirt.ErrOperationInvalid, "already running")
|
||||
if err := traced.DomainCreate(dom); !hasCode(err, libvirt.ErrOperationInvalid) {
|
||||
t.Errorf("DomainCreate error = %v, want the injected libvirt error unchanged", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTracingAPI_errorFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
inject error
|
||||
want []string
|
||||
notWant []string
|
||||
}{
|
||||
{
|
||||
name: "libvirt error carries code, name and message",
|
||||
inject: lvError(libvirt.ErrNoDomain, "Domain not found: no domain with matching name"),
|
||||
want: []string{`"error"="Domain not found`, `"libvirtError"="ErrNoDomain"`, `"libvirtCode"=42`},
|
||||
},
|
||||
{
|
||||
name: "transport error has no libvirt fields",
|
||||
inject: io.EOF,
|
||||
want: []string{`"error"="EOF"`},
|
||||
notWant: []string{`"libvirtError"`, `"libvirtCode"`},
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
inner := tracedHost(t)
|
||||
inner.fail["DomainLookupByName"] = tc.inject
|
||||
log, snapshot := captureLogger(1)
|
||||
api := &tracingAPI{inner: inner, log: log}
|
||||
if _, err := api.DomainLookupByName(testName); !errors.Is(err, tc.inject) {
|
||||
t.Fatalf("error = %v, want the injected one", err)
|
||||
}
|
||||
joined := strings.Join(snapshot(), "\n")
|
||||
for _, want := range tc.want {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("output missing %s:\n%s", want, joined)
|
||||
}
|
||||
}
|
||||
for _, absent := range tc.notWant {
|
||||
if strings.Contains(joined, absent) {
|
||||
t.Errorf("output unexpectedly contains %s:\n%s", absent, joined)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTracingAPI_detailOnlyAtV2(t *testing.T) {
|
||||
t.Parallel()
|
||||
dom := libvirt.Domain{Name: testName}
|
||||
calls := []struct {
|
||||
method string
|
||||
call func(api libvirtAPI) error
|
||||
always string
|
||||
detail string
|
||||
}{
|
||||
{"DomainDefineXML", func(api libvirtAPI) error {
|
||||
xml, err := buildDomainXML(testDomainParams())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = api.DomainDefineXML(xml)
|
||||
return err
|
||||
}, `"xmlBytes"=`, `"xml"="<domain`},
|
||||
{"StorageVolCreateXML", func(api libvirtAPI) error {
|
||||
xml, err := buildVolumeXML(rawVolume("new.iso", 3))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = api.StorageVolCreateXML(libvirt.StoragePool{Name: "default"}, xml, 0)
|
||||
return err
|
||||
}, `"xmlBytes"=`, `"xml"="<volume`},
|
||||
{"DomainGetMetadata", func(api libvirtAPI) error {
|
||||
_, err := api.DomainGetMetadata(dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, 0)
|
||||
return err
|
||||
}, `"metadataBytes"=`, `"metadata"="<instance`},
|
||||
{"DomainInterfaceAddresses", func(api libvirtAPI) error {
|
||||
_, err := api.DomainInterfaceAddresses(dom, uint32(libvirt.DomainInterfaceAddressesSrcArp), 0)
|
||||
return err
|
||||
}, `"interfaces"=1`, `"addresses"=["vnet0=192.168.1.50/24"]`},
|
||||
{"ConnectListAllDomains", func(api libvirtAPI) error {
|
||||
_, _, err := api.ConnectListAllDomains(1, 0)
|
||||
return err
|
||||
}, `"count"=2`, `"names"=[`},
|
||||
}
|
||||
for _, verbosity := range []int{1, 2} {
|
||||
for _, tc := range calls {
|
||||
t.Run(tc.method+"/v"+string(rune('0'+verbosity)), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
log, snapshot := captureLogger(verbosity)
|
||||
api := &tracingAPI{inner: tracedHost(t), log: log}
|
||||
if err := tc.call(api); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
joined := strings.Join(snapshot(), "\n")
|
||||
if !strings.Contains(joined, tc.always) {
|
||||
t.Errorf("v%d output missing %s:\n%s", verbosity, tc.always, joined)
|
||||
}
|
||||
if has := strings.Contains(joined, tc.detail); has != (verbosity >= 2) {
|
||||
t.Errorf("v%d output contains %s = %v, want %v:\n%s", verbosity, tc.detail, has, verbosity >= 2, joined)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTracingAPI_uploadLogsSizeNotContent(t *testing.T) {
|
||||
t.Parallel()
|
||||
const sentinel = "SENTINEL-seed-content-must-never-be-logged"
|
||||
log, snapshot := captureLogger(2)
|
||||
api := &tracingAPI{inner: tracedHost(t), log: log}
|
||||
vol := libvirt.StorageVol{Pool: "default", Name: seedVolumeName(testName)}
|
||||
if err := api.StorageVolUpload(vol, bytes.NewReader([]byte(sentinel)), 0, uint64(len(sentinel)), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
joined := strings.Join(snapshot(), "\n")
|
||||
if !strings.Contains(joined, `"length"=42`) {
|
||||
t.Errorf("upload line missing the length:\n%s", joined)
|
||||
}
|
||||
if strings.Contains(joined, sentinel) {
|
||||
t.Errorf("upload content leaked into logs:\n%s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTracingAPI_silentBelowV1(t *testing.T) {
|
||||
t.Parallel()
|
||||
log, snapshot := captureLogger(0)
|
||||
api := &tracingAPI{inner: tracedHost(t), log: log}
|
||||
if _, err := api.DomainLookupByName(testName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if lines := snapshot(); len(lines) != 0 {
|
||||
t.Errorf("verbosity 0 logged %d lines:\n%s", len(lines), strings.Join(lines, "\n"))
|
||||
}
|
||||
}
|
||||
@@ -50,25 +50,17 @@ func (i *instrumented) record(op string, err error) {
|
||||
i.rec.ProviderRequest(i.name, op, resultLabel(err))
|
||||
}
|
||||
|
||||
const (
|
||||
resultOK = "ok"
|
||||
resultNotFound = "not_found"
|
||||
resultQuotaExceeded = "quota_exceeded"
|
||||
resultPermanent = "permanent"
|
||||
resultTransient = "transient"
|
||||
)
|
||||
|
||||
func resultLabel(err error) string {
|
||||
switch Class(err) {
|
||||
case nil:
|
||||
return resultOK
|
||||
return "ok"
|
||||
case ErrNotFound:
|
||||
return resultNotFound
|
||||
return "not_found"
|
||||
case ErrQuotaExceeded:
|
||||
return resultQuotaExceeded
|
||||
return "quota_exceeded"
|
||||
case ErrPermanent:
|
||||
return resultPermanent
|
||||
return "permanent"
|
||||
default:
|
||||
return resultTransient
|
||||
return "transient"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
// WithTracing wraps a Provider so every call runs in a client span carrying
|
||||
// the classified result, mirroring WithMetrics. It goes through
|
||||
// tracing.StartSpan, so provider-internal logs (e.g. the gcp provider's
|
||||
// context logger) inherit traceID/spanID. Errors pass through unmodified.
|
||||
func WithTracing(name string, p Provider, opts ...tracing.Option) Provider {
|
||||
return &traced{name: name, inner: p, tracer: tracing.Tracer(opts...)}
|
||||
}
|
||||
|
||||
type traced struct {
|
||||
name string
|
||||
inner Provider
|
||||
tracer trace.Tracer
|
||||
}
|
||||
|
||||
func (t *traced) Create(ctx context.Context, req CreateRequest) (string, error) {
|
||||
ctx, span := t.start(ctx, "provider.create")
|
||||
id, err := t.inner.Create(ctx, req)
|
||||
if id != "" {
|
||||
span.SetAttributes(attribute.String("provider.id", id))
|
||||
}
|
||||
t.end(span, err)
|
||||
return id, err
|
||||
}
|
||||
|
||||
func (t *traced) Get(ctx context.Context, providerID string) (*Instance, error) {
|
||||
ctx, span := t.start(ctx, "provider.get")
|
||||
span.SetAttributes(attribute.String("provider.id", providerID))
|
||||
inst, err := t.inner.Get(ctx, providerID)
|
||||
t.end(span, err)
|
||||
return inst, err
|
||||
}
|
||||
|
||||
func (t *traced) Delete(ctx context.Context, providerID string) error {
|
||||
ctx, span := t.start(ctx, "provider.delete")
|
||||
span.SetAttributes(attribute.String("provider.id", providerID))
|
||||
err := t.inner.Delete(ctx, providerID)
|
||||
t.end(span, err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *traced) ListByTag(ctx context.Context) ([]Instance, error) {
|
||||
ctx, span := t.start(ctx, "provider.list")
|
||||
instances, err := t.inner.ListByTag(ctx)
|
||||
span.SetAttributes(attribute.Int("provider.instances", len(instances)))
|
||||
t.end(span, err)
|
||||
return instances, err
|
||||
}
|
||||
|
||||
func (t *traced) start(ctx context.Context, op string) (context.Context, trace.Span) {
|
||||
return tracing.StartSpan(ctx, t.tracer, op,
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("provider.name", t.name)))
|
||||
}
|
||||
|
||||
// end records the taxonomy class and closes the span. ErrNotFound stays Ok:
|
||||
// the reconciler polls Get to NotFound during replacement/deletion, so it is
|
||||
// an expected answer, not a failure — same reasoning as resultLabel's
|
||||
// distinct "not_found" bucket.
|
||||
func (t *traced) end(span trace.Span, err error) {
|
||||
span.SetAttributes(attribute.String("provider.result", resultLabel(err)))
|
||||
switch Class(err) {
|
||||
case nil, ErrNotFound:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
default:
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
func TestWithTracing_spanPerCall(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inner *staticProvider
|
||||
call func(p Provider) error
|
||||
wantSpan string
|
||||
wantStatus codes.Code
|
||||
wantResult string
|
||||
wantAttr attribute.KeyValue
|
||||
}{
|
||||
{
|
||||
name: "successful create",
|
||||
inner: &staticProvider{},
|
||||
call: func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
|
||||
wantSpan: "provider.create",
|
||||
wantStatus: codes.Ok,
|
||||
wantResult: resultOK,
|
||||
wantAttr: attribute.String("provider.id", "id-1"),
|
||||
},
|
||||
{
|
||||
name: "get NotFound is not a span error",
|
||||
inner: &staticProvider{getErr: Wrap(ErrNotFound, "get", "x", "id-1", nil)},
|
||||
call: func(p Provider) error { _, err := p.Get(context.Background(), "id-1"); return err },
|
||||
wantSpan: "provider.get",
|
||||
wantStatus: codes.Ok,
|
||||
wantResult: resultNotFound,
|
||||
wantAttr: attribute.String("provider.id", "id-1"),
|
||||
},
|
||||
{
|
||||
name: "delete transient error",
|
||||
inner: &staticProvider{deleteErr: Wrap(ErrTransient, "delete", "x", "id-1", errors.New("503"))},
|
||||
call: func(p Provider) error { return p.Delete(context.Background(), "id-1") },
|
||||
wantSpan: "provider.delete",
|
||||
wantStatus: codes.Error,
|
||||
wantResult: resultTransient,
|
||||
wantAttr: attribute.String("provider.id", "id-1"),
|
||||
},
|
||||
{
|
||||
name: "quota exceeded create",
|
||||
inner: &staticProvider{createErr: Wrap(ErrQuotaExceeded, "create", "x", "", nil)},
|
||||
call: func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
|
||||
wantSpan: "provider.create",
|
||||
wantStatus: codes.Error,
|
||||
wantResult: resultQuotaExceeded,
|
||||
wantAttr: attribute.String("provider.name", "x"),
|
||||
},
|
||||
{
|
||||
name: "list records instance count",
|
||||
inner: &staticProvider{},
|
||||
call: func(p Provider) error { _, err := p.ListByTag(context.Background()); return err },
|
||||
wantSpan: "provider.list",
|
||||
wantStatus: codes.Ok,
|
||||
wantResult: resultOK,
|
||||
wantAttr: attribute.Int("provider.instances", 0),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
|
||||
p := WithTracing("x", tc.inner, tracing.WithTracerProvider(tp))
|
||||
err := tc.call(p)
|
||||
|
||||
wantErr := errors.Join(tc.inner.createErr, tc.inner.getErr, tc.inner.deleteErr, tc.inner.listErr)
|
||||
if (wantErr == nil) != (err == nil) {
|
||||
t.Fatalf("decorator changed the error: got %v", err)
|
||||
}
|
||||
|
||||
ended := sr.Ended()
|
||||
if len(ended) != 1 {
|
||||
t.Fatalf("got %d spans, want 1", len(ended))
|
||||
}
|
||||
span := ended[0]
|
||||
if span.Name() != tc.wantSpan {
|
||||
t.Errorf("span name = %q, want %q", span.Name(), tc.wantSpan)
|
||||
}
|
||||
if span.SpanKind() != trace.SpanKindClient {
|
||||
t.Errorf("span kind = %v, want client", span.SpanKind())
|
||||
}
|
||||
if span.Status().Code != tc.wantStatus {
|
||||
t.Errorf("status = %v, want %v", span.Status().Code, tc.wantStatus)
|
||||
}
|
||||
attrs := span.Attributes()
|
||||
hasAttr := func(want attribute.KeyValue) bool {
|
||||
return slices.Contains(attrs, want)
|
||||
}
|
||||
if !hasAttr(attribute.String("provider.result", tc.wantResult)) {
|
||||
t.Errorf("provider.result %q missing in %v", tc.wantResult, attrs)
|
||||
}
|
||||
if !hasAttr(tc.wantAttr) {
|
||||
t.Errorf("attribute %v missing in %v", tc.wantAttr, attrs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
)
|
||||
|
||||
// HTTPMiddleware returns middleware that opens a server span per request
|
||||
// (named from the mux route pattern once routing has happened) and hands
|
||||
// handlers a request context whose logf.FromContext logger is base enriched
|
||||
// with the trace context. Incoming W3C traceparent headers are honored, so
|
||||
// clients see their own trace continue into the operator. /healthz is never
|
||||
// traced. With tracing disabled the middleware degrades to injecting base
|
||||
// unenriched — handlers can rely on logf.FromContext either way.
|
||||
func HTTPMiddleware(operation string, base logr.Logger, opts ...Option) func(http.Handler) http.Handler {
|
||||
o := newOptions(opts)
|
||||
return func(next http.Handler) http.Handler {
|
||||
inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
r2 := r.WithContext(ContextWithLogger(r.Context(), base))
|
||||
next.ServeHTTP(w, r2)
|
||||
// WithContext copies the request, so the mux recorded the matched
|
||||
// route on r2; surface it to otelhttp, which renames the span
|
||||
// from r.Pattern after the handler returns.
|
||||
r.Pattern = r2.Pattern
|
||||
})
|
||||
otelOpts := []otelhttp.Option{
|
||||
otelhttp.WithFilter(func(r *http.Request) bool { return r.URL.Path != "/healthz" }),
|
||||
// Explicit propagators: deterministic regardless of whether the
|
||||
// global propagator has been installed (tests, disabled tracing).
|
||||
otelhttp.WithPropagators(propagation.NewCompositeTextMapPropagator(
|
||||
propagation.TraceContext{}, propagation.Baggage{})),
|
||||
}
|
||||
if o.tp != nil {
|
||||
otelOpts = append(otelOpts, otelhttp.WithTracerProvider(o.tp))
|
||||
}
|
||||
return otelhttp.NewHandler(inner, operation, otelOpts...)
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
const sampleTraceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
|
||||
|
||||
func TestHTTPMiddleware(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
newServer := func(lines *[]string, tp *sdktrace.TracerProvider) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
handler := func(w http.ResponseWriter, r *http.Request) {
|
||||
logf.FromContext(r.Context()).Info("handling")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
mux.HandleFunc("GET /healthz", handler)
|
||||
mux.HandleFunc("GET /v1/things/{id}", handler)
|
||||
return HTTPMiddleware("discovery", captureLogger(lines), WithTracerProvider(tp))(mux)
|
||||
}
|
||||
|
||||
t.Run("route span, enriched handler logs, traceparent continuation", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
|
||||
var lines []string
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/things/42", nil)
|
||||
req.Header.Set("traceparent", sampleTraceparent)
|
||||
newServer(&lines, tp).ServeHTTP(rec, req)
|
||||
|
||||
ended := sr.Ended()
|
||||
if len(ended) != 1 {
|
||||
t.Fatalf("got %d spans, want 1", len(ended))
|
||||
}
|
||||
if got := ended[0].Name(); got != "GET /v1/things/{id}" {
|
||||
t.Errorf("span name = %q, want route pattern", got)
|
||||
}
|
||||
wantTrace := "4bf92f3577b34da6a3ce929d0e0e4736"
|
||||
if got := ended[0].SpanContext().TraceID().String(); got != wantTrace {
|
||||
t.Errorf("span traceID = %s, want continuation of client trace %s", got, wantTrace)
|
||||
}
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("got %d log lines, want 1: %v", len(lines), lines)
|
||||
}
|
||||
if n := strings.Count(lines[0], `"traceID"`); n != 1 {
|
||||
t.Errorf("traceID appears %d times, want 1: %s", n, lines[0])
|
||||
}
|
||||
if !strings.Contains(lines[0], wantTrace) {
|
||||
t.Errorf("handler log missing traceID %s: %s", wantTrace, lines[0])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("healthz is not traced but still gets a logger", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
|
||||
var lines []string
|
||||
rec := httptest.NewRecorder()
|
||||
newServer(&lines, tp).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
||||
|
||||
if len(sr.Ended()) != 0 {
|
||||
t.Fatalf("healthz produced %d spans, want 0", len(sr.Ended()))
|
||||
}
|
||||
if len(lines) != 1 || strings.Contains(lines[0], "traceID") {
|
||||
t.Fatalf("want one unenriched log line, got %v", lines)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
// baseLoggerKey stores the pre-enrichment logger. logr sinks never see a
|
||||
// context, so trace IDs must ride on the logger as WithValues — and zap does
|
||||
// not dedupe repeated keys, so each nested span must re-derive from the same
|
||||
// base instead of stacking traceID/spanID onto an already-enriched logger.
|
||||
type baseLoggerKey struct{}
|
||||
|
||||
// Start begins a span from the global tracer and returns a context whose
|
||||
// logf.FromContext logger carries the span's traceID/spanID. With tracing
|
||||
// disabled the span is a no-op with an invalid span context and the logger
|
||||
// is left untouched.
|
||||
//
|
||||
// Trade-off, documented: values pushed via logf.IntoContext *between* two
|
||||
// Start calls are dropped by the inner Start's re-derivation. Nothing
|
||||
// first-party does that; controller-runtime's per-reconcile logger
|
||||
// (reconcileID etc.) is captured as the base and survives.
|
||||
func Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
|
||||
return StartSpan(ctx, newOptions(nil).tracer(), name, opts...)
|
||||
}
|
||||
|
||||
// StartSpan is Start with an explicit tracer, for decorators that carry
|
||||
// their own (test-injected) TracerProvider.
|
||||
func StartSpan(ctx context.Context, tracer trace.Tracer, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
|
||||
ctx, span := tracer.Start(ctx, name, opts...)
|
||||
sc := span.SpanContext()
|
||||
if !sc.IsValid() {
|
||||
return ctx, span
|
||||
}
|
||||
base, ok := ctx.Value(baseLoggerKey{}).(logr.Logger)
|
||||
if !ok {
|
||||
base = logf.FromContext(ctx)
|
||||
ctx = context.WithValue(ctx, baseLoggerKey{}, base)
|
||||
}
|
||||
return logf.IntoContext(ctx, withSpanValues(base, sc)), span
|
||||
}
|
||||
|
||||
// ContextWithLogger seeds ctx with base as both the current logger and the
|
||||
// base for span enrichment. If ctx already carries a valid span (the HTTP
|
||||
// middleware calls this inside the otelhttp handler), the logger is
|
||||
// enriched immediately.
|
||||
//
|
||||
//nolint:logcheck // IntoContext-analogue: taking ctx and the logger to seed it with is the point.
|
||||
func ContextWithLogger(ctx context.Context, base logr.Logger) context.Context {
|
||||
ctx = context.WithValue(ctx, baseLoggerKey{}, base)
|
||||
if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
|
||||
return logf.IntoContext(ctx, withSpanValues(base, sc))
|
||||
}
|
||||
return logf.IntoContext(ctx, base)
|
||||
}
|
||||
|
||||
// withSpanValues uses lowerCamel keys to match the house style
|
||||
// (reconcileID, providerID); Grafana/Loki derived fields must match
|
||||
// "traceID", not the trace_id default.
|
||||
func withSpanValues(base logr.Logger, sc trace.SpanContext) logr.Logger {
|
||||
return base.WithValues("traceID", sc.TraceID().String(), "spanID", sc.SpanID().String())
|
||||
}
|
||||
@@ -1,136 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/go-logr/logr/funcr"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
// captureLogger records every emitted line so tests can assert on the
|
||||
// rendered key/value output — the only place duplicate zap-style keys
|
||||
// would show up.
|
||||
func captureLogger(lines *[]string) logr.Logger {
|
||||
return funcr.New(func(prefix, args string) {
|
||||
*lines = append(*lines, prefix+" "+args)
|
||||
}, funcr.Options{})
|
||||
}
|
||||
|
||||
func recordingTracer(t *testing.T) (trace.Tracer, *tracetest.SpanRecorder) {
|
||||
t.Helper()
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
return tp.Tracer("test"), sr
|
||||
}
|
||||
|
||||
func TestStartSpan_enrichesLoggerOncePerNesting(t *testing.T) {
|
||||
t.Parallel()
|
||||
tracer, sr := recordingTracer(t)
|
||||
|
||||
var lines []string
|
||||
ctx := logf.IntoContext(context.Background(), captureLogger(&lines))
|
||||
|
||||
ctx1, span1 := StartSpan(ctx, tracer, "outer")
|
||||
logf.FromContext(ctx1).Info("outer work")
|
||||
|
||||
ctx2, span2 := StartSpan(ctx1, tracer, "inner")
|
||||
logf.FromContext(ctx2).Info("inner work")
|
||||
|
||||
span2.End()
|
||||
span1.End()
|
||||
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("got %d log lines, want 2: %v", len(lines), lines)
|
||||
}
|
||||
traceID := span1.SpanContext().TraceID().String()
|
||||
for i, want := range []string{span1.SpanContext().SpanID().String(), span2.SpanContext().SpanID().String()} {
|
||||
if n := strings.Count(lines[i], `"traceID"`); n != 1 {
|
||||
t.Errorf("line %d: traceID appears %d times, want exactly 1: %s", i, n, lines[i])
|
||||
}
|
||||
if n := strings.Count(lines[i], `"spanID"`); n != 1 {
|
||||
t.Errorf("line %d: spanID appears %d times, want exactly 1: %s", i, n, lines[i])
|
||||
}
|
||||
if !strings.Contains(lines[i], traceID) {
|
||||
t.Errorf("line %d: missing traceID %s: %s", i, traceID, lines[i])
|
||||
}
|
||||
if !strings.Contains(lines[i], want) {
|
||||
t.Errorf("line %d: missing spanID %s: %s", i, want, lines[i])
|
||||
}
|
||||
}
|
||||
|
||||
ended := sr.Ended()
|
||||
if len(ended) != 2 {
|
||||
t.Fatalf("got %d spans, want 2", len(ended))
|
||||
}
|
||||
// Ended in LIFO order: inner first.
|
||||
if got := ended[0].Parent().SpanID(); got != span1.SpanContext().SpanID() {
|
||||
t.Errorf("inner span parent = %s, want %s", got, span1.SpanContext().SpanID())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartSpan_noopTracerLeavesLoggerUntouched(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var lines []string
|
||||
ctx := logf.IntoContext(context.Background(), captureLogger(&lines))
|
||||
|
||||
ctx, span := StartSpan(ctx, noop.NewTracerProvider().Tracer("test"), "op")
|
||||
defer span.End()
|
||||
logf.FromContext(ctx).Info("work")
|
||||
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("got %d log lines, want 1", len(lines))
|
||||
}
|
||||
if strings.Contains(lines[0], "traceID") {
|
||||
t.Errorf("disabled tracing must not add traceID: %s", lines[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextWithLogger(t *testing.T) {
|
||||
t.Parallel()
|
||||
tracer, _ := recordingTracer(t)
|
||||
|
||||
t.Run("no span injects base as-is", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
var lines []string
|
||||
ctx := ContextWithLogger(context.Background(), captureLogger(&lines))
|
||||
logf.FromContext(ctx).Info("plain")
|
||||
if len(lines) != 1 || strings.Contains(lines[0], "traceID") {
|
||||
t.Fatalf("want one line without traceID, got %v", lines)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("existing span enriches immediately and nested Start does not stack", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
var lines []string
|
||||
ctx, outer := tracer.Start(context.Background(), "server")
|
||||
defer outer.End()
|
||||
|
||||
ctx = ContextWithLogger(ctx, captureLogger(&lines))
|
||||
logf.FromContext(ctx).Info("handler")
|
||||
|
||||
ctx, inner := StartSpan(ctx, tracer, "child")
|
||||
defer inner.End()
|
||||
logf.FromContext(ctx).Info("nested")
|
||||
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("got %d lines, want 2: %v", len(lines), lines)
|
||||
}
|
||||
for i, line := range lines {
|
||||
if n := strings.Count(line, `"traceID"`); n != 1 {
|
||||
t.Errorf("line %d: traceID appears %d times, want 1: %s", i, n, line)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(lines[1], inner.SpanContext().SpanID().String()) {
|
||||
t.Errorf("nested line should carry the child spanID: %s", lines[1])
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// tracerName is the instrumentation scope for every span this module emits.
|
||||
const tracerName = "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator"
|
||||
|
||||
type options struct {
|
||||
tp trace.TracerProvider
|
||||
}
|
||||
|
||||
// Option configures the tracing decorators. The zero configuration uses the
|
||||
// global TracerProvider installed by Setup; tests inject their own recorder
|
||||
// via WithTracerProvider so they can run in parallel without touching
|
||||
// process-global state.
|
||||
type Option func(*options)
|
||||
|
||||
// WithTracerProvider overrides the global TracerProvider.
|
||||
func WithTracerProvider(tp trace.TracerProvider) Option {
|
||||
return func(o *options) { o.tp = tp }
|
||||
}
|
||||
|
||||
func newOptions(opts []Option) options {
|
||||
var o options
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// Tracer resolves a tracer from the given options, for decorators outside
|
||||
// this package (provider.WithTracing).
|
||||
func Tracer(opts ...Option) trace.Tracer {
|
||||
return newOptions(opts).tracer()
|
||||
}
|
||||
|
||||
// tracer resolves the configured tracer. The global path goes through
|
||||
// otel.Tracer, which delegates to whatever provider Setup installs later —
|
||||
// construction order between decorators and Setup does not matter.
|
||||
func (o options) tracer() trace.Tracer {
|
||||
if o.tp != nil {
|
||||
return o.tp.Tracer(tracerName)
|
||||
}
|
||||
return otel.Tracer(tracerName)
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
)
|
||||
|
||||
// NewReconciler wraps inner so every reconcile runs in a root span named
|
||||
// "Reconcile <kind>" and every log line under it carries the trace context.
|
||||
// Reconciles are triggered by watch events with no incoming trace to
|
||||
// continue, so each one starts a new trace.
|
||||
func NewReconciler(kind string, inner reconcile.Reconciler, opts ...Option) reconcile.Reconciler {
|
||||
return &tracedReconciler{
|
||||
spanName: "Reconcile " + kind,
|
||||
inner: inner,
|
||||
tracer: newOptions(opts).tracer(),
|
||||
}
|
||||
}
|
||||
|
||||
type tracedReconciler struct {
|
||||
spanName string
|
||||
inner reconcile.Reconciler
|
||||
tracer trace.Tracer
|
||||
}
|
||||
|
||||
func (t *tracedReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
|
||||
ctx, span := StartSpan(ctx, t.tracer, t.spanName,
|
||||
trace.WithAttributes(
|
||||
attribute.String("k8s.namespace.name", req.Namespace),
|
||||
attribute.String("k8s.object.name", req.Name),
|
||||
))
|
||||
defer span.End()
|
||||
if id := controller.ReconcileIDFromContext(ctx); id != "" {
|
||||
span.SetAttributes(attribute.String("reconcile.id", string(id)))
|
||||
}
|
||||
|
||||
res, err := t.inner.Reconcile(ctx, req)
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return res, err
|
||||
}
|
||||
span.SetStatus(codes.Ok, "")
|
||||
if res.RequeueAfter > 0 {
|
||||
span.SetAttributes(attribute.Int64("reconcile.requeue_after_ms", res.RequeueAfter.Milliseconds()))
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
)
|
||||
|
||||
type fakeReconciler struct {
|
||||
res reconcile.Result
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeReconciler) Reconcile(context.Context, reconcile.Request) (reconcile.Result, error) {
|
||||
return f.res, f.err
|
||||
}
|
||||
|
||||
func TestNewReconciler_spanPerReconcile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "p1"}}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inner *fakeReconciler
|
||||
wantStatus codes.Code
|
||||
wantAttr attribute.KeyValue
|
||||
}{
|
||||
{
|
||||
name: "success records requeue",
|
||||
inner: &fakeReconciler{res: reconcile.Result{RequeueAfter: 10 * time.Second}},
|
||||
wantStatus: codes.Ok,
|
||||
wantAttr: attribute.Int64("reconcile.requeue_after_ms", 10_000),
|
||||
},
|
||||
{
|
||||
name: "error sets error status",
|
||||
inner: &fakeReconciler{err: errors.New("boom")},
|
||||
wantStatus: codes.Error,
|
||||
wantAttr: attribute.String("k8s.object.name", "p1"),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
|
||||
r := NewReconciler("Proxy", tc.inner, WithTracerProvider(tp))
|
||||
res, err := r.Reconcile(context.Background(), req)
|
||||
if res != tc.inner.res || !errors.Is(err, tc.inner.err) {
|
||||
t.Fatalf("decorator changed the result: (%v, %v)", res, err)
|
||||
}
|
||||
|
||||
ended := sr.Ended()
|
||||
if len(ended) != 1 {
|
||||
t.Fatalf("got %d spans, want 1", len(ended))
|
||||
}
|
||||
span := ended[0]
|
||||
if span.Name() != "Reconcile Proxy" {
|
||||
t.Errorf("span name = %q, want %q", span.Name(), "Reconcile Proxy")
|
||||
}
|
||||
if span.Status().Code != tc.wantStatus {
|
||||
t.Errorf("status = %v, want %v", span.Status().Code, tc.wantStatus)
|
||||
}
|
||||
found := false
|
||||
for _, a := range span.Attributes() {
|
||||
if a == tc.wantAttr {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("attribute %v missing in %v", tc.wantAttr, span.Attributes())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
// Package tracing wires OpenTelemetry tracing into the operator: SDK setup
|
||||
// gated on standard OTEL_* environment variables, span helpers that keep the
|
||||
// logr/zap logging enriched with trace context, and decorators for the
|
||||
// reconciler, providers, HTTP server, and Kubernetes API transport.
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
||||
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
const (
|
||||
envSDKDisabled = "OTEL_SDK_DISABLED"
|
||||
envTracesExporter = "OTEL_TRACES_EXPORTER"
|
||||
envOTLPEndpoint = "OTEL_EXPORTER_OTLP_ENDPOINT"
|
||||
envOTLPTracesEndpoint = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
|
||||
envOTLPProtocol = "OTEL_EXPORTER_OTLP_PROTOCOL"
|
||||
envOTLPTracesProtocol = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"
|
||||
|
||||
exporterOTLP = "otlp"
|
||||
exporterConsole = "console"
|
||||
exporterNone = "none"
|
||||
)
|
||||
|
||||
// maxQueueSize bounds the batch processor's span buffer; the default 2048
|
||||
// is oversized for this process (3 concurrent reconciles, tickers) and the
|
||||
// manager pod runs with a 128Mi memory limit.
|
||||
const maxQueueSize = 512
|
||||
|
||||
// Setup initializes the global tracing pipeline from OTEL_* environment
|
||||
// variables and returns a shutdown func that flushes pending spans.
|
||||
//
|
||||
// Tracing stays fully off — no exporter, no global provider, spans are
|
||||
// no-ops, logs carry no traceID — unless the environment opts in by setting
|
||||
// OTEL_TRACES_EXPORTER or an OTLP endpoint. OTEL_SDK_DISABLED=true and
|
||||
// OTEL_TRACES_EXPORTER=none force it off (the Go SDK does not implement
|
||||
// OTEL_SDK_DISABLED itself). Sampling, endpoints, TLS, and headers follow
|
||||
// the standard SDK/exporter env vars (OTEL_TRACES_SAMPLER, OTEL_EXPORTER_OTLP_*).
|
||||
//
|
||||
// Setup logs via the logger in ctx (logf.FromContext) rather than a
|
||||
// parameter — the caller seeds it with logf.IntoContext.
|
||||
func Setup(ctx context.Context, service, version string) (func(context.Context) error, error) {
|
||||
log := logf.FromContext(ctx)
|
||||
noop := func(context.Context) error { return nil }
|
||||
|
||||
exporterEnv := strings.ToLower(strings.TrimSpace(os.Getenv(envTracesExporter)))
|
||||
endpointSet := os.Getenv(envOTLPEndpoint) != "" ||
|
||||
os.Getenv(envOTLPTracesEndpoint) != ""
|
||||
sdkDisabled := strings.EqualFold(strings.TrimSpace(os.Getenv(envSDKDisabled)), "true")
|
||||
|
||||
if sdkDisabled || exporterEnv == exporterNone || (exporterEnv == "" && !endpointSet) {
|
||||
log.Info("tracing disabled",
|
||||
"reason", disabledReason(sdkDisabled, exporterEnv))
|
||||
return noop, nil
|
||||
}
|
||||
|
||||
exp, expName, err := newExporter(ctx, exporterEnv)
|
||||
if err != nil {
|
||||
return noop, fmt.Errorf("tracing setup: %w", err)
|
||||
}
|
||||
|
||||
res, err := buildResource(ctx, service, version)
|
||||
if err != nil {
|
||||
if res == nil {
|
||||
return noop, fmt.Errorf("tracing setup: building resource: %w", err)
|
||||
}
|
||||
// Schema-URL conflicts between semconv versions still yield a usable
|
||||
// merged resource; keep it rather than losing tracing over metadata.
|
||||
log.V(1).Info("tracing resource merge conflict; continuing", "err", err.Error())
|
||||
}
|
||||
|
||||
tp := sdktrace.NewTracerProvider(
|
||||
sdktrace.WithBatcher(exp, sdktrace.WithMaxQueueSize(maxQueueSize)),
|
||||
sdktrace.WithResource(res),
|
||||
)
|
||||
otel.SetTracerProvider(tp)
|
||||
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
|
||||
propagation.TraceContext{}, propagation.Baggage{}))
|
||||
otelLog := log.WithName("otel")
|
||||
otel.SetLogger(otelLog)
|
||||
// V(1), not Error: an unreachable collector fails every export cycle and
|
||||
// would otherwise spam the error log every few seconds.
|
||||
otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
|
||||
otelLog.V(1).Info("otel error", "err", err.Error())
|
||||
}))
|
||||
|
||||
log.Info("tracing enabled", "exporter", expName, "service", service)
|
||||
return tp.Shutdown, nil
|
||||
}
|
||||
|
||||
func disabledReason(sdkDisabled bool, exporterEnv string) string {
|
||||
switch {
|
||||
case sdkDisabled:
|
||||
return "OTEL_SDK_DISABLED=true"
|
||||
case exporterEnv == exporterNone:
|
||||
return "OTEL_TRACES_EXPORTER=none"
|
||||
default:
|
||||
return "no OTEL_TRACES_EXPORTER or OTLP endpoint configured"
|
||||
}
|
||||
}
|
||||
|
||||
// newExporter hand-rolls the OTEL_TRACES_EXPORTER / OTEL_EXPORTER_OTLP_*
|
||||
// protocol selection instead of pulling in contrib's autoexport, which drags
|
||||
// metric/log/prometheus exporters into a trace-only binary.
|
||||
func newExporter(ctx context.Context, kind string) (sdktrace.SpanExporter, string, error) {
|
||||
switch kind {
|
||||
case "", exporterOTLP:
|
||||
proto := strings.ToLower(strings.TrimSpace(os.Getenv(envOTLPTracesProtocol)))
|
||||
if proto == "" {
|
||||
proto = strings.ToLower(strings.TrimSpace(os.Getenv(envOTLPProtocol)))
|
||||
}
|
||||
switch proto {
|
||||
case "", "http/protobuf":
|
||||
exp, err := otlptracehttp.New(ctx)
|
||||
return exp, "otlp/http", err
|
||||
case "grpc":
|
||||
exp, err := otlptracegrpc.New(ctx)
|
||||
return exp, "otlp/grpc", err
|
||||
default:
|
||||
return nil, "", fmt.Errorf("unsupported OTLP protocol %q (supported: http/protobuf, grpc)", proto)
|
||||
}
|
||||
case exporterConsole:
|
||||
exp, err := stdouttrace.New()
|
||||
return exp, exporterConsole, err
|
||||
default:
|
||||
return nil, "", fmt.Errorf("unsupported OTEL_TRACES_EXPORTER %q (supported: otlp, console, none)", kind)
|
||||
}
|
||||
}
|
||||
|
||||
// buildResource layers defaults < service identity < environment, so
|
||||
// OTEL_SERVICE_NAME / OTEL_RESOURCE_ATTRIBUTES always win.
|
||||
func buildResource(ctx context.Context, service, version string) (*resource.Resource, error) {
|
||||
base, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL,
|
||||
semconv.ServiceName(service),
|
||||
semconv.ServiceVersion(version),
|
||||
))
|
||||
if err != nil {
|
||||
return base, err
|
||||
}
|
||||
env, err := resource.New(ctx, resource.WithFromEnv())
|
||||
if err != nil {
|
||||
return base, err
|
||||
}
|
||||
return resource.Merge(base, env)
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
)
|
||||
|
||||
const testOTLPEndpoint = "http://localhost:4318"
|
||||
|
||||
// clearOTelEnv pins every env var Setup reads, so developer shells with
|
||||
// OTEL_* set can't change test outcomes. Not parallel-safe by design
|
||||
// (t.Setenv forbids t.Parallel).
|
||||
func clearOTelEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, k := range []string{
|
||||
envSDKDisabled,
|
||||
envTracesExporter,
|
||||
envOTLPEndpoint,
|
||||
envOTLPTracesEndpoint,
|
||||
envOTLPProtocol,
|
||||
envOTLPTracesProtocol,
|
||||
} {
|
||||
t.Setenv(k, "")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetup_envGating(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
wantEnabled bool
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "no env means disabled", env: nil, wantEnabled: false},
|
||||
{
|
||||
name: "endpoint enables otlp",
|
||||
env: map[string]string{envOTLPEndpoint: testOTLPEndpoint},
|
||||
wantEnabled: true,
|
||||
},
|
||||
{
|
||||
name: "traces endpoint enables otlp",
|
||||
env: map[string]string{envOTLPTracesEndpoint: testOTLPEndpoint},
|
||||
wantEnabled: true,
|
||||
},
|
||||
{
|
||||
name: "explicit console exporter",
|
||||
env: map[string]string{envTracesExporter: exporterConsole},
|
||||
wantEnabled: true,
|
||||
},
|
||||
{
|
||||
name: "grpc protocol",
|
||||
env: map[string]string{envTracesExporter: exporterOTLP, envOTLPProtocol: "grpc"},
|
||||
wantEnabled: true,
|
||||
},
|
||||
{
|
||||
name: "exporter none wins over endpoint",
|
||||
env: map[string]string{
|
||||
envTracesExporter: exporterNone,
|
||||
envOTLPEndpoint: testOTLPEndpoint,
|
||||
},
|
||||
wantEnabled: false,
|
||||
},
|
||||
{
|
||||
name: "OTEL_SDK_DISABLED wins over everything",
|
||||
env: map[string]string{
|
||||
envSDKDisabled: "true",
|
||||
envOTLPEndpoint: testOTLPEndpoint,
|
||||
},
|
||||
wantEnabled: false,
|
||||
},
|
||||
{
|
||||
name: "unsupported exporter errors",
|
||||
env: map[string]string{envTracesExporter: "jaeger"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unsupported protocol errors",
|
||||
env: map[string]string{
|
||||
envTracesExporter: exporterOTLP,
|
||||
envOTLPProtocol: "http/json",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
clearOTelEnv(t)
|
||||
for k, v := range tc.env {
|
||||
t.Setenv(k, v)
|
||||
}
|
||||
// Fresh noop global per case: disabled paths must leave it
|
||||
// untouched, and resetting avoids leaking one case's SDK
|
||||
// provider into the next (or warning-prone restores of the
|
||||
// process-default delegate).
|
||||
before := noop.NewTracerProvider()
|
||||
otel.SetTracerProvider(before)
|
||||
|
||||
shutdown, err := Setup(context.Background(), "test-svc", "abc123")
|
||||
if tc.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("want error, got nil")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Setup: %v", err)
|
||||
}
|
||||
|
||||
_, isSDK := otel.GetTracerProvider().(*sdktrace.TracerProvider)
|
||||
if isSDK != tc.wantEnabled {
|
||||
t.Errorf("global provider is SDK = %v, want %v", isSDK, tc.wantEnabled)
|
||||
}
|
||||
if tc.wantEnabled && otel.GetTracerProvider() == before {
|
||||
t.Error("enabled Setup must install a new global provider")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
if err := shutdown(ctx); err != nil {
|
||||
t.Errorf("shutdown: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"k8s.io/client-go/transport"
|
||||
)
|
||||
|
||||
// RestConfigWrapper returns a rest.Config.Wrap-compatible wrapper that adds
|
||||
// a client span to every Kubernetes API request already running inside a
|
||||
// trace. Requests with no parent span pass through untouched: informer
|
||||
// list/watch long-polls, leader-election renewals, and metrics authn would
|
||||
// otherwise each become meaningless (and in the watch case, minutes-long)
|
||||
// root spans. client-go applies Wrap innermost, so the span sees the final
|
||||
// authenticated request.
|
||||
func RestConfigWrapper(opts ...Option) transport.WrapperFunc {
|
||||
o := newOptions(opts)
|
||||
return func(rt http.RoundTripper) http.RoundTripper {
|
||||
otelOpts := []otelhttp.Option{
|
||||
// Explicit propagators: deterministic regardless of global
|
||||
// state. The apiserver ignores incoming traceparent (by
|
||||
// design), so injection is harmless there.
|
||||
otelhttp.WithPropagators(propagation.NewCompositeTextMapPropagator(
|
||||
propagation.TraceContext{}, propagation.Baggage{})),
|
||||
}
|
||||
if o.tp != nil {
|
||||
otelOpts = append(otelOpts, otelhttp.WithTracerProvider(o.tp))
|
||||
}
|
||||
return &parentGatedTransport{
|
||||
traced: otelhttp.NewTransport(rt, otelOpts...),
|
||||
plain: rt,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parentGatedTransport enforces the parent-span requirement itself rather
|
||||
// than relying on otelhttp filter semantics for transports.
|
||||
type parentGatedTransport struct {
|
||||
traced http.RoundTripper
|
||||
plain http.RoundTripper
|
||||
}
|
||||
|
||||
func (t *parentGatedTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
if trace.SpanContextFromContext(r.Context()).IsValid() {
|
||||
return t.traced.RoundTrip(r)
|
||||
}
|
||||
return t.plain.RoundTrip(r)
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func TestRestConfigWrapper_parentGated(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var gotTraceparent string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotTraceparent = r.Header.Get("traceparent")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
|
||||
rt := RestConfigWrapper(WithTracerProvider(tp))(http.DefaultTransport)
|
||||
do := func(ctx context.Context) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := rt.RoundTrip(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
|
||||
// No parent span: informer watches, leader election. Must not trace.
|
||||
do(context.Background())
|
||||
if len(sr.Ended()) != 0 {
|
||||
t.Fatalf("request without parent span produced %d spans, want 0", len(sr.Ended()))
|
||||
}
|
||||
if gotTraceparent != "" {
|
||||
t.Fatalf("request without parent span injected traceparent %q", gotTraceparent)
|
||||
}
|
||||
|
||||
// Under a parent span: one client child span, traceparent injected.
|
||||
ctx, parent := tp.Tracer("test").Start(context.Background(), "reconcile")
|
||||
do(ctx)
|
||||
parent.End()
|
||||
|
||||
var client sdktrace.ReadOnlySpan
|
||||
for _, s := range sr.Ended() {
|
||||
if s.SpanKind() == trace.SpanKindClient {
|
||||
client = s
|
||||
}
|
||||
}
|
||||
if client == nil {
|
||||
t.Fatalf("no client span recorded, got %d spans", len(sr.Ended()))
|
||||
}
|
||||
if got := client.Parent().SpanID(); got != parent.SpanContext().SpanID() {
|
||||
t.Errorf("client span parent = %s, want %s", got, parent.SpanContext().SpanID())
|
||||
}
|
||||
if gotTraceparent == "" {
|
||||
t.Error("traceparent header not injected under a parent span")
|
||||
}
|
||||
}
|
||||
@@ -1,424 +0,0 @@
|
||||
//go:build e2e
|
||||
// +build e2e
|
||||
|
||||
/*
|
||||
Copyright 2026.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/test/utils"
|
||||
)
|
||||
|
||||
// The tracing e2e proves the full pipeline against a real Tempo: operator in
|
||||
// kind → OTLP export → Tempo ingest → traces queryable with the documented
|
||||
// span topology. It is gated on TEMPO_URL (Tempo query API, e.g.
|
||||
// http://192.168.0.30:3200) and OTLP_ENDPOINT (OTLP HTTP ingest, e.g.
|
||||
// http://192.168.0.30:4318) and skips when either is unset, so the rest of
|
||||
// the suite runs anywhere.
|
||||
//
|
||||
// Every span of a run carries the resource attribute test.run.id=<runID>
|
||||
// (injected via OTEL_RESOURCE_ATTRIBUTES — no code changes), so one TraceQL
|
||||
// query finds exactly this run's traces, in the test and in Grafana alike.
|
||||
var _ = Describe("OTel tracing", Ordered, func() {
|
||||
// Proxy CRs live in default, not the operator namespace: the squid pods
|
||||
// the kubernetes provider creates carry no securityContext and would be
|
||||
// rejected by the operator namespace's restricted PSS label.
|
||||
const proxyNS = "default"
|
||||
const deploymentName = "egress-proxies-operator-controller-manager"
|
||||
const squidImage = "ubuntu/squid:6.6-24.04_edge"
|
||||
|
||||
proxyNames := []string{"proxy-tracing-e2e-1", "proxy-tracing-e2e-2"}
|
||||
|
||||
var (
|
||||
tempoURL string
|
||||
otlpEndpoint string
|
||||
runID string
|
||||
suiteStart time.Time
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
tempoURL = os.Getenv("TEMPO_URL")
|
||||
otlpEndpoint = os.Getenv("OTLP_ENDPOINT")
|
||||
if tempoURL == "" || otlpEndpoint == "" {
|
||||
Skip("TEMPO_URL / OTLP_ENDPOINT not set — skipping the Tempo-backed tracing e2e")
|
||||
}
|
||||
suiteStart = time.Now()
|
||||
runID = "e2e-" + strconv.FormatInt(suiteStart.UnixNano(), 10)
|
||||
_, _ = fmt.Fprintf(GinkgoWriter,
|
||||
"tracing e2e run id: %s — find this run in Grafana with TraceQL {resource.test.run.id=%q}\n",
|
||||
runID, runID)
|
||||
|
||||
By("creating manager namespace")
|
||||
cmd := exec.Command("kubectl", "create", "ns", namespace)
|
||||
_, err := utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to create namespace")
|
||||
|
||||
By("labeling the namespace to enforce the restricted security policy")
|
||||
cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace,
|
||||
"pod-security.kubernetes.io/enforce=restricted")
|
||||
_, err = utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy")
|
||||
|
||||
By("installing CRDs")
|
||||
cmd = exec.Command("make", "install")
|
||||
_, err = utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs")
|
||||
|
||||
By("deploying the controller-manager")
|
||||
cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", managerImage))
|
||||
_, err = utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager")
|
||||
|
||||
By("pre-pulling the squid image into kind (best effort, kills the biggest flake source)")
|
||||
if _, err := utils.Run(exec.Command("docker", "pull", squidImage)); err == nil {
|
||||
if err := utils.LoadImageToKindClusterWithName(squidImage); err != nil {
|
||||
_, _ = fmt.Fprintf(GinkgoWriter, "kind load of %s failed (continuing): %v\n", squidImage, err)
|
||||
}
|
||||
} else {
|
||||
_, _ = fmt.Fprintf(GinkgoWriter, "docker pull %s failed (continuing): %v\n", squidImage, err)
|
||||
}
|
||||
|
||||
By("preflighting the OTLP endpoint from inside the cluster")
|
||||
// Host-reachability of the OTLP endpoint does not prove
|
||||
// pod-reachability from inside kind, and the operator logs export
|
||||
// failures only at V(1) — without this, a broken path is a slow,
|
||||
// opaque search timeout instead of a clear failure.
|
||||
preflightOTLP(otlpEndpoint)
|
||||
|
||||
By("pointing the operator at the OTLP endpoint and tagging the test run")
|
||||
// OTEL_RESOURCE_ATTRIBUTES is replaced in place, which keeps it
|
||||
// listed after the downward-API POD_NAME/POD_NAMESPACE vars —
|
||||
// $(VAR) expansion only sees earlier-listed vars. exec.Command
|
||||
// passes $(...) through without shell mangling.
|
||||
cmd = exec.Command("kubectl", "set", "env",
|
||||
"deployment/"+deploymentName, "-n", namespace,
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT="+otlpEndpoint,
|
||||
fmt.Sprintf(
|
||||
"OTEL_RESOURCE_ATTRIBUTES=k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE),test.run.id=%s",
|
||||
runID))
|
||||
_, err = utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to set OTel env on the deployment")
|
||||
|
||||
cmd = exec.Command("kubectl", "rollout", "status",
|
||||
"deployment/"+deploymentName, "-n", namespace, "--timeout=3m")
|
||||
_, err = utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Rollout after set env did not finish")
|
||||
|
||||
By("verifying the operator reports tracing enabled")
|
||||
Eventually(func(g Gomega) {
|
||||
pod := newestControllerPod(g)
|
||||
out, err := utils.Run(exec.Command("kubectl", "logs", pod, "-n", namespace))
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(out).To(ContainSubstring("tracing enabled"),
|
||||
"operator did not log 'tracing enabled' after rollout")
|
||||
}, 2*time.Minute).Should(Succeed())
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
if tempoURL == "" || otlpEndpoint == "" {
|
||||
return // spec was skipped; nothing was deployed
|
||||
}
|
||||
By("cleaning up test proxies")
|
||||
args := append([]string{"delete", "proxy", "-n", proxyNS, "--ignore-not-found"}, proxyNames...)
|
||||
_, _ = utils.Run(exec.Command("kubectl", args...))
|
||||
|
||||
By("cleaning up the OTLP preflight pod")
|
||||
_, _ = utils.Run(exec.Command("kubectl", "delete", "pod", otlpProbePodName, "-n", namespace,
|
||||
"--ignore-not-found"))
|
||||
|
||||
By("undeploying the controller-manager")
|
||||
_, _ = utils.Run(exec.Command("make", "undeploy"))
|
||||
|
||||
By("uninstalling CRDs")
|
||||
_, _ = utils.Run(exec.Command("make", "uninstall"))
|
||||
|
||||
By("removing manager namespace")
|
||||
_, _ = utils.Run(exec.Command("kubectl", "delete", "ns", namespace))
|
||||
})
|
||||
|
||||
It("creates proxies and reports reconcile traces to Tempo", func() {
|
||||
By("applying two kubernetes-provider proxies")
|
||||
// Absolute paths on purpose: utils.Run chdirs the whole process.
|
||||
dir := GinkgoT().TempDir()
|
||||
for _, name := range proxyNames {
|
||||
manifest := fmt.Sprintf(`apiVersion: crawl.example.com/v1alpha1
|
||||
kind: Proxy
|
||||
metadata:
|
||||
name: %s
|
||||
namespace: %s
|
||||
spec:
|
||||
mode: Managed
|
||||
provider: kubernetes
|
||||
attributes:
|
||||
purpose: tracing-e2e
|
||||
`, name, proxyNS)
|
||||
path := filepath.Join(dir, name+".yaml")
|
||||
Expect(os.WriteFile(path, []byte(manifest), 0o644)).To(Succeed())
|
||||
_, err := utils.Run(exec.Command("kubectl", "apply", "-f", path))
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to apply %s", name)
|
||||
}
|
||||
|
||||
By("waiting for the proxies to become Ready")
|
||||
Eventually(func(g Gomega) {
|
||||
for _, name := range proxyNames {
|
||||
out, err := utils.Run(exec.Command("kubectl", "get", "proxy", name,
|
||||
"-n", proxyNS, "-o", "jsonpath={.status.phase}"))
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(out).To(Equal("Ready"), "proxy %s not Ready", name)
|
||||
}
|
||||
}, 5*time.Minute).Should(Succeed())
|
||||
|
||||
By("finding a provider.create trace for this run in Tempo")
|
||||
// Anchored on provider.create, not the root span name: a
|
||||
// "Reconcile Proxy" hit could be the finalizer-add or a drift
|
||||
// reconcile, which contain no provider call.
|
||||
traceID := eventuallyFindTrace(tempoURL, runID, "provider.create", suiteStart)
|
||||
|
||||
By("asserting the reconcile trace structure")
|
||||
spanNames, resAttrs, err := tempoTrace(tempoURL, traceID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(spanNames).To(ContainElements(
|
||||
"Reconcile Proxy", "reconcile.managed", "provider.create", "status.patch"),
|
||||
"trace %s is missing expected spans; got: %v", traceID, spanNames)
|
||||
Expect(resAttrs["service.name"]).To(Equal("egress-proxies-operator"))
|
||||
Expect(resAttrs["test.run.id"]).To(Equal(runID))
|
||||
})
|
||||
|
||||
It("traces proxy deletion", func() {
|
||||
By("deleting the proxies")
|
||||
args := append([]string{"delete", "proxy", "-n", proxyNS, "--wait=false"}, proxyNames...)
|
||||
_, err := utils.Run(exec.Command("kubectl", args...))
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to delete proxies")
|
||||
|
||||
By("waiting for the proxies to be gone (finalizer ran provider.delete)")
|
||||
Eventually(func(g Gomega) {
|
||||
out, err := utils.Run(exec.Command("kubectl", "get", "proxy", "-n", proxyNS, "-o", "name"))
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(out).NotTo(ContainSubstring("proxy-tracing-e2e"))
|
||||
}, 3*time.Minute).Should(Succeed())
|
||||
|
||||
By("finding a provider.delete trace for this run in Tempo")
|
||||
traceID := eventuallyFindTrace(tempoURL, runID, "provider.delete", suiteStart)
|
||||
|
||||
By("asserting the deletion trace structure")
|
||||
spanNames, _, err := tempoTrace(tempoURL, traceID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(spanNames).To(ContainElements("Reconcile Proxy", "reconcile.delete", "provider.delete"),
|
||||
"trace %s is missing expected spans; got: %v", traceID, spanNames)
|
||||
})
|
||||
})
|
||||
|
||||
// preflightOTLP runs a one-shot curl pod inside the cluster POSTing to the
|
||||
// OTLP HTTP ingest, and fails with a clear message when it is unreachable.
|
||||
// The pod runs in the restricted-PSS operator namespace, hence the full
|
||||
// securityContext (same shape as the curl-metrics pod).
|
||||
func preflightOTLP(otlpEndpoint string) {
|
||||
script := fmt.Sprintf(
|
||||
"for i in $(seq 1 10); do "+
|
||||
"code=$(curl -sS -o /dev/null -w '%%{http_code}' -X POST "+
|
||||
"-H 'Content-Type: application/json' -d '{}' %s/v1/traces); "+
|
||||
"echo \"attempt $i: HTTP $code\"; "+
|
||||
"[ \"$code\" = \"200\" ] && echo OTLP_OK && exit 0; sleep 2; "+
|
||||
"done; echo OTLP_UNREACHABLE; exit 1",
|
||||
otlpEndpoint)
|
||||
cmd := exec.Command("kubectl", "run", otlpProbePodName, "--restart=Never",
|
||||
"--namespace", namespace,
|
||||
"--image=curlimages/curl:latest",
|
||||
"--overrides",
|
||||
fmt.Sprintf(`{
|
||||
"spec": {
|
||||
"containers": [{
|
||||
"name": "curl",
|
||||
"image": "curlimages/curl:latest",
|
||||
"command": ["/bin/sh", "-c"],
|
||||
"args": [%q],
|
||||
"securityContext": {
|
||||
"readOnlyRootFilesystem": true,
|
||||
"allowPrivilegeEscalation": false,
|
||||
"capabilities": {
|
||||
"drop": ["ALL"]
|
||||
},
|
||||
"runAsNonRoot": true,
|
||||
"runAsUser": 1000,
|
||||
"seccompProfile": {
|
||||
"type": "RuntimeDefault"
|
||||
}
|
||||
}
|
||||
}]
|
||||
}
|
||||
}`, script))
|
||||
_, err := utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to create the OTLP preflight pod")
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
out, err := utils.Run(exec.Command("kubectl", "get", "pod", otlpProbePodName,
|
||||
"-n", namespace, "-o", "jsonpath={.status.phase}"))
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(out).To(BeElementOf("Succeeded", "Failed"), "preflight pod still running")
|
||||
}, 2*time.Minute).Should(Succeed())
|
||||
|
||||
logs, _ := utils.Run(exec.Command("kubectl", "logs", otlpProbePodName, "-n", namespace))
|
||||
Expect(logs).To(ContainSubstring("OTLP_OK"),
|
||||
"OTLP endpoint %s is not reachable from inside the kind cluster; curl output:\n%s",
|
||||
otlpEndpoint, logs)
|
||||
}
|
||||
|
||||
// otlpProbePodName mirrors the Describe-local constant for the helpers below.
|
||||
const otlpProbePodName = "curl-otlp"
|
||||
|
||||
// newestControllerPod returns the most recently created controller pod —
|
||||
// right after a rollout, an unsorted lookup may pick the terminating one.
|
||||
func newestControllerPod(g Gomega) string {
|
||||
out, err := utils.Run(exec.Command("kubectl", "get", "pods",
|
||||
"-l", "control-plane=controller-manager", "-n", namespace,
|
||||
"--sort-by=.metadata.creationTimestamp", "-o", "name"))
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
lines := utils.GetNonEmptyLines(out)
|
||||
g.Expect(lines).NotTo(BeEmpty(), "no controller pods found")
|
||||
return strings.TrimPrefix(lines[len(lines)-1], "pod/")
|
||||
}
|
||||
|
||||
// eventuallyFindTrace polls Tempo until a trace containing a span with the
|
||||
// given name exists for this run, and returns its trace ID. The batch span
|
||||
// processor flushes every ~5s, so a couple of polls is normal.
|
||||
func eventuallyFindTrace(tempoURL, runID, spanName string, since time.Time) string {
|
||||
var traceID string
|
||||
query := fmt.Sprintf(`{resource.test.run.id=%q && name=%q}`, runID, spanName)
|
||||
Eventually(func(g Gomega) {
|
||||
ids, err := tempoSearch(tempoURL, query, since)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(ids).NotTo(BeEmpty(), "no trace for %s yet (query: %s)", spanName, query)
|
||||
traceID = ids[0]
|
||||
}, 2*time.Minute).Should(Succeed())
|
||||
return traceID
|
||||
}
|
||||
|
||||
// tempoSearch runs a TraceQL query against Tempo's search API and returns
|
||||
// the matching trace IDs. start/end are Unix seconds.
|
||||
func tempoSearch(tempoURL, traceql string, since time.Time) ([]string, error) {
|
||||
u, err := url.Parse(tempoURL + "/api/search")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("q", traceql)
|
||||
q.Set("start", strconv.FormatInt(since.Add(-5*time.Minute).Unix(), 10))
|
||||
q.Set("end", strconv.FormatInt(time.Now().Add(time.Minute).Unix(), 10))
|
||||
q.Set("limit", "20")
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
body, err := tempoGet(u.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result struct {
|
||||
Traces []struct {
|
||||
TraceID string `json:"traceID"`
|
||||
} `json:"traces"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("decoding Tempo search response: %w", err)
|
||||
}
|
||||
ids := make([]string, 0, len(result.Traces))
|
||||
for _, t := range result.Traces {
|
||||
ids = append(ids, t.TraceID)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// tempoTrace fetches one trace and flattens it to a span-name list plus the
|
||||
// resource attributes. Tempo returns OTLP-JSON (batches → scopeSpans →
|
||||
// spans), not Jaeger's shape.
|
||||
func tempoTrace(tempoURL, traceID string) ([]string, map[string]string, error) {
|
||||
body, err := tempoGet(tempoURL + "/api/traces/" + traceID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var trace struct {
|
||||
Batches []struct {
|
||||
Resource struct {
|
||||
Attributes []struct {
|
||||
Key string `json:"key"`
|
||||
Value struct {
|
||||
StringValue string `json:"stringValue"`
|
||||
} `json:"value"`
|
||||
} `json:"attributes"`
|
||||
} `json:"resource"`
|
||||
ScopeSpans []struct {
|
||||
Spans []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"spans"`
|
||||
} `json:"scopeSpans"`
|
||||
} `json:"batches"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &trace); err != nil {
|
||||
return nil, nil, fmt.Errorf("decoding Tempo trace %s: %w", traceID, err)
|
||||
}
|
||||
var spanNames []string
|
||||
resAttrs := map[string]string{}
|
||||
for _, b := range trace.Batches {
|
||||
for _, a := range b.Resource.Attributes {
|
||||
if a.Value.StringValue != "" {
|
||||
resAttrs[a.Key] = a.Value.StringValue
|
||||
}
|
||||
}
|
||||
for _, ss := range b.ScopeSpans {
|
||||
for _, s := range ss.Spans {
|
||||
spanNames = append(spanNames, s.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return spanNames, resAttrs, nil
|
||||
}
|
||||
|
||||
func tempoGet(rawURL string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying Tempo: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("tempo returned %d for %s: %s", resp.StatusCode, rawURL, string(body))
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
Reference in New Issue
Block a user