Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  kacerrandClaude Fable 5.1
|
a114201ad3 | ||
|
kacerrandClaude Fable 5.1
|
7d177b4817 | ||
|
kacerrandClaude Fable 5.1
|
d171c0426e | ||
|
kacerrandClaude Fable 5.1
|
7aea14fbb3 | ||
|
kacerrandClaude Fable 5.1
|
bd633c0288 | ||
|
kacerrandClaude Fable 5.1
|
1b5ddb8349 | ||
|
kacerr
|
20ffba604b |
+6
-10
@@ -84,16 +84,12 @@
|
||||
"Bash(kubectl -n egress-proxies-operator-system get deploy egress-proxies-operator-controller-manager -o jsonpath='{.spec.template.spec.containers[0].args}')",
|
||||
"Bash(kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager)",
|
||||
"Bash(python3 -c \"import json; d=json.load\\(open\\('docs/deploy/sa_key.json'\\)\\); print\\(d.get\\('type'\\), d.get\\('client_email'\\)\\)\")",
|
||||
"Bash(tea pr *)",
|
||||
"Bash(git worktree *)",
|
||||
"Bash(python3 -c \"import yaml; yaml.safe_load\\(open\\('.gitea/workflows/build.yaml'\\)\\); print\\('YAML OK'\\)\")",
|
||||
"Bash(ruby -ryaml -e \"YAML.load_file\\('.gitea/workflows/build.yaml'\\); puts 'YAML OK'\")",
|
||||
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator status --short --branch)",
|
||||
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator log --oneline -1)",
|
||||
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator tag 0.01)",
|
||||
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator push origin 0.01)",
|
||||
"Bash(chmod +x docs/demo/run-demo.sh)",
|
||||
"Bash(bash -n docs/demo/run-demo.sh)"
|
||||
"Bash(git add *)",
|
||||
"Bash(git commit -q -m 'Add plan: KVM/libvirt provider *)",
|
||||
"Bash(go get *)",
|
||||
"Bash(go build *)",
|
||||
"Bash(go test *)",
|
||||
"Bash(git commit *)"
|
||||
],
|
||||
"additionalDirectories": [
|
||||
"/Users/jan.novak/srv/go/egress-proxies-operator/.claude",
|
||||
|
||||
@@ -90,8 +90,7 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist
|
||||
|
||||
.PHONY: test-e2e
|
||||
test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
|
||||
KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) TEMPO_URL=$(TEMPO_URL) OTLP_ENDPOINT=$(OTLP_ENDPOINT) \
|
||||
go test -tags=e2e ./test/e2e/ -v -ginkgo.v -timeout 30m
|
||||
KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v
|
||||
$(MAKE) cleanup-test-e2e
|
||||
|
||||
.PHONY: cleanup-test-e2e
|
||||
|
||||
@@ -136,81 +136,6 @@ Cloud-init from a Secret: the Secret **must** carry the label
|
||||
labelled Secrets, so an unlabelled one is invisible (the Proxy reports
|
||||
`CloudInitError`). Rotating the Secret's content triggers VM replacement.
|
||||
|
||||
## Providers
|
||||
|
||||
`Managed` proxies are provisioned by a provider — a small compute backend
|
||||
behind one minimal interface. Providers are configured in the
|
||||
`--providers-config` YAML file as **named instances**: `spec.provider` on a
|
||||
Proxy refers to an entry's `name`, not its `type`, so `gcp-eu` and `gcp-us`
|
||||
can be two differently-configured instances of the same `gcp` type (see
|
||||
[config/samples/providers-config.yaml](config/samples/providers-config.yaml)).
|
||||
|
||||
### Implemented providers
|
||||
|
||||
| Type | Creates | Per-instance config | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `kubernetes` | A real Squid pod (`ubuntu/squid:6.6-24.04_edge` by default) in the same cluster the operator runs in | `image` (optional) | Needs no cloud account — local dev, CI, and the kind quickstart. Pods share the cluster's egress IP, so it exercises the full lifecycle but not distinct egress paths. |
|
||||
| `gcp` | A Compute Engine VM with an ephemeral external IP | `project` (required), `network`, `networkTag`, `diskSizeGb` | The real egress fleet. Deliberately uses only four API calls (Insert / Get / Delete / AggregatedList), all fire-and-forget: `Create` returns as soon as the operation is submitted and the reconciler discovers progress by polling `Get`. Auth is Application Default Credentials — workload identity in-cluster, `gcloud` ADC locally; no key-file plumbing. |
|
||||
|
||||
### Adding a provider
|
||||
|
||||
A new backend (Hetzner, AWS, ...) is four pieces; the contract lives in
|
||||
[internal/provider/provider.go](internal/provider/provider.go):
|
||||
|
||||
1. **Implement the 4-method `Provider` interface** in a new
|
||||
`internal/provider/<type>/` package:
|
||||
- `Create` submits and returns — it never blocks until the VM runs, and
|
||||
must be idempotent keyed on `req.Name` (a deterministic name derived
|
||||
from the Proxy's UID), so a repeat call after a crash finds the
|
||||
existing instance instead of duplicating it.
|
||||
- `Get` returns `provider.ErrNotFound` as a *normal* outcome — the
|
||||
reconciler branches on it for replacement and adoption, so don't
|
||||
treat it as exceptional.
|
||||
- `Delete` is idempotent: deleting an already-gone instance is not an
|
||||
error.
|
||||
- `ListByTag` returns every instance the operator ever tagged, for
|
||||
orphan GC.
|
||||
2. **Tag every created resource** with `LabelManaged=true` and
|
||||
`LabelUID=<Proxy UID>`, and report `CreatedAt` — orphan GC relies on
|
||||
all three to find owned resources and skip in-flight creates.
|
||||
3. **Classify every returned error** with `provider.Wrap` into the
|
||||
four-sentinel taxonomy in
|
||||
[internal/provider/errors.go](internal/provider/errors.go)
|
||||
(`ErrNotFound` / `ErrQuotaExceeded` / `ErrTransient` / `ErrPermanent`)
|
||||
— the reconciler decides retry, slow backoff, or latching `Failed`
|
||||
purely from that classification, never from provider-specific types.
|
||||
4. **Wire it up**: add a type-specific config block in
|
||||
[internal/provider/config.go](internal/provider/config.go), and
|
||||
register the constructor in the builtins map in
|
||||
[cmd/main.go](cmd/main.go) (`"<type>": <pkg>.New`). The registry
|
||||
([internal/provider/registry](internal/provider/registry/registry.go))
|
||||
handles named instances, and the metrics wrapper is applied
|
||||
automatically.
|
||||
|
||||
Test against a fake API seam rather than the real cloud — see the
|
||||
`instancesAPI` seam in
|
||||
[internal/provider/gcp/gcp.go](internal/provider/gcp/gcp.go) for the
|
||||
pattern.
|
||||
|
||||
## Tracing
|
||||
|
||||
OpenTelemetry tracing is built in but **off by default** — the operator
|
||||
only exports spans when pointed at an OTLP receiver:
|
||||
|
||||
```bash
|
||||
# In config/manager/manager.yaml (commented-out block is already there):
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger-collector.observability:4318
|
||||
# Local dev: print spans to stdout instead
|
||||
OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml
|
||||
```
|
||||
|
||||
Every reconcile, discovery API request, and GC sweep becomes a trace, with
|
||||
child spans for provider and Kubernetes API calls; log lines inside a traced
|
||||
operation carry matching `traceID`/`spanID` fields (point Grafana/Loki
|
||||
derived fields at `traceID`). Sampling and endpoints follow the standard
|
||||
`OTEL_*` env vars; `--trace-health-probes` additionally traces each health
|
||||
probe (high volume). Details in `docs/architecture.md` §10.
|
||||
|
||||
## Caveats — read these two
|
||||
|
||||
**Changing a proxy changes its IP.** Proxies are immutable cattle: editing
|
||||
|
||||
+5
-41
@@ -42,7 +42,6 @@ import (
|
||||
"sigs.k8s.io/controller-runtime/pkg/cache"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/healthz"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log/zap"
|
||||
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
|
||||
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
|
||||
@@ -59,7 +58,6 @@ import (
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version"
|
||||
// +kubebuilder:scaffold:imports
|
||||
)
|
||||
@@ -95,7 +93,6 @@ func main() {
|
||||
var leaseCooldown, maxLeaseTTL time.Duration
|
||||
var showVersion bool
|
||||
var gcpWireFullPayloads bool
|
||||
var traceHealthProbes bool
|
||||
|
||||
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
|
||||
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
|
||||
@@ -136,9 +133,6 @@ func main() {
|
||||
"Print the commit the binary was built from and exit.")
|
||||
flag.BoolVar(&gcpWireFullPayloads, "gcp-wire-log-full-payloads", false,
|
||||
"Log GCP V(5) wire payloads verbatim instead of eliding fields larger than 1KiB.")
|
||||
flag.BoolVar(&traceHealthProbes, "trace-health-probes", false,
|
||||
"Emit one trace span per health probe. Off by default: probes run about "+
|
||||
"once per second per proxy and would dominate trace volume.")
|
||||
|
||||
opts := zap.Options{
|
||||
Development: true,
|
||||
@@ -154,17 +148,6 @@ func main() {
|
||||
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
|
||||
setupLog.Info("Starting egress-proxies-operator",
|
||||
"commit", version.Resolve(), "goVersion", goruntime.Version())
|
||||
|
||||
// Off (no-op spans, unchanged logs) unless OTEL_* env opts in; see
|
||||
// internal/tracing. Shutdown is called explicitly after mgr.Start
|
||||
// returns — the os.Exit paths below skip defers.
|
||||
tracingShutdown, err := tracing.Setup(logf.IntoContext(context.Background(), setupLog),
|
||||
"egress-proxies-operator", version.Resolve())
|
||||
if err != nil {
|
||||
setupLog.Error(err, "Failed to set up tracing")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx := ctrl.SetupSignalHandler()
|
||||
|
||||
// Providers load first and fail fast: a manager that comes up without
|
||||
@@ -179,11 +162,7 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
providers, err := registry.Build(ctx, cfg, map[string]registry.Constructor{
|
||||
"kubernetes": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
|
||||
// The kubernetes provider builds its own uncached client;
|
||||
// wrap its transport so its API calls join the caller's trace.
|
||||
return kubernetes.NewWithTransportWrapper(ctx, pc, tracing.RestConfigWrapper())
|
||||
},
|
||||
"kubernetes": kubernetes.New,
|
||||
"gcp": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
|
||||
return gcp.NewWithWireOptions(ctx, pc, gcp.WireLogOptions{FullPayloads: gcpWireFullPayloads})
|
||||
},
|
||||
@@ -194,8 +173,7 @@ func main() {
|
||||
}
|
||||
m := metrics.New()
|
||||
for name, p := range providers {
|
||||
// Tracing outermost: the span covers the metrics recording too.
|
||||
providers[name] = provider.WithTracing(name, provider.WithMetrics(name, p, m))
|
||||
providers[name] = provider.WithMetrics(name, p, m)
|
||||
}
|
||||
|
||||
// if the enable-http2 flag is false (the default), http/2 should be disabled
|
||||
@@ -258,9 +236,7 @@ func main() {
|
||||
cacheOpts.DefaultNamespaces = map[string]cache.Config{proxyNamespace: {}}
|
||||
}
|
||||
|
||||
restCfg := ctrl.GetConfigOrDie()
|
||||
restCfg.Wrap(tracing.RestConfigWrapper())
|
||||
mgr, err := ctrl.NewManager(restCfg, ctrl.Options{
|
||||
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
|
||||
Scheme: scheme,
|
||||
Metrics: metricsServerOptions,
|
||||
HealthProbeBindAddress: probeAddr,
|
||||
@@ -282,7 +258,6 @@ func main() {
|
||||
engine := health.NewEngine(mgr.GetClient())
|
||||
engine.Workers = healthWorkers
|
||||
engine.Metrics = m
|
||||
engine.TraceProbes = traceHealthProbes
|
||||
if err := mgr.Add(engine); err != nil {
|
||||
setupLog.Error(err, "Failed to add health engine")
|
||||
os.Exit(1)
|
||||
@@ -348,19 +323,8 @@ func main() {
|
||||
}
|
||||
|
||||
setupLog.Info("Starting manager")
|
||||
startErr := mgr.Start(ctx)
|
||||
|
||||
// Flush pending spans on the way out, error path included. Fresh
|
||||
// context: the signal ctx is already cancelled by the time Start
|
||||
// returns.
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if err := tracingShutdown(shutdownCtx); err != nil {
|
||||
setupLog.Error(err, "Failed to flush traces on shutdown")
|
||||
}
|
||||
cancel()
|
||||
|
||||
if startErr != nil {
|
||||
setupLog.Error(startErr, "Failed to run manager")
|
||||
if err := mgr.Start(ctx); err != nil {
|
||||
setupLog.Error(err, "Failed to run manager")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,8 +64,6 @@ spec:
|
||||
- --leader-elect
|
||||
- --health-probe-bind-address=:8081
|
||||
- --providers-config=/etc/proxy-operator/providers.yaml
|
||||
# Add --trace-health-probes to emit one span per health probe
|
||||
# (high volume: ~1/s per proxy).
|
||||
env:
|
||||
# Bearer token for the discovery API. Optional: without the
|
||||
# Secret the API serves unauthenticated (with a loud warning).
|
||||
@@ -78,33 +76,6 @@ spec:
|
||||
name: discovery-token
|
||||
key: token
|
||||
optional: true
|
||||
# --- OpenTelemetry tracing ---
|
||||
# Tracing is OFF until OTEL_EXPORTER_OTLP_ENDPOINT (or
|
||||
# OTEL_TRACES_EXPORTER) is set; without it the operator runs
|
||||
# exactly as before. The downward-API vars must stay listed
|
||||
# before OTEL_RESOURCE_ATTRIBUTES: $(VAR) expansion only sees
|
||||
# earlier-listed vars.
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: OTEL_SERVICE_NAME
|
||||
value: egress-proxies-operator
|
||||
- name: OTEL_RESOURCE_ATTRIBUTES
|
||||
value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)
|
||||
# Point at an OTLP receiver (Tempo, Jaeger, otel-collector) to
|
||||
# enable tracing; http/protobuf (4318) is the default protocol,
|
||||
# set OTEL_EXPORTER_OTLP_PROTOCOL=grpc for 4317.
|
||||
# - name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||
# value: http://jaeger-collector.observability:4318
|
||||
# - name: OTEL_TRACES_SAMPLER
|
||||
# value: parentbased_traceidratio
|
||||
# - name: OTEL_TRACES_SAMPLER_ARG
|
||||
# value: "0.1"
|
||||
image: controller:latest
|
||||
name: manager
|
||||
ports:
|
||||
|
||||
@@ -300,45 +300,6 @@ Each consuming package defines its own small recorder interface
|
||||
`metrics.Metrics` satisfies all of them structurally, so no package other
|
||||
than `cmd/main.go` imports the metrics package.
|
||||
|
||||
### 10. Tracing (`internal/tracing/`)
|
||||
|
||||
OpenTelemetry tracing, integrated with — not replacing — the logr/zap
|
||||
logging. Everything hangs off standard `OTEL_*` env vars: with no
|
||||
`OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` set, no SDK is
|
||||
installed, spans are no-ops, and logs are byte-for-byte what they were.
|
||||
`OTEL_SDK_DISABLED=true` and `OTEL_TRACES_EXPORTER=none` force it off;
|
||||
`console` prints spans to stdout for local dev; sampling follows
|
||||
`OTEL_TRACES_SAMPLER(_ARG)`.
|
||||
|
||||
Span topology — each unit of work is a new trace (watch events carry no
|
||||
incoming trace context):
|
||||
|
||||
- `Reconcile Proxy` (root, via a `reconcile.Reconciler` decorator) →
|
||||
`reconcile.managed` / `reconcile.replaceInstance` / `reconcile.delete`,
|
||||
plus `status.patch` from the deferred flush. `reconcileExternal` does no
|
||||
I/O and is unspanned.
|
||||
- `provider.create|get|delete|list` (client spans, `provider.WithTracing`
|
||||
decorator wired outermost around `WithMetrics`); the GCP SDK's own
|
||||
otelhttp transport contributes HTTP child spans automatically.
|
||||
- Kubernetes API calls become client child spans via a wrapped
|
||||
`rest.Config` transport — gated on an existing parent span, so informer
|
||||
list/watch long-polls and leader-election renewals never create root
|
||||
spans. The kube-apiserver ignores incoming `traceparent` by design;
|
||||
these spans are leaves.
|
||||
- Discovery API: one server span per request (named from the route
|
||||
pattern), incoming W3C `traceparent` honored so clients' traces continue
|
||||
into the operator; `/healthz` excluded.
|
||||
- `gc.sweep` per GC pass; `health.probe` per probe **only** with
|
||||
`--trace-health-probes` (default off — probes run ~1/s per proxy).
|
||||
|
||||
Log correlation: logr sinks never see a context, so trace IDs ride on the
|
||||
logger — `tracing.Start` re-derives the ctx logger from a captured base
|
||||
with `traceID`/`spanID` values (lowerCamel, matching `reconcileID`;
|
||||
Grafana/Loki derived-field regexes must match `traceID`, not `trace_id`).
|
||||
Re-deriving from the base rather than layering keeps zap from emitting
|
||||
duplicate keys on nested spans. GCP V(5) wire logs get the same keys from
|
||||
the slog handler's ctx.
|
||||
|
||||
## Decisions
|
||||
|
||||
Judgment calls the spec left open, and deliberate deviations — recorded so
|
||||
@@ -443,24 +404,3 @@ they read as choices, not accidents. Chronological by build step.
|
||||
logger to everything running under the manager — fighting that would
|
||||
mean two logging systems in one process. Noted as a deviation rather
|
||||
than silently ignored.
|
||||
- **Tracing is env-gated, not flag-gated:** it activates only when
|
||||
`OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` is set (user
|
||||
decision). No collector configured means no SDK installed, no-op spans,
|
||||
unchanged logs — the safe default for every existing deployment. The
|
||||
one flag is `--trace-health-probes`, off by default, because probes at
|
||||
~1/s per proxy would dominate trace volume.
|
||||
- **Trace keys are `traceID`/`spanID`**, lowerCamel like `reconcileID` and
|
||||
`providerID`, deliberately not the `trace_id` many Grafana derived-field
|
||||
examples assume — configure the derived-field regex accordingly. logr
|
||||
sinks can't read ctx, so the IDs ride on the ctx logger, re-derived from
|
||||
a captured base per span so zap never emits duplicate keys.
|
||||
- **Probe→reconcile trace links are not attempted:** the health engine's
|
||||
`GenericEvent` carries only namespace/name (no ctx), and the workqueue
|
||||
coalesces events, so any link would be a guess. A health-triggered
|
||||
reconcile starts a fresh trace; the probe that caused it is findable via
|
||||
its own (opt-in) span and shared proxy attributes.
|
||||
- **No `traceparent` toward probe targets:** probe transports stay
|
||||
uninstrumented so trace headers can never leak through a proxy to
|
||||
external sites. The kube-apiserver ignores incoming `traceparent` by
|
||||
design (public endpoint), so k8s client spans are leaves — in-process
|
||||
traces, not cross-process ones.
|
||||
|
||||
@@ -1,183 +0,0 @@
|
||||
# Execution log: OpenTelemetry tracing integrated with logr/zap logging
|
||||
|
||||
Plan: `docs/plans/2026-08-24-1025-otel-tracing.md`
|
||||
|
||||
- [x] Step 1 — Dependencies
|
||||
- [x] Step 2 — New package `internal/tracing`
|
||||
- [x] Step 3 — `provider.WithTracing` decorator
|
||||
- [x] Step 4 — `cmd/main.go` wiring
|
||||
- [x] Step 5 — Reconciler spans
|
||||
- [x] Step 6 — Discovery server
|
||||
- [x] Step 7 — GC + health
|
||||
- [x] Step 8 — GCP wire-log enrichment
|
||||
- [x] Step 9 — Manifests + docs
|
||||
|
||||
## Step 1 — Dependencies
|
||||
|
||||
Aligned the pre-existing indirect skew (otel core v1.44.0 vs otlptrace exporters
|
||||
v1.40.0) and added the new direct deps in one shot:
|
||||
|
||||
```bash
|
||||
go get go.opentelemetry.io/otel@v1.45.0 \
|
||||
go.opentelemetry.io/otel/sdk@v1.45.0 \
|
||||
go.opentelemetry.io/otel/trace@v1.45.0 \
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0 \
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp@v1.45.0 \
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace@v1.45.0 \
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
MVS side-effects worth recording: `logr v1.4.3→v1.4.4`, `httpsnoop
|
||||
v1.0.4→v1.1.0`, `grpc-gateway/v2 v2.27.7→v2.29.0`, `proto/otlp v1.9.0→v1.11.0`,
|
||||
plus a `genproto/googleapis/api` pseudo-version bump. Full `make test` passed
|
||||
against the bumped graph (`google.golang.org/api v0.292.0` and k8s v0.36
|
||||
tolerate otelhttp v0.70.0).
|
||||
|
||||
Worth noting: the first `go mod tidy` ran *before* any first-party code
|
||||
imported `otlptracehttp`/`stdouttrace`, so it silently dropped those two
|
||||
modules again; the Step 2 tidy re-added them. The plan's semconv question
|
||||
resolved to `semconv/v1.43.0` — that's what `sdk@v1.45.0/resource/builtin.go`
|
||||
imports, so first-party code uses the same version to avoid
|
||||
`ErrSchemaURLConflict` in the common path.
|
||||
|
||||
## Step 2 — `internal/tracing` package
|
||||
|
||||
Landed as planned: `tracing.go` (env-gated `Setup`, hand-rolled exporter
|
||||
selection), `logger.go` (`Start`/`StartSpan`/`ContextWithLogger` with the
|
||||
base-logger ctx key that prevents duplicate `traceID` zap fields on nested
|
||||
spans), `reconciler.go`, `transport.go`, `http.go`, `options.go`; tests for
|
||||
all of it (first use of `sdk/trace/tracetest` in the repo).
|
||||
|
||||
Two deviations from the plan's letter:
|
||||
|
||||
- **k8s transport gating is a custom RoundTripper, not `otelhttp.WithFilter`**
|
||||
(`parentGatedTransport`): requests without a parent span bypass the otel
|
||||
transport entirely, so the no-root-spans-from-informers guarantee doesn't
|
||||
depend on otelhttp filter semantics for transports.
|
||||
- **`HTTPMiddleware` must copy the route pattern back.** The logger-injecting
|
||||
inner handler wraps the request via `WithContext` (a shallow copy), so the
|
||||
mux records the matched pattern on the copy while otelhttp's post-routing
|
||||
span rename reads the original. Found by the middleware test (span named
|
||||
`"GET"` instead of `"GET /v1/things/{id}"`); fixed with `r.Pattern =
|
||||
r2.Pattern` after `next.ServeHTTP`.
|
||||
|
||||
Also: both the transport and the middleware pass explicit W3C propagators
|
||||
instead of relying on the global, so behavior is deterministic under tests
|
||||
and when tracing is disabled. `Setup` tests reset the global provider to a
|
||||
fresh noop per case — restoring otel's own default delegate triggers a
|
||||
"Setting tracer provider to its current value" warning from the SDK.
|
||||
|
||||
## Step 3 — `provider.WithTracing`
|
||||
|
||||
`internal/provider/tracing.go` mirrors `metrics.go` exactly (same wrap shape,
|
||||
same `resultLabel` classification reused as the `provider.result` span
|
||||
attribute); tests reuse `staticProvider` from `metrics_test.go`. `ErrNotFound`
|
||||
maps to span status Ok as planned — the reconciler polls `Get` to NotFound
|
||||
during replacement/deletion, so it's an answer, not a failure.
|
||||
|
||||
One small API addition to `internal/tracing` for this: exported
|
||||
`tracing.Tracer(opts...)`, since the decorator lives in package `provider`
|
||||
and couldn't reach the unexported option resolver. The `cmd/main.go` wiring
|
||||
(`WithTracing` outermost around `WithMetrics`) lands with Step 4's commit —
|
||||
same file, one commit.
|
||||
|
||||
## Step 4 — `cmd/main.go` wiring
|
||||
|
||||
As planned: `tracing.Setup` right after `ctrl.SetLogger`; manager rest.Config
|
||||
wrapped via `restCfg := ctrl.GetConfigOrDie(); restCfg.Wrap(...)` (the
|
||||
previous inline call left nowhere to wrap); providers decorated tracing-
|
||||
outermost; explicit trace flush after `mgr.Start` returns on both the error
|
||||
and clean paths, with a fresh 10s context since the signal ctx is already
|
||||
cancelled by then.
|
||||
|
||||
The kubernetes provider grew `NewWithTransportWrapper(ctx, cfg, wrap)` (its
|
||||
client is built from its own `ctrl.GetConfig()`, invisible to the manager's
|
||||
wrapped config); `New` now delegates with a nil wrapper, so `newWithClient`
|
||||
tests stayed untouched, and main registers a closure — the same pattern the
|
||||
gcp constructor already used for wire-log options.
|
||||
|
||||
Deviation: the plan put `--trace-health-probes` here, but the flag needs the
|
||||
`health.Engine.TraceProbes` field that Step 7 introduces — moved there to
|
||||
keep every commit compiling.
|
||||
|
||||
## Step 5 — Reconciler spans
|
||||
|
||||
`SetupWithManager` completes with `tracing.NewReconciler("Proxy", r)`;
|
||||
sub-spans `reconcile.managed` / `reconcile.replaceInstance` /
|
||||
`reconcile.delete` open at the top of each state machine, and `status.patch`
|
||||
opens inside the deferred flush closure so it stays within the root span
|
||||
while its error still folds into the recorded result. `reconcileExternal`
|
||||
left unspanned as planned (no I/O).
|
||||
|
||||
Two small judgment calls: the sub-spans carry no extra attributes — the
|
||||
provider decorator already records `provider.id`, and the root span carries
|
||||
the object identity, so duplicating them was noise; and `status.patch` is
|
||||
emitted every reconcile even when nothing changed (the no-op compare is the
|
||||
span's content — a real PATCH shows up as its k8s HTTP child). Tests that
|
||||
call `r.Reconcile` directly bypass the wrapper; with no global tracer set
|
||||
they see no-op spans, so the existing fake-client and envtest suites run
|
||||
unchanged.
|
||||
|
||||
## Step 6 — Discovery server
|
||||
|
||||
Middleware chain is now recover → tracing (server span + request logger) →
|
||||
request-log → body cap → auth; the request-log middleware and the three
|
||||
handler error sites log via `logf.FromContext(r.Context())` — which is
|
||||
`s.log` enriched with the request's traceID/spanID by the tracing
|
||||
middleware, not a different logger (logr sinks can't read ctx at log time,
|
||||
so per-request values must ride on the logger instance in the ctx; user
|
||||
asked, answered in-session).
|
||||
|
||||
Deviation from the plan's letter: `recoverMiddleware` keeps `s.log`. It sits
|
||||
*outside* the tracing layer, so its request ctx never has the enriched
|
||||
logger — switching it to `FromContext` would silently drop the `"discovery"`
|
||||
name and gain nothing.
|
||||
|
||||
## Step 7 — GC + health
|
||||
|
||||
`gc.sweep` runs in a root span with `gc.providers` / `gc.deleted` counters
|
||||
(a failed orphan delete no longer counts as deleted — the loop grew an
|
||||
explicit `continue`). The health engine got the deferred `--trace-health-probes`
|
||||
flag and `TraceProbes` field; the probe call moved into `runProbe`, which
|
||||
wraps it in a `health.probe` client span only when enabled, ending before
|
||||
`record` (which stays ctx-free by design). Probe transports remain
|
||||
uninstrumented so no traceparent can leak through a proxy to external
|
||||
targets. Probe→reconcile span links stay out of scope (GenericEvent carries
|
||||
no ctx; the workqueue coalesces events) — recorded in the architecture
|
||||
Decisions in Step 9.
|
||||
|
||||
## Step 8 — GCP wire-log enrichment
|
||||
|
||||
`wireFilterHandler.Handle` clones the record and appends `traceID`/`spanID`
|
||||
when the ctx carries a valid span — same keys as the logr enrichment, added
|
||||
before the elision pass (both values are well under the 1KiB threshold).
|
||||
The new `wirelog_test.go` covers the enrichment, that the JWT/token security
|
||||
filter still drops non-wire Debug records even with a span present, and that
|
||||
spanless records stay unchanged.
|
||||
|
||||
## Step 9 — Manifests, docs, lint
|
||||
|
||||
`config/manager/manager.yaml` gained the OTel env block (downward-API
|
||||
`POD_NAME`/`POD_NAMESPACE` deliberately listed *before*
|
||||
`OTEL_RESOURCE_ATTRIBUTES` — `$(VAR)` expansion only sees earlier vars),
|
||||
with the OTLP endpoint and sampler left as commented examples so deployments
|
||||
stay tracing-off by default. `docs/architecture.md` got "### 10. Tracing"
|
||||
plus five Decisions entries; README got a Tracing section.
|
||||
|
||||
`make lint` surfaced 16 new-on-branch issues (checked with
|
||||
`golangci-lint run --new-from-rev main`; the other ~43 pre-date this work):
|
||||
|
||||
- one gofmt slip, 13 goconst repeats — fixed with `env*`/`exporter*`
|
||||
constants in `internal/tracing/tracing.go` shared by the tests;
|
||||
- two logcheck hits ("function takes both a context and a logger"):
|
||||
`Setup` now reads its logger from ctx (`logf.FromContext`, caller seeds
|
||||
via `logf.IntoContext` — the linter's sanctioned pattern), and
|
||||
`ContextWithLogger` carries a justified `//nolint:logcheck`, since being
|
||||
the IntoContext-analogue is its purpose.
|
||||
|
||||
Worth noting: pre-existing lint debt (50 goconst etc. on `main`) was left
|
||||
untouched — `make lint` still fails overall; only branch-introduced issues
|
||||
were cleaned. The lint fixes changed `Setup`'s signature after Step 2/4 had
|
||||
landed, so those commits show the two-arg version; this commit is where it
|
||||
settles.
|
||||
@@ -1,68 +0,0 @@
|
||||
# Execution log: e2e test — OTel tracing against real Tempo
|
||||
|
||||
Plan: `docs/plans/2026-08-24-1224-tracing-e2e.md`
|
||||
|
||||
- [x] Step 1 — `test/e2e/tracing_test.go`
|
||||
- [x] Step 2 — Makefile
|
||||
- [x] Step 3 — Docs
|
||||
|
||||
## Steps 1–3 — spec, Makefile, docs (one commit)
|
||||
|
||||
The three steps landed together — the spec is one new file and the other
|
||||
two are its wiring. `Describe("OTel tracing", Ordered)` is fully
|
||||
self-contained (own ns create → `make install`/`make deploy` → teardown)
|
||||
because Ginkgo randomizes top-level container order, so it cannot share the
|
||||
Manager Describe's deployment. It reuses the package-level `namespace` /
|
||||
`managerImage` and the suite's idioms (`utils.Run`, curl-pod with the
|
||||
restricted-PSS overrides JSON, log-substring `Eventually`s).
|
||||
|
||||
Judgment calls beyond the plan's letter:
|
||||
|
||||
- The squid pre-pull (`docker pull` + `kind load`) is **best effort** — a
|
||||
missing docker binary logs a note and continues rather than failing the
|
||||
spec; the 5m Ready timeout still covers an in-cluster pull.
|
||||
- The OTLP preflight pod prints per-attempt HTTP codes and a final
|
||||
`OTLP_OK`/`OTLP_UNREACHABLE` marker; the assertion quotes the pod's
|
||||
output, so an unreachable endpoint names itself in the failure.
|
||||
- `kubectl set env` is passed the literal
|
||||
`OTEL_RESOURCE_ATTRIBUTES=...$(POD_NAME)...` string via `exec.Command` —
|
||||
no shell involved, kubectl stores `$()` verbatim, and the in-place update
|
||||
keeps the var after the downward-API vars it references.
|
||||
- Tempo helpers are stdlib-only; `/api/traces/<id>` is decoded as
|
||||
OTLP-JSON (`batches[].scopeSpans[].spans[].name`), which is Tempo's
|
||||
actual shape (not Jaeger's).
|
||||
|
||||
Verified so far: `go vet -tags=e2e ./...` clean.
|
||||
|
||||
## Live run against homelab Tempo
|
||||
|
||||
First attempt failed before the suite started — Docker Desktop wasn't
|
||||
running (`kind` could not create the cluster), and the failure was masked
|
||||
to exit 0 by a `| tail` pipe on the make invocation (no pipefail in that
|
||||
shell). Rerun with Docker started first and no pipe:
|
||||
|
||||
```bash
|
||||
TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e
|
||||
```
|
||||
|
||||
Result: **SUCCESS — 4/4 specs (Manager smoke + both tracing Its), 219 s**,
|
||||
kind cluster auto-deleted. Run id `e2e-1787567624494810000`; the traces are
|
||||
findable in Grafana with TraceQL
|
||||
`{resource.test.run.id="e2e-1787567624494810000"}`. Both proxies reached
|
||||
`Ready` on real squid pods; the create trace carried
|
||||
`Reconcile Proxy` / `reconcile.managed` / `provider.create` /
|
||||
`status.patch` and the expected resource attrs; the deletion trace carried
|
||||
`reconcile.delete` / `provider.delete`.
|
||||
|
||||
Worth noting: the OTLP preflight and both Tempo polls succeeded on the
|
||||
in-cluster → LAN path (kind on macOS reaches 192.168.0.30 through Docker
|
||||
Desktop's NAT), so no extra networking setup is needed on this machine.
|
||||
|
||||
The plan's negative verification also ran: with a deliberately wrong
|
||||
endpoint (`OTLP_ENDPOINT=http://192.168.0.30:9999`) the tracing spec
|
||||
failed in the BeforeAll preflight after ~52 s with
|
||||
`OTLP endpoint http://192.168.0.30:9999 is not reachable from inside the
|
||||
kind cluster; curl output: OTLP_UNREACHABLE` — a named, fast failure
|
||||
instead of a 2-minute opaque search timeout — and teardown still deleted
|
||||
the kind cluster (`make cleanup-test-e2e` run explicitly, since a failing
|
||||
`go test` skips the Makefile's cleanup step).
|
||||
@@ -0,0 +1,285 @@
|
||||
# Execution: KVM/libvirt provider
|
||||
|
||||
Plan: [2026-09-04-1404-kvm-provider.md](../plans/2026-09-04-1404-kvm-provider.md)
|
||||
|
||||
- [x] Step 0 — Save plan + branch `feat/kvm-provider`
|
||||
- [x] Step 1 — Config (`KVMConfig` & friends, `validate()` case, tests)
|
||||
- [x] Step 2 — Pure builders (`domainxml.go`, `volume.go`, `seed.go`, `id.go`, `metadata.go`)
|
||||
- [x] Step 3 — Transport + errors (`conn.go`, `errors.go`)
|
||||
- [x] Step 3b — Image cache (`image.go`)
|
||||
- [x] Step 4 — Provider methods (`kvm.go`) over a fake API
|
||||
- [ ] Step 5 — Wiring, manifests, samples, docs, CHANGELOG
|
||||
- [ ] Step 6 — Integration test against a real hypervisor
|
||||
- [ ] Step 7 — Manual validation + MR
|
||||
|
||||
## Step 0 — Save plan + branch
|
||||
|
||||
Plan committed alone on `feat/proxy-operator`, then branched:
|
||||
|
||||
```bash
|
||||
git checkout -b feat/kvm-provider
|
||||
```
|
||||
|
||||
## Step 1 — Config
|
||||
|
||||
The new structs (`KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage`) and
|
||||
`validateKVM` went into their own file, `internal/provider/config_kvm.go`,
|
||||
rather than being appended to `config.go` as the plan said: the KVM block has
|
||||
far more fields and rules than the other two providers, and keeping it
|
||||
separate leaves `config.go` readable. Tests likewise live in
|
||||
`config_kvm_test.go` instead of extending the table in `config_test.go`; the
|
||||
invalid-config table covers every required field, duplicate host names, image
|
||||
URL scheme, sha256 shape, and the disk format enum.
|
||||
|
||||
While adding the `kvm` case to `validate()`, the pairwise "type is X but a Y
|
||||
block is set" checks (which would have needed six branches with three types)
|
||||
were replaced by a `setBlocks()` helper that lists the non-nil blocks and
|
||||
rejects any that don't match `type`. The error strings are unchanged, so the
|
||||
existing GCP/kubernetes tests still pass.
|
||||
|
||||
Dependencies were fetched up front so the API signatures in the plan could be
|
||||
checked with `go doc`; they stay `// indirect` in `go.mod` until Step 2 imports
|
||||
them:
|
||||
|
||||
```bash
|
||||
go get github.com/digitalocean/go-libvirt@latest github.com/kdomanski/iso9660@v0.4.0
|
||||
# go-libvirt resolved to v0.0.0-20260814190004-1a83157e1858 (no tagged release)
|
||||
# side effect: golang.org/x/crypto 0.54.0 → 0.56.0, x/mod, x/text, x/tools bumped
|
||||
```
|
||||
|
||||
Worth noting: `go doc` showed that go-libvirt's `New(net.Conn)` is deprecated
|
||||
in favour of `NewWithDialer(socket.Dialer)`, where `Dialer` is a one-method
|
||||
`Dial() (net.Conn, error)` interface. Step 3 should wrap the SSH client's unix
|
||||
forward in that interface instead of using `New`; the plan's transport design
|
||||
is otherwise unaffected.
|
||||
|
||||
## Step 2 — Pure builders
|
||||
|
||||
`internal/provider/kvm/` now has the five builders from the plan plus
|
||||
`kvm.go` holding the package doc and `withDefaults` (disk size 10 GiB, the
|
||||
default vendor-data, `/var/run/libvirt/libvirt-sock`, pool `default`, image
|
||||
format `qcow2`). Everything is a pure function over `encoding/xml` structs or
|
||||
strings, unit-tested by unmarshalling the output back and checking fields.
|
||||
|
||||
Two design choices differ from the plan text:
|
||||
|
||||
- **Domain disks reference pool volumes, not host paths.** The plan had
|
||||
`buildDomainXML` take the overlay and seed paths (needing two
|
||||
`StorageVolGetPath` round-trips per Create). The domain XML now uses
|
||||
`<disk type="volume"><source pool=... volume=.../>`, so only the *base*
|
||||
image path is ever looked up (the overlay's `<backingStore>` needs a real
|
||||
path). Fewer RPCs and the builder's inputs are all names.
|
||||
- **Capacities are always `unit="bytes"`.** libvirt's `G` means GiB but
|
||||
`GB` means 10^9; spelling the byte count out removes the ambiguity from
|
||||
both the overlay and the seed/base volumes.
|
||||
|
||||
The metadata element is written with a default namespace
|
||||
(`<instance xmlns="…/kvm">`) because `encoding/xml` cannot emit prefixes on
|
||||
marshal. libvirt may hand it back prefixed; `parseInstanceMetadata` is
|
||||
tested against both spellings since `encoding/xml` matches on the namespace
|
||||
URI. Whether libvirt accepts the default-namespace form on `DomainDefineXML`
|
||||
is confirmed in Step 6 against a real host, not here.
|
||||
|
||||
Checked before relying on `github.com/kdomanski/iso9660` for the seed: its
|
||||
filename mangling lowercases and keeps `-`, so `user-data`/`meta-data`/
|
||||
`vendor-data` come out as `user-data;1` etc. in the directory record, which
|
||||
the Linux isofs driver (and therefore cloud-init's NoCloud mount) presents as
|
||||
`user-data`. The seed test opens the ISO with the same library's reader and
|
||||
asserts label `cidata`, all three files, and byte-identical output for
|
||||
identical inputs. `NewWriter()` stages files in a temp directory, so
|
||||
`Cleanup()` is deferred.
|
||||
|
||||
`go mod tidy` ran as part of this step; it promoted `kdomanski/iso9660` to a
|
||||
direct dependency and dropped `go-libvirt` again because nothing imports it
|
||||
yet — Step 3 brings it back.
|
||||
|
||||
Worth noting: `withDefaults` copies the `Images` map and `Hosts` slice
|
||||
instead of mutating the config in place, so the registry's shared
|
||||
`ProviderConfig` stays untouched. The default vendor-data installs
|
||||
`qemu-guest-agent` via `packages:` + `runcmd`, which means the first boot
|
||||
needs network access to the distro mirrors; a Proxy on an isolated bridge
|
||||
would still work (ARP discovery), just without the agent.
|
||||
|
||||
## Step 3 — Transport + errors
|
||||
|
||||
`conn.go` holds the `libvirtAPI` seam (17 methods, signatures as in the
|
||||
plan), a per-host connection pool, and the SSH dialer; `errors.go` holds
|
||||
`classify`, `isLibvirtError`, `hasCode` and `logAPIError`. Both are
|
||||
unit-tested; the whole package still runs in about a second under `-race`.
|
||||
|
||||
The one thing the plan did not anticipate: **go-libvirt RPCs take no
|
||||
context and never time out**. A hung hypervisor would pin a reconcile worker
|
||||
forever. So the pool exposes `call(ctx, host, fn)` rather than handing out
|
||||
raw clients: `fn` runs in a goroutine, and if `ctx` ends first the pool
|
||||
closes the SSH connection out from under it (which makes the blocked RPC
|
||||
return) and reports `ctx.Err()`. The same helper (`awaitOrCancel`) bounds
|
||||
`lv.Connect()` during dial. Two more liveness measures for the same reason:
|
||||
the SSH handshake gets a deadline (ctx's, or 30 s), and each connection runs
|
||||
an OpenSSH `keepalive@openssh.com` loop every 15 s that closes the client
|
||||
when a request errors or goes unanswered for one interval.
|
||||
|
||||
Connection-drop policy in `call`: any error that is *not* a `libvirt.Error`
|
||||
(so io.EOF, "connection closed", SSH failures) drops the cached connection
|
||||
and the next call redials; a daemon-reported `libvirt.Error` keeps it. Dials
|
||||
are serialised per host (concurrent Creates on one host share a dial) but
|
||||
hosts don't block each other.
|
||||
|
||||
As Step 1 noted, the dialer uses `libvirt.NewWithDialer` with a
|
||||
`socket.Dialer` that wraps `ssh.Client.Dial("unix", socket)` rather than the
|
||||
deprecated `New(net.Conn)`.
|
||||
|
||||
Tests: the pool is exercised through a fake dialer (reuse, redial after
|
||||
`Disconnected`, drop on transport error, keep on libvirt error, cancel
|
||||
releases a blocked call, concurrent callers share one dial, hosts
|
||||
independent). The SSH layer is tested against an in-process
|
||||
`x/crypto/ssh` server that serves `direct-streamlocal@openssh.com` channels
|
||||
by dialing the requested unix path (an echo socket standing in for
|
||||
`libvirt-sock`), with keys generated per test and a real `known_hosts` file
|
||||
written via `knownhosts.Line`. Covered: forward round-trip and the socket
|
||||
path the server saw, unknown host key rejected with `knownhosts.KeyError`,
|
||||
`insecureIgnoreHostKey`, wrong client key, cancelled ctx, missing key or
|
||||
known_hosts file, and keepalive both closing a stalled client and leaving a
|
||||
healthy one alone. The libvirt handshake (`Connect()`) itself is *not*
|
||||
covered here — faking libvirt's XDR RPC just for that is not worth it; the
|
||||
Step 6 integration test covers it against a real host.
|
||||
|
||||
`go mod tidy` promoted `go-libvirt` and `golang.org/x/crypto` to direct
|
||||
dependencies.
|
||||
|
||||
Worth noting: `classify` maps `ErrOperationInvalid` (e.g. destroying a
|
||||
domain that is already off) to Transient; Delete in Step 4 must check that
|
||||
code itself via `hasCode` and treat it as success rather than relying on
|
||||
the classifier. Nothing maps to `ErrQuotaExceeded` — libvirt has no quota
|
||||
concept; an out-of-space pool surfaces as a Transient `ErrInternalError`.
|
||||
|
||||
## Step 3b — Image cache
|
||||
|
||||
`image.go` follows the plan closely: `resolveImage` maps `placement.image`
|
||||
to a configured key or a direct http(s) URL (else Permanent);
|
||||
`imageCache.ensure` checks base volume + marker (+ marker content when a
|
||||
checksum is configured), tears down and re-fetches when anything is off,
|
||||
streams the HTTP body through `StorageVolUpload` with a `sha256` tee,
|
||||
refreshes the pool, then writes the marker last. A per-`(host, volume)`
|
||||
mutex serialises concurrent callers; the pool itself is the only cache
|
||||
state.
|
||||
|
||||
Errors from this file are pre-tagged with the taxonomy sentinel via
|
||||
`fmt.Errorf("%w: …", provider.ErrPermanent, …)` and `classify` now honours
|
||||
such tags before looking at libvirt codes. That keeps one `wrapErr` path in
|
||||
the provider instead of special-casing image errors.
|
||||
|
||||
`fake_test.go` landed with this step rather than Step 4: an in-memory
|
||||
`libvirtAPI` with pools, volumes (content, format, capacity, backing path),
|
||||
domains (state, metadata, per-source interface addresses), per-method error
|
||||
injection (`fail` always / `failOnce`) and an `uploadHook` to break a stream
|
||||
mid-way. It models the libvirt error codes the provider branches on
|
||||
(`ErrStorageVolExist`, `ErrNoStorageVol`, `ErrNoDomain`,
|
||||
`ErrOperationInvalid`, `ErrNoDomainMetadata`, `ErrAgentUnresponsive`).
|
||||
|
||||
Tests cover every case in the plan plus: no checksum configured (marker is
|
||||
still written with the computed digest; second call needs no marker read),
|
||||
403/500 alongside 404/503, missing pool → Permanent, cancelled ctx →
|
||||
Transient with nothing created, and different hosts fetching independently.
|
||||
|
||||
One test-harness lesson worth keeping: `httptest`'s server drops
|
||||
`Content-Length` and switches to chunked encoding for bodies larger than
|
||||
its 2 KiB write buffer, so the fake mirror sets the header explicitly.
|
||||
Real distro mirrors send it; a server that doesn't is reported as
|
||||
Permanent with a message naming the header.
|
||||
|
||||
Worth noting: the base volume is created with `capacity = Content-Length`
|
||||
and the *image's* format (`qcow2`), and the raw bytes are uploaded over it.
|
||||
That relies on libvirt's upload writing the file verbatim (which is how
|
||||
`virsh vol-upload` populates volumes); the following `StoragePoolRefresh`
|
||||
makes libvirt re-read the real virtual size from the qcow2 header. Step 6
|
||||
confirms this on a real host. Also, the whole download runs inside one
|
||||
`pool.call`, so cancellation aborts the HTTP body (ctx-bound) and the
|
||||
upload stream fails; the half-written base volume is deleted on the spot
|
||||
and, failing that, on the next attempt via the missing marker.
|
||||
|
||||
## Step 4 — Provider methods
|
||||
|
||||
`kvm.go` now holds `Provider`, `New` (loads the SSH key and known_hosts
|
||||
up front via `newSSHAuth`, so a bad path fails at startup), the
|
||||
`newWithDialer` seam, and `Create`/`Get`/`Delete`/`ListByTag`. Each
|
||||
method resolves its host from config, then runs the whole libvirt
|
||||
conversation inside one `hostPool.call`, so a cancelled reconcile drops
|
||||
the connection and the blocked RPC returns. Errors pass through `classify`
|
||||
in one `wrapErr`, as the Step 3b note intended.
|
||||
|
||||
Behaviour that goes beyond the plan text, or that a future reader should
|
||||
know before touching this file:
|
||||
|
||||
- **Create looks up the domain first.** If it exists, only the running
|
||||
check runs and the volumes are never touched. If not, nothing can be
|
||||
using the volumes yet: the overlay is reused when it exists (its
|
||||
creation is atomic; `ErrStorageVolExist` → reuse) but the **seed is
|
||||
always deleted and rebuilt**. The plan said "exists → skip upload"; the
|
||||
reviewer pointed out that a reconcile cancelled mid-upload leaves a
|
||||
truncated seed that the compensating delete cannot remove, because
|
||||
`hostPool.call` has already closed the connection the delete would go
|
||||
over. Skipping such a seed would boot the VM without cloud-init and no
|
||||
retry would ever repair it. Rebuilding costs one ~40 KiB upload per
|
||||
attempt before the domain exists, nothing afterwards. The table test
|
||||
breaks Create after each step and re-runs it, and a separate test
|
||||
starts from a leftover truncated seed.
|
||||
- **Get swallows address-source errors.** `DomainInterfaceAddresses` from
|
||||
ARP returns nothing until the guest has sent traffic and the agent
|
||||
source errors with `ErrAgentUnresponsive` until `qemu-guest-agent` is
|
||||
up, so both are logged at V(2) and mean "no IP yet" → `Provisioning`.
|
||||
Loopback, link-local and unspecified addresses are skipped when
|
||||
choosing the first IPv4 (the agent reports `lo` too).
|
||||
- **State mapping:** `running`/`blocked` are active; `nostate` →
|
||||
Provisioning (the reconciler answers Stopped with delete-and-recreate,
|
||||
too drastic for a state libvirt could not determine); everything else
|
||||
(paused, pmsuspended, shutdown, shutoff, crashed) → Stopped.
|
||||
- **Delete treats a missing pool as "nothing to clean"** rather than
|
||||
Permanent: failing there would pin the Proxy's finalizer on a config
|
||||
edit that renamed the pool. Domain removal is destroy (`ErrOperationInvalid`
|
||||
on a stopped domain is fine) → undefine with managed-save, snapshot
|
||||
metadata and nvram flags → overlay and seed deleted by name.
|
||||
- **ListByTag** reads each domain's metadata once (passed into the same
|
||||
`inspect` that Get uses, so no second `DomainGetMetadata` round-trip)
|
||||
and skips domains without the operator's element. A domain that cannot
|
||||
be read at all — vanished mid-sweep, corrupt metadata, state call
|
||||
failing — is *skipped*, not fatal: GC never deletes what it does not
|
||||
see, so leaving one out is safe, whereas one bad domain blocking GC for
|
||||
every other one (the first version's behaviour, caught in review) is
|
||||
not. An unreachable host still fails the whole sweep.
|
||||
`ConnectListAllDomains` is called with flags 0 (active and inactive).
|
||||
- A host named in a providerID but no longer in `kvm.hosts` is Permanent
|
||||
for Get and Delete: the operator cannot reach it, and retrying forever
|
||||
would hide the config problem.
|
||||
|
||||
Tests run the provider over a `hostDialer` that maps host names to
|
||||
`fakeAPI` instances (an unmapped host is "unreachable"). The image is
|
||||
pre-seeded in the pool for most tests; one test runs the full first-use
|
||||
download through `httptest` and confirms a second Proxy on the same host
|
||||
does not download again.
|
||||
|
||||
Found while running the package: `TestBuildSeedISO_deterministic` from
|
||||
Step 2 was wrong. `iso9660.ImageWriter` stamps `time.Now()` into the
|
||||
volume descriptor, so two builds in different seconds differ by six
|
||||
timestamp bytes; the test had only passed because both builds landed in
|
||||
the same second. Byte identity is nothing the provider depends on (an
|
||||
existing seed is never re-uploaded), so the test now compares label,
|
||||
files and size, and the comment on `buildSeedISO` no longer claims
|
||||
identical output.
|
||||
|
||||
The `go-operator-reviewer` agent ran on the first version of this step
|
||||
and found the two bugs above (truncated seed, wedged GC sweep) plus the
|
||||
double metadata read and the `nostate` mapping; all were fixed before
|
||||
committing. One finding it raised is deliberately **not** fixed here
|
||||
because it lives in Step 3b's `image.go` and changes the volume naming
|
||||
the plan chose: re-fetching a base image (checksum edited in config, or
|
||||
upstream content changed) deletes `base-<key>.<fmt>` while existing
|
||||
overlays still back onto it, breaking those VMs' disks. Content-addressed
|
||||
base names would avoid it. Flagged for the user; until decided, Step 5's
|
||||
docs must warn that an image's `sha256`/`url` must not change while
|
||||
Proxies using it exist.
|
||||
|
||||
Worth noting: `go test -cover` reports 88.2 % for the package; the
|
||||
uncovered lines are the real SSH/libvirt dial path and log-only
|
||||
branches. `go vet` and `-race` (three runs) are clean. Wiring into
|
||||
`cmd/main.go` is Step 5, so `New` is not yet reachable from a running
|
||||
operator.
|
||||
@@ -1,107 +0,0 @@
|
||||
# Plan: OpenTelemetry tracing integrated with the existing logr/zap logging
|
||||
**Created:** 2026-08-24 10:25
|
||||
|
||||
## Context
|
||||
|
||||
The operator has structured logging (controller-runtime zap → logr, `logf.FromContext(ctx)` everywhere) and Prometheus metrics, but no tracing. A single reconcile fans out into k8s API calls, provider calls (GCP VMs / squid Pods), and status patches; the discovery API serves lease requests; GC and health engines run on tickers. Today correlating one operation across those hops means grepping for `reconcileID` or `providerID`. Goal: proper OTel traces — a new trace per unit of work (reconcile, HTTP request, GC sweep, optionally health probe), child spans for k8s/provider/HTTP calls — **and every log line inside a traced operation enriched with the trace context**, without disturbing the existing logr/zap system.
|
||||
|
||||
User decisions (confirmed):
|
||||
- **Enablement:** auto from standard `OTEL_*` env vars — no collector configured ⇒ tracing fully off, zero new flags for the common case.
|
||||
- **Health probes:** not traced by default; new `--trace-health-probes` flag enables one root span per probe.
|
||||
- **Backend:** none exists yet — verification uses the console exporter and a throwaway Jaeger all-in-one in kind; manifests ship commented-out OTLP env examples.
|
||||
|
||||
## Key design facts (verified, non-obvious)
|
||||
|
||||
- **logr can't read ctx:** `logr.LogSink` never sees `context.Context`, so trace IDs cannot be added inside the zap sink. They must be attached as `WithValues` at span-creation points via `logf.IntoContext`. zap does **not** dedupe repeated keys, so enrichment must happen exactly once per ctx chain (see `tracing.Start` below).
|
||||
- Log keys: **`traceID` / `spanID`** (lowerCamel matches `reconcileID`/`providerID` house style; note in docs that Grafana/Loki derived-field regexes must match `traceID`, not the `trace_id` default).
|
||||
- **GCP calls trace themselves:** `google.golang.org/api/transport/http` already wraps its transport in `otelhttp.NewTransport` — once a global TracerProvider is set, every Compute call emits an HTTP client span. Do not add HTTP-level spans in the gcp provider; the `provider.*` decorator span becomes their parent.
|
||||
- **kube-apiserver ignores incoming `traceparent`** by design; our k8s-API client spans are leaves. Don't promise cross-process traces into the apiserver.
|
||||
- The Go SDK does **not** implement `OTEL_SDK_DISABLED` — handle it ourselves.
|
||||
- `contrib/exporters/autoexport` drags in metric/log/prometheus exporters (binary + deps on a 128Mi pod) — hand-roll trace-exporter selection (~40 lines) instead.
|
||||
- Span events are deprecated (OTel spec, Mar 2026): use `span.SetStatus` + `error.type`-style attributes, not `RecordError`/`AddEvent`.
|
||||
- `rest.Config.Wrap` puts our transport innermost (after auth), so spans see the final request — correct; but informer list/watch + leader election run outside any span ⇒ the transport must **filter out requests with no parent span** or every watch becomes a long-lived root span.
|
||||
- Manager runnables receive no logger in ctx (only HTTP servers do) — `logf.FromContext` falls back to the global logger there, which is fine; the discovery server must keep seeding from `s.log`.
|
||||
|
||||
## Steps
|
||||
|
||||
### Step 1 — Dependencies
|
||||
|
||||
`go get` (aligning the existing indirect skew: otel core v1.44 / otlptrace v1.40):
|
||||
`go.opentelemetry.io/otel@v1.45.0`, `otel/sdk@v1.45.0`, `otel/trace@v1.45.0`,
|
||||
`otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0`, `.../otlptracehttp@v1.45.0`,
|
||||
`otel/exporters/stdout/stdouttrace@v1.45.0`,
|
||||
`go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0`, plus the
|
||||
`semconv` version matching the SDK's `resource.Default()` (treat `ErrSchemaURLConflict` from `resource.New` as non-fatal if versions drift).
|
||||
`go mod tidy`; full build + `make test` to confirm `google.golang.org/api v0.292.0` / k8s v0.36 tolerate the bump (MVS picks the higher otelhttp).
|
||||
|
||||
### Step 2 — New package `internal/tracing`
|
||||
|
||||
- **`tracing.go` — `Setup(ctx, log logr.Logger, service, version string) (shutdown func(context.Context) error, err error)`**
|
||||
- Disabled (returns no-op shutdown, installs nothing) when `OTEL_SDK_DISABLED=true`, or when neither `OTEL_TRACES_EXPORTER` nor `OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is set, or `OTEL_TRACES_EXPORTER=none`. One startup Info line either way stating the effective state/exporter.
|
||||
- Exporter selection (hand-rolled, no autoexport): `OTEL_TRACES_EXPORTER` ∈ `otlp` (default) | `console`; for otlp, `OTEL_EXPORTER_OTLP_(TRACES_)PROTOCOL` ∈ `http/protobuf` (default) | `grpc` → `otlptracehttp` / `otlptracegrpc` (endpoint/TLS/headers via their native env handling).
|
||||
- `sdktrace.NewTracerProvider` with batch processor (`WithMaxQueueSize(512)` — bounded on the 128Mi pod; sampler stays the SDK default so `OTEL_TRACES_SAMPLER(_ARG)` works), resource = `resource.New(WithAttributes(service.name, service.version=commit), WithFromEnv())` with env last so it wins.
|
||||
- Globals: `otel.SetTracerProvider`, `otel.SetTextMapPropagator(TraceContext+Baggage)`, `otel.SetLogger(log.WithName("otel"))`, `otel.SetErrorHandler` → logr at `V(1)` (an unreachable collector errors every few seconds; Error level would spam).
|
||||
- **`logger.go` — the one logger-enrichment mechanism** (used by everything; prevents duplicate `traceID` keys):
|
||||
- private ctx key holding the *base* (pre-enrichment) logr.Logger.
|
||||
- `Start(ctx, spanName, opts...) (context.Context, trace.Span)`: starts a span from the global tracer; if the resulting span context is valid, capture base = stored base or `logf.FromContext(ctx)` on first call, then `logf.IntoContext(ctx, base.WithValues("traceID", ..., "spanID", ...))`. Nested `Start` re-derives from base — never stacks. Documented trade-off: `WithValues` pushed via `logf.IntoContext` *between* two `Start` calls is dropped (nothing first-party does that; controller-runtime's `reconcileID` logger is the base and survives).
|
||||
- `ContextWithLogger(ctx, base)` seeds the base explicitly (discovery middleware).
|
||||
- **`reconciler.go` — `NewReconciler(kind string, inner reconcile.Reconciler, opts...) reconcile.Reconciler`**: root span `Reconcile <kind>` (SpanKind Internal) via `Start`; attrs `k8s.namespace.name`, `k8s.object.name`, `reconcile.id` (`controller.ReconcileIDFromContext`); on return records `reconcile.requeue_after`, sets Error status from err. `WithTracerProvider` option for tests.
|
||||
- **`transport.go` — `RestConfigWrapper() transport.WrapperFunc`**: `otelhttp.NewTransport` with `WithFilter` requiring `trace.SpanContextFromContext(r.Context()).IsValid()` — k8s API spans only under an existing trace (excludes informers, leader election, metrics authn).
|
||||
- **`http.go` — `HTTPMiddleware(operation string, base logr.Logger) func(http.Handler) http.Handler`**: `otelhttp.NewHandler` (filter `/healthz`; otelhttp renames the span from `r.Pattern` after routing — no custom naming needed) + inner layer calling `ContextWithLogger`-then-enrich so handler logs carry `traceID`. Accepts incoming `traceparent` (clients continue their traces).
|
||||
|
||||
Tests (all with `sdktrace` + `tracetest.NewSpanRecorder` via the `WithTracerProvider` options, parallel; `Setup` env tests use `t.Setenv`, not parallel):
|
||||
- `Start`: nested spans → child span recorded; **funcr-captured log line contains `traceID` exactly once** (pattern: `internal/provider/gcp/gcp_test.go:277`); disabled tracer → no `traceID` in logs.
|
||||
- `Setup`: table over env combinations (unset→disabled, `none`→disabled, endpoint set→otlp, `console`, `OTEL_SDK_DISABLED`).
|
||||
- Reconciler decorator: fake inner returns result/err → span name/attrs/status.
|
||||
- Transport filter: request without parent span produces no span.
|
||||
- Middleware: `httptest` request → server span named from route pattern; `/healthz` unspanned; handler logger carries `traceID`.
|
||||
|
||||
### Step 3 — `provider.WithTracing` decorator
|
||||
|
||||
`internal/provider/tracing.go`, mirroring `metrics.go`: `WithTracing(name string, p Provider, opts...) Provider` — one Client span per call (`provider.create|get|delete|list`), attrs `provider.name`, `provider.id` (where present), result class attr from `resultLabel(err)`; status Error only for quota/permanent/transient — `ErrNotFound` stays Ok (expected during deletion polling). Returns the inner error **unmodified**. Uses `tracing.Start` so provider-internal logs (gcp `p.logger(ctx)`) inherit `traceID`. Tests in `internal/provider/tracing_test.go` reusing `staticProvider` from `metrics_test.go:19`.
|
||||
|
||||
Wiring in `cmd/main.go:174-177` — tracing outermost: `provider.WithTracing(name, provider.WithMetrics(name, p, m))`.
|
||||
|
||||
### Step 4 — `cmd/main.go` wiring
|
||||
|
||||
- `tracing.Setup(...)` immediately after `ctrl.SetLogger` (cmd/main.go:148); on error: log + `os.Exit(1)`.
|
||||
- Explicit `shutdown(context.WithTimeout(context.Background(), 10*time.Second))` (fresh ctx — signal ctx is already cancelled) after `mgr.Start` returns, on **both** the error path before `os.Exit(1)` and the clean path. Earlier `os.Exit` sites have nothing to flush.
|
||||
- `cfg := ctrl.GetConfigOrDie(); cfg.Wrap(tracing.RestConfigWrapper())` before `ctrl.NewManager` (cmd/main.go:239).
|
||||
- kubernetes provider: add `kubernetes.NewWithTransportWrapper(ctx, cfg, wrap transport.WrapperFunc)` beside `New` (internal/provider/kubernetes/kubernetes.go:42) and register a closure in main's constructor map (same pattern as gcp at cmd/main.go:166) passing `tracing.RestConfigWrapper()` — keeps `internal/tracing` out of provider packages and `newWithClient` tests untouched.
|
||||
- New flag `--trace-health-probes` (default false) → `engine.TraceProbes`.
|
||||
|
||||
### Step 5 — Reconciler spans
|
||||
|
||||
- `SetupWithManager` (proxy_controller.go:412): `b.Complete(tracing.NewReconciler("Proxy", r))`.
|
||||
- Sub-spans via `tracing.Start`: `reconcile.managed` (:116), `reconcile.replaceInstance` (:226), `reconcile.delete` (:254), and `status.patch` opened **inside the deferred flush closure** (:98-104, runs within the root span; its error already folds into `err`). `reconcileExternal` stays unspanned — no I/O, pure noise.
|
||||
- Useful attrs where cheap: phase transitions, `providerID`.
|
||||
|
||||
### Step 6 — Discovery server
|
||||
|
||||
- `handler()` (internal/discovery/server.go:151-156): chain becomes recover → `tracing.HTTPMiddleware("discovery", s.log)` → log → maxBytes → auth (otelhttp must stay outside `maxBytesMiddleware`).
|
||||
- Handlers and `logMiddleware`/`recoverMiddleware` switch `s.log` → `logf.FromContext(r.Context())` (handlers.go:85, :141, :234; server.go:163, :191) so request logs carry `traceID`. `s.log` remains for startup lines.
|
||||
- k8s `List` calls in handlers already use `r.Context()` → child spans appear via the wrapped rest transport.
|
||||
|
||||
### Step 7 — GC + health
|
||||
|
||||
- `gc.Sweeper.sweep` (internal/gc/gc.go:83): wrap in root span `gc.sweep` via `tracing.Start` (before the `.WithName("orphan-gc")` derivation so it inherits), attrs: providers swept, orphans deleted. Provider/k8s child spans come free from Steps 3/4.
|
||||
- `health.Engine`: new field `TraceProbes bool`. In the worker loop (engine.go:158) when enabled: root span `health.probe` (SpanKind Client) around `probeFn`, attrs proxy key/ok/latency/error class, ended before `record` (which stays ctx-free). No transport wrap and no propagation — never inject `traceparent` toward external targets through the proxies. Probe→reconcile span links are **out of scope**: `GenericEvent` (engine.go:330) carries no ctx and the workqueue coalesces events; documented in architecture Decisions.
|
||||
- Lease store sweep: not traced (in-memory only).
|
||||
|
||||
### Step 8 — GCP wire-log enrichment
|
||||
|
||||
`wireFilterHandler.Handle` (internal/provider/gcp/wirelog.go:46) receives the real request ctx from the SDK's slog calls: append `traceID`/`spanID` attrs when `trace.SpanContextFromContext(ctx)` is valid — V(5) wire logs become correlatable with the `provider.*` span. Add `wirelog_test.go` case.
|
||||
|
||||
### Step 9 — Manifests + docs
|
||||
|
||||
- `config/manager/manager.yaml` env (after `DISCOVERY_TOKEN`): `POD_NAME`/`POD_NAMESPACE` via downward API **first**, then `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)` ($(VAR) expansion only sees earlier vars); commented-out `OTEL_EXPORTER_OTLP_ENDPOINT` + `OTEL_TRACES_SAMPLER` examples with a "tracing is off until an endpoint is set" comment. Note `--trace-health-probes` beside the args list.
|
||||
- `docs/architecture.md`: new "### 10. Tracing" section after Metrics (:281) — span topology, enablement, `traceID` log-key choice; Decisions entries (:303): env-gated enablement, probes off by default, no probe→reconcile links (why), `traceID` naming vs Grafana defaults, no traceparent into apiserver.
|
||||
- README ops note; CHANGELOG entry (dated via `date`) once the user confirms it works.
|
||||
- Run `make lint` (CI has no lint step; `lll`/`revive` will hit new files).
|
||||
|
||||
## Verification
|
||||
|
||||
1. `make test` (unit, includes all new `tracetest` assertions) and `make lint`.
|
||||
2. Console smoke: `OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml ...` against kind — create a Proxy, confirm (a) `Reconcile Proxy` span JSON on stdout with nested `provider.create` + k8s PATCH spans, (b) the reconcile log lines carry the same `traceID` as the span, (c) no spans from informer watches or leader election.
|
||||
3. End-to-end in kind: deploy Jaeger all-in-one (`jaegertracing/all-in-one`, OTLP 4318), set `OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318` on the manager Deployment; exercise a reconcile + a `POST /v1/leases` (with and without a client `traceparent`) + wait one GC interval; in the Jaeger UI: reconcile trace shows provider+k8s children; lease trace shows route-named server span with k8s child; grep manager logs for a `traceID` and resolve it in Jaeger.
|
||||
4. Negative: run with no `OTEL_*` env — startup line says tracing disabled, no export-error spam, logs unchanged (no `traceID` keys).
|
||||
@@ -1,150 +0,0 @@
|
||||
# Plan: e2e test — OTel tracing against real Tempo
|
||||
**Created:** 2026-08-24 12:24
|
||||
|
||||
## Context
|
||||
|
||||
The tracing feature (MR #4, branch `feat/otel-tracing`) is covered by unit
|
||||
tests with an in-memory span recorder, but nothing proves the full pipeline:
|
||||
operator in a real cluster → OTLP export → Tempo ingest → queryable traces
|
||||
with the documented span topology. The homelab Tempo lives at
|
||||
`http://192.168.0.30:3200` (query API — verified live from this host:
|
||||
`/api/echo`, TraceQL `GET /api/search?q=…`, `GET /api/traces/<id>`), with
|
||||
OTLP ingest on `:4318` (HTTP, verified 200) and `:4317` (gRPC, open). User
|
||||
decisions: **Go Ginkgo e2e test** (not a shell script), **structural
|
||||
assertions** (span tree, not just trace-exists), flow per the user's sketch:
|
||||
deploy operator in kind with tracing on → create kubernetes-provider proxies
|
||||
→ delete them → shut down — with traces tagged for easy discovery per test
|
||||
run. This work continues on `feat/otel-tracing`; MR #4 stays the vehicle.
|
||||
|
||||
## Design decisions (from review, with evidence)
|
||||
|
||||
- **Proxy CRs go in the `default` namespace, not the operator namespace.**
|
||||
The squid pods the kubernetes provider creates carry no securityContext
|
||||
(internal/provider/kubernetes/pod.go:33-49) and are created in the CR's
|
||||
namespace; the operator ns is labeled `pod-security enforce=restricted`
|
||||
(test/e2e/e2e_test.go:61-62) and would reject them. Pod RBAC is
|
||||
cluster-scoped (config/rbac/role.yaml:10), so `default` works. All
|
||||
kubectl calls use explicit `-n`.
|
||||
- **Anchor assertions on `provider.create`, not the root-span search.** A
|
||||
`name="Reconcile Proxy"` hit may be the finalizer-add or a drift
|
||||
reconcile (no provider call inside). TraceQL matches span names anywhere
|
||||
in a trace, so search `{resource.test.run.id="<runID>" && name="provider.create"}`,
|
||||
fetch *that* traceID, then assert companion span names.
|
||||
- **Run identification with zero code changes:** OTel resource attribute
|
||||
`test.run.id=<runID>` via `OTEL_RESOURCE_ATTRIBUTES` (env wins —
|
||||
`resource.WithFromEnv()` merges last, internal/tracing/tracing.go). Tempo
|
||||
vParquet searches arbitrary resource attrs without config; fallback query
|
||||
documented: `{resource.service.name="egress-proxies-operator" && name="provider.create"}`
|
||||
+ start/end window (Unix **seconds**).
|
||||
- **OTLP preflight from inside the cluster:** host-reachability of
|
||||
192.168.0.30 doesn't prove pod-reachability from kind on macOS, and
|
||||
export failures log only at V(1) (invisible) — without a preflight, a
|
||||
broken path is a 2-minute opaque timeout. Reuse the suite's curl-pod
|
||||
pattern (e2e_test.go:218-248, incl. the restricted-PSS overrides JSON)
|
||||
to POST to `<OTLP_ENDPOINT>/v1/traces` and fail fast.
|
||||
- **No homelab defaults baked into the Makefile.** The spec Skips unless
|
||||
`TEMPO_URL`/`OTLP_ENDPOINT` are set (otherwise everyone without that LAN
|
||||
host inherits a 2-minute failure). The copy-pasteable invocation lives in
|
||||
docs/testing.md.
|
||||
- **Ginkgo ordering:** top-level container order is randomized, so the
|
||||
tracing Describe is fully self-contained (own ns create → make install →
|
||||
make deploy → teardown), same shape as the Manager Describe. `kubectl
|
||||
delete ns` blocks until termination, so no create/delete race between
|
||||
the two Describes.
|
||||
|
||||
## Steps
|
||||
|
||||
### Step 1 — `test/e2e/tracing_test.go` (new; `//go:build e2e`, package e2e)
|
||||
|
||||
`Describe("OTel tracing", Ordered)`:
|
||||
|
||||
- **BeforeAll:**
|
||||
1. Read `TEMPO_URL` + `OTLP_ENDPOINT`; `Skip("TEMPO_URL/OTLP_ENDPOINT not set")` when empty.
|
||||
2. `runID := "e2e-" + strconv.FormatInt(time.Now().UnixNano(), 10)`;
|
||||
print it to `GinkgoWriter` so the run is findable in Grafana by hand.
|
||||
3. Create operator ns + restricted-PSS label; `make install`;
|
||||
`make deploy IMG=<managerImage>` (same commands as e2e_test.go:54-75,
|
||||
via `utils.Run`).
|
||||
4. Pre-pull the proxy image to kill the biggest flake source:
|
||||
`docker pull ubuntu/squid:6.6-24.04_edge` + `kind load docker-image`
|
||||
(via `utils.Run`, honoring `KIND`/`KIND_CLUSTER` env like
|
||||
utils.LoadImageToKindClusterWithName).
|
||||
5. OTLP preflight: curl pod POSTing `{}` to `<OTLP_ENDPOINT>/v1/traces`,
|
||||
assert HTTP 200 in its logs (fail message names the endpoint).
|
||||
6. `kubectl set env deployment/egress-proxies-operator-controller-manager
|
||||
-n <ns> OTEL_EXPORTER_OTLP_ENDPOINT=<OTLP_ENDPOINT>
|
||||
"OTEL_RESOURCE_ATTRIBUTES=k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE),test.run.id=<runID>"`
|
||||
— in-place update keeps the var after the downward-API vars ($(VAR)
|
||||
expansion needs that ordering); `exec.Command` means no shell mangling
|
||||
of `$()`. Then `kubectl rollout status`.
|
||||
7. `Eventually`: newest controller pod (sorted by creationTimestamp — a
|
||||
terminating old pod may otherwise be picked) logs contain
|
||||
`"tracing enabled"`.
|
||||
- **It "creates proxies and reports reconcile traces":**
|
||||
- Write 2 Proxy manifests (names `proxy-tracing-e2e-1/2`,
|
||||
`provider: kubernetes`, modeled on config/samples/proxy_kubernetes.yaml)
|
||||
to `GinkgoT().TempDir()` (absolute paths — `utils.Run` chdirs the
|
||||
process); `kubectl apply -n default`.
|
||||
- `Eventually` (5m — first squid start) `.status.phase == "Ready"`
|
||||
(confirmed field, api/v1alpha1/proxy_types.go:41).
|
||||
- `Eventually`: Tempo search
|
||||
`{resource.test.run.id="<runID>" && name="provider.create"}` ≥1 hit
|
||||
(BSP flushes every ~5s; 2m default timeout is ample).
|
||||
- Fetch that traceID via `/api/traces/<id>`; assert span names include
|
||||
`Reconcile Proxy`, `reconcile.managed`, `provider.create`,
|
||||
`status.patch`, and resource attr `service.name=egress-proxies-operator`.
|
||||
- **It "traces deletion":**
|
||||
- `kubectl delete proxy -n default …`; `Eventually` CRs gone (finalizer
|
||||
→ `provider.delete`, DeletionPoll 10s).
|
||||
- `Eventually`: Tempo search finds `provider.delete` for the runID; fetch
|
||||
and assert `reconcile.delete` in the same trace.
|
||||
- **AfterAll** (= the sketch's "shut down operator"): delete leftover
|
||||
proxies in `default`; `make undeploy` (SIGTERM → `tracingShutdown`
|
||||
flushes); `make uninstall`; delete operator ns — errors discarded
|
||||
(`_, _ = utils.Run(...)`), mirroring e2e_test.go:79-95.
|
||||
- **Tempo client helpers** (same file, stdlib only — the test process runs
|
||||
on the host, which reaches Tempo directly):
|
||||
- `tempoSearch(traceql string)` → `GET {TEMPO_URL}/api/search?q=…&start=…&end=…`
|
||||
(Unix seconds, window = suite start − 5m → now); minimal struct
|
||||
`{Traces []struct{ TraceID string }}`.
|
||||
- `tempoTrace(id string)` → `/api/traces/<id>`, decoded as **OTLP-JSON**
|
||||
(`batches[].scopeSpans[].spans[].name`, resource attrs as
|
||||
`{key, value:{stringValue}}`) — not Jaeger's shape. Helper flattens to
|
||||
a span-name set + resource attr map.
|
||||
|
||||
### Step 2 — Makefile
|
||||
|
||||
- `test-e2e`: pass `TEMPO_URL`/`OTLP_ENDPOINT` through to `go test` env
|
||||
(no defaults) and add `-timeout 30m` — the suite already runs
|
||||
docker-build + kind-load in BeforeSuite plus 3–5m Eventuallys, and this
|
||||
adds a second full deploy cycle + image pulls; the 10m default will be
|
||||
exceeded on cold caches.
|
||||
|
||||
### Step 3 — Docs
|
||||
|
||||
- `docs/testing.md` (§ e2e, currently "scaffold… only asserts manager runs
|
||||
and serves metrics"): document the tracing spec, its Skip gate, and the
|
||||
invocation:
|
||||
`TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`.
|
||||
- Execution-log entries per step in `docs/plans-executions/2026-08-24-1224-tracing-e2e.md`.
|
||||
- CHANGELOG entry only after the user confirms a green run (house rule).
|
||||
|
||||
## Not doing (and why)
|
||||
|
||||
- No dedicated flush assertion on shutdown — teardown exercises the flush
|
||||
path, but attributing a specific late span to it is guesswork.
|
||||
- No gRPC (4317) variant — http/protobuf is the operator default and the
|
||||
verified path; a variant run is a one-env-var change if ever wanted.
|
||||
- No changes to `internal/` — the whole test works through public surface
|
||||
(env vars, kubectl, Tempo API), which is the point of an e2e test.
|
||||
|
||||
## Verification
|
||||
|
||||
1. `TEMPO_URL=http://192.168.0.30:3200 OTLP_ENDPOINT=http://192.168.0.30:4318 make test-e2e`
|
||||
→ tracing spec passes (both Its); without the env vars → spec reports
|
||||
Skipped, remainder of suite unaffected.
|
||||
2. In Grafana: TraceQL `{resource.test.run.id="<runID printed in test log>"}`
|
||||
shows the run's traces with the expected tree.
|
||||
3. `go vet -tags=e2e ./...` clean (documented routine).
|
||||
4. A deliberately wrong `OTLP_ENDPOINT` (e.g. port 9) fails fast in the
|
||||
preflight step with a clear message, not a 2-minute search timeout.
|
||||
@@ -0,0 +1,313 @@
|
||||
# Plan: KVM/libvirt provider
|
||||
|
||||
**Created:** 2026-09-04 14:04
|
||||
|
||||
## Context
|
||||
|
||||
The operator provisions egress proxies through named provider instances (today
|
||||
`gcp` and `kubernetes`). The user has several KVM hypervisors on the home
|
||||
network and wants proxy VMs created there, using stock Debian/Ubuntu cloud
|
||||
images so the VM stays a normal machine they can customise through their own
|
||||
cloud-init user-data. Longer term the operator may become a generic "run a VM
|
||||
or container with a set of scripts" tool; the KVM provider should not
|
||||
foreclose that.
|
||||
|
||||
The provider contract (`internal/provider/provider.go`: `Create`/`Get`/`Delete`/
|
||||
`ListByTag`, keyed by `NameFromUID`, cloud-init passed verbatim, GC-by-tag) fits
|
||||
libvirt with **no CRD or reconciler changes**. All work is a new provider
|
||||
package plus config/wiring/docs, following the GCP provider's shape.
|
||||
|
||||
Decisions taken with the user (2026-09-04):
|
||||
|
||||
- **Transport:** SSH to each hypervisor, forwarding the libvirt unix socket
|
||||
(`ssh.Client.Dial("unix", "/var/run/libvirt/libvirt-sock")` → `libvirt.New(conn)`).
|
||||
Pure Go, no cgo, no libvirt libs in the image. Verified: `x/crypto/ssh`
|
||||
`Client.Dial` has a `"unix"` case (`dialStreamLocal`), go-libvirt `New(net.Conn)`.
|
||||
- **Topology:** one provider instance can span several hosts from day one
|
||||
(config has a `hosts` list; providerID encodes the host), so adding hosts
|
||||
later is a config edit with no providerID migration. With one host configured
|
||||
`placement.zone` may be omitted.
|
||||
- **Networking:** VM NIC on a host bridge to the LAN, IP from the router's
|
||||
DHCP. IP discovery via `DomainInterfaceAddresses` (ARP source first, then
|
||||
guest agent). Provider-injected `vendor-data` installs `qemu-guest-agent`
|
||||
so agent discovery works on stock images without touching user-data.
|
||||
- **Base image:** a stock Debian/Ubuntu cloud image (qcow2, cloud-init
|
||||
preinstalled) that **the operator fetches itself** on first use per host:
|
||||
`placement.image` names an entry in the provider's `images:` map (url +
|
||||
optional sha256) or is an `https://` URL directly. The provider streams the
|
||||
download through libvirt's `StorageVolUpload` into the host's pool, so
|
||||
nothing is pre-placed and the operator needs no filesystem access on the
|
||||
host. Per-VM disk is a qcow2 overlay with the cached base as backing store.
|
||||
- **Seed:** NoCloud ISO (`user-data`, `meta-data`, `vendor-data`) built in
|
||||
pure Go with `github.com/kdomanski/iso9660` (`WriteTo(w, "cidata")`) and
|
||||
streamed to the pool with `StorageVolUpload(vol, io.Reader, ...)`. Nothing
|
||||
to install on the hypervisors.
|
||||
|
||||
## Design
|
||||
|
||||
### Config (`internal/provider/config.go`)
|
||||
|
||||
```yaml
|
||||
- name: kvm-home
|
||||
type: kvm
|
||||
kvm:
|
||||
sshKeyFile: /etc/proxy-operator/kvm/id_ed25519 # required
|
||||
knownHostsFile: /etc/proxy-operator/kvm/known_hosts # required unless insecureIgnoreHostKey
|
||||
insecureIgnoreHostKey: false
|
||||
diskSizeGb: 10 # overlay capacity; cloud-init growpart expands root fs
|
||||
vendorData: | # optional override; default installs qemu-guest-agent
|
||||
#cloud-config
|
||||
...
|
||||
images: # optional; placement.image names a key (or is an https URL)
|
||||
debian-13:
|
||||
url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||
sha256: "…" # optional; verified after download, re-fetch when it changes
|
||||
format: qcow2 # default qcow2
|
||||
ubuntu-24.04:
|
||||
url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
|
||||
machineTypes: # required, ≥1; placement.machineType names a key
|
||||
small: {vcpus: 1, memoryMiB: 1024}
|
||||
medium: {vcpus: 2, memoryMiB: 2048}
|
||||
hosts: # required, ≥1, unique names
|
||||
- name: nas1 # placement.zone value
|
||||
address: nas1.home.lan:22 # required
|
||||
user: virt # required; must be in the libvirt group
|
||||
socket: /var/run/libvirt/libvirt-sock # default
|
||||
pool: default # default "default"
|
||||
bridge: br0 # required
|
||||
```
|
||||
|
||||
Add `KVMConfig`, `KVMHost`, `KVMMachineType`, `KVMImage` structs; a `"kvm"`
|
||||
case in `validate()` (cross-block checks like the others, required fields
|
||||
above, unique host names, image URLs must be `http(s)`, sha256 if set must be
|
||||
64 hex chars). Tests in `config_test.go` extend the existing table.
|
||||
|
||||
### Placement mapping
|
||||
|
||||
| Placement field | KVM meaning |
|
||||
|---|---|
|
||||
| `zone` | host name from `hosts`; empty → the single configured host, else `ErrPermanent` |
|
||||
| `machineType` | key in `machineTypes`; unknown → `ErrPermanent` |
|
||||
| `image` | key in `images:`, or an `https://` URL; anything else → `ErrPermanent`. Fetched into the host's pool on first use (see *Image cache*) |
|
||||
| `region` | ignored |
|
||||
|
||||
### providerID
|
||||
|
||||
`hosts/<host>/domains/<name>` with `formatProviderID`/`parseProviderID`
|
||||
(same self-contained style as GCP's `zones/<zone>/instances/<name>`).
|
||||
|
||||
### GC contract without labels
|
||||
|
||||
Domain XML gets a `<metadata>` child in a private namespace, e.g.
|
||||
`xmlns="https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm"`,
|
||||
carrying `<managed>true</managed><uid>…</uid><createdAt>RFC3339</createdAt>`.
|
||||
`Get` reads it via `DomainGetMetadata(dom, DomainMetadataElement, uri)`;
|
||||
`ListByTag` does `ConnectListAllDomains` on every host and keeps domains where
|
||||
that call succeeds and `managed == "true"`. `CreatedAt` comes from the metadata
|
||||
(libvirt has no creation time). A host that is unreachable makes `ListByTag`
|
||||
return an error (GC skips the sweep that round rather than acting on a partial view).
|
||||
|
||||
### Connection management (`internal/provider/kvm/conn.go`)
|
||||
|
||||
Per-host lazily dialed `*ssh.Client` + `*libvirt.Libvirt`, cached under a
|
||||
mutex, re-dialed when `Disconnected()` is closed or a call fails with a
|
||||
transport error. SSH auth: private key from `sshKeyFile`; host key check via
|
||||
`knownhosts.New(knownHostsFile)` unless `insecureIgnoreHostKey`. Dial timeout
|
||||
from `ctx`. The provider calls libvirt through a narrow interface:
|
||||
|
||||
```go
|
||||
type libvirtAPI interface {
|
||||
DomainLookupByName(name string) (libvirt.Domain, error)
|
||||
DomainDefineXML(xml string) (libvirt.Domain, error)
|
||||
DomainCreate(d libvirt.Domain) error
|
||||
DomainDestroy(d libvirt.Domain) error
|
||||
DomainUndefineFlags(d libvirt.Domain, f libvirt.DomainUndefineFlagsValues) error
|
||||
DomainGetState(d libvirt.Domain, flags uint32) (int32, int32, error)
|
||||
DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, f libvirt.DomainModificationImpact) (string, error)
|
||||
DomainInterfaceAddresses(d libvirt.Domain, src uint32, flags uint32) ([]libvirt.DomainInterface, error)
|
||||
ConnectListAllDomains(need int32, f libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error)
|
||||
StoragePoolLookupByName(name string) (libvirt.StoragePool, error)
|
||||
StorageVolLookupByName(p libvirt.StoragePool, name string) (libvirt.StorageVol, error)
|
||||
StorageVolGetPath(v libvirt.StorageVol) (string, error)
|
||||
StorageVolCreateXML(p libvirt.StoragePool, xml string, f libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error)
|
||||
StorageVolUpload(v libvirt.StorageVol, r io.Reader, off, n uint64, f libvirt.StorageVolUploadFlags) error
|
||||
StorageVolDownload(v libvirt.StorageVol, w io.Writer, off, n uint64, f libvirt.StorageVolDownloadFlags) error
|
||||
StorageVolDelete(v libvirt.StorageVol, f libvirt.StorageVolDeleteFlags) error
|
||||
StoragePoolRefresh(p libvirt.StoragePool, flags uint32) error
|
||||
}
|
||||
```
|
||||
|
||||
(All signatures verified against go-libvirt `main` on 2026-09-04.) A
|
||||
`hostDialer func(ctx, KVMHost) (libvirtAPI, io.Closer, error)` is the test seam;
|
||||
`newWithDialer(cfg, dialer)` mirrors GCP's `newWithAPI`.
|
||||
|
||||
### Image cache (`internal/provider/kvm/image.go`)
|
||||
|
||||
`ensureImage(ctx, host, api, img) (basePath string, err error)` makes the base
|
||||
image present in the host's pool, once per host per image:
|
||||
|
||||
- Volume names: `base-<key>.<format>` for an `images:` key, or
|
||||
`base-<sha256(url)[:16]>.<format>` for a URL given directly in
|
||||
`placement.image`. Marker volume `<base>.sha256` (raw, 64 bytes) holds the
|
||||
digest of the verified upload.
|
||||
- **Ready check:** base volume exists *and* marker exists *and* (no `sha256`
|
||||
configured, or marker content == configured `sha256`). Read the marker with
|
||||
`StorageVolDownload`. Ready → return `StorageVolGetPath(base)`.
|
||||
- **Not ready:** delete both volumes if present (partial upload from a crash,
|
||||
or a changed checksum), then `GET url` with the ctx, require a
|
||||
`Content-Length` (`ErrPermanent` otherwise), `StorageVolCreateXML` the base
|
||||
with `<capacity unit="bytes">Content-Length</capacity>` and the configured
|
||||
format, `StorageVolUpload(base, io.TeeReader(body, sha256), 0, 0, 0)` (length
|
||||
0 = until EOF, so nothing is buffered in the operator), then compare the
|
||||
digest with the configured one (mismatch → delete base, `ErrPermanent`),
|
||||
`StoragePoolRefresh` so libvirt re-reads the real qcow2 capacity, create the
|
||||
marker volume and upload the digest last.
|
||||
- HTTP 4xx → `ErrPermanent`; 5xx, network, or upload transport errors →
|
||||
`ErrTransient` (partial volumes are cleaned on the next attempt).
|
||||
- A per-`(host, image)` `sync.Mutex` map serialises concurrent Creates for the
|
||||
same image on the same host; the second caller finds it ready.
|
||||
- **Judgment call:** the fetch runs synchronously inside `Create`, so the first
|
||||
Proxy for a given image on a given host holds a reconcile worker for the
|
||||
download (LAN speed, a few hundred MB, once). This keeps `Create` idempotent
|
||||
and needs no background state. Async fetch with a `Provisioning`-style
|
||||
progress status is a possible later improvement, noted in the docs.
|
||||
|
||||
### Create (idempotent, fire-and-forget after the image is cached)
|
||||
|
||||
1. Resolve host, machine type, pool, and image source (`images:` key or URL;
|
||||
else `ErrPermanent`). `ensureImage` → base path.
|
||||
2. Overlay volume `<name>.qcow2`: `StorageVolCreateXML` with
|
||||
`<capacity unit="G">diskSizeGb</capacity>`, `<target><format type="qcow2"/></target>`,
|
||||
`<backingStore><path>base</path><format type="qcow2"/></backingStore>`.
|
||||
`ErrStorageVolExist` → reuse.
|
||||
3. Seed volume `<name>-seed.iso`: build ISO in memory (`user-data` = req.CloudInit,
|
||||
`meta-data` = `instance-id: <name>\nlocal-hostname: <name>\n`, `vendor-data` =
|
||||
cfg or default), `StorageVolCreateXML` with exact capacity + `<format type="raw"/>`,
|
||||
then `StorageVolUpload`. Exists → skip upload (content is deterministic per spec hash;
|
||||
spec changes recreate the whole instance anyway).
|
||||
4. `DomainLookupByName(name)`; if `ErrNoDomain` → `DomainDefineXML(buildDomainXML(...))`.
|
||||
5. `DomainGetState`; if not running → `DomainCreate`. Return providerID.
|
||||
|
||||
`buildDomainXML` is a pure function over `encoding/xml` structs: `type="kvm"`,
|
||||
q35 machine, `host-passthrough` CPU, vcpus/memory from the profile, virtio disk
|
||||
(overlay) + sata cdrom (seed), `interface type="bridge"` with virtio model,
|
||||
`org.qemu.guest_agent.0` virtio channel, serial console, virtio rng, no
|
||||
graphics, and the `<metadata>` block. Unit-tested field by field like
|
||||
`buildInsertRequest`.
|
||||
|
||||
### Get
|
||||
|
||||
`parseProviderID` → host → `DomainLookupByName` (`ErrNoDomain` → `ErrNotFound`)
|
||||
→ `DomainGetState` + `DomainGetMetadata` + `DomainInterfaceAddresses`
|
||||
(ARP, then agent; first IPv4 wins; agent errors while the agent isn't up yet
|
||||
are swallowed). Mapping:
|
||||
|
||||
| libvirt state | IP | Instance.State |
|
||||
|---|---|---|
|
||||
| running | yes | Running |
|
||||
| running | no | Provisioning |
|
||||
| shutoff / crashed / paused / pmsuspended | – | Stopped |
|
||||
| no domain | – | `ErrNotFound` |
|
||||
|
||||
### Delete (idempotent)
|
||||
|
||||
Destroy if running (`ErrOperationInvalid` on a stopped domain is fine) →
|
||||
`DomainUndefineFlags(managed-save | snapshots-metadata | nvram)` → delete
|
||||
`<name>.qcow2` and `<name>-seed.iso` (`ErrNoStorageVol` fine). Volumes are
|
||||
deleted by name even when the domain is already gone, so a crash between
|
||||
volume creation and define still gets cleaned up on the next reconcile.
|
||||
Known gap (documented, not solved now): volumes orphaned with no domain *and*
|
||||
no CR are invisible to `ListByTag`.
|
||||
|
||||
### Errors (`internal/provider/kvm/errors.go`)
|
||||
|
||||
`classify(err)` on `libvirt.Error.Code`: `ErrNoDomain`, `ErrNoStorageVol` →
|
||||
`ErrNotFound`; `ErrNoStoragePool`, `ErrXMLError`, `ErrConfigUnsupported`,
|
||||
`ErrNoSupport`, `ErrAuthFailed`, `ErrInvalidArg` → `ErrPermanent`; SSH dial /
|
||||
host-key / transport errors and everything else → `ErrTransient`. Image
|
||||
download: HTTP 4xx, missing Content-Length, checksum mismatch → `ErrPermanent`;
|
||||
5xx and network errors → `ErrTransient`. Quota is not detectable from libvirt;
|
||||
left as transient (revisit with a capacity check later).
|
||||
|
||||
### Wiring & manifests
|
||||
|
||||
- `cmd/main.go`: add `"kvm": kvm.New` to the builtins map.
|
||||
- `config/manager/manager.yaml`: mount Secret `kvm-ssh` (optional) at
|
||||
`/etc/proxy-operator/kvm` (keys `id_ed25519`, `known_hosts`); document the
|
||||
`kubectl create secret generic kvm-ssh --from-file=...` command.
|
||||
- `config/samples/providers-config.yaml`: add the `kvm-home` block with the
|
||||
`images:` map (Debian 13 and Ubuntu 24.04 official cloud-image URLs);
|
||||
`config/samples/proxy_kvm.yaml`: Proxy with `placement: {zone: nas1,
|
||||
machineType: small, image: debian-13}` and the same squid cloud-init as the
|
||||
GCP sample plus `ssh_authorized_keys`.
|
||||
- `hack/providers-dev.yaml` stays kubernetes-only (needs no secrets).
|
||||
- Deps: `github.com/digitalocean/go-libvirt` (latest pseudo-version),
|
||||
`github.com/kdomanski/iso9660 v0.4.0`, promote `golang.org/x/crypto` to direct.
|
||||
|
||||
### Docs
|
||||
|
||||
- New `docs/kvm-provider.md`: hypervisor prerequisites (user in `libvirt`
|
||||
group, a bridge to the LAN, a storage pool with enough space), the config
|
||||
reference including `images:`, how the image cache works (`base-*` and
|
||||
`*.sha256` volumes in the pool, when a re-download happens, how to evict
|
||||
with `virsh vol-delete`), the placement table, how to log into a VM
|
||||
(`ssh_authorized_keys` in user-data, `virsh console` on the host), and the
|
||||
known gaps (no quota detection, orphan volumes without a domain,
|
||||
synchronous first-time image fetch).
|
||||
- Update the provider tables in `README.md`, `docs/architecture.md` §5,
|
||||
`docs/testing.md` (integration test section).
|
||||
|
||||
### Towards the generic runner (not built now, just kept open)
|
||||
|
||||
Provider knows nothing about squid. Future hooks: `vendor-data` for injecting an
|
||||
agent, `QEMUDomainAgentCommand` for reading exit status/output, and a future
|
||||
`Placement`-independent "job" mode in the CRD. None of this changes the KVM
|
||||
provider's shape.
|
||||
|
||||
## Steps
|
||||
|
||||
0. **Save plan + branch.** Copy this file to `docs/plans/<ts>-kvm-provider.md`,
|
||||
commit alone, then `git checkout -b feat/kvm-provider` off `feat/proxy-operator`
|
||||
(the working main for this project).
|
||||
1. **Config.** `KVMConfig` & friends, `validate()` case, table tests.
|
||||
2. **Pure builders.** `internal/provider/kvm/{domainxml.go,volume.go,seed.go,id.go,metadata.go}`
|
||||
+ tests: domain XML fields, volume XML, seed contents (unpack the ISO with
|
||||
the same library's reader and assert the three files + label `cidata`),
|
||||
providerID round-trip, metadata marshal/parse.
|
||||
3. **Transport + errors.** `conn.go` (SSH dial, host-key policy, per-host cache,
|
||||
reconnect), `errors.go` + classification table test. SSH dialer tested
|
||||
against an in-process `x/crypto/ssh` server that serves a unix socket
|
||||
forward (cheap, keeps the dial code covered).
|
||||
3b. **Image cache.** `image.go` + `image_test.go`: `httptest.Server` serving a
|
||||
small fake image over the `fakeAPI` (which records uploaded bytes);
|
||||
cases: fresh fetch, already ready, marker missing (partial upload → redo),
|
||||
checksum changed → redo, checksum mismatch → Permanent + cleanup, 404 →
|
||||
Permanent, 503 → Transient, two concurrent callers → one download.
|
||||
4. **Provider methods.** `kvm.go`: `New`, `newWithDialer`, `Create/Get/Delete/ListByTag`
|
||||
over `libvirtAPI`; `kvm_test.go` with a hand-written `fakeAPI` (domains and
|
||||
volumes in maps, per-call error injection) covering idempotent re-Create
|
||||
after each partial step, state/IP mapping, delete-when-gone, ListByTag
|
||||
across two hosts and with one host down.
|
||||
5. **Wiring, manifests, samples, docs, CHANGELOG.**
|
||||
6. **Integration test** (`//go:build integration`, `kvm_integration_test.go`):
|
||||
skipped unless `KVM_TEST_HOST`, `KVM_TEST_USER`, `KVM_TEST_SSH_KEY`,
|
||||
`KVM_TEST_BRIDGE`, `KVM_TEST_IMAGE_URL` are set; fetches the image (or finds
|
||||
it cached), creates, waits for Running+IP, deletes, asserts per-VM volumes
|
||||
gone and the base image still cached. Deviation from the testcontainers convention,
|
||||
called out in `docs/testing.md`: libvirt+KVM can't run inside a container
|
||||
without `/dev/kvm`, so this targets a real hypervisor.
|
||||
7. **Manual validation** on one real hypervisor, results in
|
||||
`docs/plans-executions/` (like `docs/gcp-vm-validation.md`), then MR via `tea`.
|
||||
|
||||
## Verification
|
||||
|
||||
- `make test` green (`-race`, envtest suite untouched but must still pass).
|
||||
- `go test -short ./internal/provider/...` covers builders, fake-API provider, errors, config.
|
||||
- Integration: `KVM_TEST_HOST=nas1.home.lan … go test -tags=integration -run TestKVM ./internal/provider/kvm/`.
|
||||
- End to end: deploy with the `kvm-ssh` Secret and `kvm-home` config against a
|
||||
host with no cached image, apply `config/samples/proxy_kvm.yaml`, watch the
|
||||
operator log the image fetch and `virsh vol-list` show `base-debian-13.qcow2`
|
||||
plus its `.sha256` marker, watch `kubectl get px -w` reach `Ready` with a LAN
|
||||
IP, apply a second Proxy and confirm no second download, confirm the VM shows up in `GET /v1/proxies` and works with
|
||||
`hack/` demo egress-IP check, delete the Proxy and confirm `virsh list --all`
|
||||
and `virsh vol-list default` are clean.
|
||||
+2
-32
@@ -92,38 +92,8 @@ compiles only under `-tags=e2e`, manages its own kind cluster
|
||||
(`make test-e2e` / `make cleanup-test-e2e`), and has been kept compiling
|
||||
(`go vet -tags=e2e ./...` is part of the routine) but is **not part of
|
||||
`make test` and was not used for the release verification** — the manual
|
||||
kind run below covers strictly more.
|
||||
|
||||
### The OTel tracing spec (Tempo-gated)
|
||||
|
||||
`test/e2e/tracing_test.go` proves the full tracing pipeline against a
|
||||
**real Tempo**: deploy in kind with tracing enabled, create two
|
||||
kubernetes-provider proxies, delete them, and assert in Tempo that the
|
||||
traces exist with the documented span topology (`Reconcile Proxy` →
|
||||
`reconcile.managed` / `provider.create` / `status.patch`, and
|
||||
`reconcile.delete` / `provider.delete` on the way out).
|
||||
|
||||
It **skips unless both env vars are set** (so the rest of the suite runs
|
||||
anywhere). Homelab invocation:
|
||||
|
||||
```bash
|
||||
TEMPO_URL=http://192.168.0.30:3200 \
|
||||
OTLP_ENDPOINT=http://192.168.0.30:4318 \
|
||||
make test-e2e
|
||||
```
|
||||
|
||||
Worth knowing:
|
||||
|
||||
- Every span of a run carries the resource attribute
|
||||
`test.run.id=e2e-<nanos>` (injected via `OTEL_RESOURCE_ATTRIBUTES`, no
|
||||
code involved); the run ID is printed in the test log, and
|
||||
`{resource.test.run.id="<id>"}` in Grafana shows exactly that run.
|
||||
- The spec preflights the OTLP endpoint **from inside the cluster** with a
|
||||
curl pod and fails fast with a clear message if it's unreachable —
|
||||
export failures are otherwise only visible at `-zap-log-level=1`.
|
||||
- Proxy CRs are created in `default`, not the operator namespace: the
|
||||
squid pods carry no securityContext and the operator namespace enforces
|
||||
restricted PSS.
|
||||
kind run below covers strictly more. Treat it as scaffold to grow into if
|
||||
CI wants an automated cluster smoke test.
|
||||
|
||||
## The kind verification run (the real end-to-end)
|
||||
|
||||
|
||||
@@ -4,17 +4,13 @@ go 1.26.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute v1.65.0
|
||||
github.com/go-logr/logr v1.4.4
|
||||
github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858
|
||||
github.com/go-logr/logr v1.4.3
|
||||
github.com/kdomanski/iso9660 v0.4.0
|
||||
github.com/onsi/ginkgo/v2 v2.27.4
|
||||
github.com/onsi/gomega v1.39.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
|
||||
go.opentelemetry.io/otel v1.45.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0
|
||||
go.opentelemetry.io/otel/sdk v1.45.0
|
||||
go.opentelemetry.io/otel/trace v1.45.0
|
||||
golang.org/x/crypto v0.56.0
|
||||
google.golang.org/api v0.292.0
|
||||
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
|
||||
k8s.io/api v0.36.0
|
||||
@@ -38,7 +34,7 @@ require (
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/emicklei/go-restful/v3 v3.13.0 // indirect
|
||||
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
@@ -55,7 +51,7 @@ require (
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.23.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
@@ -73,27 +69,31 @@ require (
|
||||
github.com/stoewer/go-strcase v1.3.0 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.54.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
|
||||
golang.org/x/mod v0.37.0 // indirect
|
||||
golang.org/x/mod v0.38.0 // indirect
|
||||
golang.org/x/net v0.57.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/term v0.45.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/tools v0.47.0 // indirect
|
||||
golang.org/x/tools v0.48.0 // indirect
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
|
||||
google.golang.org/grpc v1.83.0 // indirect
|
||||
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
|
||||
|
||||
@@ -28,14 +28,16 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858 h1:8xCFt73OddCD5WXxoYidJBWp3bJMC+CpE7Zwf5tcwk8=
|
||||
github.com/digitalocean/go-libvirt v0.0.0-20260814190004-1a83157e1858/go.mod h1:qb0Ofa71d3oXARQf633h2tNaeBxLsVxuDp+jcsVO2+4=
|
||||
github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
|
||||
github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
|
||||
github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
|
||||
github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
|
||||
github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
|
||||
github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
|
||||
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
|
||||
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
|
||||
@@ -47,8 +49,8 @@ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6O
|
||||
github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
|
||||
github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
|
||||
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
|
||||
@@ -86,8 +88,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.19 h1:mMOE7DN2+p76/EdIrm
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k=
|
||||
github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE=
|
||||
github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
@@ -96,6 +98,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE
|
||||
github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg=
|
||||
github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
@@ -170,28 +174,24 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
|
||||
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
|
||||
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=
|
||||
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
|
||||
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
|
||||
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
|
||||
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
|
||||
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
|
||||
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
@@ -202,12 +202,12 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
|
||||
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
|
||||
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
||||
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
||||
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
|
||||
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
@@ -218,12 +218,12 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
|
||||
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
|
||||
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
|
||||
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
@@ -232,8 +232,8 @@ google.golang.org/api v0.292.0 h1:Ewiwo/GTtiaPZSNAZQUcWLh8AYDEoPmIXyJfeoTSMHU=
|
||||
google.golang.org/api v0.292.0/go.mod h1:07kjmMnFGm2RQuCza2EZM/5N68G/fVvFb1xKjWqoFA0=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
|
||||
|
||||
@@ -41,7 +41,6 @@ import (
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
// HealthSnapshotter provides the current probe verdict for a proxy. The
|
||||
@@ -97,14 +96,9 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
|
||||
}
|
||||
base := p.DeepCopy()
|
||||
defer func() {
|
||||
// Runs inside the root reconcile span (this defer fires before the
|
||||
// tracing.NewReconciler wrapper sees the return), and its error is
|
||||
// folded into err, which that wrapper records.
|
||||
pctx, span := tracing.Start(ctx, "status.patch")
|
||||
defer span.End()
|
||||
// NotFound is expected when this reconcile just removed the last
|
||||
// finalizer and the object is already gone.
|
||||
if perr := r.patchStatusIfChanged(pctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
|
||||
if perr := r.patchStatusIfChanged(ctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
|
||||
err = errors.Join(err, perr)
|
||||
}
|
||||
}()
|
||||
@@ -120,8 +114,6 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
|
||||
}
|
||||
|
||||
func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
|
||||
ctx, span := tracing.Start(ctx, "reconcile.managed")
|
||||
defer span.End()
|
||||
log := logf.FromContext(ctx)
|
||||
|
||||
if controllerutil.AddFinalizer(p, crawlv1alpha1.FinalizerName) {
|
||||
@@ -232,8 +224,6 @@ func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1
|
||||
// exists" — hence: delete, poll to NotFound, only then advance the hash and
|
||||
// let the create branch run.
|
||||
func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.Proxy, prov provider.Provider, hash string) (ctrl.Result, error) {
|
||||
ctx, span := tracing.Start(ctx, "reconcile.replaceInstance")
|
||||
defer span.End()
|
||||
_, err := prov.Get(ctx, p.Status.ProviderID)
|
||||
if provider.Class(err) == provider.ErrNotFound {
|
||||
// Old instance is gone. The Update inside setSpecHash refreshes p
|
||||
@@ -262,8 +252,6 @@ func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.
|
||||
}
|
||||
|
||||
func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
|
||||
ctx, span := tracing.Start(ctx, "reconcile.delete")
|
||||
defer span.End()
|
||||
if !controllerutil.ContainsFinalizer(p, crawlv1alpha1.FinalizerName) {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
@@ -421,10 +409,7 @@ func (r *ProxyReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
if r.HealthEvents != nil {
|
||||
b = b.WatchesRawSource(source.Channel(r.HealthEvents, &handler.EnqueueRequestForObject{}))
|
||||
}
|
||||
// Root span per reconcile; sub-reconcilers and the status flush hang
|
||||
// their spans off it. Tests calling r.Reconcile directly bypass the
|
||||
// wrapper and see no-op spans — the global tracer is never set there.
|
||||
return b.Complete(tracing.NewReconciler("Proxy", r))
|
||||
return b.Complete(r)
|
||||
}
|
||||
|
||||
func (r *ProxyReconciler) applyDefaults() {
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
|
||||
@@ -83,10 +82,7 @@ func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
var list crawlv1alpha1.ProxyList
|
||||
if err := s.Reader.List(r.Context(), &list); err != nil {
|
||||
// The ctx logger is s.log enriched with this request's
|
||||
// traceID/spanID by the tracing middleware (logr sinks can't read
|
||||
// ctx at log time, so per-request values ride on the logger).
|
||||
logf.FromContext(r.Context()).Error(err, "listing proxies")
|
||||
s.log.Error(err, "listing proxies")
|
||||
writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
|
||||
return
|
||||
}
|
||||
@@ -142,7 +138,7 @@ func (s *Server) handleAcquireLease(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
var list crawlv1alpha1.ProxyList
|
||||
if err := s.Reader.List(r.Context(), &list); err != nil {
|
||||
logf.FromContext(r.Context()).Error(err, "listing proxies for lease")
|
||||
s.log.Error(err, "listing proxies for lease")
|
||||
writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
|
||||
return
|
||||
}
|
||||
@@ -235,7 +231,7 @@ func (s *Server) handleReportLease(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusNotFound, "unknown_lease", "no such lease")
|
||||
return
|
||||
}
|
||||
logf.FromContext(r.Context()).Error(err, "reporting lease", "leaseID", r.PathValue("id"))
|
||||
s.log.Error(err, "reporting lease", "leaseID", r.PathValue("id"))
|
||||
writeError(w, http.StatusInternalServerError, "internal", "report failed")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
// LeaseStore is what the handlers need from a lease backend. Defined here,
|
||||
@@ -137,9 +136,7 @@ func (s *Server) Start(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// handler assembles the mux and the middleware chain, outermost first:
|
||||
// recover → tracing (server span + request logger) → request-log →
|
||||
// body-size cap → bearer auth. Everything inside the tracing layer logs via
|
||||
// logf.FromContext(r.Context()) and so carries traceID/spanID.
|
||||
// recover → request-log → body-size cap → bearer auth.
|
||||
func (s *Server) handler() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
@@ -155,7 +152,6 @@ func (s *Server) handler() http.Handler {
|
||||
h = s.authMiddleware(h)
|
||||
h = maxBytesMiddleware(h)
|
||||
h = s.logMiddleware(h)
|
||||
h = tracing.HTTPMiddleware("discovery", s.log)(h)
|
||||
h = s.recoverMiddleware(h)
|
||||
return h
|
||||
}
|
||||
@@ -192,7 +188,7 @@ func (s *Server) logMiddleware(next http.Handler) http.Handler {
|
||||
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
|
||||
start := time.Now()
|
||||
next.ServeHTTP(rec, r)
|
||||
logf.FromContext(r.Context()).Info("request",
|
||||
s.log.Info("request",
|
||||
"method", r.Method, "path", r.URL.Path,
|
||||
"status", rec.status, "duration", time.Since(start).String())
|
||||
})
|
||||
|
||||
@@ -12,11 +12,8 @@ import (
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
// Sweeper is the manager Runnable running the sweep loop.
|
||||
@@ -84,8 +81,6 @@ func (s *Sweeper) Start(ctx context.Context) error {
|
||||
// that deletion, and GC racing it would double-delete. A UID becomes
|
||||
// orphan-eligible only once the object is fully gone.
|
||||
func (s *Sweeper) sweep(ctx context.Context) {
|
||||
ctx, span := tracing.Start(ctx, "gc.sweep")
|
||||
defer span.End()
|
||||
log := logf.FromContext(ctx).WithName("orphan-gc")
|
||||
|
||||
var list crawlv1alpha1.ProxyList
|
||||
@@ -100,7 +95,6 @@ func (s *Sweeper) sweep(ctx context.Context) {
|
||||
live[string(list.Items[i].UID)] = true
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
for name, prov := range s.Providers {
|
||||
instances, err := prov.ListByTag(ctx)
|
||||
if err != nil {
|
||||
@@ -125,13 +119,7 @@ func (s *Sweeper) sweep(ctx context.Context) {
|
||||
if err := prov.Delete(ctx, inst.ID); err != nil {
|
||||
log.Error(err, "deleting orphaned instance",
|
||||
"provider", name, "providerID", inst.ID, "uid", inst.UID)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
}
|
||||
span.SetAttributes(
|
||||
attribute.Int("gc.providers", len(s.Providers)),
|
||||
attribute.Int("gc.deleted", deleted),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -10,9 +10,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
@@ -21,7 +18,6 @@ import (
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
// Snapshot is the engine's current verdict for one proxy, read by the
|
||||
@@ -103,10 +99,6 @@ type Engine struct {
|
||||
// transition-only by design, so metrics are where high-frequency
|
||||
// signal (true probe recency, every latency sample) lives.
|
||||
Metrics ProbeMetrics
|
||||
// TraceProbes emits one root span per probe (--trace-health-probes).
|
||||
// Off by default: probes run about once per second per proxy and would
|
||||
// dominate trace volume.
|
||||
TraceProbes bool
|
||||
|
||||
probeFn func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult
|
||||
|
||||
@@ -163,7 +155,8 @@ func (e *Engine) Start(ctx context.Context) error {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case job := <-jobs:
|
||||
e.record(job, e.runProbe(ctx, job), time.Now())
|
||||
res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
|
||||
e.record(job, res, time.Now())
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -271,29 +264,6 @@ func newState(p *crawlv1alpha1.Proxy, now time.Time, interval time.Duration) *st
|
||||
return st
|
||||
}
|
||||
|
||||
// runProbe executes one probe, inside its own root span when TraceProbes is
|
||||
// on. The span ends before record folds the result (record stays ctx-free),
|
||||
// and the probe transport is deliberately uninstrumented — no traceparent
|
||||
// must ever leak through a proxy toward external targets.
|
||||
func (e *Engine) runProbe(ctx context.Context, job probeJob) probeResult {
|
||||
if !e.TraceProbes {
|
||||
return e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
|
||||
}
|
||||
ctx, span := tracing.Start(ctx, "health.probe",
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("proxy", job.key.String())))
|
||||
defer span.End()
|
||||
res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
|
||||
span.SetAttributes(
|
||||
attribute.Bool("probe.ok", res.ok),
|
||||
attribute.Int64("probe.latency_ms", res.latency.Milliseconds()),
|
||||
)
|
||||
if !res.ok {
|
||||
span.SetStatus(codes.Error, res.err.Error())
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// record folds one probe result into the proxy's threshold state and emits
|
||||
// an event when the result is status-affecting: a first-ever verdict, a
|
||||
// threshold-crossing flip, or a material latency change (beyond
|
||||
|
||||
+36
-13
@@ -21,6 +21,23 @@ type ProviderConfig struct {
|
||||
Type string `json:"type"`
|
||||
Kubernetes *KubernetesConfig `json:"kubernetes,omitempty"`
|
||||
GCP *GCPConfig `json:"gcp,omitempty"`
|
||||
KVM *KVMConfig `json:"kvm,omitempty"`
|
||||
}
|
||||
|
||||
// setBlocks names the type-specific blocks that are non-nil, so validate can
|
||||
// reject a block that doesn't match Type without a pairwise check per type.
|
||||
func (p ProviderConfig) setBlocks() []string {
|
||||
var out []string
|
||||
if p.Kubernetes != nil {
|
||||
out = append(out, "kubernetes")
|
||||
}
|
||||
if p.GCP != nil {
|
||||
out = append(out, "gcp")
|
||||
}
|
||||
if p.KVM != nil {
|
||||
out = append(out, "kvm")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// KubernetesConfig configures the kubernetes-pod provider, which creates
|
||||
@@ -81,22 +98,28 @@ func (c *Config) validate() error {
|
||||
}
|
||||
seen[p.Name] = true
|
||||
|
||||
prefix := fmt.Sprintf("providers[%d] %q", i, p.Name)
|
||||
switch p.Type {
|
||||
case "kubernetes":
|
||||
if p.GCP != nil {
|
||||
return fmt.Errorf("providers[%d] %q: type is kubernetes but a gcp block is set", i, p.Name)
|
||||
}
|
||||
case "gcp":
|
||||
if p.Kubernetes != nil {
|
||||
return fmt.Errorf("providers[%d] %q: type is gcp but a kubernetes block is set", i, p.Name)
|
||||
}
|
||||
if p.GCP == nil || p.GCP.Project == "" {
|
||||
return fmt.Errorf("providers[%d] %q: gcp.project is required", i, p.Name)
|
||||
}
|
||||
case "kubernetes", "gcp", "kvm":
|
||||
case "":
|
||||
return fmt.Errorf("providers[%d] %q: type is required", i, p.Name)
|
||||
return fmt.Errorf("%s: type is required", prefix)
|
||||
default:
|
||||
return fmt.Errorf("providers[%d] %q: unknown provider type %q", i, p.Name, p.Type)
|
||||
return fmt.Errorf("%s: unknown provider type %q", prefix, p.Type)
|
||||
}
|
||||
for _, block := range p.setBlocks() {
|
||||
if block != p.Type {
|
||||
return fmt.Errorf("%s: type is %s but a %s block is set", prefix, p.Type, block)
|
||||
}
|
||||
}
|
||||
switch p.Type {
|
||||
case "gcp":
|
||||
if p.GCP == nil || p.GCP.Project == "" {
|
||||
return fmt.Errorf("%s: gcp.project is required", prefix)
|
||||
}
|
||||
case "kvm":
|
||||
if err := validateKVM(prefix, p.KVM); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// KVMConfig configures a kvm provider instance: one or more libvirt/KVM
|
||||
// hypervisors reached over SSH (the libvirt unix socket is forwarded through
|
||||
// the SSH connection, so hosts need no libvirtd TCP/TLS listener).
|
||||
// placement.zone on a Proxy selects a host by name; with a single host it may
|
||||
// be omitted.
|
||||
type KVMConfig struct {
|
||||
// SSHKeyFile is the path to the private key used for every host. Required.
|
||||
SSHKeyFile string `json:"sshKeyFile"`
|
||||
// KnownHostsFile is an OpenSSH known_hosts file used to verify host keys.
|
||||
// Required unless InsecureIgnoreHostKey is set.
|
||||
KnownHostsFile string `json:"knownHostsFile,omitempty"`
|
||||
// InsecureIgnoreHostKey skips host-key verification entirely.
|
||||
InsecureIgnoreHostKey bool `json:"insecureIgnoreHostKey,omitempty"`
|
||||
// DiskSizeGB is the capacity of each VM's overlay disk. Default 10. The
|
||||
// guest's cloud-init growpart expands the root filesystem into it.
|
||||
DiskSizeGB int64 `json:"diskSizeGb,omitempty"`
|
||||
// VendorData is cloud-init vendor-data placed on every seed next to the
|
||||
// Proxy's own user-data. Default: installs and starts qemu-guest-agent so
|
||||
// the provider can read the VM's IP even before it shows up in ARP.
|
||||
VendorData string `json:"vendorData,omitempty"`
|
||||
// Images are named base images; placement.image names a key here (or is
|
||||
// an http(s) URL directly). The provider downloads each image into a
|
||||
// host's storage pool on first use and caches it there.
|
||||
Images map[string]KVMImage `json:"images,omitempty"`
|
||||
// MachineTypes are named size profiles; placement.machineType names a
|
||||
// key here. Required, at least one.
|
||||
MachineTypes map[string]KVMMachineType `json:"machineTypes"`
|
||||
// Hosts are the hypervisors. Required, at least one, unique names.
|
||||
Hosts []KVMHost `json:"hosts"`
|
||||
}
|
||||
|
||||
// KVMImage is one downloadable base image (a distro cloud image with
|
||||
// cloud-init preinstalled, e.g. Debian's generic qcow2).
|
||||
type KVMImage struct {
|
||||
// URL is where to fetch the image from. Required, http or https.
|
||||
URL string `json:"url"`
|
||||
// SHA256 is the expected hex digest of the file. Optional; when set it is
|
||||
// verified after download and a later change triggers a re-download.
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
// Format is the image's disk format, "qcow2" (default) or "raw".
|
||||
Format string `json:"format,omitempty"`
|
||||
}
|
||||
|
||||
// KVMMachineType is a VM size profile.
|
||||
type KVMMachineType struct {
|
||||
VCPUs int32 `json:"vcpus"`
|
||||
MemoryMiB int64 `json:"memoryMiB"`
|
||||
}
|
||||
|
||||
// KVMHost is one hypervisor.
|
||||
type KVMHost struct {
|
||||
// Name is the placement.zone value that selects this host. Required.
|
||||
Name string `json:"name"`
|
||||
// Address is the SSH endpoint, host:port. Required.
|
||||
Address string `json:"address"`
|
||||
// User is the SSH user; it must be allowed to use the libvirt socket
|
||||
// (typically a member of the libvirt group). Required.
|
||||
User string `json:"user"`
|
||||
// Socket is the libvirt unix socket path on the host.
|
||||
// Default /var/run/libvirt/libvirt-sock.
|
||||
Socket string `json:"socket,omitempty"`
|
||||
// Pool is the libvirt storage pool for base images, overlay disks and
|
||||
// seed ISOs. Default "default".
|
||||
Pool string `json:"pool,omitempty"`
|
||||
// Bridge is the host bridge the VM's NIC attaches to; it must lead to a
|
||||
// network the cluster can reach and that hands out DHCP leases. Required.
|
||||
Bridge string `json:"bridge"`
|
||||
}
|
||||
|
||||
var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
|
||||
|
||||
func validateKVM(prefix string, c *KVMConfig) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("%s: kvm block is required", prefix)
|
||||
}
|
||||
if c.SSHKeyFile == "" {
|
||||
return fmt.Errorf("%s: kvm.sshKeyFile is required", prefix)
|
||||
}
|
||||
if c.KnownHostsFile == "" && !c.InsecureIgnoreHostKey {
|
||||
return fmt.Errorf("%s: kvm.knownHostsFile is required unless kvm.insecureIgnoreHostKey is true", prefix)
|
||||
}
|
||||
if c.DiskSizeGB < 0 {
|
||||
return fmt.Errorf("%s: kvm.diskSizeGb must not be negative", prefix)
|
||||
}
|
||||
for name, img := range c.Images {
|
||||
p := fmt.Sprintf("%s: kvm.images[%q]", prefix, name)
|
||||
u, err := url.Parse(img.URL)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return fmt.Errorf("%s.url must be an http(s) URL (got %q)", p, img.URL)
|
||||
}
|
||||
if img.SHA256 != "" && !sha256Hex.MatchString(img.SHA256) {
|
||||
return fmt.Errorf("%s.sha256 must be 64 hex characters", p)
|
||||
}
|
||||
if img.Format != "" && img.Format != "qcow2" && img.Format != "raw" {
|
||||
return fmt.Errorf("%s.format must be qcow2 or raw (got %q)", p, img.Format)
|
||||
}
|
||||
}
|
||||
if len(c.MachineTypes) == 0 {
|
||||
return fmt.Errorf("%s: kvm.machineTypes must define at least one profile", prefix)
|
||||
}
|
||||
for name, mt := range c.MachineTypes {
|
||||
if mt.VCPUs <= 0 || mt.MemoryMiB <= 0 {
|
||||
return fmt.Errorf("%s: kvm.machineTypes[%q] needs vcpus > 0 and memoryMiB > 0", prefix, name)
|
||||
}
|
||||
}
|
||||
if len(c.Hosts) == 0 {
|
||||
return fmt.Errorf("%s: kvm.hosts must list at least one host", prefix)
|
||||
}
|
||||
seen := make(map[string]bool, len(c.Hosts))
|
||||
for i, h := range c.Hosts {
|
||||
p := fmt.Sprintf("%s: kvm.hosts[%d]", prefix, i)
|
||||
if h.Name == "" {
|
||||
return fmt.Errorf("%s: name is required", p)
|
||||
}
|
||||
if seen[h.Name] {
|
||||
return fmt.Errorf("%s: duplicate host name %q", p, h.Name)
|
||||
}
|
||||
seen[h.Name] = true
|
||||
if h.Address == "" {
|
||||
return fmt.Errorf("%s %q: address is required", p, h.Name)
|
||||
}
|
||||
if h.User == "" {
|
||||
return fmt.Errorf("%s %q: user is required", p, h.Name)
|
||||
}
|
||||
if h.Bridge == "" {
|
||||
return fmt.Errorf("%s %q: bridge is required", p, h.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const validKVMYAML = `
|
||||
providers:
|
||||
- name: kvm-home
|
||||
type: kvm
|
||||
kvm:
|
||||
sshKeyFile: /etc/proxy-operator/kvm/id_ed25519
|
||||
knownHostsFile: /etc/proxy-operator/kvm/known_hosts
|
||||
diskSizeGb: 20
|
||||
images:
|
||||
debian-13:
|
||||
url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||
sha256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
|
||||
ubuntu:
|
||||
url: http://mirror.local/noble.img
|
||||
format: raw
|
||||
machineTypes:
|
||||
small: {vcpus: 1, memoryMiB: 1024}
|
||||
medium: {vcpus: 2, memoryMiB: 2048}
|
||||
hosts:
|
||||
- name: nas1
|
||||
address: nas1.home.lan:22
|
||||
user: virt
|
||||
bridge: br0
|
||||
- name: lab2
|
||||
address: 10.0.0.2:2222
|
||||
user: root
|
||||
socket: /run/libvirt/virtqemud-sock
|
||||
pool: fast
|
||||
bridge: br1
|
||||
`
|
||||
|
||||
func TestLoadConfig_kvmValid(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg, err := LoadConfig([]byte(validKVMYAML))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig() error = %v, want nil", err)
|
||||
}
|
||||
k := cfg.Providers[0].KVM
|
||||
if k == nil {
|
||||
t.Fatal("providers[0].kvm = nil")
|
||||
}
|
||||
if k.SSHKeyFile != "/etc/proxy-operator/kvm/id_ed25519" || k.DiskSizeGB != 20 {
|
||||
t.Errorf("kvm = %+v, want sshKeyFile and diskSizeGb parsed", k)
|
||||
}
|
||||
if got := k.Images["debian-13"].SHA256; !strings.HasPrefix(got, "0123") {
|
||||
t.Errorf("images[debian-13].sha256 = %q", got)
|
||||
}
|
||||
if k.Images["ubuntu"].Format != "raw" {
|
||||
t.Errorf("images[ubuntu].format = %q, want raw", k.Images["ubuntu"].Format)
|
||||
}
|
||||
if mt := k.MachineTypes["medium"]; mt.VCPUs != 2 || mt.MemoryMiB != 2048 {
|
||||
t.Errorf("machineTypes[medium] = %+v, want {2 2048}", mt)
|
||||
}
|
||||
if len(k.Hosts) != 2 || k.Hosts[1].Socket != "/run/libvirt/virtqemud-sock" || k.Hosts[1].Pool != "fast" {
|
||||
t.Errorf("hosts = %+v, want two hosts with lab2's socket and pool parsed", k.Hosts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_kvmInsecureHostKeyNeedsNoKnownHosts(t *testing.T) {
|
||||
t.Parallel()
|
||||
yaml := strings.Replace(validKVMYAML,
|
||||
"knownHostsFile: /etc/proxy-operator/kvm/known_hosts",
|
||||
"insecureIgnoreHostKey: true", 1)
|
||||
if _, err := LoadConfig([]byte(yaml)); err != nil {
|
||||
t.Fatalf("LoadConfig() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_kvmInvalid(t *testing.T) {
|
||||
t.Parallel()
|
||||
minimal := func(kvmBody string) string {
|
||||
return "providers:\n - name: kvm-home\n type: kvm\n kvm:\n" + indent(kvmBody, 6)
|
||||
}
|
||||
hostsAndTypes := `
|
||||
machineTypes:
|
||||
small: {vcpus: 1, memoryMiB: 1024}
|
||||
hosts:
|
||||
- {name: nas1, address: nas1:22, user: virt, bridge: br0}
|
||||
`
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
wantErrSub string
|
||||
}{
|
||||
{
|
||||
name: "missing kvm block",
|
||||
yaml: "providers:\n - name: kvm-home\n type: kvm\n",
|
||||
wantErrSub: "kvm block is required",
|
||||
},
|
||||
{
|
||||
name: "missing sshKeyFile",
|
||||
yaml: minimal("knownHostsFile: /kh\n" + hostsAndTypes),
|
||||
wantErrSub: "kvm.sshKeyFile is required",
|
||||
},
|
||||
{
|
||||
name: "missing knownHostsFile without insecure flag",
|
||||
yaml: minimal("sshKeyFile: /key\n" + hostsAndTypes),
|
||||
wantErrSub: "kvm.knownHostsFile is required unless",
|
||||
},
|
||||
{
|
||||
name: "negative disk size",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\ndiskSizeGb: -1\n" + hostsAndTypes),
|
||||
wantErrSub: "diskSizeGb must not be negative",
|
||||
},
|
||||
{
|
||||
name: "no machine types",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nhosts:\n - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
|
||||
wantErrSub: "machineTypes must define at least one",
|
||||
},
|
||||
{
|
||||
name: "machine type with zero memory",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n small: {vcpus: 1}\nhosts:\n - {name: nas1, address: nas1:22, user: virt, bridge: br0}\n"),
|
||||
wantErrSub: `machineTypes["small"] needs vcpus > 0 and memoryMiB > 0`,
|
||||
},
|
||||
{
|
||||
name: "no hosts",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n small: {vcpus: 1, memoryMiB: 1024}\n"),
|
||||
wantErrSub: "kvm.hosts must list at least one host",
|
||||
},
|
||||
{
|
||||
name: "duplicate host name",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n - {name: nas1, address: a:22, user: virt, bridge: br0}\n - {name: nas1, address: b:22, user: virt, bridge: br0}\n"),
|
||||
wantErrSub: `duplicate host name "nas1"`,
|
||||
},
|
||||
{
|
||||
name: "host missing address",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n - {name: nas1, user: virt, bridge: br0}\n"),
|
||||
wantErrSub: "address is required",
|
||||
},
|
||||
{
|
||||
name: "host missing user",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n - {name: nas1, address: a:22, bridge: br0}\n"),
|
||||
wantErrSub: "user is required",
|
||||
},
|
||||
{
|
||||
name: "host missing bridge",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nmachineTypes:\n small: {vcpus: 1, memoryMiB: 1024}\nhosts:\n - {name: nas1, address: a:22, user: virt}\n"),
|
||||
wantErrSub: "bridge is required",
|
||||
},
|
||||
{
|
||||
name: "image with non-http url",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n d: {url: file:///tmp/x.qcow2}\n" + hostsAndTypes),
|
||||
wantErrSub: `images["d"].url must be an http(s) URL`,
|
||||
},
|
||||
{
|
||||
name: "image with bad sha256",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n d: {url: https://x/y.qcow2, sha256: abc}\n" + hostsAndTypes),
|
||||
wantErrSub: `images["d"].sha256 must be 64 hex characters`,
|
||||
},
|
||||
{
|
||||
name: "image with unknown format",
|
||||
yaml: minimal("sshKeyFile: /key\nknownHostsFile: /kh\nimages:\n d: {url: https://x/y.vmdk, format: vmdk}\n" + hostsAndTypes),
|
||||
wantErrSub: `images["d"].format must be qcow2 or raw`,
|
||||
},
|
||||
{
|
||||
name: "kvm type with gcp block",
|
||||
yaml: `
|
||||
providers:
|
||||
- name: p1
|
||||
type: kvm
|
||||
gcp:
|
||||
project: my-project`,
|
||||
wantErrSub: "type is kvm but a gcp block is set",
|
||||
},
|
||||
{
|
||||
name: "gcp type with kvm block",
|
||||
yaml: `
|
||||
providers:
|
||||
- name: p1
|
||||
type: gcp
|
||||
gcp:
|
||||
project: my-project
|
||||
kvm:
|
||||
sshKeyFile: /key`,
|
||||
wantErrSub: "type is gcp but a kvm block is set",
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, err := LoadConfig([]byte(tc.yaml))
|
||||
if err == nil {
|
||||
t.Fatalf("LoadConfig() error = nil, want error containing %q", tc.wantErrSub)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.wantErrSub) {
|
||||
t.Errorf("LoadConfig() error = %q, want substring %q", err.Error(), tc.wantErrSub)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func indent(s string, n int) string {
|
||||
pad := strings.Repeat(" ", n)
|
||||
lines := strings.Split(strings.TrimLeft(s, "\n"), "\n")
|
||||
for i, l := range lines {
|
||||
if l != "" {
|
||||
lines[i] = pad + l
|
||||
}
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// wireLogMaxFieldBytes is the elision threshold for string fields in wire
|
||||
@@ -48,15 +47,6 @@ func (h *wireFilterHandler) Handle(ctx context.Context, rec slog.Record) error {
|
||||
if rec.Level <= slog.LevelDebug && rec.Message != "api request" && rec.Message != "api response" {
|
||||
return nil
|
||||
}
|
||||
// The SDK logs with the request ctx, so wire records can carry the
|
||||
// surrounding provider span — the one place slog's ctx-aware handlers
|
||||
// beat logr, and the same keys the logr enrichment uses.
|
||||
if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
|
||||
rec = rec.Clone()
|
||||
rec.AddAttrs(
|
||||
slog.String("traceID", sc.TraceID().String()),
|
||||
slog.String("spanID", sc.SpanID().String()))
|
||||
}
|
||||
if h.fullPayloads {
|
||||
return h.inner.Handle(ctx, rec)
|
||||
}
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
package gcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-logr/logr/funcr"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
)
|
||||
|
||||
func TestWireFilterHandler_addsTraceContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var lines []string
|
||||
base := funcr.New(func(prefix, args string) {
|
||||
lines = append(lines, prefix+" "+args)
|
||||
}, funcr.Options{Verbosity: 5})
|
||||
log := wireLogger(base, WireLogOptions{})
|
||||
|
||||
tp := sdktrace.NewTracerProvider()
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
ctx, span := tp.Tracer("test").Start(context.Background(), "provider.create")
|
||||
defer span.End()
|
||||
|
||||
log.DebugContext(ctx, "api request", "url", "https://compute.googleapis.com/x")
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("got %d lines, want 1: %v", len(lines), lines)
|
||||
}
|
||||
traceID := span.SpanContext().TraceID().String()
|
||||
if !strings.Contains(lines[0], "traceID") || !strings.Contains(lines[0], traceID) {
|
||||
t.Errorf("wire record missing trace context %s: %s", traceID, lines[0])
|
||||
}
|
||||
|
||||
// The security filter must still win: non-wire Debug records are
|
||||
// dropped even when a span is present.
|
||||
log.DebugContext(ctx, "token exchange", "assertion", "secret-jwt")
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("filtered record leaked: %v", lines[1:])
|
||||
}
|
||||
|
||||
// No span in ctx: record passes through without trace keys.
|
||||
log.DebugContext(context.Background(), "api response", "status", 200)
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("got %d lines, want 2", len(lines))
|
||||
}
|
||||
if strings.Contains(lines[1], "traceID") {
|
||||
t.Errorf("spanless record must not carry traceID: %s", lines[1])
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/transport"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
@@ -40,22 +39,11 @@ type Provider struct {
|
||||
|
||||
// New builds a kubernetes Provider from its config block. Satisfies
|
||||
// registry.Constructor.
|
||||
func New(ctx context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
|
||||
return NewWithTransportWrapper(ctx, cfg, nil)
|
||||
}
|
||||
|
||||
// NewWithTransportWrapper is New with an optional transport wrapper applied
|
||||
// to the provider's own rest.Config (this client is built independently of
|
||||
// the manager's, so the composition root must wrap it separately for
|
||||
// tracing). A nil wrapper means a plain client.
|
||||
func NewWithTransportWrapper(_ context.Context, cfg provider.ProviderConfig, wrap transport.WrapperFunc) (provider.Provider, error) {
|
||||
func New(_ context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
|
||||
restCfg, err := ctrl.GetConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kubernetes provider %q: loading kubeconfig: %w", cfg.Name, err)
|
||||
}
|
||||
if wrap != nil {
|
||||
restCfg.Wrap(wrap)
|
||||
}
|
||||
scheme := runtime.NewScheme()
|
||||
if err := corev1.AddToScheme(scheme); err != nil {
|
||||
return nil, fmt.Errorf("kubernetes provider %q: %w", cfg.Name, err)
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultHandshakeTimeout = 30 * time.Second
|
||||
keepaliveInterval = 15 * time.Second
|
||||
)
|
||||
|
||||
// libvirtAPI is the slice of *libvirt.Libvirt the provider uses, and the
|
||||
// test seam: the fake in kvm_test.go implements exactly this.
|
||||
type libvirtAPI interface {
|
||||
DomainLookupByName(name string) (libvirt.Domain, error)
|
||||
DomainDefineXML(xml string) (libvirt.Domain, error)
|
||||
DomainCreate(d libvirt.Domain) error
|
||||
DomainDestroy(d libvirt.Domain) error
|
||||
DomainUndefineFlags(d libvirt.Domain, flags libvirt.DomainUndefineFlagsValues) error
|
||||
DomainGetState(d libvirt.Domain, flags uint32) (state int32, reason int32, err error)
|
||||
DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, flags libvirt.DomainModificationImpact) (string, error)
|
||||
DomainInterfaceAddresses(d libvirt.Domain, source uint32, flags uint32) ([]libvirt.DomainInterface, error)
|
||||
ConnectListAllDomains(needResults int32, flags libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error)
|
||||
StoragePoolLookupByName(name string) (libvirt.StoragePool, error)
|
||||
StoragePoolRefresh(pool libvirt.StoragePool, flags uint32) error
|
||||
StorageVolLookupByName(pool libvirt.StoragePool, name string) (libvirt.StorageVol, error)
|
||||
StorageVolGetPath(vol libvirt.StorageVol) (string, error)
|
||||
StorageVolCreateXML(pool libvirt.StoragePool, xml string, flags libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error)
|
||||
StorageVolUpload(vol libvirt.StorageVol, r io.Reader, offset, length uint64, flags libvirt.StorageVolUploadFlags) error
|
||||
StorageVolDownload(vol libvirt.StorageVol, w io.Writer, offset, length uint64, flags libvirt.StorageVolDownloadFlags) error
|
||||
StorageVolDelete(vol libvirt.StorageVol, flags libvirt.StorageVolDeleteFlags) error
|
||||
}
|
||||
|
||||
// hostClient is one live connection to a hypervisor.
|
||||
type hostClient struct {
|
||||
api libvirtAPI
|
||||
// done is closed once the connection is known to be lost. A nil channel
|
||||
// never fires, which suits fakes.
|
||||
done <-chan struct{}
|
||||
close func()
|
||||
}
|
||||
|
||||
func (c *hostClient) alive() bool {
|
||||
select {
|
||||
case <-c.done:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// hostDialer opens a connection to one host. The SSH implementation is the
|
||||
// real one; tests inject their own.
|
||||
type hostDialer func(ctx context.Context, host provider.KVMHost) (*hostClient, error)
|
||||
|
||||
// hostPool caches one connection per host, dialing lazily and redialing
|
||||
// after the connection drops. Dials for different hosts don't block each
|
||||
// other; concurrent callers for the same host share a single dial.
|
||||
type hostPool struct {
|
||||
dial hostDialer
|
||||
|
||||
mu sync.Mutex
|
||||
entries map[string]*hostEntry
|
||||
}
|
||||
|
||||
type hostEntry struct {
|
||||
mu sync.Mutex
|
||||
client *hostClient
|
||||
}
|
||||
|
||||
func newHostPool(dial hostDialer) *hostPool {
|
||||
return &hostPool{dial: dial, entries: make(map[string]*hostEntry)}
|
||||
}
|
||||
|
||||
func (p *hostPool) entry(name string) *hostEntry {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
e, ok := p.entries[name]
|
||||
if !ok {
|
||||
e = &hostEntry{}
|
||||
p.entries[name] = e
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func (p *hostPool) get(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
|
||||
e := p.entry(host.Name)
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if e.client != nil {
|
||||
if e.client.alive() {
|
||||
return e.client, nil
|
||||
}
|
||||
e.client.close()
|
||||
e.client = nil
|
||||
}
|
||||
c, err := p.dial(ctx, host)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connecting to host %s (%s): %w", host.Name, host.Address, err)
|
||||
}
|
||||
e.client = c
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// drop closes and forgets c if it is still the cached client for host, so a
|
||||
// connection that was already replaced by a redial is left alone.
|
||||
func (p *hostPool) drop(host string, c *hostClient) {
|
||||
e := p.entry(host)
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if e.client == c {
|
||||
e.client = nil
|
||||
}
|
||||
c.close()
|
||||
}
|
||||
|
||||
// call runs fn against host's connection. go-libvirt RPCs take no context
|
||||
// and block until the daemon answers, so when ctx ends first the connection
|
||||
// is closed — which unblocks fn with an error — and ctx.Err() is returned.
|
||||
// A transport error (anything that is not a libvirt.Error) also drops the
|
||||
// cached connection so the next call redials instead of reusing a dead
|
||||
// link.
|
||||
func (p *hostPool) call(ctx context.Context, host provider.KVMHost, fn func(api libvirtAPI) error) error {
|
||||
c, err := p.get(ctx, host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = awaitOrCancel(ctx, func() error { return fn(c.api) }, func() { p.drop(host.Name, c) })
|
||||
if err != nil && !isLibvirtError(err) {
|
||||
p.drop(host.Name, c)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// awaitOrCancel runs fn in the background and returns its result, unless ctx
|
||||
// ends first: then abort is invoked (it must make fn return eventually, e.g.
|
||||
// by closing the connection fn is blocked on) and ctx.Err() is returned. The
|
||||
// abandoned fn's result is discarded.
|
||||
func awaitOrCancel(ctx context.Context, fn func() error, abort func()) error {
|
||||
errc := make(chan error, 1)
|
||||
go func() { errc <- fn() }()
|
||||
select {
|
||||
case err := <-errc:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
abort()
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// sshAuth holds the pieces of the SSH client config shared by every host:
|
||||
// the private key and the host-key policy, both loaded once at startup so a
|
||||
// missing or unreadable file fails the operator fast rather than on the
|
||||
// first Create.
|
||||
type sshAuth struct {
|
||||
signer ssh.Signer
|
||||
hostKey ssh.HostKeyCallback
|
||||
}
|
||||
|
||||
func newSSHAuth(cfg provider.KVMConfig) (*sshAuth, error) {
|
||||
key, err := os.ReadFile(cfg.SSHKeyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading kvm.sshKeyFile: %w", err)
|
||||
}
|
||||
signer, err := ssh.ParsePrivateKey(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing kvm.sshKeyFile: %w", err)
|
||||
}
|
||||
hostKey := ssh.InsecureIgnoreHostKey() //nolint:gosec // opt-in via config, documented as insecure
|
||||
if !cfg.InsecureIgnoreHostKey {
|
||||
hostKey, err = knownhosts.New(cfg.KnownHostsFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("loading kvm.knownHostsFile: %w", err)
|
||||
}
|
||||
}
|
||||
return &sshAuth{signer: signer, hostKey: hostKey}, nil
|
||||
}
|
||||
|
||||
// dialer returns the production hostDialer: SSH to the host, forward its
|
||||
// libvirt unix socket through the connection, and open a libvirt session
|
||||
// over that.
|
||||
func (a *sshAuth) dialer() hostDialer {
|
||||
return func(ctx context.Context, host provider.KVMHost) (*hostClient, error) {
|
||||
client, err := a.dialSSH(ctx, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hc, err := connectLibvirt(ctx, client, host.Socket)
|
||||
if err != nil {
|
||||
client.Close()
|
||||
return nil, err
|
||||
}
|
||||
return hc, nil
|
||||
}
|
||||
}
|
||||
|
||||
// dialSSH establishes the SSH connection. The handshake is bounded by ctx's
|
||||
// deadline, or defaultHandshakeTimeout when ctx has none, since a
|
||||
// half-open hypervisor must not pin a reconcile worker indefinitely.
|
||||
func (a *sshAuth) dialSSH(ctx context.Context, host provider.KVMHost) (*ssh.Client, error) {
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
deadline = time.Now().Add(defaultHandshakeTimeout)
|
||||
}
|
||||
var d net.Dialer
|
||||
nc, err := d.DialContext(ctx, "tcp", host.Address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ssh dial: %w", err)
|
||||
}
|
||||
_ = nc.SetDeadline(deadline)
|
||||
conn, chans, reqs, err := ssh.NewClientConn(nc, host.Address, &ssh.ClientConfig{
|
||||
User: host.User,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(a.signer)},
|
||||
HostKeyCallback: a.hostKey,
|
||||
})
|
||||
if err != nil {
|
||||
nc.Close()
|
||||
return nil, fmt.Errorf("ssh handshake: %w", err)
|
||||
}
|
||||
_ = nc.SetDeadline(time.Time{})
|
||||
return ssh.NewClient(conn, chans, reqs), nil
|
||||
}
|
||||
|
||||
// unixForward adapts an SSH client to go-libvirt's socket.Dialer: each Dial
|
||||
// opens a direct-streamlocal channel to the libvirt socket on the host.
|
||||
type unixForward struct {
|
||||
client *ssh.Client
|
||||
path string
|
||||
}
|
||||
|
||||
func (u unixForward) Dial() (net.Conn, error) {
|
||||
return u.client.Dial("unix", u.path)
|
||||
}
|
||||
|
||||
// connectLibvirt opens the libvirt session over the SSH client and starts a
|
||||
// keepalive loop so a dead link is noticed within roughly two intervals
|
||||
// rather than at the TCP stack's leisure. Closing the returned client tears
|
||||
// down the SSH connection, which in turn ends the libvirt session.
|
||||
func connectLibvirt(ctx context.Context, client *ssh.Client, socket string) (*hostClient, error) {
|
||||
lv := libvirt.NewWithDialer(unixForward{client: client, path: socket})
|
||||
if err := awaitOrCancel(ctx, lv.Connect, func() { client.Close() }); err != nil {
|
||||
return nil, fmt.Errorf("libvirt connect via %s: %w", socket, err)
|
||||
}
|
||||
stop := make(chan struct{})
|
||||
var once sync.Once
|
||||
go keepalive(client, keepaliveInterval, stop)
|
||||
return &hostClient{
|
||||
api: lv,
|
||||
done: lv.Disconnected(),
|
||||
close: func() {
|
||||
once.Do(func() {
|
||||
close(stop)
|
||||
client.Close()
|
||||
})
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// keepalive sends OpenSSH keepalive requests until stop closes; a failed or
|
||||
// unanswered request closes the client, which surfaces as Disconnected on
|
||||
// the libvirt side and makes the pool redial.
|
||||
func keepalive(client *ssh.Client, interval time.Duration, stop <-chan struct{}) {
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
replied := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, err := client.SendRequest("keepalive@openssh.com", true, nil)
|
||||
replied <- err
|
||||
}()
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case err := <-replied:
|
||||
if err != nil {
|
||||
client.Close()
|
||||
return
|
||||
}
|
||||
case <-time.After(interval):
|
||||
client.Close()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,569 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// --- hostPool over a fake dialer ---
|
||||
|
||||
type fakeClient struct {
|
||||
done chan struct{}
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func (c *fakeClient) close() {
|
||||
if c.closed.CompareAndSwap(false, true) {
|
||||
close(c.done)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeDialer struct {
|
||||
mu sync.Mutex
|
||||
dials int
|
||||
err error
|
||||
delay time.Duration
|
||||
clients []*fakeClient
|
||||
}
|
||||
|
||||
func (d *fakeDialer) dial(ctx context.Context, _ provider.KVMHost) (*hostClient, error) {
|
||||
if d.delay > 0 {
|
||||
select {
|
||||
case <-time.After(d.delay):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.dials++
|
||||
if d.err != nil {
|
||||
return nil, d.err
|
||||
}
|
||||
fc := &fakeClient{done: make(chan struct{})}
|
||||
d.clients = append(d.clients, fc)
|
||||
return &hostClient{api: nil, done: fc.done, close: fc.close}, nil
|
||||
}
|
||||
|
||||
func (d *fakeDialer) dialCount() int {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.dials
|
||||
}
|
||||
|
||||
var testHost = provider.KVMHost{Name: "nas1", Address: "nas1:22", User: "virt", Socket: defaultSocket, Pool: "default", Bridge: "br0"}
|
||||
|
||||
func noop(libvirtAPI) error { return nil }
|
||||
|
||||
func TestHostPool_call_reusesConnection(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
for range 3 {
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("call() error = %v", err)
|
||||
}
|
||||
}
|
||||
if d.dialCount() != 1 {
|
||||
t.Errorf("dials = %d, want 1", d.dialCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPool_call_redialsAfterDisconnect(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("call() error = %v", err)
|
||||
}
|
||||
d.clients[0].close()
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("second call() error = %v", err)
|
||||
}
|
||||
if d.dialCount() != 2 {
|
||||
t.Errorf("dials = %d, want 2 after the first connection dropped", d.dialCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPool_call_transportErrorDropsConnection(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
err := p.call(context.Background(), testHost, func(libvirtAPI) error { return io.ErrUnexpectedEOF })
|
||||
if !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
t.Fatalf("call() error = %v, want the transport error passed through", err)
|
||||
}
|
||||
if !d.clients[0].closed.Load() {
|
||||
t.Error("connection not closed after a transport error")
|
||||
}
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("call() after transport error = %v", err)
|
||||
}
|
||||
if d.dialCount() != 2 {
|
||||
t.Errorf("dials = %d, want 2", d.dialCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPool_call_libvirtErrorKeepsConnection(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
err := p.call(context.Background(), testHost, func(libvirtAPI) error { return lvErr(libvirt.ErrNoDomain) })
|
||||
if !hasCode(err, libvirt.ErrNoDomain) {
|
||||
t.Fatalf("call() error = %v, want the libvirt error passed through", err)
|
||||
}
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("call() error = %v", err)
|
||||
}
|
||||
if d.dialCount() != 1 {
|
||||
t.Errorf("dials = %d, want 1: a daemon-reported error must not cost the connection", d.dialCount())
|
||||
}
|
||||
if d.clients[0].closed.Load() {
|
||||
t.Error("connection closed after a libvirt error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPool_call_dialErrorPropagates(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{err: errors.New("ssh: handshake failed")}
|
||||
p := newHostPool(d.dial)
|
||||
err := p.call(context.Background(), testHost, noop)
|
||||
if err == nil || !strings.Contains(err.Error(), "nas1") || !strings.Contains(err.Error(), "handshake failed") {
|
||||
t.Errorf("call() error = %v, want the dial error naming the host", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPool_call_cancelAbandonsBlockedCall(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
released := make(chan struct{})
|
||||
go func() {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
err := p.call(ctx, testHost, func(libvirtAPI) error {
|
||||
// Simulates an RPC stuck on a hung hypervisor: it only returns once
|
||||
// the pool closes the connection out from under it.
|
||||
<-d.clients[0].done
|
||||
close(released)
|
||||
return io.EOF
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("call() error = %v, want context.Canceled", err)
|
||||
}
|
||||
select {
|
||||
case <-released:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("blocked call was not released by closing the connection")
|
||||
}
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Fatalf("call() after cancel = %v", err)
|
||||
}
|
||||
if d.dialCount() != 2 {
|
||||
t.Errorf("dials = %d, want 2", d.dialCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPool_call_concurrentCallersShareOneDial(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{delay: 30 * time.Millisecond}
|
||||
p := newHostPool(d.dial)
|
||||
var wg sync.WaitGroup
|
||||
for range 8 {
|
||||
wg.Go(func() {
|
||||
if err := p.call(context.Background(), testHost, noop); err != nil {
|
||||
t.Errorf("call() error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
if d.dialCount() != 1 {
|
||||
t.Errorf("dials = %d, want 1", d.dialCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPool_call_hostsAreIndependent(t *testing.T) {
|
||||
t.Parallel()
|
||||
d := &fakeDialer{}
|
||||
p := newHostPool(d.dial)
|
||||
other := testHost
|
||||
other.Name = "lab2"
|
||||
for _, h := range []provider.KVMHost{testHost, other, testHost, other} {
|
||||
if err := p.call(context.Background(), h, noop); err != nil {
|
||||
t.Fatalf("call(%s) error = %v", h.Name, err)
|
||||
}
|
||||
}
|
||||
if d.dialCount() != 2 {
|
||||
t.Errorf("dials = %d, want one per host", d.dialCount())
|
||||
}
|
||||
}
|
||||
|
||||
// --- SSH dial against an in-process server ---
|
||||
|
||||
func newKeyPair(t *testing.T) (ssh.Signer, []byte) {
|
||||
t.Helper()
|
||||
_, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
signer, err := ssh.NewSignerFromKey(priv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
block, err := ssh.MarshalPrivateKey(priv, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return signer, pem.EncodeToMemory(block)
|
||||
}
|
||||
|
||||
// testSSHServer accepts one public key and serves
|
||||
// direct-streamlocal@openssh.com channels by dialing the requested unix
|
||||
// path, which is exactly what OpenSSH does for ssh.Client.Dial("unix").
|
||||
type testSSHServer struct {
|
||||
addr string
|
||||
hostKey ssh.PublicKey
|
||||
|
||||
mu sync.Mutex
|
||||
paths []string
|
||||
}
|
||||
|
||||
func startSSHServer(t *testing.T, clientKey ssh.PublicKey, answerGlobalRequests bool) *testSSHServer {
|
||||
t.Helper()
|
||||
hostSigner, _ := newKeyPair(t)
|
||||
cfg := &ssh.ServerConfig{
|
||||
PublicKeyCallback: func(_ ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
|
||||
if bytes.Equal(key.Marshal(), clientKey.Marshal()) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, errors.New("unknown public key")
|
||||
},
|
||||
}
|
||||
cfg.AddHostKey(hostSigner)
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
s := &testSSHServer{addr: ln.Addr().String(), hostKey: hostSigner.PublicKey()}
|
||||
go func() {
|
||||
for {
|
||||
nc, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go s.serve(nc, cfg, answerGlobalRequests)
|
||||
}
|
||||
}()
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *testSSHServer) serve(nc net.Conn, cfg *ssh.ServerConfig, answerGlobalRequests bool) {
|
||||
_, chans, reqs, err := ssh.NewServerConn(nc, cfg)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if answerGlobalRequests {
|
||||
go ssh.DiscardRequests(reqs)
|
||||
} else {
|
||||
go func() {
|
||||
for range reqs { //nolint:revive // deliberately never replies, to stall keepalives
|
||||
}
|
||||
}()
|
||||
}
|
||||
for newCh := range chans {
|
||||
if newCh.ChannelType() != "direct-streamlocal@openssh.com" {
|
||||
_ = newCh.Reject(ssh.UnknownChannelType, "test server only forwards unix sockets")
|
||||
continue
|
||||
}
|
||||
var payload struct {
|
||||
Path string
|
||||
Reserved0 string
|
||||
Reserved1 uint32
|
||||
}
|
||||
if err := ssh.Unmarshal(newCh.ExtraData(), &payload); err != nil {
|
||||
_ = newCh.Reject(ssh.ConnectionFailed, err.Error())
|
||||
continue
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.paths = append(s.paths, payload.Path)
|
||||
s.mu.Unlock()
|
||||
target, err := net.Dial("unix", payload.Path)
|
||||
if err != nil {
|
||||
_ = newCh.Reject(ssh.ConnectionFailed, err.Error())
|
||||
continue
|
||||
}
|
||||
ch, chReqs, err := newCh.Accept()
|
||||
if err != nil {
|
||||
target.Close()
|
||||
continue
|
||||
}
|
||||
go ssh.DiscardRequests(chReqs)
|
||||
go func() { _, _ = io.Copy(ch, target); ch.Close() }()
|
||||
go func() { _, _ = io.Copy(target, ch); target.Close() }()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *testSSHServer) requestedPaths() []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]string(nil), s.paths...)
|
||||
}
|
||||
|
||||
// startEchoSocket serves a unix socket that echoes everything back, standing
|
||||
// in for libvirt-sock.
|
||||
func startEchoSocket(t *testing.T) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "libvirt-sock")
|
||||
ln, err := net.Listen("unix", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() { _, _ = io.Copy(c, c); c.Close() }()
|
||||
}
|
||||
}()
|
||||
return path
|
||||
}
|
||||
|
||||
type sshFixture struct {
|
||||
server *testSSHServer
|
||||
cfg provider.KVMConfig
|
||||
host provider.KVMHost
|
||||
keyFile string
|
||||
}
|
||||
|
||||
func newSSHFixture(t *testing.T, answerGlobalRequests bool) *sshFixture {
|
||||
t.Helper()
|
||||
clientSigner, clientPEM := newKeyPair(t)
|
||||
server := startSSHServer(t, clientSigner.PublicKey(), answerGlobalRequests)
|
||||
dir := t.TempDir()
|
||||
keyFile := filepath.Join(dir, "id_ed25519")
|
||||
if err := os.WriteFile(keyFile, clientPEM, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
khFile := filepath.Join(dir, "known_hosts")
|
||||
line := knownhosts.Line([]string{server.addr}, server.hostKey)
|
||||
if err := os.WriteFile(khFile, []byte(line+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &sshFixture{
|
||||
server: server,
|
||||
keyFile: keyFile,
|
||||
cfg: provider.KVMConfig{SSHKeyFile: keyFile, KnownHostsFile: khFile},
|
||||
host: provider.KVMHost{Name: "nas1", Address: server.addr, User: "virt", Socket: startEchoSocket(t)},
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHAuth_dialSSH_forwardsUnixSocket(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
client, err := auth.dialSSH(ctx, f.host)
|
||||
if err != nil {
|
||||
t.Fatalf("dialSSH() error = %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
conn, err := unixForward{client: client, path: f.host.Socket}.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("unixForward.Dial() error = %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := conn.Write([]byte("ping")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buf := make([]byte, 4)
|
||||
if _, err := io.ReadFull(conn, buf); err != nil || string(buf) != "ping" {
|
||||
t.Fatalf("echo through the forward = %q, %v; want ping", buf, err)
|
||||
}
|
||||
if paths := f.server.requestedPaths(); len(paths) != 1 || paths[0] != f.host.Socket {
|
||||
t.Errorf("server saw forward requests %v, want [%s]", paths, f.host.Socket)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHAuth_dialSSH_rejectsUnknownHostKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
impostor, _ := newKeyPair(t)
|
||||
line := knownhosts.Line([]string{f.server.addr}, impostor.PublicKey())
|
||||
if err := os.WriteFile(f.cfg.KnownHostsFile, []byte(line+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
client, err := auth.dialSSH(context.Background(), f.host)
|
||||
if err == nil {
|
||||
client.Close()
|
||||
t.Fatal("dialSSH() succeeded against a host whose key is not in known_hosts")
|
||||
}
|
||||
var keyErr *knownhosts.KeyError
|
||||
if !errors.As(err, &keyErr) {
|
||||
t.Errorf("dialSSH() error = %v, want a knownhosts.KeyError", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHAuth_dialSSH_insecureIgnoreHostKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
f.cfg.KnownHostsFile = ""
|
||||
f.cfg.InsecureIgnoreHostKey = true
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
client, err := auth.dialSSH(context.Background(), f.host)
|
||||
if err != nil {
|
||||
t.Fatalf("dialSSH() error = %v", err)
|
||||
}
|
||||
client.Close()
|
||||
}
|
||||
|
||||
func TestSSHAuth_dialSSH_rejectsWrongClientKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
_, otherPEM := newKeyPair(t)
|
||||
if err := os.WriteFile(f.keyFile, otherPEM, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
client, err := auth.dialSSH(context.Background(), f.host)
|
||||
if err == nil {
|
||||
client.Close()
|
||||
t.Fatal("dialSSH() succeeded with a key the server does not know")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "ssh handshake") {
|
||||
t.Errorf("dialSSH() error = %v, want an auth failure from the handshake", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHAuth_dialSSH_cancelledContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if client, err := auth.dialSSH(ctx, f.host); err == nil {
|
||||
client.Close()
|
||||
t.Fatal("dialSSH() succeeded with an already-cancelled context")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSSHAuth_missingFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, keyPEM := newKeyPair(t)
|
||||
keyFile := filepath.Join(t.TempDir(), "key")
|
||||
if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg provider.KVMConfig
|
||||
wantSub string
|
||||
}{
|
||||
{"key file missing", provider.KVMConfig{SSHKeyFile: "/nonexistent/key", InsecureIgnoreHostKey: true}, "sshKeyFile"},
|
||||
{"key file garbage", provider.KVMConfig{SSHKeyFile: keyFile + ".bad", InsecureIgnoreHostKey: true}, "sshKeyFile"},
|
||||
{"known_hosts missing", provider.KVMConfig{SSHKeyFile: keyFile, KnownHostsFile: "/nonexistent/kh"}, "knownHostsFile"},
|
||||
}
|
||||
if err := os.WriteFile(keyFile+".bad", []byte("not a key"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, err := newSSHAuth(tc.cfg)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantSub) {
|
||||
t.Errorf("newSSHAuth() error = %v, want one mentioning %s", err, tc.wantSub)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeepalive_closesClientWhenServerStalls(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, false)
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
client, err := auth.dialSSH(context.Background(), f.host)
|
||||
if err != nil {
|
||||
t.Fatalf("dialSSH() error = %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
stop := make(chan struct{})
|
||||
defer close(stop)
|
||||
go keepalive(client, 20*time.Millisecond, stop)
|
||||
|
||||
closed := make(chan struct{})
|
||||
go func() { _ = client.Wait(); close(closed) }()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("client still open after keepalives went unanswered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeepalive_keepsHealthyClientOpen(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newSSHFixture(t, true)
|
||||
auth, err := newSSHAuth(f.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newSSHAuth() error = %v", err)
|
||||
}
|
||||
client, err := auth.dialSSH(context.Background(), f.host)
|
||||
if err != nil {
|
||||
t.Fatalf("dialSSH() error = %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
stop := make(chan struct{})
|
||||
defer close(stop)
|
||||
go keepalive(client, 10*time.Millisecond, stop)
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
fwd := unixForward{client: client, path: f.host.Socket}
|
||||
conn, err := fwd.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("forward after several keepalives failed: %v", err)
|
||||
}
|
||||
conn.Close()
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
const guestAgentChannel = "org.qemu.guest_agent.0"
|
||||
|
||||
// domainParams is everything buildDomainXML needs, already resolved from the
|
||||
// config and placement so the builder stays a pure function.
|
||||
type domainParams struct {
|
||||
Name string
|
||||
UID string
|
||||
CreatedAt time.Time
|
||||
VCPUs int32
|
||||
MemoryMiB int64
|
||||
Pool string
|
||||
Bridge string
|
||||
DiskVol string
|
||||
SeedVol string
|
||||
}
|
||||
|
||||
// buildDomainXML renders a q35 KVM guest: host-passthrough CPU, a virtio
|
||||
// overlay disk and a SATA cdrom for the seed (both referenced as pool
|
||||
// volumes so no host paths are needed), one virtio NIC on the host bridge,
|
||||
// a serial console, the guest-agent channel, a virtio RNG, no graphics, and
|
||||
// the operator's metadata element.
|
||||
func buildDomainXML(p domainParams) (string, error) {
|
||||
d := domainXML{
|
||||
Type: "kvm",
|
||||
Name: p.Name,
|
||||
Metadata: domainMetadataXML{Instance: newInstanceMetadata(p.UID, p.CreatedAt)},
|
||||
Memory: sizeXML{Unit: "MiB", Value: p.MemoryMiB},
|
||||
VCPU: p.VCPUs,
|
||||
OS: osXML{
|
||||
Type: osTypeXML{Arch: "x86_64", Machine: "q35", Value: "hvm"},
|
||||
Boot: bootXML{Dev: "hd"},
|
||||
},
|
||||
Features: featuresXML{ACPI: &struct{}{}, APIC: &struct{}{}},
|
||||
CPU: cpuXML{Mode: "host-passthrough"},
|
||||
Clock: clockXML{Offset: "utc"},
|
||||
OnPoweroff: "destroy",
|
||||
OnReboot: "restart",
|
||||
OnCrash: "destroy",
|
||||
Devices: devicesXML{
|
||||
Disks: []diskXML{
|
||||
{
|
||||
Type: "volume",
|
||||
Device: "disk",
|
||||
Driver: driverXML{Name: "qemu", Type: formatQcow2},
|
||||
Source: diskSourceXML{Pool: p.Pool, Volume: p.DiskVol},
|
||||
Target: targetXML{Dev: "vda", Bus: "virtio"},
|
||||
},
|
||||
{
|
||||
Type: "volume",
|
||||
Device: "cdrom",
|
||||
Driver: driverXML{Name: "qemu", Type: formatRaw},
|
||||
Source: diskSourceXML{Pool: p.Pool, Volume: p.SeedVol},
|
||||
Target: targetXML{Dev: "sda", Bus: "sata"},
|
||||
ReadOnly: &struct{}{},
|
||||
},
|
||||
},
|
||||
Interfaces: []interfaceXML{{
|
||||
Type: "bridge",
|
||||
Source: ifaceSourceXML{Bridge: p.Bridge},
|
||||
Model: modelXML{Type: "virtio"},
|
||||
}},
|
||||
Serials: []serialXML{{Type: "pty", Target: portXML{Port: 0}}},
|
||||
Consoles: []consoleXML{{Type: "pty", Target: consoleTargetXML{Type: "serial", Port: 0}}},
|
||||
Channels: []channelXML{{
|
||||
Type: "unix",
|
||||
Target: channelTargetXML{Type: "virtio", Name: guestAgentChannel},
|
||||
}},
|
||||
RNG: rngXML{Model: "virtio", Backend: rngBackendXML{Model: "random", Value: "/dev/urandom"}},
|
||||
},
|
||||
}
|
||||
out, err := xml.Marshal(d)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshalling domain xml: %w", err)
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
type domainXML struct {
|
||||
XMLName xml.Name `xml:"domain"`
|
||||
Type string `xml:"type,attr"`
|
||||
Name string `xml:"name"`
|
||||
Metadata domainMetadataXML `xml:"metadata"`
|
||||
Memory sizeXML `xml:"memory"`
|
||||
VCPU int32 `xml:"vcpu"`
|
||||
OS osXML `xml:"os"`
|
||||
Features featuresXML `xml:"features"`
|
||||
CPU cpuXML `xml:"cpu"`
|
||||
Clock clockXML `xml:"clock"`
|
||||
OnPoweroff string `xml:"on_poweroff"`
|
||||
OnReboot string `xml:"on_reboot"`
|
||||
OnCrash string `xml:"on_crash"`
|
||||
Devices devicesXML `xml:"devices"`
|
||||
}
|
||||
|
||||
type domainMetadataXML struct {
|
||||
Instance instanceMetadata
|
||||
}
|
||||
|
||||
type sizeXML struct {
|
||||
Unit string `xml:"unit,attr"`
|
||||
Value int64 `xml:",chardata"`
|
||||
}
|
||||
|
||||
type osXML struct {
|
||||
Type osTypeXML `xml:"type"`
|
||||
Boot bootXML `xml:"boot"`
|
||||
}
|
||||
|
||||
type osTypeXML struct {
|
||||
Arch string `xml:"arch,attr"`
|
||||
Machine string `xml:"machine,attr"`
|
||||
Value string `xml:",chardata"`
|
||||
}
|
||||
|
||||
type bootXML struct {
|
||||
Dev string `xml:"dev,attr"`
|
||||
}
|
||||
|
||||
type featuresXML struct {
|
||||
ACPI *struct{} `xml:"acpi"`
|
||||
APIC *struct{} `xml:"apic"`
|
||||
}
|
||||
|
||||
type cpuXML struct {
|
||||
Mode string `xml:"mode,attr"`
|
||||
}
|
||||
|
||||
type clockXML struct {
|
||||
Offset string `xml:"offset,attr"`
|
||||
}
|
||||
|
||||
type devicesXML struct {
|
||||
Disks []diskXML `xml:"disk"`
|
||||
Interfaces []interfaceXML `xml:"interface"`
|
||||
Serials []serialXML `xml:"serial"`
|
||||
Consoles []consoleXML `xml:"console"`
|
||||
Channels []channelXML `xml:"channel"`
|
||||
RNG rngXML `xml:"rng"`
|
||||
}
|
||||
|
||||
type diskXML struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Device string `xml:"device,attr"`
|
||||
Driver driverXML `xml:"driver"`
|
||||
Source diskSourceXML `xml:"source"`
|
||||
Target targetXML `xml:"target"`
|
||||
ReadOnly *struct{} `xml:"readonly"`
|
||||
}
|
||||
|
||||
type driverXML struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Type string `xml:"type,attr"`
|
||||
}
|
||||
|
||||
type diskSourceXML struct {
|
||||
Pool string `xml:"pool,attr"`
|
||||
Volume string `xml:"volume,attr"`
|
||||
}
|
||||
|
||||
type targetXML struct {
|
||||
Dev string `xml:"dev,attr"`
|
||||
Bus string `xml:"bus,attr"`
|
||||
}
|
||||
|
||||
type interfaceXML struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Source ifaceSourceXML `xml:"source"`
|
||||
Model modelXML `xml:"model"`
|
||||
}
|
||||
|
||||
type ifaceSourceXML struct {
|
||||
Bridge string `xml:"bridge,attr"`
|
||||
}
|
||||
|
||||
type modelXML struct {
|
||||
Type string `xml:"type,attr"`
|
||||
}
|
||||
|
||||
type serialXML struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Target portXML `xml:"target"`
|
||||
}
|
||||
|
||||
type portXML struct {
|
||||
Port int `xml:"port,attr"`
|
||||
}
|
||||
|
||||
type consoleXML struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Target consoleTargetXML `xml:"target"`
|
||||
}
|
||||
|
||||
type consoleTargetXML struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Port int `xml:"port,attr"`
|
||||
}
|
||||
|
||||
type channelXML struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Target channelTargetXML `xml:"target"`
|
||||
}
|
||||
|
||||
type channelTargetXML struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Name string `xml:"name,attr"`
|
||||
}
|
||||
|
||||
type rngXML struct {
|
||||
Model string `xml:"model,attr"`
|
||||
Backend rngBackendXML `xml:"backend"`
|
||||
}
|
||||
|
||||
type rngBackendXML struct {
|
||||
Model string `xml:"model,attr"`
|
||||
Value string `xml:",chardata"`
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testDomainParams() domainParams {
|
||||
return domainParams{
|
||||
Name: "proxy-abc123def456ghij",
|
||||
UID: "11111111-2222-3333-4444-555555555555",
|
||||
CreatedAt: time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC),
|
||||
VCPUs: 2,
|
||||
MemoryMiB: 2048,
|
||||
Pool: "default",
|
||||
Bridge: "br0",
|
||||
DiskVol: "proxy-abc123def456ghij.qcow2",
|
||||
SeedVol: "proxy-abc123def456ghij-seed.iso",
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDomainXML_fieldByField(t *testing.T) {
|
||||
t.Parallel()
|
||||
raw, err := buildDomainXML(testDomainParams())
|
||||
if err != nil {
|
||||
t.Fatalf("buildDomainXML() error = %v", err)
|
||||
}
|
||||
var d domainXML
|
||||
if err := xml.Unmarshal([]byte(raw), &d); err != nil {
|
||||
t.Fatalf("output is not well-formed XML: %v\n%s", err, raw)
|
||||
}
|
||||
|
||||
if d.Type != "kvm" || d.Name != "proxy-abc123def456ghij" {
|
||||
t.Errorf("type/name = %s/%s", d.Type, d.Name)
|
||||
}
|
||||
if d.VCPU != 2 || d.Memory.Value != 2048 || d.Memory.Unit != "MiB" {
|
||||
t.Errorf("vcpu/memory = %d/%d %s, want 2/2048 MiB", d.VCPU, d.Memory.Value, d.Memory.Unit)
|
||||
}
|
||||
if d.OS.Type.Value != "hvm" || d.OS.Type.Arch != "x86_64" || d.OS.Type.Machine != "q35" || d.OS.Boot.Dev != "hd" {
|
||||
t.Errorf("os = %+v", d.OS)
|
||||
}
|
||||
if d.Features.ACPI == nil || d.Features.APIC == nil {
|
||||
t.Errorf("features = %+v, want acpi and apic", d.Features)
|
||||
}
|
||||
if d.CPU.Mode != "host-passthrough" {
|
||||
t.Errorf("cpu mode = %q", d.CPU.Mode)
|
||||
}
|
||||
if d.OnPoweroff != "destroy" || d.OnReboot != "restart" || d.OnCrash != "destroy" {
|
||||
t.Errorf("lifecycle = %s/%s/%s", d.OnPoweroff, d.OnReboot, d.OnCrash)
|
||||
}
|
||||
|
||||
m := d.Metadata.Instance
|
||||
if !m.managed() || m.UID != "11111111-2222-3333-4444-555555555555" || m.CreatedAt != "2026-09-04T12:00:00Z" {
|
||||
t.Errorf("metadata = %+v", m)
|
||||
}
|
||||
if !strings.Contains(raw, `xmlns="`+metadataURI+`"`) {
|
||||
t.Errorf("metadata element is not namespaced; libvirt rejects that:\n%s", raw)
|
||||
}
|
||||
|
||||
if len(d.Devices.Disks) != 2 {
|
||||
t.Fatalf("disks = %d, want 2", len(d.Devices.Disks))
|
||||
}
|
||||
disk, seed := d.Devices.Disks[0], d.Devices.Disks[1]
|
||||
if disk.Type != "volume" || disk.Device != "disk" || disk.Driver.Type != "qcow2" ||
|
||||
disk.Source.Pool != "default" || disk.Source.Volume != "proxy-abc123def456ghij.qcow2" ||
|
||||
disk.Target.Dev != "vda" || disk.Target.Bus != "virtio" || disk.ReadOnly != nil {
|
||||
t.Errorf("overlay disk = %+v", disk)
|
||||
}
|
||||
if seed.Type != "volume" || seed.Device != "cdrom" || seed.Driver.Type != "raw" ||
|
||||
seed.Source.Pool != "default" || seed.Source.Volume != "proxy-abc123def456ghij-seed.iso" ||
|
||||
seed.Target.Bus != "sata" || seed.ReadOnly == nil {
|
||||
t.Errorf("seed cdrom = %+v", seed)
|
||||
}
|
||||
|
||||
if len(d.Devices.Interfaces) != 1 {
|
||||
t.Fatalf("interfaces = %d, want 1", len(d.Devices.Interfaces))
|
||||
}
|
||||
if nic := d.Devices.Interfaces[0]; nic.Type != "bridge" || nic.Source.Bridge != "br0" || nic.Model.Type != "virtio" {
|
||||
t.Errorf("interface = %+v", nic)
|
||||
}
|
||||
|
||||
if len(d.Devices.Channels) != 1 || d.Devices.Channels[0].Target.Name != guestAgentChannel || d.Devices.Channels[0].Target.Type != "virtio" {
|
||||
t.Errorf("channels = %+v, want one virtio %s channel", d.Devices.Channels, guestAgentChannel)
|
||||
}
|
||||
if len(d.Devices.Serials) != 1 || len(d.Devices.Consoles) != 1 || d.Devices.Consoles[0].Target.Type != "serial" {
|
||||
t.Errorf("serial/console = %+v/%+v", d.Devices.Serials, d.Devices.Consoles)
|
||||
}
|
||||
if d.Devices.RNG.Model != "virtio" || d.Devices.RNG.Backend.Value != "/dev/urandom" {
|
||||
t.Errorf("rng = %+v", d.Devices.RNG)
|
||||
}
|
||||
if strings.Contains(raw, "<graphics") {
|
||||
t.Errorf("domain has a graphics device; headless VMs must not:\n%s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDomainXML_escapesUserValues(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := testDomainParams()
|
||||
p.Bridge = `br0" onload="x`
|
||||
raw, err := buildDomainXML(p)
|
||||
if err != nil {
|
||||
t.Fatalf("buildDomainXML() error = %v", err)
|
||||
}
|
||||
var d domainXML
|
||||
if err := xml.Unmarshal([]byte(raw), &d); err != nil {
|
||||
t.Fatalf("output is not well-formed XML: %v", err)
|
||||
}
|
||||
if d.Devices.Interfaces[0].Source.Bridge != p.Bridge {
|
||||
t.Errorf("bridge = %q, want the raw value back", d.Devices.Interfaces[0].Source.Bridge)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
"github.com/go-logr/logr"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// classify maps an error onto the provider taxonomy. Errors the package
|
||||
// already tagged with a sentinel (image download failures, placement
|
||||
// errors) keep their class. For libvirt errors: missing domain or volume →
|
||||
// NotFound; anything the host will keep rejecting (bad XML, unsupported
|
||||
// config, denied access, missing pool) → Permanent; every other libvirt
|
||||
// code, plus SSH/transport failures and cancellation, → Transient, because
|
||||
// retrying is safer than latching Failed on an error nobody taught this
|
||||
// function to recognise. libvirt exposes no quota signal, so nothing maps
|
||||
// to QuotaExceeded.
|
||||
func classify(err error) error {
|
||||
for _, class := range []error{provider.ErrNotFound, provider.ErrQuotaExceeded, provider.ErrPermanent, provider.ErrTransient} {
|
||||
if errors.Is(err, class) {
|
||||
return class
|
||||
}
|
||||
}
|
||||
var lerr libvirt.Error
|
||||
if !errors.As(err, &lerr) {
|
||||
return provider.ErrTransient
|
||||
}
|
||||
switch libvirt.ErrorNumber(lerr.Code) {
|
||||
case libvirt.ErrNoDomain, libvirt.ErrNoStorageVol:
|
||||
return provider.ErrNotFound
|
||||
case libvirt.ErrNoStoragePool,
|
||||
libvirt.ErrXMLError, libvirt.ErrXMLDetail, libvirt.ErrXMLInvalidSchema,
|
||||
libvirt.ErrConfigUnsupported, libvirt.ErrNoSupport,
|
||||
libvirt.ErrOperationUnsupported, libvirt.ErrArgumentUnsupported,
|
||||
libvirt.ErrInvalidArg,
|
||||
libvirt.ErrAuthFailed, libvirt.ErrAccessDenied, libvirt.ErrOperationDenied:
|
||||
return provider.ErrPermanent
|
||||
default:
|
||||
return provider.ErrTransient
|
||||
}
|
||||
}
|
||||
|
||||
// isLibvirtError reports whether err came back from the libvirt daemon (as
|
||||
// opposed to the SSH or RPC transport), which is what decides whether the
|
||||
// cached connection is still trustworthy.
|
||||
func isLibvirtError(err error) bool {
|
||||
var lerr libvirt.Error
|
||||
return errors.As(err, &lerr)
|
||||
}
|
||||
|
||||
func hasCode(err error, code libvirt.ErrorNumber) bool {
|
||||
var lerr libvirt.Error
|
||||
return errors.As(err, &lerr) && libvirt.ErrorNumber(lerr.Code) == code
|
||||
}
|
||||
|
||||
// logAPIError records the raw libvirt code and message at V(1); classify
|
||||
// collapses them onto the coarser taxonomy, so this is the only place the
|
||||
// original detail survives.
|
||||
func logAPIError(log logr.Logger, op string, err error) {
|
||||
if !log.V(1).Enabled() {
|
||||
return
|
||||
}
|
||||
kv := []any{"op", op, "error", err.Error()}
|
||||
var lerr libvirt.Error
|
||||
if errors.As(err, &lerr) {
|
||||
kv = append(kv, "libvirtCode", lerr.Code, "libvirtMessage", lerr.Message)
|
||||
}
|
||||
log.V(1).Info("libvirt call failed", kv...)
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
func lvErr(code libvirt.ErrorNumber) error {
|
||||
return libvirt.Error{Code: uint32(code), Message: code.String()}
|
||||
}
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want error
|
||||
}{
|
||||
{"no domain", lvErr(libvirt.ErrNoDomain), provider.ErrNotFound},
|
||||
{"no storage vol", lvErr(libvirt.ErrNoStorageVol), provider.ErrNotFound},
|
||||
{"no storage pool", lvErr(libvirt.ErrNoStoragePool), provider.ErrPermanent},
|
||||
{"xml error", lvErr(libvirt.ErrXMLError), provider.ErrPermanent},
|
||||
{"xml detail", lvErr(libvirt.ErrXMLDetail), provider.ErrPermanent},
|
||||
{"config unsupported", lvErr(libvirt.ErrConfigUnsupported), provider.ErrPermanent},
|
||||
{"no support", lvErr(libvirt.ErrNoSupport), provider.ErrPermanent},
|
||||
{"invalid arg", lvErr(libvirt.ErrInvalidArg), provider.ErrPermanent},
|
||||
{"auth failed", lvErr(libvirt.ErrAuthFailed), provider.ErrPermanent},
|
||||
{"access denied", lvErr(libvirt.ErrAccessDenied), provider.ErrPermanent},
|
||||
{"operation invalid", lvErr(libvirt.ErrOperationInvalid), provider.ErrTransient},
|
||||
{"operation failed", lvErr(libvirt.ErrOperationFailed), provider.ErrTransient},
|
||||
{"internal error", lvErr(libvirt.ErrInternalError), provider.ErrTransient},
|
||||
{"storage vol exists", lvErr(libvirt.ErrStorageVolExist), provider.ErrTransient},
|
||||
{"wrapped libvirt error", fmt.Errorf("defining: %w", lvErr(libvirt.ErrNoDomain)), provider.ErrNotFound},
|
||||
{"pre-tagged permanent", fmt.Errorf("%w: image returned HTTP 404", provider.ErrPermanent), provider.ErrPermanent},
|
||||
{"pre-tagged transient wrapping libvirt not-found", fmt.Errorf("%w: %w", provider.ErrTransient, lvErr(libvirt.ErrNoDomain)), provider.ErrTransient},
|
||||
{"transport eof", io.EOF, provider.ErrTransient},
|
||||
{"context cancelled", context.Canceled, provider.ErrTransient},
|
||||
{"plain error", errors.New("ssh: handshake failed"), provider.ErrTransient},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := classify(tc.err); !errors.Is(got, tc.want) {
|
||||
t.Errorf("classify(%v) = %v, want %v", tc.err, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasCode(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := fmt.Errorf("lookup: %w", lvErr(libvirt.ErrNoDomain))
|
||||
if !hasCode(err, libvirt.ErrNoDomain) {
|
||||
t.Error("hasCode(ErrNoDomain) = false through a wrap")
|
||||
}
|
||||
if hasCode(err, libvirt.ErrNoStorageVol) {
|
||||
t.Error("hasCode(ErrNoStorageVol) = true for an ErrNoDomain error")
|
||||
}
|
||||
if hasCode(io.EOF, libvirt.ErrNoDomain) || isLibvirtError(io.EOF) {
|
||||
t.Error("io.EOF classified as a libvirt error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,408 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"sync"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
)
|
||||
|
||||
// fakeAPI is an in-memory libvirtAPI: pools of volumes and a set of
|
||||
// domains, with per-method error injection. It models just enough libvirt
|
||||
// semantics for the provider's decisions to be observable — ErrStorageVolExist
|
||||
// on duplicate create, ErrNoDomain/ErrNoStorageVol on missing lookups,
|
||||
// ErrOperationInvalid on destroying a stopped domain, and so on.
|
||||
type fakeAPI struct {
|
||||
mu sync.Mutex
|
||||
poolPath string
|
||||
pools map[string]map[string]*fakeVol
|
||||
domains map[string]*fakeDomain
|
||||
calls []string
|
||||
fail map[string]error
|
||||
failOnce map[string]error
|
||||
// uploadHook, when set, sees every StorageVolUpload before the fake
|
||||
// reads the stream; it can return an error to simulate a broken upload.
|
||||
uploadHook func(vol string, r io.Reader) error
|
||||
}
|
||||
|
||||
type fakeVol struct {
|
||||
name string
|
||||
format string
|
||||
capacity int64
|
||||
backing string
|
||||
data []byte
|
||||
}
|
||||
|
||||
type fakeDomain struct {
|
||||
name string
|
||||
xml string
|
||||
metadata string
|
||||
running bool
|
||||
// addrs is what DomainInterfaceAddresses returns per source.
|
||||
addrs map[uint32][]libvirt.DomainInterface
|
||||
}
|
||||
|
||||
func newFakeAPI(pools ...string) *fakeAPI {
|
||||
f := &fakeAPI{
|
||||
poolPath: "/var/lib/libvirt/images",
|
||||
pools: map[string]map[string]*fakeVol{},
|
||||
domains: map[string]*fakeDomain{},
|
||||
fail: map[string]error{},
|
||||
failOnce: map[string]error{},
|
||||
}
|
||||
for _, p := range pools {
|
||||
f.pools[p] = map[string]*fakeVol{}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// enter records the call and returns any injected error for method.
|
||||
func (f *fakeAPI) enter(method string) error {
|
||||
f.calls = append(f.calls, method)
|
||||
if err, ok := f.failOnce[method]; ok {
|
||||
delete(f.failOnce, method)
|
||||
return err
|
||||
}
|
||||
return f.fail[method]
|
||||
}
|
||||
|
||||
func (f *fakeAPI) callCount(method string) int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
n := 0
|
||||
for _, c := range f.calls {
|
||||
if c == method {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (f *fakeAPI) hasCall(method string) bool { return f.callCount(method) > 0 }
|
||||
|
||||
func (f *fakeAPI) vol(pool, name string) *fakeVol {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.pools[pool][name]
|
||||
}
|
||||
|
||||
func (f *fakeAPI) putVol(pool, name, format string, data []byte) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.pools[pool][name] = &fakeVol{name: name, format: format, capacity: int64(len(data)), data: data}
|
||||
}
|
||||
|
||||
func (f *fakeAPI) volNames(pool string) []string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []string
|
||||
for n := range f.pools[pool] {
|
||||
out = append(out, n)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fakeAPI) domain(name string) *fakeDomain {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.domains[name]
|
||||
}
|
||||
|
||||
func (f *fakeAPI) putDomain(d *fakeDomain) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.domains[d.name] = d
|
||||
}
|
||||
|
||||
func lvError(code libvirt.ErrorNumber, format string, args ...any) error {
|
||||
return libvirt.Error{Code: uint32(code), Message: fmt.Sprintf(format, args...)}
|
||||
}
|
||||
|
||||
// --- domains ---
|
||||
|
||||
func (f *fakeAPI) DomainLookupByName(name string) (libvirt.Domain, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("DomainLookupByName"); err != nil {
|
||||
return libvirt.Domain{}, err
|
||||
}
|
||||
if _, ok := f.domains[name]; !ok {
|
||||
return libvirt.Domain{}, lvError(libvirt.ErrNoDomain, "Domain not found: no domain with matching name '%s'", name)
|
||||
}
|
||||
return libvirt.Domain{Name: name}, nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) DomainDefineXML(raw string) (libvirt.Domain, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("DomainDefineXML"); err != nil {
|
||||
return libvirt.Domain{}, err
|
||||
}
|
||||
var d domainXML
|
||||
if err := xml.Unmarshal([]byte(raw), &d); err != nil {
|
||||
return libvirt.Domain{}, lvError(libvirt.ErrXMLError, "%v", err)
|
||||
}
|
||||
meta, err := xml.Marshal(d.Metadata.Instance)
|
||||
if err != nil {
|
||||
return libvirt.Domain{}, lvError(libvirt.ErrXMLError, "%v", err)
|
||||
}
|
||||
existing := f.domains[d.Name]
|
||||
nd := &fakeDomain{name: d.Name, xml: raw, metadata: string(meta)}
|
||||
if existing != nil {
|
||||
nd.running = existing.running
|
||||
nd.addrs = existing.addrs
|
||||
}
|
||||
f.domains[d.Name] = nd
|
||||
return libvirt.Domain{Name: d.Name}, nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) DomainCreate(d libvirt.Domain) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("DomainCreate"); err != nil {
|
||||
return err
|
||||
}
|
||||
dom, ok := f.domains[d.Name]
|
||||
if !ok {
|
||||
return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
|
||||
}
|
||||
if dom.running {
|
||||
return lvError(libvirt.ErrOperationInvalid, "domain is already running")
|
||||
}
|
||||
dom.running = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) DomainDestroy(d libvirt.Domain) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("DomainDestroy"); err != nil {
|
||||
return err
|
||||
}
|
||||
dom, ok := f.domains[d.Name]
|
||||
if !ok {
|
||||
return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
|
||||
}
|
||||
if !dom.running {
|
||||
return lvError(libvirt.ErrOperationInvalid, "domain is not running")
|
||||
}
|
||||
dom.running = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) DomainUndefineFlags(d libvirt.Domain, _ libvirt.DomainUndefineFlagsValues) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("DomainUndefineFlags"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, ok := f.domains[d.Name]; !ok {
|
||||
return lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
|
||||
}
|
||||
delete(f.domains, d.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) DomainGetState(d libvirt.Domain, _ uint32) (int32, int32, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("DomainGetState"); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
dom, ok := f.domains[d.Name]
|
||||
if !ok {
|
||||
return 0, 0, lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
|
||||
}
|
||||
if dom.running {
|
||||
return int32(libvirt.DomainRunning), 0, nil
|
||||
}
|
||||
return int32(libvirt.DomainShutoff), 0, nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) DomainGetMetadata(d libvirt.Domain, typ int32, uri libvirt.OptString, _ libvirt.DomainModificationImpact) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("DomainGetMetadata"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
dom, ok := f.domains[d.Name]
|
||||
if !ok {
|
||||
return "", lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
|
||||
}
|
||||
if typ != int32(libvirt.DomainMetadataElement) || len(uri) != 1 || uri[0] != metadataURI || dom.metadata == "" {
|
||||
return "", lvError(libvirt.ErrNoDomainMetadata, "Requested metadata element is not present")
|
||||
}
|
||||
return dom.metadata, nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) DomainInterfaceAddresses(d libvirt.Domain, source uint32, _ uint32) ([]libvirt.DomainInterface, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("DomainInterfaceAddresses"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dom, ok := f.domains[d.Name]
|
||||
if !ok {
|
||||
return nil, lvError(libvirt.ErrNoDomain, "no domain %s", d.Name)
|
||||
}
|
||||
if !dom.running {
|
||||
return nil, lvError(libvirt.ErrOperationInvalid, "domain is not running")
|
||||
}
|
||||
if source == uint32(libvirt.DomainInterfaceAddressesSrcAgent) && dom.addrs[source] == nil {
|
||||
return nil, lvError(libvirt.ErrAgentUnresponsive, "Guest agent is not responding")
|
||||
}
|
||||
return dom.addrs[source], nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) ConnectListAllDomains(_ int32, _ libvirt.ConnectListAllDomainsFlags) ([]libvirt.Domain, uint32, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("ConnectListAllDomains"); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
names := make([]string, 0, len(f.domains))
|
||||
for n := range f.domains {
|
||||
names = append(names, n)
|
||||
}
|
||||
slices.Sort(names)
|
||||
out := make([]libvirt.Domain, 0, len(names))
|
||||
for _, n := range names {
|
||||
out = append(out, libvirt.Domain{Name: n})
|
||||
}
|
||||
return out, uint32(len(out)), nil
|
||||
}
|
||||
|
||||
// --- storage ---
|
||||
|
||||
func (f *fakeAPI) StoragePoolLookupByName(name string) (libvirt.StoragePool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("StoragePoolLookupByName"); err != nil {
|
||||
return libvirt.StoragePool{}, err
|
||||
}
|
||||
if _, ok := f.pools[name]; !ok {
|
||||
return libvirt.StoragePool{}, lvError(libvirt.ErrNoStoragePool, "Storage pool not found: no storage pool with matching name '%s'", name)
|
||||
}
|
||||
return libvirt.StoragePool{Name: name}, nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) StoragePoolRefresh(pool libvirt.StoragePool, _ uint32) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("StoragePoolRefresh"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, ok := f.pools[pool.Name]; !ok {
|
||||
return lvError(libvirt.ErrNoStoragePool, "no pool %s", pool.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) StorageVolLookupByName(pool libvirt.StoragePool, name string) (libvirt.StorageVol, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("StorageVolLookupByName"); err != nil {
|
||||
return libvirt.StorageVol{}, err
|
||||
}
|
||||
if _, ok := f.pools[pool.Name][name]; !ok {
|
||||
return libvirt.StorageVol{}, lvError(libvirt.ErrNoStorageVol, "Storage volume not found: no storage vol with matching name '%s'", name)
|
||||
}
|
||||
return libvirt.StorageVol{Pool: pool.Name, Name: name, Key: f.poolPath + "/" + name}, nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) StorageVolGetPath(vol libvirt.StorageVol) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("StorageVolGetPath"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, ok := f.pools[vol.Pool][vol.Name]; !ok {
|
||||
return "", lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
|
||||
}
|
||||
return f.poolPath + "/" + vol.Name, nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) StorageVolCreateXML(pool libvirt.StoragePool, raw string, _ libvirt.StorageVolCreateFlags) (libvirt.StorageVol, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("StorageVolCreateXML"); err != nil {
|
||||
return libvirt.StorageVol{}, err
|
||||
}
|
||||
vols, ok := f.pools[pool.Name]
|
||||
if !ok {
|
||||
return libvirt.StorageVol{}, lvError(libvirt.ErrNoStoragePool, "no pool %s", pool.Name)
|
||||
}
|
||||
var v volumeXML
|
||||
if err := xml.Unmarshal([]byte(raw), &v); err != nil {
|
||||
return libvirt.StorageVol{}, lvError(libvirt.ErrXMLError, "%v", err)
|
||||
}
|
||||
if _, exists := vols[v.Name]; exists {
|
||||
return libvirt.StorageVol{}, lvError(libvirt.ErrStorageVolExist, "storage volume '%s' exists already", v.Name)
|
||||
}
|
||||
fv := &fakeVol{name: v.Name, format: v.Target.Format.Type, capacity: v.Capacity.Value}
|
||||
if v.BackingStore != nil {
|
||||
fv.backing = v.BackingStore.Path
|
||||
}
|
||||
vols[v.Name] = fv
|
||||
return libvirt.StorageVol{Pool: pool.Name, Name: v.Name, Key: f.poolPath + "/" + v.Name}, nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) StorageVolUpload(vol libvirt.StorageVol, r io.Reader, _, _ uint64, _ libvirt.StorageVolUploadFlags) error {
|
||||
f.mu.Lock()
|
||||
if err := f.enter("StorageVolUpload"); err != nil {
|
||||
f.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
fv, ok := f.pools[vol.Pool][vol.Name]
|
||||
hook := f.uploadHook
|
||||
f.mu.Unlock()
|
||||
if !ok {
|
||||
return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
|
||||
}
|
||||
if hook != nil {
|
||||
if err := hook(vol.Name, r); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Read outside the lock: the stream may be a slow HTTP body.
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading upload stream: %w", err)
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
fv.data = data
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeAPI) StorageVolDownload(vol libvirt.StorageVol, w io.Writer, _, _ uint64, _ libvirt.StorageVolDownloadFlags) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("StorageVolDownload"); err != nil {
|
||||
return err
|
||||
}
|
||||
fv, ok := f.pools[vol.Pool][vol.Name]
|
||||
if !ok {
|
||||
return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
|
||||
}
|
||||
_, err := io.Copy(w, bytes.NewReader(fv.data))
|
||||
return err
|
||||
}
|
||||
|
||||
func (f *fakeAPI) StorageVolDelete(vol libvirt.StorageVol, _ libvirt.StorageVolDeleteFlags) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.enter("StorageVolDelete"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, ok := f.pools[vol.Pool][vol.Name]; !ok {
|
||||
return lvError(libvirt.ErrNoStorageVol, "no vol %s", vol.Name)
|
||||
}
|
||||
delete(f.pools[vol.Pool], vol.Name)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The providerID carries the host, so Get/Delete stay correct after a
|
||||
// placement.zone edit mid-replacement — re-reading the spec then would look
|
||||
// at the wrong hypervisor.
|
||||
func formatProviderID(host, name string) string {
|
||||
return fmt.Sprintf("hosts/%s/domains/%s", host, name)
|
||||
}
|
||||
|
||||
func parseProviderID(id string) (host, name string, err error) {
|
||||
parts := strings.Split(id, "/")
|
||||
if len(parts) != 4 || parts[0] != "hosts" || parts[2] != "domains" || parts[1] == "" || parts[3] == "" {
|
||||
return "", "", fmt.Errorf("malformed kvm providerID %q, want hosts/<host>/domains/<name>", id)
|
||||
}
|
||||
return parts[1], parts[3], nil
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package kvm
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseProviderID_roundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
id := formatProviderID("nas1", "proxy-abc123def456ghij")
|
||||
if id != "hosts/nas1/domains/proxy-abc123def456ghij" {
|
||||
t.Fatalf("formatProviderID() = %q", id)
|
||||
}
|
||||
host, name, err := parseProviderID(id)
|
||||
if err != nil {
|
||||
t.Fatalf("parseProviderID(%q) error = %v", id, err)
|
||||
}
|
||||
if host != "nas1" || name != "proxy-abc123def456ghij" {
|
||||
t.Errorf("parseProviderID(%q) = %q, %q", id, host, name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseProviderID_malformed(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
id string
|
||||
}{
|
||||
{"empty", ""},
|
||||
{"gcp style", "zones/europe-west1-b/instances/proxy-x"},
|
||||
{"missing name", "hosts/nas1/domains/"},
|
||||
{"missing host", "hosts//domains/proxy-x"},
|
||||
{"too many segments", "hosts/nas1/domains/proxy-x/extra"},
|
||||
{"bare name", "proxy-x"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if _, _, err := parseProviderID(tc.id); err == nil {
|
||||
t.Errorf("parseProviderID(%q) error = nil, want error", tc.id)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
"github.com/go-logr/logr"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// imageSource is a resolved placement.image: either an entry from the
|
||||
// provider's images map (Key set) or a URL given directly.
|
||||
type imageSource struct {
|
||||
Key string
|
||||
URL string
|
||||
SHA256 string
|
||||
Format string
|
||||
}
|
||||
|
||||
// resolveImage maps placement.image to a source. Anything that is neither a
|
||||
// configured key nor an http(s) URL is a spec error the reconciler should
|
||||
// surface, not retry.
|
||||
func resolveImage(cfg provider.KVMConfig, image string) (imageSource, error) {
|
||||
if img, ok := cfg.Images[image]; ok {
|
||||
return imageSource{Key: image, URL: img.URL, SHA256: img.SHA256, Format: img.Format}, nil
|
||||
}
|
||||
u, err := url.Parse(image)
|
||||
if err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != "" {
|
||||
return imageSource{URL: image, Format: formatQcow2}, nil
|
||||
}
|
||||
return imageSource{}, fmt.Errorf("%w: placement.image %q is neither a key in kvm.images nor an http(s) URL",
|
||||
provider.ErrPermanent, image)
|
||||
}
|
||||
|
||||
// volumeName is the cached base image's name in the pool: readable for
|
||||
// configured keys, a URL hash for ad-hoc URLs.
|
||||
func (s imageSource) volumeName() string {
|
||||
if s.Key != "" {
|
||||
return "base-" + s.Key + "." + s.Format
|
||||
}
|
||||
sum := sha256.Sum256([]byte(s.URL))
|
||||
return "base-" + hex.EncodeToString(sum[:8]) + "." + s.Format
|
||||
}
|
||||
|
||||
// markerName is the sidecar volume holding the sha256 of the verified
|
||||
// upload. Its presence is what says "the base volume is complete".
|
||||
func (s imageSource) markerName() string {
|
||||
return s.volumeName() + ".sha256"
|
||||
}
|
||||
|
||||
// imageCache fetches base images into a host's pool on first use. It keeps
|
||||
// no state about what is cached — the pool is the source of truth, checked
|
||||
// on every call — only a per-(host, image) lock so concurrent Creates for
|
||||
// the same image on the same host result in one download.
|
||||
type imageCache struct {
|
||||
http *http.Client
|
||||
|
||||
mu sync.Mutex
|
||||
locks map[string]*sync.Mutex
|
||||
}
|
||||
|
||||
func newImageCache(client *http.Client) *imageCache {
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
return &imageCache{http: client, locks: make(map[string]*sync.Mutex)}
|
||||
}
|
||||
|
||||
func (c *imageCache) lockFor(key string) *sync.Mutex {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
m, ok := c.locks[key]
|
||||
if !ok {
|
||||
m = &sync.Mutex{}
|
||||
c.locks[key] = m
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// ensure makes src present in pool and returns the base volume's path on
|
||||
// the host. The image is downloaded synchronously through the libvirt
|
||||
// stream when the pool does not already hold a verified copy.
|
||||
func (c *imageCache) ensure(ctx context.Context, log logr.Logger, api libvirtAPI, hostName string, pool libvirt.StoragePool, src imageSource) (string, error) {
|
||||
m := c.lockFor(hostName + "/" + src.volumeName())
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
base, ready, err := c.lookup(api, pool, src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ready {
|
||||
log.V(1).Info("base image already cached", "host", hostName, "volume", src.volumeName())
|
||||
return api.StorageVolGetPath(base)
|
||||
}
|
||||
if err := deleteIfExists(api, pool, src.markerName()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := deleteIfExists(api, pool, src.volumeName()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
base, err = c.fetch(ctx, log, api, hostName, pool, src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return api.StorageVolGetPath(base)
|
||||
}
|
||||
|
||||
// lookup reports whether the pool holds a complete, verified base volume:
|
||||
// the volume and its marker both exist and, when a checksum is configured,
|
||||
// the marker records that checksum.
|
||||
func (c *imageCache) lookup(api libvirtAPI, pool libvirt.StoragePool, src imageSource) (libvirt.StorageVol, bool, error) {
|
||||
base, err := api.StorageVolLookupByName(pool, src.volumeName())
|
||||
if hasCode(err, libvirt.ErrNoStorageVol) {
|
||||
return libvirt.StorageVol{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, false, fmt.Errorf("looking up base volume: %w", err)
|
||||
}
|
||||
marker, err := api.StorageVolLookupByName(pool, src.markerName())
|
||||
if hasCode(err, libvirt.ErrNoStorageVol) {
|
||||
return base, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, false, fmt.Errorf("looking up image marker: %w", err)
|
||||
}
|
||||
if src.SHA256 == "" {
|
||||
return base, true, nil
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := api.StorageVolDownload(marker, &buf, 0, 0, 0); err != nil {
|
||||
return libvirt.StorageVol{}, false, fmt.Errorf("reading image marker: %w", err)
|
||||
}
|
||||
return base, strings.EqualFold(strings.TrimSpace(buf.String()), src.SHA256), nil
|
||||
}
|
||||
|
||||
// fetch streams the image from its URL straight into a new base volume,
|
||||
// hashing on the way, then writes the marker. Nothing is buffered in the
|
||||
// operator beyond the HTTP body's read buffer.
|
||||
func (c *imageCache) fetch(ctx context.Context, log logr.Logger, api libvirtAPI, hostName string, pool libvirt.StoragePool, src imageSource) (libvirt.StorageVol, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, src.URL, nil)
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, fmt.Errorf("%w: building image request: %v", provider.ErrPermanent, err)
|
||||
}
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, fmt.Errorf("%w: fetching image %s: %v", provider.ErrTransient, src.URL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
switch {
|
||||
case resp.StatusCode >= 200 && resp.StatusCode < 300:
|
||||
case resp.StatusCode >= 400 && resp.StatusCode < 500:
|
||||
return libvirt.StorageVol{}, fmt.Errorf("%w: image %s returned HTTP %d", provider.ErrPermanent, src.URL, resp.StatusCode)
|
||||
default:
|
||||
return libvirt.StorageVol{}, fmt.Errorf("%w: image %s returned HTTP %d", provider.ErrTransient, src.URL, resp.StatusCode)
|
||||
}
|
||||
if resp.ContentLength < 0 {
|
||||
return libvirt.StorageVol{}, fmt.Errorf("%w: image %s has no Content-Length; the base volume needs a known size", provider.ErrPermanent, src.URL)
|
||||
}
|
||||
|
||||
log.Info("fetching base image into host pool",
|
||||
"host", hostName, "pool", pool.Name, "volume", src.volumeName(),
|
||||
"url", src.URL, "bytes", resp.ContentLength)
|
||||
start := time.Now()
|
||||
|
||||
baseXML, err := buildVolumeXML(volumeSpec{Name: src.volumeName(), SizeBytes: resp.ContentLength, Format: src.Format})
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, err
|
||||
}
|
||||
base, err := api.StorageVolCreateXML(pool, baseXML, 0)
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, fmt.Errorf("creating base volume: %w", err)
|
||||
}
|
||||
hasher := sha256.New()
|
||||
if err := api.StorageVolUpload(base, io.TeeReader(resp.Body, hasher), 0, 0, 0); err != nil {
|
||||
_ = api.StorageVolDelete(base, 0)
|
||||
return libvirt.StorageVol{}, fmt.Errorf("uploading image to base volume: %w", err)
|
||||
}
|
||||
digest := hex.EncodeToString(hasher.Sum(nil))
|
||||
if src.SHA256 != "" && !strings.EqualFold(digest, src.SHA256) {
|
||||
_ = api.StorageVolDelete(base, 0)
|
||||
return libvirt.StorageVol{}, fmt.Errorf("%w: image %s sha256 is %s, config expects %s",
|
||||
provider.ErrPermanent, src.URL, digest, src.SHA256)
|
||||
}
|
||||
// libvirt sized the volume from Content-Length; a refresh makes it
|
||||
// re-read the real virtual size from the uploaded qcow2 header.
|
||||
if err := api.StoragePoolRefresh(pool, 0); err != nil {
|
||||
return libvirt.StorageVol{}, fmt.Errorf("refreshing pool after upload: %w", err)
|
||||
}
|
||||
|
||||
markerXML, err := buildVolumeXML(rawVolume(src.markerName(), int64(len(digest))))
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, err
|
||||
}
|
||||
marker, err := api.StorageVolCreateXML(pool, markerXML, 0)
|
||||
if err != nil {
|
||||
return libvirt.StorageVol{}, fmt.Errorf("creating image marker: %w", err)
|
||||
}
|
||||
if err := api.StorageVolUpload(marker, strings.NewReader(digest), 0, 0, 0); err != nil {
|
||||
_ = api.StorageVolDelete(marker, 0)
|
||||
return libvirt.StorageVol{}, fmt.Errorf("writing image marker: %w", err)
|
||||
}
|
||||
log.Info("base image cached",
|
||||
"host", hostName, "volume", src.volumeName(), "sha256", digest,
|
||||
"bytes", resp.ContentLength, "duration", time.Since(start).Round(time.Second))
|
||||
return base, nil
|
||||
}
|
||||
|
||||
func deleteIfExists(api libvirtAPI, pool libvirt.StoragePool, name string) error {
|
||||
vol, err := api.StorageVolLookupByName(pool, name)
|
||||
if hasCode(err, libvirt.ErrNoStorageVol) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("looking up volume %s: %w", name, err)
|
||||
}
|
||||
if err := api.StorageVolDelete(vol, 0); err != nil && !hasCode(err, libvirt.ErrNoStorageVol) {
|
||||
return fmt.Errorf("deleting volume %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
var testImage = bytes.Repeat([]byte("QFI\xfb fake qcow2 payload "), 1024)
|
||||
|
||||
func testImageSHA() string {
|
||||
sum := sha256.Sum256(testImage)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// imageServer serves testImage and counts hits. Status overrides the
|
||||
// response, noLength forces chunked encoding (no Content-Length), delay
|
||||
// slows each response so concurrent callers overlap.
|
||||
type imageServer struct {
|
||||
*httptest.Server
|
||||
hits atomic.Int32
|
||||
status int
|
||||
noLength bool
|
||||
delay time.Duration
|
||||
body []byte
|
||||
}
|
||||
|
||||
func newImageServer(t *testing.T) *imageServer {
|
||||
t.Helper()
|
||||
s := &imageServer{status: http.StatusOK, body: testImage}
|
||||
s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
s.hits.Add(1)
|
||||
time.Sleep(s.delay)
|
||||
if s.status != http.StatusOK {
|
||||
w.WriteHeader(s.status)
|
||||
return
|
||||
}
|
||||
// Bodies larger than the server's write buffer would otherwise go
|
||||
// out chunked without a Content-Length, unlike real image mirrors.
|
||||
if s.noLength {
|
||||
w.(http.Flusher).Flush()
|
||||
} else {
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(s.body)))
|
||||
}
|
||||
_, _ = w.Write(s.body)
|
||||
}))
|
||||
t.Cleanup(s.Close)
|
||||
return s
|
||||
}
|
||||
|
||||
func testSource(srv *imageServer, sha string) imageSource {
|
||||
return imageSource{Key: "debian-13", URL: srv.URL + "/debian.qcow2", SHA256: sha, Format: formatQcow2}
|
||||
}
|
||||
|
||||
func ensureTest(t *testing.T, api *fakeAPI, src imageSource) (string, error) {
|
||||
t.Helper()
|
||||
c := newImageCache(nil)
|
||||
pool, err := api.StoragePoolLookupByName("default")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, src)
|
||||
}
|
||||
|
||||
func TestResolveImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := withDefaults(provider.KVMConfig{Images: map[string]provider.KVMImage{
|
||||
"debian-13": {URL: "https://cloud.debian.org/d.qcow2", SHA256: "abc"},
|
||||
}})
|
||||
tests := []struct {
|
||||
name string
|
||||
image string
|
||||
want imageSource
|
||||
wantErr bool
|
||||
}{
|
||||
{"configured key", "debian-13", imageSource{Key: "debian-13", URL: "https://cloud.debian.org/d.qcow2", SHA256: "abc", Format: "qcow2"}, false},
|
||||
{"direct https url", "https://x.example/img.qcow2", imageSource{URL: "https://x.example/img.qcow2", Format: "qcow2"}, false},
|
||||
{"direct http url", "http://mirror/img.qcow2", imageSource{URL: "http://mirror/img.qcow2", Format: "qcow2"}, false},
|
||||
{"unknown key", "ubuntu", imageSource{}, true},
|
||||
{"file url", "file:///tmp/x.qcow2", imageSource{}, true},
|
||||
{"empty", "", imageSource{}, true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := resolveImage(cfg, tc.image)
|
||||
if tc.wantErr {
|
||||
if !errors.Is(err, provider.ErrPermanent) {
|
||||
t.Fatalf("resolveImage(%q) error = %v, want ErrPermanent", tc.image, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("resolveImage(%q) error = %v", tc.image, err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Errorf("resolveImage(%q) = %+v, want %+v", tc.image, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageSource_volumeNames(t *testing.T) {
|
||||
t.Parallel()
|
||||
keyed := imageSource{Key: "debian-13", URL: "https://a/b.qcow2", Format: "qcow2"}
|
||||
if keyed.volumeName() != "base-debian-13.qcow2" || keyed.markerName() != "base-debian-13.qcow2.sha256" {
|
||||
t.Errorf("keyed names = %s / %s", keyed.volumeName(), keyed.markerName())
|
||||
}
|
||||
direct := imageSource{URL: "https://a/b.qcow2", Format: "qcow2"}
|
||||
name := direct.volumeName()
|
||||
if !strings.HasPrefix(name, "base-") || !strings.HasSuffix(name, ".qcow2") || len(name) != len("base-")+16+len(".qcow2") {
|
||||
t.Errorf("direct-url name = %q, want base-<16 hex>.qcow2", name)
|
||||
}
|
||||
other := imageSource{URL: "https://a/c.qcow2", Format: "qcow2"}
|
||||
if other.volumeName() == name {
|
||||
t.Error("different URLs produced the same volume name")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_freshFetch(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
path, err := ensureTest(t, api, testSource(srv, testImageSHA()))
|
||||
if err != nil {
|
||||
t.Fatalf("ensure() error = %v", err)
|
||||
}
|
||||
if path != "/var/lib/libvirt/images/base-debian-13.qcow2" {
|
||||
t.Errorf("path = %q", path)
|
||||
}
|
||||
base := api.vol("default", "base-debian-13.qcow2")
|
||||
if base == nil || !bytes.Equal(base.data, testImage) {
|
||||
t.Fatalf("base volume missing or content differs")
|
||||
}
|
||||
if base.format != "qcow2" || base.capacity != int64(len(testImage)) {
|
||||
t.Errorf("base volume format/capacity = %s/%d, want qcow2/%d", base.format, base.capacity, len(testImage))
|
||||
}
|
||||
marker := api.vol("default", "base-debian-13.qcow2.sha256")
|
||||
if marker == nil || string(marker.data) != testImageSHA() {
|
||||
t.Fatalf("marker missing or wrong: %+v", marker)
|
||||
}
|
||||
if !api.hasCall("StoragePoolRefresh") {
|
||||
t.Error("pool was not refreshed after the upload")
|
||||
}
|
||||
if srv.hits.Load() != 1 {
|
||||
t.Errorf("http hits = %d, want 1", srv.hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_noChecksumConfigured(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
if _, err := ensureTest(t, api, testSource(srv, "")); err != nil {
|
||||
t.Fatalf("ensure() error = %v", err)
|
||||
}
|
||||
if marker := api.vol("default", "base-debian-13.qcow2.sha256"); marker == nil || string(marker.data) != testImageSHA() {
|
||||
t.Errorf("marker should still record the computed digest, got %+v", marker)
|
||||
}
|
||||
// Second call: ready without a checksum to compare, so no marker read.
|
||||
if _, err := ensureTest(t, api, testSource(srv, "")); err != nil {
|
||||
t.Fatalf("second ensure() error = %v", err)
|
||||
}
|
||||
if srv.hits.Load() != 1 {
|
||||
t.Errorf("http hits = %d, want 1", srv.hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_alreadyCached(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
api.putVol("default", "base-debian-13.qcow2", "qcow2", testImage)
|
||||
api.putVol("default", "base-debian-13.qcow2.sha256", "raw", []byte(testImageSHA()))
|
||||
path, err := ensureTest(t, api, testSource(srv, testImageSHA()))
|
||||
if err != nil {
|
||||
t.Fatalf("ensure() error = %v", err)
|
||||
}
|
||||
if path != "/var/lib/libvirt/images/base-debian-13.qcow2" {
|
||||
t.Errorf("path = %q", path)
|
||||
}
|
||||
if srv.hits.Load() != 0 {
|
||||
t.Errorf("http hits = %d, want 0 for a cached image", srv.hits.Load())
|
||||
}
|
||||
if api.hasCall("StorageVolCreateXML") || api.hasCall("StorageVolDelete") {
|
||||
t.Errorf("cached image was touched: calls = %v", api.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_markerMissingRedownloads(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
api.putVol("default", "base-debian-13.qcow2", "qcow2", []byte("half an image"))
|
||||
if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
|
||||
t.Fatalf("ensure() error = %v", err)
|
||||
}
|
||||
if srv.hits.Load() != 1 {
|
||||
t.Errorf("http hits = %d, want 1: a base without its marker is a partial upload", srv.hits.Load())
|
||||
}
|
||||
if base := api.vol("default", "base-debian-13.qcow2"); base == nil || !bytes.Equal(base.data, testImage) {
|
||||
t.Error("partial base volume was not replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_checksumChangedRedownloads(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
api.putVol("default", "base-debian-13.qcow2", "qcow2", []byte("an older release"))
|
||||
api.putVol("default", "base-debian-13.qcow2.sha256", "raw", []byte(strings.Repeat("0", 64)))
|
||||
if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
|
||||
t.Fatalf("ensure() error = %v", err)
|
||||
}
|
||||
if srv.hits.Load() != 1 {
|
||||
t.Errorf("http hits = %d, want 1 after the configured checksum changed", srv.hits.Load())
|
||||
}
|
||||
if marker := api.vol("default", "base-debian-13.qcow2.sha256"); marker == nil || string(marker.data) != testImageSHA() {
|
||||
t.Errorf("marker not updated: %+v", marker)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_checksumMismatchIsPermanent(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
_, err := ensureTest(t, api, testSource(srv, strings.Repeat("a", 64)))
|
||||
if !errors.Is(err, provider.ErrPermanent) {
|
||||
t.Fatalf("ensure() error = %v, want ErrPermanent", err)
|
||||
}
|
||||
if names := api.volNames("default"); len(names) != 0 {
|
||||
t.Errorf("volumes left behind after a checksum mismatch: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_httpErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
status int
|
||||
want error
|
||||
}{
|
||||
{"404 is permanent", http.StatusNotFound, provider.ErrPermanent},
|
||||
{"403 is permanent", http.StatusForbidden, provider.ErrPermanent},
|
||||
{"503 is transient", http.StatusServiceUnavailable, provider.ErrTransient},
|
||||
{"500 is transient", http.StatusInternalServerError, provider.ErrTransient},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
srv.status = tc.status
|
||||
api := newFakeAPI("default")
|
||||
_, err := ensureTest(t, api, testSource(srv, ""))
|
||||
if !errors.Is(err, tc.want) {
|
||||
t.Fatalf("ensure() error = %v, want %v", err, tc.want)
|
||||
}
|
||||
if names := api.volNames("default"); len(names) != 0 {
|
||||
t.Errorf("volumes created despite HTTP %d: %v", tc.status, names)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_noContentLengthIsPermanent(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
srv.noLength = true
|
||||
api := newFakeAPI("default")
|
||||
_, err := ensureTest(t, api, testSource(srv, ""))
|
||||
if !errors.Is(err, provider.ErrPermanent) || !strings.Contains(err.Error(), "Content-Length") {
|
||||
t.Fatalf("ensure() error = %v, want ErrPermanent about Content-Length", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_uploadFailureCleansUp(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
api.uploadHook = func(_ string, r io.Reader) error {
|
||||
_, _ = io.CopyN(io.Discard, r, 100)
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
_, err := ensureTest(t, api, testSource(srv, testImageSHA()))
|
||||
if !errors.Is(err, provider.ErrTransient) && classify(err) != provider.ErrTransient {
|
||||
t.Fatalf("ensure() error = %v, want a transient failure", err)
|
||||
}
|
||||
if names := api.volNames("default"); len(names) != 0 {
|
||||
t.Errorf("volumes left behind after a failed upload: %v", names)
|
||||
}
|
||||
// Next attempt starts clean and succeeds.
|
||||
api.uploadHook = nil
|
||||
if _, err := ensureTest(t, api, testSource(srv, testImageSHA())); err != nil {
|
||||
t.Fatalf("retry ensure() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_missingPoolIsPermanent(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
c := newImageCache(nil)
|
||||
pool, _ := api.StoragePoolLookupByName("default")
|
||||
delete(api.pools, "default")
|
||||
_, err := c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, testSource(srv, ""))
|
||||
if classify(err) != provider.ErrPermanent {
|
||||
t.Fatalf("ensure() error = %v, want one classifying as Permanent", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_concurrentCallersDownloadOnce(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
srv.delay = 50 * time.Millisecond
|
||||
api := newFakeAPI("default")
|
||||
c := newImageCache(nil)
|
||||
pool, _ := api.StoragePoolLookupByName("default")
|
||||
src := testSource(srv, testImageSHA())
|
||||
var wg sync.WaitGroup
|
||||
paths := make([]string, 4)
|
||||
errs := make([]error, 4)
|
||||
for i := range 4 {
|
||||
wg.Go(func() {
|
||||
paths[i], errs[i] = c.ensure(context.Background(), logr.Discard(), api, "nas1", pool, src)
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
for i := range 4 {
|
||||
if errs[i] != nil || paths[i] != "/var/lib/libvirt/images/base-debian-13.qcow2" {
|
||||
t.Errorf("caller %d: path=%q err=%v", i, paths[i], errs[i])
|
||||
}
|
||||
}
|
||||
if srv.hits.Load() != 1 {
|
||||
t.Errorf("http hits = %d, want 1 for four concurrent callers", srv.hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_differentHostsFetchIndependently(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
c := newImageCache(nil)
|
||||
src := testSource(srv, "")
|
||||
for _, host := range []string{"nas1", "lab2"} {
|
||||
api := newFakeAPI("default")
|
||||
pool, _ := api.StoragePoolLookupByName("default")
|
||||
if _, err := c.ensure(context.Background(), logr.Discard(), api, host, pool, src); err != nil {
|
||||
t.Fatalf("ensure(%s) error = %v", host, err)
|
||||
}
|
||||
}
|
||||
if srv.hits.Load() != 2 {
|
||||
t.Errorf("http hits = %d, want one per host", srv.hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageCache_ensure_cancelledContextIsTransient(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
c := newImageCache(nil)
|
||||
pool, _ := api.StoragePoolLookupByName("default")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err := c.ensure(ctx, logr.Discard(), api, "nas1", pool, testSource(srv, ""))
|
||||
if classify(err) != provider.ErrTransient {
|
||||
t.Fatalf("ensure() error = %v, want transient", err)
|
||||
}
|
||||
if names := api.volNames("default"); len(names) != 0 {
|
||||
t.Errorf("volumes created despite cancelled context: %v", names)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,560 @@
|
||||
// Package kvm implements the provider contract on libvirt/KVM hypervisors.
|
||||
// Each configured host is reached over SSH with the libvirt unix socket
|
||||
// forwarded through the connection, so the hypervisors need no libvirtd TCP
|
||||
// or TLS listener and the operator image needs no libvirt libraries. VMs
|
||||
// boot a stock cloud image (fetched into the host's storage pool on first
|
||||
// use) through a qcow2 overlay, with the Proxy's cloud-init delivered on a
|
||||
// NoCloud seed ISO built in-process.
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
"github.com/go-logr/logr"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultDiskSizeGB = 10
|
||||
defaultSocket = "/var/run/libvirt/libvirt-sock"
|
||||
defaultPool = "default"
|
||||
formatQcow2 = "qcow2"
|
||||
formatRaw = "raw"
|
||||
|
||||
// undefineFlags removes everything libvirt may keep alongside a domain
|
||||
// definition, so an undefine never fails on leftover state.
|
||||
undefineFlags = libvirt.DomainUndefineManagedSave |
|
||||
libvirt.DomainUndefineSnapshotsMetadata |
|
||||
libvirt.DomainUndefineNvram
|
||||
)
|
||||
|
||||
func withDefaults(cfg provider.KVMConfig) provider.KVMConfig {
|
||||
if cfg.DiskSizeGB == 0 {
|
||||
cfg.DiskSizeGB = defaultDiskSizeGB
|
||||
}
|
||||
if cfg.VendorData == "" {
|
||||
cfg.VendorData = defaultVendorData
|
||||
}
|
||||
images := make(map[string]provider.KVMImage, len(cfg.Images))
|
||||
for k, img := range cfg.Images {
|
||||
if img.Format == "" {
|
||||
img.Format = formatQcow2
|
||||
}
|
||||
images[k] = img
|
||||
}
|
||||
cfg.Images = images
|
||||
hosts := make([]provider.KVMHost, len(cfg.Hosts))
|
||||
for i, h := range cfg.Hosts {
|
||||
if h.Socket == "" {
|
||||
h.Socket = defaultSocket
|
||||
}
|
||||
if h.Pool == "" {
|
||||
h.Pool = defaultPool
|
||||
}
|
||||
hosts[i] = h
|
||||
}
|
||||
cfg.Hosts = hosts
|
||||
return cfg
|
||||
}
|
||||
|
||||
// Provider implements provider.Provider on one or more libvirt hosts.
|
||||
type Provider struct {
|
||||
name string
|
||||
cfg provider.KVMConfig
|
||||
hosts *hostPool
|
||||
images *imageCache
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// New builds a Provider that reaches its hosts over SSH. The key and
|
||||
// known_hosts files are read here so a bad path fails at startup rather
|
||||
// than on the first Create. Satisfies registry.Constructor.
|
||||
func New(_ context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
|
||||
cfg := provider.KVMConfig{}
|
||||
if pc.KVM != nil {
|
||||
cfg = *pc.KVM
|
||||
}
|
||||
auth, err := newSSHAuth(cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kvm provider %q: %w", pc.Name, err)
|
||||
}
|
||||
return newWithDialer(pc, auth.dialer(), nil), nil
|
||||
}
|
||||
|
||||
// newWithDialer is the test seam: the dialer decides what libvirtAPI each
|
||||
// host resolves to, and the HTTP client is what image downloads go through.
|
||||
func newWithDialer(pc provider.ProviderConfig, dial hostDialer, httpClient *http.Client) *Provider {
|
||||
cfg := provider.KVMConfig{}
|
||||
if pc.KVM != nil {
|
||||
cfg = *pc.KVM
|
||||
}
|
||||
return &Provider{
|
||||
name: pc.Name,
|
||||
cfg: withDefaults(cfg),
|
||||
hosts: newHostPool(dial),
|
||||
images: newImageCache(httpClient),
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) logger(ctx context.Context) logr.Logger {
|
||||
return logf.FromContext(ctx).WithName("kvm").WithValues("provider", p.name)
|
||||
}
|
||||
|
||||
func (p *Provider) wrapErr(op, id string, err error) error {
|
||||
return provider.Wrap(classify(err), op, p.name, id, err)
|
||||
}
|
||||
|
||||
// host resolves placement.zone (or a providerID's host segment) to a
|
||||
// configured host. An empty zone is allowed only when exactly one host is
|
||||
// configured; anything else is a spec error.
|
||||
func (p *Provider) host(zone string) (provider.KVMHost, error) {
|
||||
if zone == "" {
|
||||
switch len(p.cfg.Hosts) {
|
||||
case 0:
|
||||
return provider.KVMHost{}, fmt.Errorf("%w: kvm.hosts is empty", provider.ErrPermanent)
|
||||
case 1:
|
||||
return p.cfg.Hosts[0], nil
|
||||
default:
|
||||
return provider.KVMHost{}, fmt.Errorf("%w: placement.zone is required when kvm.hosts lists more than one host",
|
||||
provider.ErrPermanent)
|
||||
}
|
||||
}
|
||||
for _, h := range p.cfg.Hosts {
|
||||
if h.Name == zone {
|
||||
return h, nil
|
||||
}
|
||||
}
|
||||
return provider.KVMHost{}, fmt.Errorf("%w: host %q is not in kvm.hosts", provider.ErrPermanent, zone)
|
||||
}
|
||||
|
||||
// Create makes the VM exist and running on the placed host, and is
|
||||
// idempotent: a defined domain is only started if needed, its volumes
|
||||
// untouched. Before the domain exists nothing can be using the volumes,
|
||||
// so an existing overlay is reused (its creation is atomic) but the seed
|
||||
// is rebuilt from scratch — a previous attempt may have been cut off mid
|
||||
// upload, and a truncated seed would boot a VM without cloud-init. The
|
||||
// first Create for an image on a host also downloads that image into the
|
||||
// host's pool, synchronously.
|
||||
func (p *Provider) Create(ctx context.Context, req provider.CreateRequest) (string, error) {
|
||||
pl := req.Placement
|
||||
host, err := p.host(pl.Zone)
|
||||
if err != nil {
|
||||
return "", p.wrapErr("create", "", err)
|
||||
}
|
||||
mt, ok := p.cfg.MachineTypes[pl.MachineType]
|
||||
if !ok {
|
||||
return "", p.wrapErr("create", "", fmt.Errorf("%w: placement.machineType %q is not in kvm.machineTypes",
|
||||
provider.ErrPermanent, pl.MachineType))
|
||||
}
|
||||
src, err := resolveImage(p.cfg, pl.Image)
|
||||
if err != nil {
|
||||
return "", p.wrapErr("create", "", err)
|
||||
}
|
||||
id := formatProviderID(host.Name, req.Name)
|
||||
log := p.logger(ctx).WithValues("host", host.Name, "name", req.Name)
|
||||
log.V(2).Info("KVM create resolved",
|
||||
"pool", host.Pool, "bridge", host.Bridge, "machineType", pl.MachineType,
|
||||
"vcpus", mt.VCPUs, "memoryMiB", mt.MemoryMiB, "diskSizeGb", p.cfg.DiskSizeGB,
|
||||
"image", src.volumeName(), "cloudInitBytes", len(req.CloudInit))
|
||||
|
||||
err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
|
||||
dom, err := api.DomainLookupByName(req.Name)
|
||||
if err == nil {
|
||||
log.V(1).Info("domain already defined")
|
||||
return p.ensureRunning(log, api, dom)
|
||||
}
|
||||
if !hasCode(err, libvirt.ErrNoDomain) {
|
||||
return fmt.Errorf("looking up domain: %w", err)
|
||||
}
|
||||
pool, err := api.StoragePoolLookupByName(host.Pool)
|
||||
if err != nil {
|
||||
return fmt.Errorf("looking up pool %s: %w", host.Pool, err)
|
||||
}
|
||||
basePath, err := p.images.ensure(ctx, log, api, host.Name, pool, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.ensureOverlay(log, api, pool, req.Name, basePath, src.Format); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.writeSeed(log, api, pool, req.Name, req.CloudInit); err != nil {
|
||||
return err
|
||||
}
|
||||
dom, err = p.defineDomain(log, api, host, req, mt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return p.ensureRunning(log, api, dom)
|
||||
})
|
||||
if err != nil {
|
||||
logAPIError(log, "create", err)
|
||||
return "", p.wrapErr("create", id, err)
|
||||
}
|
||||
log.V(1).Info("KVM domain ensured running", "id", id)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (p *Provider) ensureOverlay(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, basePath, baseFormat string) error {
|
||||
xml, err := buildVolumeXML(overlayVolume(name, p.cfg.DiskSizeGB, basePath, baseFormat))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = api.StorageVolCreateXML(pool, xml, 0)
|
||||
switch {
|
||||
case err == nil:
|
||||
log.V(1).Info("overlay volume created", "volume", overlayVolumeName(name), "base", basePath)
|
||||
case hasCode(err, libvirt.ErrStorageVolExist):
|
||||
log.V(1).Info("overlay volume already exists", "volume", overlayVolumeName(name))
|
||||
default:
|
||||
return fmt.Errorf("creating overlay volume: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeSeed replaces the seed volume with a freshly built NoCloud ISO.
|
||||
// Only called while no domain references it.
|
||||
func (p *Provider) writeSeed(log logr.Logger, api libvirtAPI, pool libvirt.StoragePool, name, userData string) error {
|
||||
seedName := seedVolumeName(name)
|
||||
if err := deleteIfExists(api, pool, seedName); err != nil {
|
||||
return err
|
||||
}
|
||||
iso, err := buildSeedISO(name, userData, p.cfg.VendorData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", provider.ErrPermanent, err)
|
||||
}
|
||||
xml, err := buildVolumeXML(rawVolume(seedName, int64(len(iso))))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vol, err := api.StorageVolCreateXML(pool, xml, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating seed volume: %w", err)
|
||||
}
|
||||
if err := api.StorageVolUpload(vol, bytes.NewReader(iso), 0, 0, 0); err != nil {
|
||||
return fmt.Errorf("uploading seed volume: %w", err)
|
||||
}
|
||||
log.V(1).Info("seed volume uploaded", "volume", seedName, "bytes", len(iso))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Provider) defineDomain(log logr.Logger, api libvirtAPI, host provider.KVMHost, req provider.CreateRequest, mt provider.KVMMachineType) (libvirt.Domain, error) {
|
||||
xml, err := buildDomainXML(domainParams{
|
||||
Name: req.Name,
|
||||
UID: req.UID,
|
||||
CreatedAt: p.now(),
|
||||
VCPUs: mt.VCPUs,
|
||||
MemoryMiB: mt.MemoryMiB,
|
||||
Pool: host.Pool,
|
||||
Bridge: host.Bridge,
|
||||
DiskVol: overlayVolumeName(req.Name),
|
||||
SeedVol: seedVolumeName(req.Name),
|
||||
})
|
||||
if err != nil {
|
||||
return libvirt.Domain{}, err
|
||||
}
|
||||
dom, err := api.DomainDefineXML(xml)
|
||||
if err != nil {
|
||||
return libvirt.Domain{}, fmt.Errorf("defining domain: %w", err)
|
||||
}
|
||||
log.V(1).Info("domain defined", "uid", req.UID)
|
||||
return dom, nil
|
||||
}
|
||||
|
||||
func (p *Provider) ensureRunning(log logr.Logger, api libvirtAPI, dom libvirt.Domain) error {
|
||||
state, _, err := api.DomainGetState(dom, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading domain state: %w", err)
|
||||
}
|
||||
if isActive(libvirt.DomainState(state)) {
|
||||
return nil
|
||||
}
|
||||
err = api.DomainCreate(dom)
|
||||
switch {
|
||||
case err == nil:
|
||||
log.V(1).Info("domain started")
|
||||
case hasCode(err, libvirt.ErrOperationInvalid):
|
||||
log.V(1).Info("domain already running, start treated as success")
|
||||
default:
|
||||
return fmt.Errorf("starting domain: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get reads the domain's state, metadata and addresses. The providerID
|
||||
// carries the host, so this looks at the right hypervisor even after a
|
||||
// placement.zone edit.
|
||||
func (p *Provider) Get(ctx context.Context, providerID string) (*provider.Instance, error) {
|
||||
hostName, name, err := parseProviderID(providerID)
|
||||
if err != nil {
|
||||
return nil, provider.Wrap(provider.ErrPermanent, "get", p.name, providerID, err)
|
||||
}
|
||||
host, err := p.host(hostName)
|
||||
if err != nil {
|
||||
return nil, p.wrapErr("get", providerID, err)
|
||||
}
|
||||
log := p.logger(ctx).WithValues("host", host.Name, "name", name)
|
||||
var inst *provider.Instance
|
||||
err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
|
||||
dom, err := api.DomainLookupByName(name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("looking up domain: %w", err)
|
||||
}
|
||||
meta, err := p.metadata(api, dom)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inst, err = p.inspect(log, api, host.Name, dom, meta)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
logAPIError(log, "get", err)
|
||||
return nil, p.wrapErr("get", providerID, err)
|
||||
}
|
||||
log.V(1).Info("KVM domain fetched", "state", inst.State, "ip", inst.IP, "uid", inst.UID)
|
||||
return inst, nil
|
||||
}
|
||||
|
||||
// inspect assembles an Instance from a domain that is known to exist and
|
||||
// whose metadata has already been read (the zero value for a domain
|
||||
// without the operator's element, which reports an empty UID as the
|
||||
// contract requires).
|
||||
func (p *Provider) inspect(log logr.Logger, api libvirtAPI, hostName string, dom libvirt.Domain, meta instanceMetadata) (*provider.Instance, error) {
|
||||
state, _, err := api.DomainGetState(dom, 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading domain state: %w", err)
|
||||
}
|
||||
var ip string
|
||||
if isActive(libvirt.DomainState(state)) {
|
||||
ip = p.address(log, api, dom)
|
||||
}
|
||||
return &provider.Instance{
|
||||
ID: formatProviderID(hostName, dom.Name),
|
||||
IP: ip,
|
||||
State: mapState(libvirt.DomainState(state), ip),
|
||||
UID: meta.UID,
|
||||
CreatedAt: meta.createdAt(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// metadata returns the operator's element, or the zero value when the
|
||||
// domain has none (it was not created by this operator).
|
||||
func (p *Provider) metadata(api libvirtAPI, dom libvirt.Domain) (instanceMetadata, error) {
|
||||
raw, err := api.DomainGetMetadata(dom, int32(libvirt.DomainMetadataElement), libvirt.OptString{metadataURI}, libvirt.DomainAffectCurrent)
|
||||
if hasCode(err, libvirt.ErrNoDomainMetadata) {
|
||||
return instanceMetadata{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return instanceMetadata{}, fmt.Errorf("reading domain metadata: %w", err)
|
||||
}
|
||||
meta, err := parseInstanceMetadata(raw)
|
||||
if err != nil {
|
||||
return instanceMetadata{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
// address returns the domain's first routable IPv4, asking the host's ARP
|
||||
// table first and the guest agent second. Both sources fail routinely while
|
||||
// the guest is still booting (no traffic seen yet, agent not installed
|
||||
// yet), so their errors mean "no address yet", not "Get failed".
|
||||
func (p *Provider) address(log logr.Logger, api libvirtAPI, dom libvirt.Domain) string {
|
||||
for _, source := range []libvirt.DomainInterfaceAddressesSource{
|
||||
libvirt.DomainInterfaceAddressesSrcArp,
|
||||
libvirt.DomainInterfaceAddressesSrcAgent,
|
||||
} {
|
||||
ifaces, err := api.DomainInterfaceAddresses(dom, uint32(source), 0)
|
||||
if err != nil {
|
||||
log.V(2).Info("no addresses from source", "source", source.String(), "error", err.Error())
|
||||
continue
|
||||
}
|
||||
if ip := firstIPv4(ifaces); ip != "" {
|
||||
return ip
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstIPv4(ifaces []libvirt.DomainInterface) string {
|
||||
for _, iface := range ifaces {
|
||||
for _, a := range iface.Addrs {
|
||||
if a.Type != int32(libvirt.IPAddrTypeIpv4) {
|
||||
continue
|
||||
}
|
||||
ip := net.ParseIP(a.Addr)
|
||||
if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsUnspecified() {
|
||||
continue
|
||||
}
|
||||
return a.Addr
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// isActive reports whether the guest is executing (or blocked on the
|
||||
// hypervisor, which is still "up" from the outside).
|
||||
func isActive(s libvirt.DomainState) bool {
|
||||
return s == libvirt.DomainRunning || s == libvirt.DomainBlocked
|
||||
}
|
||||
|
||||
// mapState collapses libvirt's states onto the provider's. An active
|
||||
// domain without an address is still Provisioning: an empty IP must never
|
||||
// be published as Running. "No state" is also Provisioning — the
|
||||
// reconciler answers Stopped with delete-and-recreate, too drastic for a
|
||||
// state libvirt could not determine; the next poll decides. Everything
|
||||
// else — paused, suspended, shut off, crashed, shutting down — is Stopped.
|
||||
func mapState(s libvirt.DomainState, ip string) provider.InstanceState {
|
||||
if s == libvirt.DomainNostate {
|
||||
return provider.StateProvisioning
|
||||
}
|
||||
if !isActive(s) {
|
||||
return provider.StateStopped
|
||||
}
|
||||
if ip == "" {
|
||||
return provider.StateProvisioning
|
||||
}
|
||||
return provider.StateRunning
|
||||
}
|
||||
|
||||
// Delete tears the VM down and is idempotent at every step: a stopped
|
||||
// domain is not destroyed again, a missing domain is not undefined, and
|
||||
// the per-VM volumes are removed by name even when the domain is already
|
||||
// gone, so a crash between volume creation and define is cleaned up on the
|
||||
// next reconcile.
|
||||
func (p *Provider) Delete(ctx context.Context, providerID string) error {
|
||||
hostName, name, err := parseProviderID(providerID)
|
||||
if err != nil {
|
||||
return provider.Wrap(provider.ErrPermanent, "delete", p.name, providerID, err)
|
||||
}
|
||||
host, err := p.host(hostName)
|
||||
if err != nil {
|
||||
return p.wrapErr("delete", providerID, err)
|
||||
}
|
||||
log := p.logger(ctx).WithValues("host", host.Name, "name", name)
|
||||
err = p.hosts.call(ctx, host, func(api libvirtAPI) error {
|
||||
if err := p.removeDomain(log, api, name); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.removeVolumes(log, api, host.Pool, name)
|
||||
})
|
||||
if err != nil {
|
||||
logAPIError(log, "delete", err)
|
||||
return p.wrapErr("delete", providerID, err)
|
||||
}
|
||||
log.V(1).Info("KVM domain deleted")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Provider) removeDomain(log logr.Logger, api libvirtAPI, name string) error {
|
||||
dom, err := api.DomainLookupByName(name)
|
||||
if hasCode(err, libvirt.ErrNoDomain) {
|
||||
log.V(1).Info("domain already gone")
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("looking up domain: %w", err)
|
||||
}
|
||||
err = api.DomainDestroy(dom)
|
||||
switch {
|
||||
case err == nil:
|
||||
log.V(1).Info("domain destroyed")
|
||||
case hasCode(err, libvirt.ErrOperationInvalid):
|
||||
log.V(1).Info("domain not running, destroy skipped")
|
||||
case hasCode(err, libvirt.ErrNoDomain):
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("destroying domain: %w", err)
|
||||
}
|
||||
if err := api.DomainUndefineFlags(dom, undefineFlags); err != nil && !hasCode(err, libvirt.ErrNoDomain) {
|
||||
return fmt.Errorf("undefining domain: %w", err)
|
||||
}
|
||||
log.V(1).Info("domain undefined")
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeVolumes deletes the overlay and seed. A missing pool means there
|
||||
// is nothing left to clean, not a failure: refusing to finish a Delete
|
||||
// over a pool that no longer exists would pin the Proxy's finalizer.
|
||||
func (p *Provider) removeVolumes(log logr.Logger, api libvirtAPI, poolName, name string) error {
|
||||
pool, err := api.StoragePoolLookupByName(poolName)
|
||||
if hasCode(err, libvirt.ErrNoStoragePool) {
|
||||
log.Info("pool not found, no volumes to delete", "pool", poolName)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("looking up pool %s: %w", poolName, err)
|
||||
}
|
||||
for _, vol := range []string{overlayVolumeName(name), seedVolumeName(name)} {
|
||||
if err := deleteIfExists(api, pool, vol); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
log.V(1).Info("per-VM volumes deleted")
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListByTag inspects every domain on every host and keeps those carrying
|
||||
// the operator's managed metadata. One unreachable host fails the whole
|
||||
// sweep, so orphan GC never acts on a partial view of the fleet; a single
|
||||
// domain that cannot be read is skipped instead, which is safe in the
|
||||
// same direction — a domain missing from the list is never collected.
|
||||
func (p *Provider) ListByTag(ctx context.Context) ([]provider.Instance, error) {
|
||||
log := p.logger(ctx)
|
||||
var out []provider.Instance
|
||||
for _, host := range p.cfg.Hosts {
|
||||
hlog := log.WithValues("host", host.Name)
|
||||
err := p.hosts.call(ctx, host, func(api libvirtAPI) error {
|
||||
doms, _, err := api.ConnectListAllDomains(1, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing domains: %w", err)
|
||||
}
|
||||
for _, dom := range doms {
|
||||
if inst := p.managedInstance(hlog, api, host.Name, dom); inst != nil {
|
||||
out = append(out, *inst)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
logAPIError(hlog, "list", err)
|
||||
return nil, p.wrapErr("list", "", err)
|
||||
}
|
||||
}
|
||||
log.V(1).Info("KVM domains listed", "hosts", len(p.cfg.Hosts), "managed", len(out))
|
||||
for _, inst := range out {
|
||||
log.V(2).Info("KVM listed instance", "id", inst.ID, "state", inst.State, "uid", inst.UID, "createdAt", inst.CreatedAt)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// managedInstance returns nil for domains the operator does not own, for
|
||||
// domains that vanished between the listing and the inspection, and for
|
||||
// domains whose metadata or state cannot be read (corrupt element, mid
|
||||
// teardown): one bad domain must not block GC for every other one. A
|
||||
// transport failure would also land here, but it fails the next RPC in
|
||||
// the same sweep too, so the host still counts as unreachable.
|
||||
func (p *Provider) managedInstance(log logr.Logger, api libvirtAPI, hostName string, dom libvirt.Domain) *provider.Instance {
|
||||
log = log.WithValues("name", dom.Name)
|
||||
meta, err := p.metadata(api, dom)
|
||||
if err != nil {
|
||||
log.V(1).Info("skipping domain with unreadable metadata", "error", err.Error())
|
||||
return nil
|
||||
}
|
||||
if !meta.managed() {
|
||||
return nil
|
||||
}
|
||||
inst, err := p.inspect(log, api, hostName, dom, meta)
|
||||
if err != nil {
|
||||
log.V(1).Info("skipping domain that could not be inspected", "error", err.Error())
|
||||
return nil
|
||||
}
|
||||
return inst
|
||||
}
|
||||
@@ -0,0 +1,880 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
libvirt "github.com/digitalocean/go-libvirt"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
func TestWithDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := withDefaults(provider.KVMConfig{
|
||||
Images: map[string]provider.KVMImage{
|
||||
"debian": {URL: "https://x/d.qcow2"},
|
||||
"raw": {URL: "https://x/r.img", Format: "raw"},
|
||||
},
|
||||
Hosts: []provider.KVMHost{
|
||||
{Name: "a", Address: "a:22", User: "virt", Bridge: "br0"},
|
||||
{Name: "b", Address: "b:22", User: "virt", Bridge: "br1", Socket: "/run/s", Pool: "fast"},
|
||||
},
|
||||
})
|
||||
if cfg.DiskSizeGB != defaultDiskSizeGB {
|
||||
t.Errorf("diskSizeGb = %d, want %d", cfg.DiskSizeGB, defaultDiskSizeGB)
|
||||
}
|
||||
if cfg.VendorData != defaultVendorData {
|
||||
t.Errorf("vendorData = %q, want the default", cfg.VendorData)
|
||||
}
|
||||
if cfg.Images["debian"].Format != "qcow2" || cfg.Images["raw"].Format != "raw" {
|
||||
t.Errorf("image formats = %q/%q, want qcow2 default and raw kept", cfg.Images["debian"].Format, cfg.Images["raw"].Format)
|
||||
}
|
||||
if a := cfg.Hosts[0]; a.Socket != defaultSocket || a.Pool != defaultPool {
|
||||
t.Errorf("host a = %+v, want default socket and pool", a)
|
||||
}
|
||||
if b := cfg.Hosts[1]; b.Socket != "/run/s" || b.Pool != "fast" {
|
||||
t.Errorf("host b = %+v, want explicit socket and pool kept", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithDefaults_keepsExplicitValues(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := withDefaults(provider.KVMConfig{DiskSizeGB: 42, VendorData: "#cloud-config\n"})
|
||||
if cfg.DiskSizeGB != 42 || cfg.VendorData != "#cloud-config\n" {
|
||||
t.Errorf("cfg = %+v, want explicit values kept", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
// --- provider over fake hosts ---
|
||||
|
||||
const (
|
||||
testName = "proxy-abc123def456ghij"
|
||||
testUID = "11111111-2222-3333-4444-555555555555"
|
||||
testImageURL = "https://images.example/debian-13.qcow2"
|
||||
testCloudInit = "#cloud-config\npackages: [squid]\n"
|
||||
)
|
||||
|
||||
var (
|
||||
testNow = time.Date(2026, 9, 8, 10, 0, 0, 0, time.UTC)
|
||||
errHostOff = errors.New("ssh dial: connection refused")
|
||||
)
|
||||
|
||||
// hostFakes routes each configured host name to its own fakeAPI; a host
|
||||
// with no entry is unreachable.
|
||||
type hostFakes map[string]*fakeAPI
|
||||
|
||||
func (h hostFakes) dial(_ context.Context, host provider.KVMHost) (*hostClient, error) {
|
||||
api, ok := h[host.Name]
|
||||
if !ok {
|
||||
return nil, errHostOff
|
||||
}
|
||||
return &hostClient{api: api, close: func() {}}, nil
|
||||
}
|
||||
|
||||
func testConfig(hosts ...string) provider.ProviderConfig {
|
||||
cfg := &provider.KVMConfig{
|
||||
SSHKeyFile: "/k",
|
||||
KnownHostsFile: "/kh",
|
||||
Images: map[string]provider.KVMImage{"debian-13": {URL: testImageURL}},
|
||||
MachineTypes: map[string]provider.KVMMachineType{"small": {VCPUs: 2, MemoryMiB: 2048}},
|
||||
}
|
||||
for _, h := range hosts {
|
||||
cfg.Hosts = append(cfg.Hosts, provider.KVMHost{Name: h, Address: h + ":22", User: "virt", Bridge: "br0"})
|
||||
}
|
||||
return provider.ProviderConfig{Name: "kvm-home", Type: "kvm", KVM: cfg}
|
||||
}
|
||||
|
||||
func newTestProvider(fakes hostFakes, hosts ...string) *Provider {
|
||||
p := newWithDialer(testConfig(hosts...), fakes.dial, nil)
|
||||
p.now = func() time.Time { return testNow }
|
||||
return p
|
||||
}
|
||||
|
||||
// cacheImage pre-populates the pool with the configured image and its
|
||||
// marker so Create never touches the network.
|
||||
func cacheImage(api *fakeAPI) {
|
||||
src := imageSource{Key: "debian-13", URL: testImageURL, Format: formatQcow2}
|
||||
api.putVol("default", src.volumeName(), formatQcow2, []byte("qcow2"))
|
||||
api.putVol("default", src.markerName(), formatRaw, []byte(testImageSHA()))
|
||||
}
|
||||
|
||||
func readyHost() *fakeAPI {
|
||||
api := newFakeAPI("default")
|
||||
cacheImage(api)
|
||||
return api
|
||||
}
|
||||
|
||||
func testRequest() provider.CreateRequest {
|
||||
return provider.CreateRequest{
|
||||
Name: testName,
|
||||
UID: testUID,
|
||||
Namespace: "default",
|
||||
ProxyName: "home-proxy-1",
|
||||
Placement: provider.Placement{Zone: "nas1", MachineType: "small", Image: "debian-13"},
|
||||
CloudInit: testCloudInit,
|
||||
Port: 3128,
|
||||
}
|
||||
}
|
||||
|
||||
func metadataXML(t *testing.T, uid string, created time.Time) string {
|
||||
t.Helper()
|
||||
out, err := xml.Marshal(newInstanceMetadata(uid, created))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func ipv4(addr string) libvirt.DomainIPAddr {
|
||||
return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv4), Addr: addr, Prefix: 24}
|
||||
}
|
||||
|
||||
func ipv6(addr string) libvirt.DomainIPAddr {
|
||||
return libvirt.DomainIPAddr{Type: int32(libvirt.IPAddrTypeIpv6), Addr: addr, Prefix: 64}
|
||||
}
|
||||
|
||||
// managedDomain seeds a domain as Create would have left it.
|
||||
func managedDomain(t *testing.T, api *fakeAPI, name, uid string, running bool, addrs map[uint32][]libvirt.DomainInterface) {
|
||||
t.Helper()
|
||||
api.putDomain(&fakeDomain{name: name, metadata: metadataXML(t, uid, testNow), running: running, addrs: addrs})
|
||||
api.putVol("default", overlayVolumeName(name), formatQcow2, nil)
|
||||
api.putVol("default", seedVolumeName(name), formatRaw, []byte("iso"))
|
||||
}
|
||||
|
||||
func wantClass(t *testing.T, err error, class error) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatalf("got nil error, want %v", class)
|
||||
}
|
||||
if !errors.Is(err, class) {
|
||||
t.Fatalf("error class = %v, want %v (err: %v)", provider.Class(err), class, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertCreated(t *testing.T, api *fakeAPI) {
|
||||
t.Helper()
|
||||
dom := api.domain(testName)
|
||||
if dom == nil {
|
||||
t.Fatal("domain not defined")
|
||||
}
|
||||
if !dom.running {
|
||||
t.Error("domain not running")
|
||||
}
|
||||
overlay := api.vol("default", overlayVolumeName(testName))
|
||||
if overlay == nil {
|
||||
t.Fatal("overlay volume missing")
|
||||
}
|
||||
if overlay.backing != "/var/lib/libvirt/images/base-debian-13.qcow2" || overlay.format != formatQcow2 {
|
||||
t.Errorf("overlay = %+v, want qcow2 backed by the cached base", overlay)
|
||||
}
|
||||
if overlay.capacity != defaultDiskSizeGB*bytesPerGiB {
|
||||
t.Errorf("overlay capacity = %d, want %d", overlay.capacity, defaultDiskSizeGB*bytesPerGiB)
|
||||
}
|
||||
seed := api.vol("default", seedVolumeName(testName))
|
||||
if seed == nil {
|
||||
t.Fatal("seed volume missing")
|
||||
}
|
||||
if seed.format != formatRaw || len(seed.data) == 0 || int64(len(seed.data)) != seed.capacity {
|
||||
t.Errorf("seed = format %s, %d bytes, capacity %d; want raw with capacity == uploaded size", seed.format, len(seed.data), seed.capacity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_freshHost(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
|
||||
id, err := p.Create(context.Background(), testRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
if want := "hosts/nas1/domains/" + testName; id != want {
|
||||
t.Errorf("providerID = %s, want %s", id, want)
|
||||
}
|
||||
assertCreated(t, api)
|
||||
|
||||
var d domainXML
|
||||
if err := xml.Unmarshal([]byte(api.domain(testName).xml), &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.VCPU != 2 || d.Memory.Value != 2048 || d.Memory.Unit != "MiB" {
|
||||
t.Errorf("domain size = %d vcpu / %d %s, want 2 / 2048 MiB", d.VCPU, d.Memory.Value, d.Memory.Unit)
|
||||
}
|
||||
if d.Devices.Interfaces[0].Source.Bridge != "br0" {
|
||||
t.Errorf("bridge = %q, want br0", d.Devices.Interfaces[0].Source.Bridge)
|
||||
}
|
||||
if m := d.Metadata.Instance; m.UID != testUID || !m.managed() || !m.createdAt().Equal(testNow) {
|
||||
t.Errorf("metadata = %+v, want managed, uid %s, createdAt %s", m, testUID, testNow)
|
||||
}
|
||||
if api.hasCall("StorageVolDownload") || api.callCount("StorageVolUpload") != 1 {
|
||||
t.Errorf("calls = %v, want no image download and exactly one upload (the seed)", api.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_fetchesImageOnFirstUse(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := newImageServer(t)
|
||||
api := newFakeAPI("default")
|
||||
cfg := testConfig("nas1")
|
||||
cfg.KVM.Images["debian-13"] = provider.KVMImage{URL: srv.URL + "/debian.qcow2", SHA256: testImageSHA()}
|
||||
p := newWithDialer(cfg, hostFakes{"nas1": api}.dial, nil)
|
||||
|
||||
if _, err := p.Create(context.Background(), testRequest()); err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
if srv.hits.Load() != 1 {
|
||||
t.Errorf("image server hits = %d, want 1", srv.hits.Load())
|
||||
}
|
||||
base := api.vol("default", "base-debian-13.qcow2")
|
||||
if base == nil || string(base.data) != string(testImage) {
|
||||
t.Fatal("base image not cached in the pool")
|
||||
}
|
||||
if api.domain(testName) == nil || !api.domain(testName).running {
|
||||
t.Error("domain not running after image fetch")
|
||||
}
|
||||
|
||||
// A second Proxy with the same image on the same host reuses the cache.
|
||||
req := testRequest()
|
||||
req.Name, req.UID = "proxy-second", "uid-2"
|
||||
if _, err := p.Create(context.Background(), req); err != nil {
|
||||
t.Fatalf("second Create: %v", err)
|
||||
}
|
||||
if srv.hits.Load() != 1 {
|
||||
t.Errorf("image server hits after second Create = %d, want still 1", srv.hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_isIdempotentWhenComplete(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
if _, err := p.Create(context.Background(), testRequest()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := len(api.calls)
|
||||
|
||||
id, err := p.Create(context.Background(), testRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("second Create: %v", err)
|
||||
}
|
||||
if id != "hosts/nas1/domains/"+testName {
|
||||
t.Errorf("providerID = %s", id)
|
||||
}
|
||||
for _, m := range []string{"DomainDefineXML", "DomainCreate", "StorageVolUpload"} {
|
||||
if n := api.callCount(m); n != 1 {
|
||||
t.Errorf("%s called %d times across two Creates, want 1", m, n)
|
||||
}
|
||||
}
|
||||
if len(api.calls) == before {
|
||||
t.Error("second Create made no calls at all; expected lookups")
|
||||
}
|
||||
}
|
||||
|
||||
// Each case breaks Create after a different step, then re-runs it and
|
||||
// expects the second attempt to pick up where the first stopped without
|
||||
// duplicating what already exists.
|
||||
func TestCreate_resumesAfterPartialFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
sabotage func(api *fakeAPI)
|
||||
class error
|
||||
check func(t *testing.T, api *fakeAPI)
|
||||
}{
|
||||
{
|
||||
name: "seed upload fails after overlay exists",
|
||||
sabotage: func(api *fakeAPI) {
|
||||
api.uploadHook = func(vol string, _ io.Reader) error {
|
||||
api.uploadHook = nil
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
},
|
||||
class: provider.ErrTransient,
|
||||
check: func(t *testing.T, api *fakeAPI) {
|
||||
if n := api.callCount("StorageVolUpload"); n != 2 {
|
||||
t.Errorf("seed uploads = %d, want 2 (failed, then redone)", n)
|
||||
}
|
||||
if n := api.callCount("StorageVolCreateXML"); n != 4 {
|
||||
t.Errorf("volume creates = %d, want 4 (overlay, seed; overlay rejected as existing, seed again)", n)
|
||||
}
|
||||
if names := api.volNames("default"); len(names) != 4 {
|
||||
t.Errorf("pool holds %v, want base, marker, one overlay, one seed", names)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "define fails after volumes exist",
|
||||
sabotage: func(api *fakeAPI) { api.failOnce["DomainDefineXML"] = lvError(libvirt.ErrInternalError, "boom") },
|
||||
class: provider.ErrTransient,
|
||||
check: func(t *testing.T, api *fakeAPI) {
|
||||
if n := api.callCount("StorageVolUpload"); n != 2 {
|
||||
t.Errorf("seed uploads = %d, want 2 (seed rebuilt while no domain exists)", n)
|
||||
}
|
||||
if n := api.callCount("DomainDefineXML"); n != 2 {
|
||||
t.Errorf("defines = %d, want 2", n)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "start fails after define",
|
||||
sabotage: func(api *fakeAPI) { api.failOnce["DomainCreate"] = lvError(libvirt.ErrInternalError, "boom") },
|
||||
class: provider.ErrTransient,
|
||||
check: func(t *testing.T, api *fakeAPI) {
|
||||
if n := api.callCount("DomainDefineXML"); n != 1 {
|
||||
t.Errorf("defines = %d, want 1 (existing domain reused)", n)
|
||||
}
|
||||
if n := api.callCount("DomainCreate"); n != 2 {
|
||||
t.Errorf("starts = %d, want 2", n)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "transport drops mid-create",
|
||||
sabotage: func(api *fakeAPI) { api.failOnce["DomainLookupByName"] = io.EOF },
|
||||
class: provider.ErrTransient,
|
||||
check: func(t *testing.T, api *fakeAPI) {
|
||||
if n := api.callCount("DomainDefineXML"); n != 1 {
|
||||
t.Errorf("defines = %d, want 1", n)
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
tc.sabotage(api)
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
|
||||
_, err := p.Create(context.Background(), testRequest())
|
||||
wantClass(t, err, tc.class)
|
||||
if _, err := p.Create(context.Background(), testRequest()); err != nil {
|
||||
t.Fatalf("retry Create: %v", err)
|
||||
}
|
||||
assertCreated(t, api)
|
||||
tc.check(t, api)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A reconcile cancelled mid-upload leaves a seed volume whose content is
|
||||
// short; the compensating delete cannot run because the pool has already
|
||||
// closed the connection. The next Create must not trust that volume.
|
||||
func TestCreate_replacesLeftoverSeed(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
api.putVol("default", seedVolumeName(testName), formatRaw, []byte("trunc"))
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
|
||||
if _, err := p.Create(context.Background(), testRequest()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertCreated(t, api)
|
||||
if seed := api.vol("default", seedVolumeName(testName)); string(seed.data) == "trunc" {
|
||||
t.Error("leftover seed was kept")
|
||||
}
|
||||
if n := api.callCount("StorageVolUpload"); n != 1 {
|
||||
t.Errorf("uploads = %d, want 1 (the rebuilt seed)", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_startsStoppedDomain(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
managedDomain(t, api, testName, testUID, false, nil)
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
|
||||
if _, err := p.Create(context.Background(), testRequest()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !api.domain(testName).running {
|
||||
t.Error("domain still stopped")
|
||||
}
|
||||
if api.hasCall("DomainDefineXML") || api.hasCall("StorageVolUpload") || api.hasCall("StorageVolDelete") {
|
||||
t.Errorf("calls = %v, want no define, upload or volume delete for an existing domain", api.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_singleHostNeedsNoZone(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
req := testRequest()
|
||||
req.Placement.Zone = ""
|
||||
|
||||
id, err := p.Create(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id != "hosts/nas1/domains/"+testName {
|
||||
t.Errorf("providerID = %s, want the single host filled in", id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_placementErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
hosts []string
|
||||
edit func(*provider.CreateRequest)
|
||||
want string
|
||||
}{
|
||||
{"unknown host", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Zone = "nas9" }, "not in kvm.hosts"},
|
||||
{"empty zone with two hosts", []string{"nas1", "nas2"}, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "placement.zone is required"},
|
||||
{"empty zone with no hosts", nil, func(r *provider.CreateRequest) { r.Placement.Zone = "" }, "kvm.hosts is empty"},
|
||||
{"unknown machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "huge" }, "not in kvm.machineTypes"},
|
||||
{"empty machine type", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.MachineType = "" }, "not in kvm.machineTypes"},
|
||||
{"unknown image", []string{"nas1"}, func(r *provider.CreateRequest) { r.Placement.Image = "windows" }, "neither a key"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fakes := hostFakes{}
|
||||
for _, h := range tc.hosts {
|
||||
fakes[h] = readyHost()
|
||||
}
|
||||
p := newTestProvider(fakes, tc.hosts...)
|
||||
req := testRequest()
|
||||
tc.edit(&req)
|
||||
|
||||
_, err := p.Create(context.Background(), req)
|
||||
wantClass(t, err, provider.ErrPermanent)
|
||||
if !strings.Contains(err.Error(), tc.want) {
|
||||
t.Errorf("error = %q, want it to mention %q", err, tc.want)
|
||||
}
|
||||
for _, api := range fakes {
|
||||
if len(api.calls) != 0 {
|
||||
t.Errorf("host was called: %v", api.calls)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_hostFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
setup func() (hostFakes, string)
|
||||
class error
|
||||
}{
|
||||
{
|
||||
name: "host unreachable",
|
||||
setup: func() (hostFakes, string) { return hostFakes{}, "nas1" },
|
||||
class: provider.ErrTransient,
|
||||
},
|
||||
{
|
||||
name: "pool missing",
|
||||
setup: func() (hostFakes, string) {
|
||||
api := newFakeAPI("other")
|
||||
return hostFakes{"nas1": api}, "nas1"
|
||||
},
|
||||
class: provider.ErrPermanent,
|
||||
},
|
||||
{
|
||||
name: "domain xml rejected",
|
||||
setup: func() (hostFakes, string) {
|
||||
api := readyHost()
|
||||
api.fail["DomainDefineXML"] = lvError(libvirt.ErrConfigUnsupported, "no q35 here")
|
||||
return hostFakes{"nas1": api}, "nas1"
|
||||
},
|
||||
class: provider.ErrPermanent,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
fakes, host := tc.setup()
|
||||
p := newTestProvider(fakes, host)
|
||||
_, err := p.Create(context.Background(), testRequest())
|
||||
wantClass(t, err, tc.class)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_cancelledContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
_, err := p.Create(ctx, testRequest())
|
||||
wantClass(t, err, provider.ErrTransient)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("error = %v, want it to unwrap to context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet(t *testing.T) {
|
||||
t.Parallel()
|
||||
arp := map[uint32][]libvirt.DomainInterface{
|
||||
uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Name: "vnet0", Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.50")}}},
|
||||
}
|
||||
agentOnly := map[uint32][]libvirt.DomainInterface{
|
||||
uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
|
||||
{Name: "lo", Addrs: []libvirt.DomainIPAddr{ipv4("127.0.0.1"), ipv6("::1")}},
|
||||
{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv6("fe80::1"), ipv4("192.168.1.51")}},
|
||||
},
|
||||
}
|
||||
linkLocalOnly := map[uint32][]libvirt.DomainInterface{
|
||||
uint32(libvirt.DomainInterfaceAddressesSrcAgent): {
|
||||
{Name: "enp1s0", Addrs: []libvirt.DomainIPAddr{ipv4("169.254.10.10")}},
|
||||
},
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
running bool
|
||||
addrs map[uint32][]libvirt.DomainInterface
|
||||
state provider.InstanceState
|
||||
ip string
|
||||
}{
|
||||
{"running with arp address", true, arp, provider.StateRunning, "192.168.1.50"},
|
||||
{"running, agent only, skips loopback and v6", true, agentOnly, provider.StateRunning, "192.168.1.51"},
|
||||
{"running, no address yet", true, nil, provider.StateProvisioning, ""},
|
||||
{"running, only link-local", true, linkLocalOnly, provider.StateProvisioning, ""},
|
||||
{"shut off", false, arp, provider.StateStopped, ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
managedDomain(t, api, testName, testUID, tc.running, tc.addrs)
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
|
||||
inst, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
want := provider.Instance{ID: "hosts/nas1/domains/" + testName, IP: tc.ip, State: tc.state, UID: testUID, CreatedAt: testNow}
|
||||
if *inst != want {
|
||||
t.Errorf("instance = %+v, want %+v", *inst, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_unmanagedDomainHasNoUID(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
api.putDomain(&fakeDomain{name: "hand-made", running: true})
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
|
||||
inst, err := p.Get(context.Background(), "hosts/nas1/domains/hand-made")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if inst.UID != "" || !inst.CreatedAt.IsZero() {
|
||||
t.Errorf("instance = %+v, want empty UID and zero CreatedAt", inst)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_errors(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
id string
|
||||
setup func(t *testing.T, api *fakeAPI)
|
||||
class error
|
||||
}{
|
||||
{"missing domain", "hosts/nas1/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrNotFound},
|
||||
{"malformed id", "zones/x/instances/y", func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
|
||||
{"host not configured", "hosts/nas9/domains/" + testName, func(*testing.T, *fakeAPI) {}, provider.ErrPermanent},
|
||||
{
|
||||
"state read fails", "hosts/nas1/domains/" + testName,
|
||||
func(t *testing.T, api *fakeAPI) {
|
||||
managedDomain(t, api, testName, testUID, true, nil)
|
||||
api.fail["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
|
||||
},
|
||||
provider.ErrTransient,
|
||||
},
|
||||
{
|
||||
"corrupt metadata", "hosts/nas1/domains/" + testName,
|
||||
func(_ *testing.T, api *fakeAPI) {
|
||||
api.putDomain(&fakeDomain{name: testName, metadata: "<instance", running: true})
|
||||
},
|
||||
provider.ErrPermanent,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
tc.setup(t, api)
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
_, err := p.Get(context.Background(), tc.id)
|
||||
wantClass(t, err, tc.class)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_hostUnreachable(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := newTestProvider(hostFakes{}, "nas1")
|
||||
_, err := p.Get(context.Background(), "hosts/nas1/domains/"+testName)
|
||||
wantClass(t, err, provider.ErrTransient)
|
||||
}
|
||||
|
||||
func assertGone(t *testing.T, api *fakeAPI) {
|
||||
t.Helper()
|
||||
if api.domain(testName) != nil {
|
||||
t.Error("domain still defined")
|
||||
}
|
||||
if names := api.volNames("default"); len(names) != 2 {
|
||||
t.Errorf("pool holds %v, want only the cached base image and its marker", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
setup func(t *testing.T, api *fakeAPI)
|
||||
check func(t *testing.T, api *fakeAPI)
|
||||
}{
|
||||
{
|
||||
name: "running domain",
|
||||
setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, true, nil) },
|
||||
check: func(t *testing.T, api *fakeAPI) {
|
||||
if !api.hasCall("DomainDestroy") || !api.hasCall("DomainUndefineFlags") {
|
||||
t.Errorf("calls = %v, want destroy and undefine", api.calls)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "stopped domain",
|
||||
setup: func(t *testing.T, api *fakeAPI) { managedDomain(t, api, testName, testUID, false, nil) },
|
||||
check: func(t *testing.T, api *fakeAPI) {
|
||||
if !api.hasCall("DomainUndefineFlags") {
|
||||
t.Errorf("calls = %v, want undefine despite destroy being invalid", api.calls)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "domain gone, volumes left",
|
||||
setup: func(_ *testing.T, api *fakeAPI) {
|
||||
api.putVol("default", overlayVolumeName(testName), formatQcow2, nil)
|
||||
api.putVol("default", seedVolumeName(testName), formatRaw, nil)
|
||||
},
|
||||
check: func(t *testing.T, api *fakeAPI) {
|
||||
if api.hasCall("DomainDestroy") || api.hasCall("DomainUndefineFlags") {
|
||||
t.Errorf("calls = %v, want no domain operations", api.calls)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nothing exists",
|
||||
setup: func(*testing.T, *fakeAPI) {},
|
||||
check: func(t *testing.T, api *fakeAPI) {
|
||||
if api.hasCall("StorageVolDelete") {
|
||||
t.Errorf("calls = %v, want no deletes", api.calls)
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := readyHost()
|
||||
tc.setup(t, api)
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
|
||||
if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
assertGone(t, api)
|
||||
tc.check(t, api)
|
||||
if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
|
||||
t.Fatalf("second Delete: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_missingPoolIsNotAnError(t *testing.T) {
|
||||
t.Parallel()
|
||||
api := newFakeAPI("other")
|
||||
api.putDomain(&fakeDomain{name: testName, metadata: metadataXML(t, testUID, testNow), running: true})
|
||||
p := newTestProvider(hostFakes{"nas1": api}, "nas1")
|
||||
|
||||
if err := p.Delete(context.Background(), "hosts/nas1/domains/"+testName); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
if api.domain(testName) != nil {
|
||||
t.Error("domain still defined")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete_errors(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
id string
|
||||
fakes func(t *testing.T) hostFakes
|
||||
class error
|
||||
}{
|
||||
{"malformed id", "nope", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
|
||||
{"host not configured", "hosts/nas9/domains/x", func(*testing.T) hostFakes { return hostFakes{"nas1": readyHost()} }, provider.ErrPermanent},
|
||||
{"host unreachable", "hosts/nas1/domains/x", func(*testing.T) hostFakes { return hostFakes{} }, provider.ErrTransient},
|
||||
{
|
||||
"undefine fails", "hosts/nas1/domains/" + testName,
|
||||
func(t *testing.T) hostFakes {
|
||||
api := readyHost()
|
||||
managedDomain(t, api, testName, testUID, true, nil)
|
||||
api.fail["DomainUndefineFlags"] = lvError(libvirt.ErrInternalError, "boom")
|
||||
return hostFakes{"nas1": api}
|
||||
},
|
||||
provider.ErrTransient,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := newTestProvider(tc.fakes(t), "nas1")
|
||||
wantClass(t, p.Delete(context.Background(), tc.id), tc.class)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListByTag_acrossHosts(t *testing.T) {
|
||||
t.Parallel()
|
||||
nas1, nas2 := readyHost(), readyHost()
|
||||
managedDomain(t, nas1, "proxy-one", "uid-1", true, map[uint32][]libvirt.DomainInterface{
|
||||
uint32(libvirt.DomainInterfaceAddressesSrcArp): {{Addrs: []libvirt.DomainIPAddr{ipv4("192.168.1.10")}}},
|
||||
})
|
||||
nas1.putDomain(&fakeDomain{name: "hand-made", running: true})
|
||||
managedDomain(t, nas2, "proxy-two", "uid-2", false, nil)
|
||||
nas2.putDomain(&fakeDomain{name: "foreign", metadata: `<instance xmlns="` + metadataURI + `"><managed>false</managed></instance>`})
|
||||
p := newTestProvider(hostFakes{"nas1": nas1, "nas2": nas2}, "nas1", "nas2")
|
||||
|
||||
got, err := p.ListByTag(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ListByTag: %v", err)
|
||||
}
|
||||
want := []provider.Instance{
|
||||
{ID: "hosts/nas1/domains/proxy-one", IP: "192.168.1.10", State: provider.StateRunning, UID: "uid-1", CreatedAt: testNow},
|
||||
{ID: "hosts/nas2/domains/proxy-two", State: provider.StateStopped, UID: "uid-2", CreatedAt: testNow},
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %d instances %+v, want %d", len(got), got, len(want))
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Errorf("instance[%d] = %+v, want %+v", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListByTag_oneHostDownFailsSweep(t *testing.T) {
|
||||
t.Parallel()
|
||||
nas1 := readyHost()
|
||||
managedDomain(t, nas1, "proxy-one", "uid-1", true, nil)
|
||||
p := newTestProvider(hostFakes{"nas1": nas1}, "nas1", "nas2")
|
||||
|
||||
got, err := p.ListByTag(context.Background())
|
||||
wantClass(t, err, provider.ErrTransient)
|
||||
if got != nil {
|
||||
t.Errorf("got %+v alongside the error, want nil (no partial view)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A domain that cannot be read is left out of the sweep rather than
|
||||
// failing it: GC never deletes what it does not see, so skipping is the
|
||||
// safe direction, while one bad domain blocking GC for all others is not.
|
||||
func TestListByTag_skipsUnreadableDomain(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
setup func(t *testing.T, api *fakeAPI)
|
||||
}{
|
||||
{
|
||||
"deleted mid-sweep",
|
||||
func(t *testing.T, api *fakeAPI) {
|
||||
managedDomain(t, api, "proxy-one", "uid-1", true, nil)
|
||||
api.failOnce["DomainGetMetadata"] = lvError(libvirt.ErrNoDomain, "gone")
|
||||
},
|
||||
},
|
||||
{
|
||||
"corrupt metadata",
|
||||
func(_ *testing.T, api *fakeAPI) {
|
||||
api.putDomain(&fakeDomain{name: "proxy-one", metadata: "<instance", running: true})
|
||||
},
|
||||
},
|
||||
{
|
||||
"state unreadable",
|
||||
func(t *testing.T, api *fakeAPI) {
|
||||
managedDomain(t, api, "proxy-one", "uid-1", true, nil)
|
||||
api.failOnce["DomainGetState"] = lvError(libvirt.ErrInternalError, "boom")
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
nas1 := readyHost()
|
||||
tc.setup(t, nas1)
|
||||
managedDomain(t, nas1, "proxy-two", "uid-2", false, nil)
|
||||
p := newTestProvider(hostFakes{"nas1": nas1}, "nas1")
|
||||
|
||||
got, err := p.ListByTag(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ListByTag: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].UID != "uid-2" {
|
||||
t.Errorf("got %+v, want only proxy-two", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListByTag_noHosts(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := newTestProvider(hostFakes{})
|
||||
got, err := p.ListByTag(context.Background())
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Errorf("got %v, %v; want empty, nil", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapState(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
state libvirt.DomainState
|
||||
ip string
|
||||
want provider.InstanceState
|
||||
}{
|
||||
{libvirt.DomainRunning, "10.0.0.1", provider.StateRunning},
|
||||
{libvirt.DomainRunning, "", provider.StateProvisioning},
|
||||
{libvirt.DomainBlocked, "10.0.0.1", provider.StateRunning},
|
||||
{libvirt.DomainPaused, "10.0.0.1", provider.StateStopped},
|
||||
{libvirt.DomainShutdown, "", provider.StateStopped},
|
||||
{libvirt.DomainShutoff, "", provider.StateStopped},
|
||||
{libvirt.DomainCrashed, "", provider.StateStopped},
|
||||
{libvirt.DomainPmsuspended, "", provider.StateStopped},
|
||||
{libvirt.DomainNostate, "", provider.StateProvisioning},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(fmt.Sprintf("%s/%q", tc.state, tc.ip), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := mapState(tc.state, tc.ip); got != tc.want {
|
||||
t.Errorf("mapState(%s, %q) = %s, want %s", tc.state, tc.ip, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew_badKeyPathFailsFast(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := testConfig("nas1")
|
||||
cfg.KVM.SSHKeyFile = t.TempDir() + "/missing"
|
||||
if _, err := New(context.Background(), cfg); err == nil {
|
||||
t.Fatal("New succeeded with a missing key file")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
|
||||
)
|
||||
|
||||
// metadataURI namespaces the operator's element inside the domain's
|
||||
// <metadata>. libvirt has no labels or tags, but it stores any namespaced XML
|
||||
// there and hands it back through DomainGetMetadata, which is how the GC
|
||||
// contract (managed flag + owning UID) and the creation time survive across
|
||||
// operator restarts.
|
||||
const metadataURI = "https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm"
|
||||
|
||||
type instanceMetadata struct {
|
||||
XMLName xml.Name `xml:"https://gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/kvm instance"`
|
||||
Managed string `xml:"managed"`
|
||||
UID string `xml:"uid"`
|
||||
CreatedAt string `xml:"createdAt"`
|
||||
}
|
||||
|
||||
func newInstanceMetadata(uid string, createdAt time.Time) instanceMetadata {
|
||||
return instanceMetadata{
|
||||
Managed: provider.LabelManagedYes,
|
||||
UID: uid,
|
||||
CreatedAt: createdAt.UTC().Format(time.RFC3339),
|
||||
}
|
||||
}
|
||||
|
||||
// parseInstanceMetadata decodes the element as DomainGetMetadata returns it.
|
||||
// libvirt may re-serialise it with a namespace prefix rather than the default
|
||||
// namespace it was defined with; encoding/xml matches on namespace URI, so
|
||||
// both spellings decode.
|
||||
func parseInstanceMetadata(s string) (instanceMetadata, error) {
|
||||
var m instanceMetadata
|
||||
if err := xml.Unmarshal([]byte(s), &m); err != nil {
|
||||
return instanceMetadata{}, fmt.Errorf("parsing domain metadata: %w", err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m instanceMetadata) managed() bool {
|
||||
return m.Managed == provider.LabelManagedYes
|
||||
}
|
||||
|
||||
// createdAt returns the zero time on a malformed timestamp: orphan GC treats
|
||||
// zero as "old", so a corrupt value can never protect a domain forever.
|
||||
func (m instanceMetadata) createdAt() time.Time {
|
||||
t, err := time.Parse(time.RFC3339, m.CreatedAt)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return t
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestInstanceMetadata_roundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
created := time.Date(2026, 9, 4, 12, 30, 0, 0, time.FixedZone("CEST", 2*3600))
|
||||
raw, err := xml.Marshal(newInstanceMetadata("uid-1", created))
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(string(raw), `xmlns="`+metadataURI+`"`) {
|
||||
t.Errorf("marshalled metadata lacks the namespace declaration: %s", raw)
|
||||
}
|
||||
m, err := parseInstanceMetadata(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseInstanceMetadata() error = %v", err)
|
||||
}
|
||||
if !m.managed() || m.UID != "uid-1" {
|
||||
t.Errorf("parsed = %+v, want managed with uid-1", m)
|
||||
}
|
||||
if got := m.createdAt(); !got.Equal(created) || got.Location() != time.UTC {
|
||||
t.Errorf("createdAt() = %v, want %v in UTC", got, created)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseInstanceMetadata_libvirtPrefixedForm(t *testing.T) {
|
||||
t.Parallel()
|
||||
// libvirt re-serialises the stored element with whatever prefix it
|
||||
// chooses; only the namespace URI is contractual.
|
||||
in := `<op:instance xmlns:op="` + metadataURI + `">` +
|
||||
`<op:managed>true</op:managed><op:uid>uid-2</op:uid>` +
|
||||
`<op:createdAt>2026-09-04T10:00:00Z</op:createdAt></op:instance>`
|
||||
m, err := parseInstanceMetadata(in)
|
||||
if err != nil {
|
||||
t.Fatalf("parseInstanceMetadata() error = %v", err)
|
||||
}
|
||||
if !m.managed() || m.UID != "uid-2" || m.createdAt().Hour() != 10 {
|
||||
t.Errorf("parsed = %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseInstanceMetadata_notManaged(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
wantErr bool
|
||||
}{
|
||||
{"managed false", `<instance xmlns="` + metadataURI + `"><managed>false</managed><uid>u</uid></instance>`, false},
|
||||
{"foreign element", `<foo xmlns="urn:other"><managed>true</managed></foo>`, true},
|
||||
{"garbage", `not xml`, true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, err := parseInstanceMetadata(tc.in)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("error = %v, wantErr %v", err, tc.wantErr)
|
||||
}
|
||||
if m.managed() {
|
||||
t.Errorf("managed() = true for %q", tc.in)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstanceMetadata_createdAtMalformedIsZero(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := instanceMetadata{CreatedAt: "yesterday"}
|
||||
if !m.createdAt().IsZero() {
|
||||
t.Errorf("createdAt() = %v, want zero time", m.createdAt())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/kdomanski/iso9660"
|
||||
)
|
||||
|
||||
// seedLabel is the volume label cloud-init's NoCloud datasource looks for.
|
||||
const seedLabel = "cidata"
|
||||
|
||||
// defaultVendorData is the cloud-init vendor-data shipped on every seed
|
||||
// unless the config overrides it. cloud-init merges it under the Proxy's own
|
||||
// user-data, so stock images grow a guest agent (which lets the provider
|
||||
// read the VM's IP before ARP sees traffic) without the user's cloud-init
|
||||
// having to know about it.
|
||||
const defaultVendorData = `#cloud-config
|
||||
packages:
|
||||
- qemu-guest-agent
|
||||
runcmd:
|
||||
- [systemctl, enable, --now, qemu-guest-agent]
|
||||
`
|
||||
|
||||
// buildSeedISO renders a NoCloud seed: user-data verbatim, vendor-data as
|
||||
// given, and a meta-data whose instance-id is the deterministic VM name, so
|
||||
// cloud-init never mistakes a reboot for a new instance. The ISO's volume
|
||||
// descriptor carries the build time, so two builds differ in a few
|
||||
// timestamp bytes; the files and label are identical.
|
||||
func buildSeedISO(name, userData, vendorData string) ([]byte, error) {
|
||||
w, err := iso9660.NewWriter()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating iso writer: %w", err)
|
||||
}
|
||||
defer w.Cleanup() //nolint:errcheck // temp-dir cleanup; nothing to do on failure
|
||||
|
||||
files := []struct{ name, body string }{
|
||||
{"user-data", userData},
|
||||
{"meta-data", fmt.Sprintf("instance-id: %s\nlocal-hostname: %s\n", name, name)},
|
||||
{"vendor-data", vendorData},
|
||||
}
|
||||
for _, f := range files {
|
||||
if err := w.AddFile(strings.NewReader(f.body), f.name); err != nil {
|
||||
return nil, fmt.Errorf("adding %s to seed iso: %w", f.name, err)
|
||||
}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := w.WriteTo(&buf, seedLabel); err != nil {
|
||||
return nil, fmt.Errorf("writing seed iso: %w", err)
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"maps"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kdomanski/iso9660"
|
||||
)
|
||||
|
||||
func readSeed(t *testing.T, iso []byte) (label string, files map[string]string) {
|
||||
t.Helper()
|
||||
img, err := iso9660.OpenImage(bytes.NewReader(iso))
|
||||
if err != nil {
|
||||
t.Fatalf("OpenImage() error = %v", err)
|
||||
}
|
||||
label, err = img.Label()
|
||||
if err != nil {
|
||||
t.Fatalf("Label() error = %v", err)
|
||||
}
|
||||
root, err := img.RootDir()
|
||||
if err != nil {
|
||||
t.Fatalf("RootDir() error = %v", err)
|
||||
}
|
||||
children, err := root.GetChildren()
|
||||
if err != nil {
|
||||
t.Fatalf("GetChildren() error = %v", err)
|
||||
}
|
||||
files = make(map[string]string, len(children))
|
||||
for _, c := range children {
|
||||
body, err := io.ReadAll(c.Reader())
|
||||
if err != nil {
|
||||
t.Fatalf("reading %s: %v", c.Name(), err)
|
||||
}
|
||||
files[c.Name()] = string(body)
|
||||
}
|
||||
return label, files
|
||||
}
|
||||
|
||||
func TestBuildSeedISO_contents(t *testing.T) {
|
||||
t.Parallel()
|
||||
userData := "#cloud-config\npackages: [squid]\n"
|
||||
iso, err := buildSeedISO("proxy-abc123def456ghij", userData, defaultVendorData)
|
||||
if err != nil {
|
||||
t.Fatalf("buildSeedISO() error = %v", err)
|
||||
}
|
||||
label, files := readSeed(t, iso)
|
||||
if !strings.EqualFold(label, seedLabel) {
|
||||
t.Errorf("label = %q, want %q", label, seedLabel)
|
||||
}
|
||||
if len(files) != 3 {
|
||||
t.Errorf("files = %v, want exactly user-data, meta-data, vendor-data", files)
|
||||
}
|
||||
if files["user-data"] != userData {
|
||||
t.Errorf("user-data = %q, want verbatim cloud-init", files["user-data"])
|
||||
}
|
||||
if files["vendor-data"] != defaultVendorData {
|
||||
t.Errorf("vendor-data = %q", files["vendor-data"])
|
||||
}
|
||||
if got, want := files["meta-data"], "instance-id: proxy-abc123def456ghij\nlocal-hostname: proxy-abc123def456ghij\n"; got != want {
|
||||
t.Errorf("meta-data = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The writer stamps the build time into the volume descriptor, so two
|
||||
// builds are not byte-identical; what must not change between them is
|
||||
// what cloud-init reads: the label and the three files.
|
||||
func TestBuildSeedISO_deterministicContent(t *testing.T) {
|
||||
t.Parallel()
|
||||
a, err := buildSeedISO("proxy-x", "#cloud-config\n", "")
|
||||
if err != nil {
|
||||
t.Fatalf("first buildSeedISO() error = %v", err)
|
||||
}
|
||||
b, err := buildSeedISO("proxy-x", "#cloud-config\n", "")
|
||||
if err != nil {
|
||||
t.Fatalf("second buildSeedISO() error = %v", err)
|
||||
}
|
||||
labelA, filesA := readSeed(t, a)
|
||||
labelB, filesB := readSeed(t, b)
|
||||
if labelA != labelB || !maps.Equal(filesA, filesB) {
|
||||
t.Errorf("two builds for the same inputs differ: %q %v vs %q %v", labelA, filesA, labelB, filesB)
|
||||
}
|
||||
if len(a) != len(b) {
|
||||
t.Errorf("sizes differ: %d vs %d; the seed volume is created with the ISO's exact size", len(a), len(b))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSeedISO_emptyUserData(t *testing.T) {
|
||||
t.Parallel()
|
||||
iso, err := buildSeedISO("proxy-x", "", defaultVendorData)
|
||||
if err != nil {
|
||||
t.Fatalf("buildSeedISO() error = %v", err)
|
||||
}
|
||||
_, files := readSeed(t, iso)
|
||||
if body, ok := files["user-data"]; !ok || body != "" {
|
||||
t.Errorf("user-data = %q (present=%v), want an empty file to still exist", body, ok)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const bytesPerGiB = 1 << 30
|
||||
|
||||
func overlayVolumeName(name string) string { return name + ".qcow2" }
|
||||
|
||||
func seedVolumeName(name string) string { return name + "-seed.iso" }
|
||||
|
||||
// volumeSpec describes a storage volume to create. A non-empty BackingPath
|
||||
// makes it a copy-on-write overlay of that file.
|
||||
type volumeSpec struct {
|
||||
Name string
|
||||
SizeBytes int64
|
||||
Format string
|
||||
BackingPath string
|
||||
BackingFormat string
|
||||
}
|
||||
|
||||
func overlayVolume(name string, sizeGB int64, basePath, baseFormat string) volumeSpec {
|
||||
return volumeSpec{
|
||||
Name: overlayVolumeName(name),
|
||||
SizeBytes: sizeGB * bytesPerGiB,
|
||||
Format: formatQcow2,
|
||||
BackingPath: basePath,
|
||||
BackingFormat: baseFormat,
|
||||
}
|
||||
}
|
||||
|
||||
func rawVolume(name string, sizeBytes int64) volumeSpec {
|
||||
return volumeSpec{Name: name, SizeBytes: sizeBytes, Format: formatRaw}
|
||||
}
|
||||
|
||||
// buildVolumeXML renders the volume for StorageVolCreateXML. Capacity is
|
||||
// always given in bytes so no libvirt unit-suffix ambiguity (G vs GiB) can
|
||||
// creep in.
|
||||
func buildVolumeXML(v volumeSpec) (string, error) {
|
||||
x := volumeXML{
|
||||
Type: "file",
|
||||
Name: v.Name,
|
||||
Capacity: sizeXML{Unit: "bytes", Value: v.SizeBytes},
|
||||
Target: volumeTargetXML{Format: formatXML{Type: v.Format}},
|
||||
}
|
||||
if v.BackingPath != "" {
|
||||
x.BackingStore = &backingStoreXML{Path: v.BackingPath, Format: formatXML{Type: v.BackingFormat}}
|
||||
}
|
||||
out, err := xml.Marshal(x)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshalling volume xml: %w", err)
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
type volumeXML struct {
|
||||
XMLName xml.Name `xml:"volume"`
|
||||
Type string `xml:"type,attr"`
|
||||
Name string `xml:"name"`
|
||||
Capacity sizeXML `xml:"capacity"`
|
||||
Target volumeTargetXML `xml:"target"`
|
||||
BackingStore *backingStoreXML `xml:"backingStore"`
|
||||
}
|
||||
|
||||
type volumeTargetXML struct {
|
||||
Format formatXML `xml:"format"`
|
||||
}
|
||||
|
||||
type formatXML struct {
|
||||
Type string `xml:"type,attr"`
|
||||
}
|
||||
|
||||
type backingStoreXML struct {
|
||||
Path string `xml:"path"`
|
||||
Format formatXML `xml:"format"`
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package kvm
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestVolumeNames(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := overlayVolumeName("proxy-x"); got != "proxy-x.qcow2" {
|
||||
t.Errorf("overlayVolumeName = %q", got)
|
||||
}
|
||||
if got := seedVolumeName("proxy-x"); got != "proxy-x-seed.iso" {
|
||||
t.Errorf("seedVolumeName = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildVolumeXML_overlay(t *testing.T) {
|
||||
t.Parallel()
|
||||
raw, err := buildVolumeXML(overlayVolume("proxy-x", 10, "/var/lib/libvirt/images/base-debian-13.qcow2", "qcow2"))
|
||||
if err != nil {
|
||||
t.Fatalf("buildVolumeXML() error = %v", err)
|
||||
}
|
||||
var v volumeXML
|
||||
if err := xml.Unmarshal([]byte(raw), &v); err != nil {
|
||||
t.Fatalf("output is not well-formed XML: %v", err)
|
||||
}
|
||||
if v.Type != "file" || v.Name != "proxy-x.qcow2" {
|
||||
t.Errorf("type/name = %s/%s", v.Type, v.Name)
|
||||
}
|
||||
if v.Capacity.Unit != "bytes" || v.Capacity.Value != 10*bytesPerGiB {
|
||||
t.Errorf("capacity = %d %s, want %d bytes", v.Capacity.Value, v.Capacity.Unit, 10*bytesPerGiB)
|
||||
}
|
||||
if v.Target.Format.Type != "qcow2" {
|
||||
t.Errorf("format = %q", v.Target.Format.Type)
|
||||
}
|
||||
if v.BackingStore == nil {
|
||||
t.Fatalf("backingStore missing:\n%s", raw)
|
||||
}
|
||||
if v.BackingStore.Path != "/var/lib/libvirt/images/base-debian-13.qcow2" || v.BackingStore.Format.Type != "qcow2" {
|
||||
t.Errorf("backingStore = %+v", v.BackingStore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildVolumeXML_rawHasNoBackingStore(t *testing.T) {
|
||||
t.Parallel()
|
||||
raw, err := buildVolumeXML(rawVolume("proxy-x-seed.iso", 372736))
|
||||
if err != nil {
|
||||
t.Fatalf("buildVolumeXML() error = %v", err)
|
||||
}
|
||||
if strings.Contains(raw, "backingStore") {
|
||||
t.Errorf("raw volume must not declare a backing store:\n%s", raw)
|
||||
}
|
||||
var v volumeXML
|
||||
if err := xml.Unmarshal([]byte(raw), &v); err != nil {
|
||||
t.Fatalf("output is not well-formed XML: %v", err)
|
||||
}
|
||||
if v.Name != "proxy-x-seed.iso" || v.Capacity.Value != 372736 || v.Target.Format.Type != "raw" {
|
||||
t.Errorf("volume = %+v", v)
|
||||
}
|
||||
}
|
||||
@@ -50,25 +50,17 @@ func (i *instrumented) record(op string, err error) {
|
||||
i.rec.ProviderRequest(i.name, op, resultLabel(err))
|
||||
}
|
||||
|
||||
const (
|
||||
resultOK = "ok"
|
||||
resultNotFound = "not_found"
|
||||
resultQuotaExceeded = "quota_exceeded"
|
||||
resultPermanent = "permanent"
|
||||
resultTransient = "transient"
|
||||
)
|
||||
|
||||
func resultLabel(err error) string {
|
||||
switch Class(err) {
|
||||
case nil:
|
||||
return resultOK
|
||||
return "ok"
|
||||
case ErrNotFound:
|
||||
return resultNotFound
|
||||
return "not_found"
|
||||
case ErrQuotaExceeded:
|
||||
return resultQuotaExceeded
|
||||
return "quota_exceeded"
|
||||
case ErrPermanent:
|
||||
return resultPermanent
|
||||
return "permanent"
|
||||
default:
|
||||
return resultTransient
|
||||
return "transient"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
// WithTracing wraps a Provider so every call runs in a client span carrying
|
||||
// the classified result, mirroring WithMetrics. It goes through
|
||||
// tracing.StartSpan, so provider-internal logs (e.g. the gcp provider's
|
||||
// context logger) inherit traceID/spanID. Errors pass through unmodified.
|
||||
func WithTracing(name string, p Provider, opts ...tracing.Option) Provider {
|
||||
return &traced{name: name, inner: p, tracer: tracing.Tracer(opts...)}
|
||||
}
|
||||
|
||||
type traced struct {
|
||||
name string
|
||||
inner Provider
|
||||
tracer trace.Tracer
|
||||
}
|
||||
|
||||
func (t *traced) Create(ctx context.Context, req CreateRequest) (string, error) {
|
||||
ctx, span := t.start(ctx, "provider.create")
|
||||
id, err := t.inner.Create(ctx, req)
|
||||
if id != "" {
|
||||
span.SetAttributes(attribute.String("provider.id", id))
|
||||
}
|
||||
t.end(span, err)
|
||||
return id, err
|
||||
}
|
||||
|
||||
func (t *traced) Get(ctx context.Context, providerID string) (*Instance, error) {
|
||||
ctx, span := t.start(ctx, "provider.get")
|
||||
span.SetAttributes(attribute.String("provider.id", providerID))
|
||||
inst, err := t.inner.Get(ctx, providerID)
|
||||
t.end(span, err)
|
||||
return inst, err
|
||||
}
|
||||
|
||||
func (t *traced) Delete(ctx context.Context, providerID string) error {
|
||||
ctx, span := t.start(ctx, "provider.delete")
|
||||
span.SetAttributes(attribute.String("provider.id", providerID))
|
||||
err := t.inner.Delete(ctx, providerID)
|
||||
t.end(span, err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *traced) ListByTag(ctx context.Context) ([]Instance, error) {
|
||||
ctx, span := t.start(ctx, "provider.list")
|
||||
instances, err := t.inner.ListByTag(ctx)
|
||||
span.SetAttributes(attribute.Int("provider.instances", len(instances)))
|
||||
t.end(span, err)
|
||||
return instances, err
|
||||
}
|
||||
|
||||
func (t *traced) start(ctx context.Context, op string) (context.Context, trace.Span) {
|
||||
return tracing.StartSpan(ctx, t.tracer, op,
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("provider.name", t.name)))
|
||||
}
|
||||
|
||||
// end records the taxonomy class and closes the span. ErrNotFound stays Ok:
|
||||
// the reconciler polls Get to NotFound during replacement/deletion, so it is
|
||||
// an expected answer, not a failure — same reasoning as resultLabel's
|
||||
// distinct "not_found" bucket.
|
||||
func (t *traced) end(span trace.Span, err error) {
|
||||
span.SetAttributes(attribute.String("provider.result", resultLabel(err)))
|
||||
switch Class(err) {
|
||||
case nil, ErrNotFound:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
default:
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
|
||||
)
|
||||
|
||||
func TestWithTracing_spanPerCall(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inner *staticProvider
|
||||
call func(p Provider) error
|
||||
wantSpan string
|
||||
wantStatus codes.Code
|
||||
wantResult string
|
||||
wantAttr attribute.KeyValue
|
||||
}{
|
||||
{
|
||||
name: "successful create",
|
||||
inner: &staticProvider{},
|
||||
call: func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
|
||||
wantSpan: "provider.create",
|
||||
wantStatus: codes.Ok,
|
||||
wantResult: resultOK,
|
||||
wantAttr: attribute.String("provider.id", "id-1"),
|
||||
},
|
||||
{
|
||||
name: "get NotFound is not a span error",
|
||||
inner: &staticProvider{getErr: Wrap(ErrNotFound, "get", "x", "id-1", nil)},
|
||||
call: func(p Provider) error { _, err := p.Get(context.Background(), "id-1"); return err },
|
||||
wantSpan: "provider.get",
|
||||
wantStatus: codes.Ok,
|
||||
wantResult: resultNotFound,
|
||||
wantAttr: attribute.String("provider.id", "id-1"),
|
||||
},
|
||||
{
|
||||
name: "delete transient error",
|
||||
inner: &staticProvider{deleteErr: Wrap(ErrTransient, "delete", "x", "id-1", errors.New("503"))},
|
||||
call: func(p Provider) error { return p.Delete(context.Background(), "id-1") },
|
||||
wantSpan: "provider.delete",
|
||||
wantStatus: codes.Error,
|
||||
wantResult: resultTransient,
|
||||
wantAttr: attribute.String("provider.id", "id-1"),
|
||||
},
|
||||
{
|
||||
name: "quota exceeded create",
|
||||
inner: &staticProvider{createErr: Wrap(ErrQuotaExceeded, "create", "x", "", nil)},
|
||||
call: func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
|
||||
wantSpan: "provider.create",
|
||||
wantStatus: codes.Error,
|
||||
wantResult: resultQuotaExceeded,
|
||||
wantAttr: attribute.String("provider.name", "x"),
|
||||
},
|
||||
{
|
||||
name: "list records instance count",
|
||||
inner: &staticProvider{},
|
||||
call: func(p Provider) error { _, err := p.ListByTag(context.Background()); return err },
|
||||
wantSpan: "provider.list",
|
||||
wantStatus: codes.Ok,
|
||||
wantResult: resultOK,
|
||||
wantAttr: attribute.Int("provider.instances", 0),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
|
||||
p := WithTracing("x", tc.inner, tracing.WithTracerProvider(tp))
|
||||
err := tc.call(p)
|
||||
|
||||
wantErr := errors.Join(tc.inner.createErr, tc.inner.getErr, tc.inner.deleteErr, tc.inner.listErr)
|
||||
if (wantErr == nil) != (err == nil) {
|
||||
t.Fatalf("decorator changed the error: got %v", err)
|
||||
}
|
||||
|
||||
ended := sr.Ended()
|
||||
if len(ended) != 1 {
|
||||
t.Fatalf("got %d spans, want 1", len(ended))
|
||||
}
|
||||
span := ended[0]
|
||||
if span.Name() != tc.wantSpan {
|
||||
t.Errorf("span name = %q, want %q", span.Name(), tc.wantSpan)
|
||||
}
|
||||
if span.SpanKind() != trace.SpanKindClient {
|
||||
t.Errorf("span kind = %v, want client", span.SpanKind())
|
||||
}
|
||||
if span.Status().Code != tc.wantStatus {
|
||||
t.Errorf("status = %v, want %v", span.Status().Code, tc.wantStatus)
|
||||
}
|
||||
attrs := span.Attributes()
|
||||
hasAttr := func(want attribute.KeyValue) bool {
|
||||
return slices.Contains(attrs, want)
|
||||
}
|
||||
if !hasAttr(attribute.String("provider.result", tc.wantResult)) {
|
||||
t.Errorf("provider.result %q missing in %v", tc.wantResult, attrs)
|
||||
}
|
||||
if !hasAttr(tc.wantAttr) {
|
||||
t.Errorf("attribute %v missing in %v", tc.wantAttr, attrs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
)
|
||||
|
||||
// HTTPMiddleware returns middleware that opens a server span per request
|
||||
// (named from the mux route pattern once routing has happened) and hands
|
||||
// handlers a request context whose logf.FromContext logger is base enriched
|
||||
// with the trace context. Incoming W3C traceparent headers are honored, so
|
||||
// clients see their own trace continue into the operator. /healthz is never
|
||||
// traced. With tracing disabled the middleware degrades to injecting base
|
||||
// unenriched — handlers can rely on logf.FromContext either way.
|
||||
func HTTPMiddleware(operation string, base logr.Logger, opts ...Option) func(http.Handler) http.Handler {
|
||||
o := newOptions(opts)
|
||||
return func(next http.Handler) http.Handler {
|
||||
inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
r2 := r.WithContext(ContextWithLogger(r.Context(), base))
|
||||
next.ServeHTTP(w, r2)
|
||||
// WithContext copies the request, so the mux recorded the matched
|
||||
// route on r2; surface it to otelhttp, which renames the span
|
||||
// from r.Pattern after the handler returns.
|
||||
r.Pattern = r2.Pattern
|
||||
})
|
||||
otelOpts := []otelhttp.Option{
|
||||
otelhttp.WithFilter(func(r *http.Request) bool { return r.URL.Path != "/healthz" }),
|
||||
// Explicit propagators: deterministic regardless of whether the
|
||||
// global propagator has been installed (tests, disabled tracing).
|
||||
otelhttp.WithPropagators(propagation.NewCompositeTextMapPropagator(
|
||||
propagation.TraceContext{}, propagation.Baggage{})),
|
||||
}
|
||||
if o.tp != nil {
|
||||
otelOpts = append(otelOpts, otelhttp.WithTracerProvider(o.tp))
|
||||
}
|
||||
return otelhttp.NewHandler(inner, operation, otelOpts...)
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
const sampleTraceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
|
||||
|
||||
func TestHTTPMiddleware(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
newServer := func(lines *[]string, tp *sdktrace.TracerProvider) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
handler := func(w http.ResponseWriter, r *http.Request) {
|
||||
logf.FromContext(r.Context()).Info("handling")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
mux.HandleFunc("GET /healthz", handler)
|
||||
mux.HandleFunc("GET /v1/things/{id}", handler)
|
||||
return HTTPMiddleware("discovery", captureLogger(lines), WithTracerProvider(tp))(mux)
|
||||
}
|
||||
|
||||
t.Run("route span, enriched handler logs, traceparent continuation", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
|
||||
var lines []string
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/things/42", nil)
|
||||
req.Header.Set("traceparent", sampleTraceparent)
|
||||
newServer(&lines, tp).ServeHTTP(rec, req)
|
||||
|
||||
ended := sr.Ended()
|
||||
if len(ended) != 1 {
|
||||
t.Fatalf("got %d spans, want 1", len(ended))
|
||||
}
|
||||
if got := ended[0].Name(); got != "GET /v1/things/{id}" {
|
||||
t.Errorf("span name = %q, want route pattern", got)
|
||||
}
|
||||
wantTrace := "4bf92f3577b34da6a3ce929d0e0e4736"
|
||||
if got := ended[0].SpanContext().TraceID().String(); got != wantTrace {
|
||||
t.Errorf("span traceID = %s, want continuation of client trace %s", got, wantTrace)
|
||||
}
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("got %d log lines, want 1: %v", len(lines), lines)
|
||||
}
|
||||
if n := strings.Count(lines[0], `"traceID"`); n != 1 {
|
||||
t.Errorf("traceID appears %d times, want 1: %s", n, lines[0])
|
||||
}
|
||||
if !strings.Contains(lines[0], wantTrace) {
|
||||
t.Errorf("handler log missing traceID %s: %s", wantTrace, lines[0])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("healthz is not traced but still gets a logger", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
|
||||
var lines []string
|
||||
rec := httptest.NewRecorder()
|
||||
newServer(&lines, tp).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
||||
|
||||
if len(sr.Ended()) != 0 {
|
||||
t.Fatalf("healthz produced %d spans, want 0", len(sr.Ended()))
|
||||
}
|
||||
if len(lines) != 1 || strings.Contains(lines[0], "traceID") {
|
||||
t.Fatalf("want one unenriched log line, got %v", lines)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
// baseLoggerKey stores the pre-enrichment logger. logr sinks never see a
|
||||
// context, so trace IDs must ride on the logger as WithValues — and zap does
|
||||
// not dedupe repeated keys, so each nested span must re-derive from the same
|
||||
// base instead of stacking traceID/spanID onto an already-enriched logger.
|
||||
type baseLoggerKey struct{}
|
||||
|
||||
// Start begins a span from the global tracer and returns a context whose
|
||||
// logf.FromContext logger carries the span's traceID/spanID. With tracing
|
||||
// disabled the span is a no-op with an invalid span context and the logger
|
||||
// is left untouched.
|
||||
//
|
||||
// Trade-off, documented: values pushed via logf.IntoContext *between* two
|
||||
// Start calls are dropped by the inner Start's re-derivation. Nothing
|
||||
// first-party does that; controller-runtime's per-reconcile logger
|
||||
// (reconcileID etc.) is captured as the base and survives.
|
||||
func Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
|
||||
return StartSpan(ctx, newOptions(nil).tracer(), name, opts...)
|
||||
}
|
||||
|
||||
// StartSpan is Start with an explicit tracer, for decorators that carry
|
||||
// their own (test-injected) TracerProvider.
|
||||
func StartSpan(ctx context.Context, tracer trace.Tracer, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
|
||||
ctx, span := tracer.Start(ctx, name, opts...)
|
||||
sc := span.SpanContext()
|
||||
if !sc.IsValid() {
|
||||
return ctx, span
|
||||
}
|
||||
base, ok := ctx.Value(baseLoggerKey{}).(logr.Logger)
|
||||
if !ok {
|
||||
base = logf.FromContext(ctx)
|
||||
ctx = context.WithValue(ctx, baseLoggerKey{}, base)
|
||||
}
|
||||
return logf.IntoContext(ctx, withSpanValues(base, sc)), span
|
||||
}
|
||||
|
||||
// ContextWithLogger seeds ctx with base as both the current logger and the
|
||||
// base for span enrichment. If ctx already carries a valid span (the HTTP
|
||||
// middleware calls this inside the otelhttp handler), the logger is
|
||||
// enriched immediately.
|
||||
//
|
||||
//nolint:logcheck // IntoContext-analogue: taking ctx and the logger to seed it with is the point.
|
||||
func ContextWithLogger(ctx context.Context, base logr.Logger) context.Context {
|
||||
ctx = context.WithValue(ctx, baseLoggerKey{}, base)
|
||||
if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
|
||||
return logf.IntoContext(ctx, withSpanValues(base, sc))
|
||||
}
|
||||
return logf.IntoContext(ctx, base)
|
||||
}
|
||||
|
||||
// withSpanValues uses lowerCamel keys to match the house style
|
||||
// (reconcileID, providerID); Grafana/Loki derived fields must match
|
||||
// "traceID", not the trace_id default.
|
||||
func withSpanValues(base logr.Logger, sc trace.SpanContext) logr.Logger {
|
||||
return base.WithValues("traceID", sc.TraceID().String(), "spanID", sc.SpanID().String())
|
||||
}
|
||||
@@ -1,136 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/go-logr/logr/funcr"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
// captureLogger records every emitted line so tests can assert on the
|
||||
// rendered key/value output — the only place duplicate zap-style keys
|
||||
// would show up.
|
||||
func captureLogger(lines *[]string) logr.Logger {
|
||||
return funcr.New(func(prefix, args string) {
|
||||
*lines = append(*lines, prefix+" "+args)
|
||||
}, funcr.Options{})
|
||||
}
|
||||
|
||||
func recordingTracer(t *testing.T) (trace.Tracer, *tracetest.SpanRecorder) {
|
||||
t.Helper()
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
return tp.Tracer("test"), sr
|
||||
}
|
||||
|
||||
func TestStartSpan_enrichesLoggerOncePerNesting(t *testing.T) {
|
||||
t.Parallel()
|
||||
tracer, sr := recordingTracer(t)
|
||||
|
||||
var lines []string
|
||||
ctx := logf.IntoContext(context.Background(), captureLogger(&lines))
|
||||
|
||||
ctx1, span1 := StartSpan(ctx, tracer, "outer")
|
||||
logf.FromContext(ctx1).Info("outer work")
|
||||
|
||||
ctx2, span2 := StartSpan(ctx1, tracer, "inner")
|
||||
logf.FromContext(ctx2).Info("inner work")
|
||||
|
||||
span2.End()
|
||||
span1.End()
|
||||
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("got %d log lines, want 2: %v", len(lines), lines)
|
||||
}
|
||||
traceID := span1.SpanContext().TraceID().String()
|
||||
for i, want := range []string{span1.SpanContext().SpanID().String(), span2.SpanContext().SpanID().String()} {
|
||||
if n := strings.Count(lines[i], `"traceID"`); n != 1 {
|
||||
t.Errorf("line %d: traceID appears %d times, want exactly 1: %s", i, n, lines[i])
|
||||
}
|
||||
if n := strings.Count(lines[i], `"spanID"`); n != 1 {
|
||||
t.Errorf("line %d: spanID appears %d times, want exactly 1: %s", i, n, lines[i])
|
||||
}
|
||||
if !strings.Contains(lines[i], traceID) {
|
||||
t.Errorf("line %d: missing traceID %s: %s", i, traceID, lines[i])
|
||||
}
|
||||
if !strings.Contains(lines[i], want) {
|
||||
t.Errorf("line %d: missing spanID %s: %s", i, want, lines[i])
|
||||
}
|
||||
}
|
||||
|
||||
ended := sr.Ended()
|
||||
if len(ended) != 2 {
|
||||
t.Fatalf("got %d spans, want 2", len(ended))
|
||||
}
|
||||
// Ended in LIFO order: inner first.
|
||||
if got := ended[0].Parent().SpanID(); got != span1.SpanContext().SpanID() {
|
||||
t.Errorf("inner span parent = %s, want %s", got, span1.SpanContext().SpanID())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartSpan_noopTracerLeavesLoggerUntouched(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var lines []string
|
||||
ctx := logf.IntoContext(context.Background(), captureLogger(&lines))
|
||||
|
||||
ctx, span := StartSpan(ctx, noop.NewTracerProvider().Tracer("test"), "op")
|
||||
defer span.End()
|
||||
logf.FromContext(ctx).Info("work")
|
||||
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("got %d log lines, want 1", len(lines))
|
||||
}
|
||||
if strings.Contains(lines[0], "traceID") {
|
||||
t.Errorf("disabled tracing must not add traceID: %s", lines[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextWithLogger(t *testing.T) {
|
||||
t.Parallel()
|
||||
tracer, _ := recordingTracer(t)
|
||||
|
||||
t.Run("no span injects base as-is", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
var lines []string
|
||||
ctx := ContextWithLogger(context.Background(), captureLogger(&lines))
|
||||
logf.FromContext(ctx).Info("plain")
|
||||
if len(lines) != 1 || strings.Contains(lines[0], "traceID") {
|
||||
t.Fatalf("want one line without traceID, got %v", lines)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("existing span enriches immediately and nested Start does not stack", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
var lines []string
|
||||
ctx, outer := tracer.Start(context.Background(), "server")
|
||||
defer outer.End()
|
||||
|
||||
ctx = ContextWithLogger(ctx, captureLogger(&lines))
|
||||
logf.FromContext(ctx).Info("handler")
|
||||
|
||||
ctx, inner := StartSpan(ctx, tracer, "child")
|
||||
defer inner.End()
|
||||
logf.FromContext(ctx).Info("nested")
|
||||
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("got %d lines, want 2: %v", len(lines), lines)
|
||||
}
|
||||
for i, line := range lines {
|
||||
if n := strings.Count(line, `"traceID"`); n != 1 {
|
||||
t.Errorf("line %d: traceID appears %d times, want 1: %s", i, n, line)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(lines[1], inner.SpanContext().SpanID().String()) {
|
||||
t.Errorf("nested line should carry the child spanID: %s", lines[1])
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// tracerName is the instrumentation scope for every span this module emits.
|
||||
const tracerName = "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator"
|
||||
|
||||
type options struct {
|
||||
tp trace.TracerProvider
|
||||
}
|
||||
|
||||
// Option configures the tracing decorators. The zero configuration uses the
|
||||
// global TracerProvider installed by Setup; tests inject their own recorder
|
||||
// via WithTracerProvider so they can run in parallel without touching
|
||||
// process-global state.
|
||||
type Option func(*options)
|
||||
|
||||
// WithTracerProvider overrides the global TracerProvider.
|
||||
func WithTracerProvider(tp trace.TracerProvider) Option {
|
||||
return func(o *options) { o.tp = tp }
|
||||
}
|
||||
|
||||
func newOptions(opts []Option) options {
|
||||
var o options
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// Tracer resolves a tracer from the given options, for decorators outside
|
||||
// this package (provider.WithTracing).
|
||||
func Tracer(opts ...Option) trace.Tracer {
|
||||
return newOptions(opts).tracer()
|
||||
}
|
||||
|
||||
// tracer resolves the configured tracer. The global path goes through
|
||||
// otel.Tracer, which delegates to whatever provider Setup installs later —
|
||||
// construction order between decorators and Setup does not matter.
|
||||
func (o options) tracer() trace.Tracer {
|
||||
if o.tp != nil {
|
||||
return o.tp.Tracer(tracerName)
|
||||
}
|
||||
return otel.Tracer(tracerName)
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
)
|
||||
|
||||
// NewReconciler wraps inner so every reconcile runs in a root span named
|
||||
// "Reconcile <kind>" and every log line under it carries the trace context.
|
||||
// Reconciles are triggered by watch events with no incoming trace to
|
||||
// continue, so each one starts a new trace.
|
||||
func NewReconciler(kind string, inner reconcile.Reconciler, opts ...Option) reconcile.Reconciler {
|
||||
return &tracedReconciler{
|
||||
spanName: "Reconcile " + kind,
|
||||
inner: inner,
|
||||
tracer: newOptions(opts).tracer(),
|
||||
}
|
||||
}
|
||||
|
||||
type tracedReconciler struct {
|
||||
spanName string
|
||||
inner reconcile.Reconciler
|
||||
tracer trace.Tracer
|
||||
}
|
||||
|
||||
func (t *tracedReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
|
||||
ctx, span := StartSpan(ctx, t.tracer, t.spanName,
|
||||
trace.WithAttributes(
|
||||
attribute.String("k8s.namespace.name", req.Namespace),
|
||||
attribute.String("k8s.object.name", req.Name),
|
||||
))
|
||||
defer span.End()
|
||||
if id := controller.ReconcileIDFromContext(ctx); id != "" {
|
||||
span.SetAttributes(attribute.String("reconcile.id", string(id)))
|
||||
}
|
||||
|
||||
res, err := t.inner.Reconcile(ctx, req)
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return res, err
|
||||
}
|
||||
span.SetStatus(codes.Ok, "")
|
||||
if res.RequeueAfter > 0 {
|
||||
span.SetAttributes(attribute.Int64("reconcile.requeue_after_ms", res.RequeueAfter.Milliseconds()))
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
)
|
||||
|
||||
type fakeReconciler struct {
|
||||
res reconcile.Result
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeReconciler) Reconcile(context.Context, reconcile.Request) (reconcile.Result, error) {
|
||||
return f.res, f.err
|
||||
}
|
||||
|
||||
func TestNewReconciler_spanPerReconcile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "p1"}}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inner *fakeReconciler
|
||||
wantStatus codes.Code
|
||||
wantAttr attribute.KeyValue
|
||||
}{
|
||||
{
|
||||
name: "success records requeue",
|
||||
inner: &fakeReconciler{res: reconcile.Result{RequeueAfter: 10 * time.Second}},
|
||||
wantStatus: codes.Ok,
|
||||
wantAttr: attribute.Int64("reconcile.requeue_after_ms", 10_000),
|
||||
},
|
||||
{
|
||||
name: "error sets error status",
|
||||
inner: &fakeReconciler{err: errors.New("boom")},
|
||||
wantStatus: codes.Error,
|
||||
wantAttr: attribute.String("k8s.object.name", "p1"),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
|
||||
r := NewReconciler("Proxy", tc.inner, WithTracerProvider(tp))
|
||||
res, err := r.Reconcile(context.Background(), req)
|
||||
if res != tc.inner.res || !errors.Is(err, tc.inner.err) {
|
||||
t.Fatalf("decorator changed the result: (%v, %v)", res, err)
|
||||
}
|
||||
|
||||
ended := sr.Ended()
|
||||
if len(ended) != 1 {
|
||||
t.Fatalf("got %d spans, want 1", len(ended))
|
||||
}
|
||||
span := ended[0]
|
||||
if span.Name() != "Reconcile Proxy" {
|
||||
t.Errorf("span name = %q, want %q", span.Name(), "Reconcile Proxy")
|
||||
}
|
||||
if span.Status().Code != tc.wantStatus {
|
||||
t.Errorf("status = %v, want %v", span.Status().Code, tc.wantStatus)
|
||||
}
|
||||
found := false
|
||||
for _, a := range span.Attributes() {
|
||||
if a == tc.wantAttr {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("attribute %v missing in %v", tc.wantAttr, span.Attributes())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
// Package tracing wires OpenTelemetry tracing into the operator: SDK setup
|
||||
// gated on standard OTEL_* environment variables, span helpers that keep the
|
||||
// logr/zap logging enriched with trace context, and decorators for the
|
||||
// reconciler, providers, HTTP server, and Kubernetes API transport.
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
||||
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
const (
|
||||
envSDKDisabled = "OTEL_SDK_DISABLED"
|
||||
envTracesExporter = "OTEL_TRACES_EXPORTER"
|
||||
envOTLPEndpoint = "OTEL_EXPORTER_OTLP_ENDPOINT"
|
||||
envOTLPTracesEndpoint = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
|
||||
envOTLPProtocol = "OTEL_EXPORTER_OTLP_PROTOCOL"
|
||||
envOTLPTracesProtocol = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"
|
||||
|
||||
exporterOTLP = "otlp"
|
||||
exporterConsole = "console"
|
||||
exporterNone = "none"
|
||||
)
|
||||
|
||||
// maxQueueSize bounds the batch processor's span buffer; the default 2048
|
||||
// is oversized for this process (3 concurrent reconciles, tickers) and the
|
||||
// manager pod runs with a 128Mi memory limit.
|
||||
const maxQueueSize = 512
|
||||
|
||||
// Setup initializes the global tracing pipeline from OTEL_* environment
|
||||
// variables and returns a shutdown func that flushes pending spans.
|
||||
//
|
||||
// Tracing stays fully off — no exporter, no global provider, spans are
|
||||
// no-ops, logs carry no traceID — unless the environment opts in by setting
|
||||
// OTEL_TRACES_EXPORTER or an OTLP endpoint. OTEL_SDK_DISABLED=true and
|
||||
// OTEL_TRACES_EXPORTER=none force it off (the Go SDK does not implement
|
||||
// OTEL_SDK_DISABLED itself). Sampling, endpoints, TLS, and headers follow
|
||||
// the standard SDK/exporter env vars (OTEL_TRACES_SAMPLER, OTEL_EXPORTER_OTLP_*).
|
||||
//
|
||||
// Setup logs via the logger in ctx (logf.FromContext) rather than a
|
||||
// parameter — the caller seeds it with logf.IntoContext.
|
||||
func Setup(ctx context.Context, service, version string) (func(context.Context) error, error) {
|
||||
log := logf.FromContext(ctx)
|
||||
noop := func(context.Context) error { return nil }
|
||||
|
||||
exporterEnv := strings.ToLower(strings.TrimSpace(os.Getenv(envTracesExporter)))
|
||||
endpointSet := os.Getenv(envOTLPEndpoint) != "" ||
|
||||
os.Getenv(envOTLPTracesEndpoint) != ""
|
||||
sdkDisabled := strings.EqualFold(strings.TrimSpace(os.Getenv(envSDKDisabled)), "true")
|
||||
|
||||
if sdkDisabled || exporterEnv == exporterNone || (exporterEnv == "" && !endpointSet) {
|
||||
log.Info("tracing disabled",
|
||||
"reason", disabledReason(sdkDisabled, exporterEnv))
|
||||
return noop, nil
|
||||
}
|
||||
|
||||
exp, expName, err := newExporter(ctx, exporterEnv)
|
||||
if err != nil {
|
||||
return noop, fmt.Errorf("tracing setup: %w", err)
|
||||
}
|
||||
|
||||
res, err := buildResource(ctx, service, version)
|
||||
if err != nil {
|
||||
if res == nil {
|
||||
return noop, fmt.Errorf("tracing setup: building resource: %w", err)
|
||||
}
|
||||
// Schema-URL conflicts between semconv versions still yield a usable
|
||||
// merged resource; keep it rather than losing tracing over metadata.
|
||||
log.V(1).Info("tracing resource merge conflict; continuing", "err", err.Error())
|
||||
}
|
||||
|
||||
tp := sdktrace.NewTracerProvider(
|
||||
sdktrace.WithBatcher(exp, sdktrace.WithMaxQueueSize(maxQueueSize)),
|
||||
sdktrace.WithResource(res),
|
||||
)
|
||||
otel.SetTracerProvider(tp)
|
||||
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
|
||||
propagation.TraceContext{}, propagation.Baggage{}))
|
||||
otelLog := log.WithName("otel")
|
||||
otel.SetLogger(otelLog)
|
||||
// V(1), not Error: an unreachable collector fails every export cycle and
|
||||
// would otherwise spam the error log every few seconds.
|
||||
otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
|
||||
otelLog.V(1).Info("otel error", "err", err.Error())
|
||||
}))
|
||||
|
||||
log.Info("tracing enabled", "exporter", expName, "service", service)
|
||||
return tp.Shutdown, nil
|
||||
}
|
||||
|
||||
func disabledReason(sdkDisabled bool, exporterEnv string) string {
|
||||
switch {
|
||||
case sdkDisabled:
|
||||
return "OTEL_SDK_DISABLED=true"
|
||||
case exporterEnv == exporterNone:
|
||||
return "OTEL_TRACES_EXPORTER=none"
|
||||
default:
|
||||
return "no OTEL_TRACES_EXPORTER or OTLP endpoint configured"
|
||||
}
|
||||
}
|
||||
|
||||
// newExporter hand-rolls the OTEL_TRACES_EXPORTER / OTEL_EXPORTER_OTLP_*
|
||||
// protocol selection instead of pulling in contrib's autoexport, which drags
|
||||
// metric/log/prometheus exporters into a trace-only binary.
|
||||
func newExporter(ctx context.Context, kind string) (sdktrace.SpanExporter, string, error) {
|
||||
switch kind {
|
||||
case "", exporterOTLP:
|
||||
proto := strings.ToLower(strings.TrimSpace(os.Getenv(envOTLPTracesProtocol)))
|
||||
if proto == "" {
|
||||
proto = strings.ToLower(strings.TrimSpace(os.Getenv(envOTLPProtocol)))
|
||||
}
|
||||
switch proto {
|
||||
case "", "http/protobuf":
|
||||
exp, err := otlptracehttp.New(ctx)
|
||||
return exp, "otlp/http", err
|
||||
case "grpc":
|
||||
exp, err := otlptracegrpc.New(ctx)
|
||||
return exp, "otlp/grpc", err
|
||||
default:
|
||||
return nil, "", fmt.Errorf("unsupported OTLP protocol %q (supported: http/protobuf, grpc)", proto)
|
||||
}
|
||||
case exporterConsole:
|
||||
exp, err := stdouttrace.New()
|
||||
return exp, exporterConsole, err
|
||||
default:
|
||||
return nil, "", fmt.Errorf("unsupported OTEL_TRACES_EXPORTER %q (supported: otlp, console, none)", kind)
|
||||
}
|
||||
}
|
||||
|
||||
// buildResource layers defaults < service identity < environment, so
|
||||
// OTEL_SERVICE_NAME / OTEL_RESOURCE_ATTRIBUTES always win.
|
||||
func buildResource(ctx context.Context, service, version string) (*resource.Resource, error) {
|
||||
base, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL,
|
||||
semconv.ServiceName(service),
|
||||
semconv.ServiceVersion(version),
|
||||
))
|
||||
if err != nil {
|
||||
return base, err
|
||||
}
|
||||
env, err := resource.New(ctx, resource.WithFromEnv())
|
||||
if err != nil {
|
||||
return base, err
|
||||
}
|
||||
return resource.Merge(base, env)
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
)
|
||||
|
||||
const testOTLPEndpoint = "http://localhost:4318"
|
||||
|
||||
// clearOTelEnv pins every env var Setup reads, so developer shells with
|
||||
// OTEL_* set can't change test outcomes. Not parallel-safe by design
|
||||
// (t.Setenv forbids t.Parallel).
|
||||
func clearOTelEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, k := range []string{
|
||||
envSDKDisabled,
|
||||
envTracesExporter,
|
||||
envOTLPEndpoint,
|
||||
envOTLPTracesEndpoint,
|
||||
envOTLPProtocol,
|
||||
envOTLPTracesProtocol,
|
||||
} {
|
||||
t.Setenv(k, "")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetup_envGating(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
wantEnabled bool
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "no env means disabled", env: nil, wantEnabled: false},
|
||||
{
|
||||
name: "endpoint enables otlp",
|
||||
env: map[string]string{envOTLPEndpoint: testOTLPEndpoint},
|
||||
wantEnabled: true,
|
||||
},
|
||||
{
|
||||
name: "traces endpoint enables otlp",
|
||||
env: map[string]string{envOTLPTracesEndpoint: testOTLPEndpoint},
|
||||
wantEnabled: true,
|
||||
},
|
||||
{
|
||||
name: "explicit console exporter",
|
||||
env: map[string]string{envTracesExporter: exporterConsole},
|
||||
wantEnabled: true,
|
||||
},
|
||||
{
|
||||
name: "grpc protocol",
|
||||
env: map[string]string{envTracesExporter: exporterOTLP, envOTLPProtocol: "grpc"},
|
||||
wantEnabled: true,
|
||||
},
|
||||
{
|
||||
name: "exporter none wins over endpoint",
|
||||
env: map[string]string{
|
||||
envTracesExporter: exporterNone,
|
||||
envOTLPEndpoint: testOTLPEndpoint,
|
||||
},
|
||||
wantEnabled: false,
|
||||
},
|
||||
{
|
||||
name: "OTEL_SDK_DISABLED wins over everything",
|
||||
env: map[string]string{
|
||||
envSDKDisabled: "true",
|
||||
envOTLPEndpoint: testOTLPEndpoint,
|
||||
},
|
||||
wantEnabled: false,
|
||||
},
|
||||
{
|
||||
name: "unsupported exporter errors",
|
||||
env: map[string]string{envTracesExporter: "jaeger"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unsupported protocol errors",
|
||||
env: map[string]string{
|
||||
envTracesExporter: exporterOTLP,
|
||||
envOTLPProtocol: "http/json",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
clearOTelEnv(t)
|
||||
for k, v := range tc.env {
|
||||
t.Setenv(k, v)
|
||||
}
|
||||
// Fresh noop global per case: disabled paths must leave it
|
||||
// untouched, and resetting avoids leaking one case's SDK
|
||||
// provider into the next (or warning-prone restores of the
|
||||
// process-default delegate).
|
||||
before := noop.NewTracerProvider()
|
||||
otel.SetTracerProvider(before)
|
||||
|
||||
shutdown, err := Setup(context.Background(), "test-svc", "abc123")
|
||||
if tc.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("want error, got nil")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Setup: %v", err)
|
||||
}
|
||||
|
||||
_, isSDK := otel.GetTracerProvider().(*sdktrace.TracerProvider)
|
||||
if isSDK != tc.wantEnabled {
|
||||
t.Errorf("global provider is SDK = %v, want %v", isSDK, tc.wantEnabled)
|
||||
}
|
||||
if tc.wantEnabled && otel.GetTracerProvider() == before {
|
||||
t.Error("enabled Setup must install a new global provider")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
if err := shutdown(ctx); err != nil {
|
||||
t.Errorf("shutdown: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"k8s.io/client-go/transport"
|
||||
)
|
||||
|
||||
// RestConfigWrapper returns a rest.Config.Wrap-compatible wrapper that adds
|
||||
// a client span to every Kubernetes API request already running inside a
|
||||
// trace. Requests with no parent span pass through untouched: informer
|
||||
// list/watch long-polls, leader-election renewals, and metrics authn would
|
||||
// otherwise each become meaningless (and in the watch case, minutes-long)
|
||||
// root spans. client-go applies Wrap innermost, so the span sees the final
|
||||
// authenticated request.
|
||||
func RestConfigWrapper(opts ...Option) transport.WrapperFunc {
|
||||
o := newOptions(opts)
|
||||
return func(rt http.RoundTripper) http.RoundTripper {
|
||||
otelOpts := []otelhttp.Option{
|
||||
// Explicit propagators: deterministic regardless of global
|
||||
// state. The apiserver ignores incoming traceparent (by
|
||||
// design), so injection is harmless there.
|
||||
otelhttp.WithPropagators(propagation.NewCompositeTextMapPropagator(
|
||||
propagation.TraceContext{}, propagation.Baggage{})),
|
||||
}
|
||||
if o.tp != nil {
|
||||
otelOpts = append(otelOpts, otelhttp.WithTracerProvider(o.tp))
|
||||
}
|
||||
return &parentGatedTransport{
|
||||
traced: otelhttp.NewTransport(rt, otelOpts...),
|
||||
plain: rt,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parentGatedTransport enforces the parent-span requirement itself rather
|
||||
// than relying on otelhttp filter semantics for transports.
|
||||
type parentGatedTransport struct {
|
||||
traced http.RoundTripper
|
||||
plain http.RoundTripper
|
||||
}
|
||||
|
||||
func (t *parentGatedTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
if trace.SpanContextFromContext(r.Context()).IsValid() {
|
||||
return t.traced.RoundTrip(r)
|
||||
}
|
||||
return t.plain.RoundTrip(r)
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func TestRestConfigWrapper_parentGated(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var gotTraceparent string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotTraceparent = r.Header.Get("traceparent")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
sr := tracetest.NewSpanRecorder()
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
|
||||
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
|
||||
|
||||
rt := RestConfigWrapper(WithTracerProvider(tp))(http.DefaultTransport)
|
||||
do := func(ctx context.Context) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := rt.RoundTrip(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
|
||||
// No parent span: informer watches, leader election. Must not trace.
|
||||
do(context.Background())
|
||||
if len(sr.Ended()) != 0 {
|
||||
t.Fatalf("request without parent span produced %d spans, want 0", len(sr.Ended()))
|
||||
}
|
||||
if gotTraceparent != "" {
|
||||
t.Fatalf("request without parent span injected traceparent %q", gotTraceparent)
|
||||
}
|
||||
|
||||
// Under a parent span: one client child span, traceparent injected.
|
||||
ctx, parent := tp.Tracer("test").Start(context.Background(), "reconcile")
|
||||
do(ctx)
|
||||
parent.End()
|
||||
|
||||
var client sdktrace.ReadOnlySpan
|
||||
for _, s := range sr.Ended() {
|
||||
if s.SpanKind() == trace.SpanKindClient {
|
||||
client = s
|
||||
}
|
||||
}
|
||||
if client == nil {
|
||||
t.Fatalf("no client span recorded, got %d spans", len(sr.Ended()))
|
||||
}
|
||||
if got := client.Parent().SpanID(); got != parent.SpanContext().SpanID() {
|
||||
t.Errorf("client span parent = %s, want %s", got, parent.SpanContext().SpanID())
|
||||
}
|
||||
if gotTraceparent == "" {
|
||||
t.Error("traceparent header not injected under a parent span")
|
||||
}
|
||||
}
|
||||
@@ -1,424 +0,0 @@
|
||||
//go:build e2e
|
||||
// +build e2e
|
||||
|
||||
/*
|
||||
Copyright 2026.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/test/utils"
|
||||
)
|
||||
|
||||
// The tracing e2e proves the full pipeline against a real Tempo: operator in
|
||||
// kind → OTLP export → Tempo ingest → traces queryable with the documented
|
||||
// span topology. It is gated on TEMPO_URL (Tempo query API, e.g.
|
||||
// http://192.168.0.30:3200) and OTLP_ENDPOINT (OTLP HTTP ingest, e.g.
|
||||
// http://192.168.0.30:4318) and skips when either is unset, so the rest of
|
||||
// the suite runs anywhere.
|
||||
//
|
||||
// Every span of a run carries the resource attribute test.run.id=<runID>
|
||||
// (injected via OTEL_RESOURCE_ATTRIBUTES — no code changes), so one TraceQL
|
||||
// query finds exactly this run's traces, in the test and in Grafana alike.
|
||||
var _ = Describe("OTel tracing", Ordered, func() {
|
||||
// Proxy CRs live in default, not the operator namespace: the squid pods
|
||||
// the kubernetes provider creates carry no securityContext and would be
|
||||
// rejected by the operator namespace's restricted PSS label.
|
||||
const proxyNS = "default"
|
||||
const deploymentName = "egress-proxies-operator-controller-manager"
|
||||
const squidImage = "ubuntu/squid:6.6-24.04_edge"
|
||||
|
||||
proxyNames := []string{"proxy-tracing-e2e-1", "proxy-tracing-e2e-2"}
|
||||
|
||||
var (
|
||||
tempoURL string
|
||||
otlpEndpoint string
|
||||
runID string
|
||||
suiteStart time.Time
|
||||
)
|
||||
|
||||
BeforeAll(func() {
|
||||
tempoURL = os.Getenv("TEMPO_URL")
|
||||
otlpEndpoint = os.Getenv("OTLP_ENDPOINT")
|
||||
if tempoURL == "" || otlpEndpoint == "" {
|
||||
Skip("TEMPO_URL / OTLP_ENDPOINT not set — skipping the Tempo-backed tracing e2e")
|
||||
}
|
||||
suiteStart = time.Now()
|
||||
runID = "e2e-" + strconv.FormatInt(suiteStart.UnixNano(), 10)
|
||||
_, _ = fmt.Fprintf(GinkgoWriter,
|
||||
"tracing e2e run id: %s — find this run in Grafana with TraceQL {resource.test.run.id=%q}\n",
|
||||
runID, runID)
|
||||
|
||||
By("creating manager namespace")
|
||||
cmd := exec.Command("kubectl", "create", "ns", namespace)
|
||||
_, err := utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to create namespace")
|
||||
|
||||
By("labeling the namespace to enforce the restricted security policy")
|
||||
cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace,
|
||||
"pod-security.kubernetes.io/enforce=restricted")
|
||||
_, err = utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy")
|
||||
|
||||
By("installing CRDs")
|
||||
cmd = exec.Command("make", "install")
|
||||
_, err = utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs")
|
||||
|
||||
By("deploying the controller-manager")
|
||||
cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", managerImage))
|
||||
_, err = utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager")
|
||||
|
||||
By("pre-pulling the squid image into kind (best effort, kills the biggest flake source)")
|
||||
if _, err := utils.Run(exec.Command("docker", "pull", squidImage)); err == nil {
|
||||
if err := utils.LoadImageToKindClusterWithName(squidImage); err != nil {
|
||||
_, _ = fmt.Fprintf(GinkgoWriter, "kind load of %s failed (continuing): %v\n", squidImage, err)
|
||||
}
|
||||
} else {
|
||||
_, _ = fmt.Fprintf(GinkgoWriter, "docker pull %s failed (continuing): %v\n", squidImage, err)
|
||||
}
|
||||
|
||||
By("preflighting the OTLP endpoint from inside the cluster")
|
||||
// Host-reachability of the OTLP endpoint does not prove
|
||||
// pod-reachability from inside kind, and the operator logs export
|
||||
// failures only at V(1) — without this, a broken path is a slow,
|
||||
// opaque search timeout instead of a clear failure.
|
||||
preflightOTLP(otlpEndpoint)
|
||||
|
||||
By("pointing the operator at the OTLP endpoint and tagging the test run")
|
||||
// OTEL_RESOURCE_ATTRIBUTES is replaced in place, which keeps it
|
||||
// listed after the downward-API POD_NAME/POD_NAMESPACE vars —
|
||||
// $(VAR) expansion only sees earlier-listed vars. exec.Command
|
||||
// passes $(...) through without shell mangling.
|
||||
cmd = exec.Command("kubectl", "set", "env",
|
||||
"deployment/"+deploymentName, "-n", namespace,
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT="+otlpEndpoint,
|
||||
fmt.Sprintf(
|
||||
"OTEL_RESOURCE_ATTRIBUTES=k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE),test.run.id=%s",
|
||||
runID))
|
||||
_, err = utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to set OTel env on the deployment")
|
||||
|
||||
cmd = exec.Command("kubectl", "rollout", "status",
|
||||
"deployment/"+deploymentName, "-n", namespace, "--timeout=3m")
|
||||
_, err = utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Rollout after set env did not finish")
|
||||
|
||||
By("verifying the operator reports tracing enabled")
|
||||
Eventually(func(g Gomega) {
|
||||
pod := newestControllerPod(g)
|
||||
out, err := utils.Run(exec.Command("kubectl", "logs", pod, "-n", namespace))
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(out).To(ContainSubstring("tracing enabled"),
|
||||
"operator did not log 'tracing enabled' after rollout")
|
||||
}, 2*time.Minute).Should(Succeed())
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
if tempoURL == "" || otlpEndpoint == "" {
|
||||
return // spec was skipped; nothing was deployed
|
||||
}
|
||||
By("cleaning up test proxies")
|
||||
args := append([]string{"delete", "proxy", "-n", proxyNS, "--ignore-not-found"}, proxyNames...)
|
||||
_, _ = utils.Run(exec.Command("kubectl", args...))
|
||||
|
||||
By("cleaning up the OTLP preflight pod")
|
||||
_, _ = utils.Run(exec.Command("kubectl", "delete", "pod", otlpProbePodName, "-n", namespace,
|
||||
"--ignore-not-found"))
|
||||
|
||||
By("undeploying the controller-manager")
|
||||
_, _ = utils.Run(exec.Command("make", "undeploy"))
|
||||
|
||||
By("uninstalling CRDs")
|
||||
_, _ = utils.Run(exec.Command("make", "uninstall"))
|
||||
|
||||
By("removing manager namespace")
|
||||
_, _ = utils.Run(exec.Command("kubectl", "delete", "ns", namespace))
|
||||
})
|
||||
|
||||
It("creates proxies and reports reconcile traces to Tempo", func() {
|
||||
By("applying two kubernetes-provider proxies")
|
||||
// Absolute paths on purpose: utils.Run chdirs the whole process.
|
||||
dir := GinkgoT().TempDir()
|
||||
for _, name := range proxyNames {
|
||||
manifest := fmt.Sprintf(`apiVersion: crawl.example.com/v1alpha1
|
||||
kind: Proxy
|
||||
metadata:
|
||||
name: %s
|
||||
namespace: %s
|
||||
spec:
|
||||
mode: Managed
|
||||
provider: kubernetes
|
||||
attributes:
|
||||
purpose: tracing-e2e
|
||||
`, name, proxyNS)
|
||||
path := filepath.Join(dir, name+".yaml")
|
||||
Expect(os.WriteFile(path, []byte(manifest), 0o644)).To(Succeed())
|
||||
_, err := utils.Run(exec.Command("kubectl", "apply", "-f", path))
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to apply %s", name)
|
||||
}
|
||||
|
||||
By("waiting for the proxies to become Ready")
|
||||
Eventually(func(g Gomega) {
|
||||
for _, name := range proxyNames {
|
||||
out, err := utils.Run(exec.Command("kubectl", "get", "proxy", name,
|
||||
"-n", proxyNS, "-o", "jsonpath={.status.phase}"))
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(out).To(Equal("Ready"), "proxy %s not Ready", name)
|
||||
}
|
||||
}, 5*time.Minute).Should(Succeed())
|
||||
|
||||
By("finding a provider.create trace for this run in Tempo")
|
||||
// Anchored on provider.create, not the root span name: a
|
||||
// "Reconcile Proxy" hit could be the finalizer-add or a drift
|
||||
// reconcile, which contain no provider call.
|
||||
traceID := eventuallyFindTrace(tempoURL, runID, "provider.create", suiteStart)
|
||||
|
||||
By("asserting the reconcile trace structure")
|
||||
spanNames, resAttrs, err := tempoTrace(tempoURL, traceID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(spanNames).To(ContainElements(
|
||||
"Reconcile Proxy", "reconcile.managed", "provider.create", "status.patch"),
|
||||
"trace %s is missing expected spans; got: %v", traceID, spanNames)
|
||||
Expect(resAttrs["service.name"]).To(Equal("egress-proxies-operator"))
|
||||
Expect(resAttrs["test.run.id"]).To(Equal(runID))
|
||||
})
|
||||
|
||||
It("traces proxy deletion", func() {
|
||||
By("deleting the proxies")
|
||||
args := append([]string{"delete", "proxy", "-n", proxyNS, "--wait=false"}, proxyNames...)
|
||||
_, err := utils.Run(exec.Command("kubectl", args...))
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to delete proxies")
|
||||
|
||||
By("waiting for the proxies to be gone (finalizer ran provider.delete)")
|
||||
Eventually(func(g Gomega) {
|
||||
out, err := utils.Run(exec.Command("kubectl", "get", "proxy", "-n", proxyNS, "-o", "name"))
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(out).NotTo(ContainSubstring("proxy-tracing-e2e"))
|
||||
}, 3*time.Minute).Should(Succeed())
|
||||
|
||||
By("finding a provider.delete trace for this run in Tempo")
|
||||
traceID := eventuallyFindTrace(tempoURL, runID, "provider.delete", suiteStart)
|
||||
|
||||
By("asserting the deletion trace structure")
|
||||
spanNames, _, err := tempoTrace(tempoURL, traceID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(spanNames).To(ContainElements("Reconcile Proxy", "reconcile.delete", "provider.delete"),
|
||||
"trace %s is missing expected spans; got: %v", traceID, spanNames)
|
||||
})
|
||||
})
|
||||
|
||||
// preflightOTLP runs a one-shot curl pod inside the cluster POSTing to the
|
||||
// OTLP HTTP ingest, and fails with a clear message when it is unreachable.
|
||||
// The pod runs in the restricted-PSS operator namespace, hence the full
|
||||
// securityContext (same shape as the curl-metrics pod).
|
||||
func preflightOTLP(otlpEndpoint string) {
|
||||
script := fmt.Sprintf(
|
||||
"for i in $(seq 1 10); do "+
|
||||
"code=$(curl -sS -o /dev/null -w '%%{http_code}' -X POST "+
|
||||
"-H 'Content-Type: application/json' -d '{}' %s/v1/traces); "+
|
||||
"echo \"attempt $i: HTTP $code\"; "+
|
||||
"[ \"$code\" = \"200\" ] && echo OTLP_OK && exit 0; sleep 2; "+
|
||||
"done; echo OTLP_UNREACHABLE; exit 1",
|
||||
otlpEndpoint)
|
||||
cmd := exec.Command("kubectl", "run", otlpProbePodName, "--restart=Never",
|
||||
"--namespace", namespace,
|
||||
"--image=curlimages/curl:latest",
|
||||
"--overrides",
|
||||
fmt.Sprintf(`{
|
||||
"spec": {
|
||||
"containers": [{
|
||||
"name": "curl",
|
||||
"image": "curlimages/curl:latest",
|
||||
"command": ["/bin/sh", "-c"],
|
||||
"args": [%q],
|
||||
"securityContext": {
|
||||
"readOnlyRootFilesystem": true,
|
||||
"allowPrivilegeEscalation": false,
|
||||
"capabilities": {
|
||||
"drop": ["ALL"]
|
||||
},
|
||||
"runAsNonRoot": true,
|
||||
"runAsUser": 1000,
|
||||
"seccompProfile": {
|
||||
"type": "RuntimeDefault"
|
||||
}
|
||||
}
|
||||
}]
|
||||
}
|
||||
}`, script))
|
||||
_, err := utils.Run(cmd)
|
||||
Expect(err).NotTo(HaveOccurred(), "Failed to create the OTLP preflight pod")
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
out, err := utils.Run(exec.Command("kubectl", "get", "pod", otlpProbePodName,
|
||||
"-n", namespace, "-o", "jsonpath={.status.phase}"))
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(out).To(BeElementOf("Succeeded", "Failed"), "preflight pod still running")
|
||||
}, 2*time.Minute).Should(Succeed())
|
||||
|
||||
logs, _ := utils.Run(exec.Command("kubectl", "logs", otlpProbePodName, "-n", namespace))
|
||||
Expect(logs).To(ContainSubstring("OTLP_OK"),
|
||||
"OTLP endpoint %s is not reachable from inside the kind cluster; curl output:\n%s",
|
||||
otlpEndpoint, logs)
|
||||
}
|
||||
|
||||
// otlpProbePodName mirrors the Describe-local constant for the helpers below.
|
||||
const otlpProbePodName = "curl-otlp"
|
||||
|
||||
// newestControllerPod returns the most recently created controller pod —
|
||||
// right after a rollout, an unsorted lookup may pick the terminating one.
|
||||
func newestControllerPod(g Gomega) string {
|
||||
out, err := utils.Run(exec.Command("kubectl", "get", "pods",
|
||||
"-l", "control-plane=controller-manager", "-n", namespace,
|
||||
"--sort-by=.metadata.creationTimestamp", "-o", "name"))
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
lines := utils.GetNonEmptyLines(out)
|
||||
g.Expect(lines).NotTo(BeEmpty(), "no controller pods found")
|
||||
return strings.TrimPrefix(lines[len(lines)-1], "pod/")
|
||||
}
|
||||
|
||||
// eventuallyFindTrace polls Tempo until a trace containing a span with the
|
||||
// given name exists for this run, and returns its trace ID. The batch span
|
||||
// processor flushes every ~5s, so a couple of polls is normal.
|
||||
func eventuallyFindTrace(tempoURL, runID, spanName string, since time.Time) string {
|
||||
var traceID string
|
||||
query := fmt.Sprintf(`{resource.test.run.id=%q && name=%q}`, runID, spanName)
|
||||
Eventually(func(g Gomega) {
|
||||
ids, err := tempoSearch(tempoURL, query, since)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
g.Expect(ids).NotTo(BeEmpty(), "no trace for %s yet (query: %s)", spanName, query)
|
||||
traceID = ids[0]
|
||||
}, 2*time.Minute).Should(Succeed())
|
||||
return traceID
|
||||
}
|
||||
|
||||
// tempoSearch runs a TraceQL query against Tempo's search API and returns
|
||||
// the matching trace IDs. start/end are Unix seconds.
|
||||
func tempoSearch(tempoURL, traceql string, since time.Time) ([]string, error) {
|
||||
u, err := url.Parse(tempoURL + "/api/search")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("q", traceql)
|
||||
q.Set("start", strconv.FormatInt(since.Add(-5*time.Minute).Unix(), 10))
|
||||
q.Set("end", strconv.FormatInt(time.Now().Add(time.Minute).Unix(), 10))
|
||||
q.Set("limit", "20")
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
body, err := tempoGet(u.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result struct {
|
||||
Traces []struct {
|
||||
TraceID string `json:"traceID"`
|
||||
} `json:"traces"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, fmt.Errorf("decoding Tempo search response: %w", err)
|
||||
}
|
||||
ids := make([]string, 0, len(result.Traces))
|
||||
for _, t := range result.Traces {
|
||||
ids = append(ids, t.TraceID)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// tempoTrace fetches one trace and flattens it to a span-name list plus the
|
||||
// resource attributes. Tempo returns OTLP-JSON (batches → scopeSpans →
|
||||
// spans), not Jaeger's shape.
|
||||
func tempoTrace(tempoURL, traceID string) ([]string, map[string]string, error) {
|
||||
body, err := tempoGet(tempoURL + "/api/traces/" + traceID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var trace struct {
|
||||
Batches []struct {
|
||||
Resource struct {
|
||||
Attributes []struct {
|
||||
Key string `json:"key"`
|
||||
Value struct {
|
||||
StringValue string `json:"stringValue"`
|
||||
} `json:"value"`
|
||||
} `json:"attributes"`
|
||||
} `json:"resource"`
|
||||
ScopeSpans []struct {
|
||||
Spans []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"spans"`
|
||||
} `json:"scopeSpans"`
|
||||
} `json:"batches"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &trace); err != nil {
|
||||
return nil, nil, fmt.Errorf("decoding Tempo trace %s: %w", traceID, err)
|
||||
}
|
||||
var spanNames []string
|
||||
resAttrs := map[string]string{}
|
||||
for _, b := range trace.Batches {
|
||||
for _, a := range b.Resource.Attributes {
|
||||
if a.Value.StringValue != "" {
|
||||
resAttrs[a.Key] = a.Value.StringValue
|
||||
}
|
||||
}
|
||||
for _, ss := range b.ScopeSpans {
|
||||
for _, s := range ss.Spans {
|
||||
spanNames = append(spanNames, s.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return spanNames, resAttrs, nil
|
||||
}
|
||||
|
||||
func tempoGet(rawURL string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying Tempo: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("tempo returned %d for %s: %s", resp.StatusCode, rawURL, string(body))
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
Reference in New Issue
Block a user