18 Commits

Author SHA1 Message Date
aeb4115c72 Add tracing manifests and docs; clean up branch lint findings
Manager env block (downward-API resource attrs, commented OTLP
examples), architecture §10 + Decisions entries, README section.
Lint: goconst constants, gofmt, logcheck (Setup now takes its logger
from ctx via logf.FromContext).

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-24 11:30:27 +02:00
53c0d77ef5 Enrich GCP wire logs with trace context from the request ctx
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-24 11:09:48 +02:00
bca32f10d3 Trace GC sweeps and (opt-in) health probes
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-24 11:07:13 +02:00
ec7267b8de Trace discovery API requests and enrich request logs
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-24 10:59:48 +02:00
896bf89a19 Trace reconciles: root span wrapper plus state-machine sub-spans
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-24 10:53:52 +02:00
8e4f3b9b13 Wire tracing into the composition root
Setup after SetLogger, wrapped rest configs (manager + kubernetes
provider), tracing-outermost provider decorators, and an explicit
trace flush after mgr.Start returns (os.Exit skips defers).

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-24 10:51:24 +02:00
d15b06ec45 Add provider.WithTracing decorator mirroring WithMetrics
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-24 10:50:13 +02:00
6b34f68469 Add internal/tracing: env-gated OTel setup, span/log helpers, decorators
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-24 10:43:55 +02:00
b1a16774bf Align OTel deps at v1.45.0 and add trace exporter modules
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-24 10:41:06 +02:00
328877e000 Add plan: OTel tracing integrated with logr/zap logging
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-24 10:26:21 +02:00
5ccd317bec Extend Claude permission allowlist with session-approved commands
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-12 00:18:09 +02:00
6ff9eebefc Document providers and how to add one in README
New Providers section: named-instance config model, a table of the
implemented kubernetes and gcp providers, and a four-step guide for
adding a new backend (interface contract, GC tagging, error taxonomy,
wiring in config.go and main.go).

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-12 00:16:54 +02:00
f6b67006dc Add tmux demo driver and table variant of egress IP check
run-demo.sh opens a 2x2 tmux grid: egress-IP table looping in a
netshoot pod, kubectl get px watch, and both create scripts running
with COUNT (default 4) proxies. show-egress-ips-table.sh is the
pane-sized one-line-per-proxy variant used by the driver.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-11 23:10:23 +02:00
0d68111bc2 Add demo scripts: egress IP check and bulk proxy creation
docs/demo/show-egress-ips.sh probes each healthy proxy from the
discovery API against an IP-echo site; create-kubernetes-proxies.sh and
create-gcp-proxies.sh bulk-create demo Proxies, the gcp one spreading
them across randomly picked EU zones.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-11 22:40:24 +02:00
e1abac3e8f Add plan: demo script for egress IP check via proxies
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-11 22:21:28 +02:00
f3ff6a0ca2 Use $BASE_URL variable in docs/api.md curl examples
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-11 22:01:50 +02:00
09845e4eaf Document the discovery API in docs/api.md
Full client-facing reference: auth, all routes with schemas and curl
examples, selection/cooldown semantics, caveats. README and
architecture.md link to it.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-11 21:55:16 +02:00
e7fdae0859 Add plan: discovery API documentation
Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-11 21:53:02 +02:00
41 changed files with 2698 additions and 60 deletions

View File

@@ -83,7 +83,17 @@
"Bash(kubectl -n egress-proxies-operator-system get pods -o wide)", "Bash(kubectl -n egress-proxies-operator-system get pods -o wide)",
"Bash(kubectl -n egress-proxies-operator-system get deploy egress-proxies-operator-controller-manager -o jsonpath='{.spec.template.spec.containers[0].args}')", "Bash(kubectl -n egress-proxies-operator-system get deploy egress-proxies-operator-controller-manager -o jsonpath='{.spec.template.spec.containers[0].args}')",
"Bash(kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager)", "Bash(kubectl -n egress-proxies-operator-system logs deploy/egress-proxies-operator-controller-manager)",
"Bash(python3 -c \"import json; d=json.load\\(open\\('docs/deploy/sa_key.json'\\)\\); print\\(d.get\\('type'\\), d.get\\('client_email'\\)\\)\")" "Bash(python3 -c \"import json; d=json.load\\(open\\('docs/deploy/sa_key.json'\\)\\); print\\(d.get\\('type'\\), d.get\\('client_email'\\)\\)\")",
"Bash(tea pr *)",
"Bash(git worktree *)",
"Bash(python3 -c \"import yaml; yaml.safe_load\\(open\\('.gitea/workflows/build.yaml'\\)\\); print\\('YAML OK'\\)\")",
"Bash(ruby -ryaml -e \"YAML.load_file\\('.gitea/workflows/build.yaml'\\); puts 'YAML OK'\")",
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator status --short --branch)",
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator log --oneline -1)",
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator tag 0.01)",
"Bash(git -C /Users/jan.novak/srv/go/egress-proxies-operator push origin 0.01)",
"Bash(chmod +x docs/demo/run-demo.sh)",
"Bash(bash -n docs/demo/run-demo.sh)"
], ],
"additionalDirectories": [ "additionalDirectories": [
"/Users/jan.novak/srv/go/egress-proxies-operator/.claude", "/Users/jan.novak/srv/go/egress-proxies-operator/.claude",

View File

@@ -59,7 +59,8 @@ kubectl get px -w
# proxy-kubernetes-sample Managed kubernetes Ready 10.244.x.x True # proxy-kubernetes-sample Managed kubernetes Ready 10.244.x.x True
``` ```
Once it's `Ready`, port-forward the discovery API and use it: Once it's `Ready`, port-forward the discovery API and use it (full
reference with schemas and error codes: [docs/api.md](docs/api.md)):
```sh ```sh
kubectl -n egress-proxies-operator-system port-forward \ kubectl -n egress-proxies-operator-system port-forward \
@@ -135,6 +136,81 @@ Cloud-init from a Secret: the Secret **must** carry the label
labelled Secrets, so an unlabelled one is invisible (the Proxy reports labelled Secrets, so an unlabelled one is invisible (the Proxy reports
`CloudInitError`). Rotating the Secret's content triggers VM replacement. `CloudInitError`). Rotating the Secret's content triggers VM replacement.
## Providers
`Managed` proxies are provisioned by a provider — a small compute backend
behind one minimal interface. Providers are configured in the
`--providers-config` YAML file as **named instances**: `spec.provider` on a
Proxy refers to an entry's `name`, not its `type`, so `gcp-eu` and `gcp-us`
can be two differently-configured instances of the same `gcp` type (see
[config/samples/providers-config.yaml](config/samples/providers-config.yaml)).
### Implemented providers
| Type | Creates | Per-instance config | Notes |
| --- | --- | --- | --- |
| `kubernetes` | A real Squid pod (`ubuntu/squid:6.6-24.04_edge` by default) in the same cluster the operator runs in | `image` (optional) | Needs no cloud account — local dev, CI, and the kind quickstart. Pods share the cluster's egress IP, so it exercises the full lifecycle but not distinct egress paths. |
| `gcp` | A Compute Engine VM with an ephemeral external IP | `project` (required), `network`, `networkTag`, `diskSizeGb` | The real egress fleet. Deliberately uses only four API calls (Insert / Get / Delete / AggregatedList), all fire-and-forget: `Create` returns as soon as the operation is submitted and the reconciler discovers progress by polling `Get`. Auth is Application Default Credentials — workload identity in-cluster, `gcloud` ADC locally; no key-file plumbing. |
### Adding a provider
A new backend (Hetzner, AWS, ...) is four pieces; the contract lives in
[internal/provider/provider.go](internal/provider/provider.go):
1. **Implement the 4-method `Provider` interface** in a new
`internal/provider/<type>/` package:
- `Create` submits and returns — it never blocks until the VM runs, and
must be idempotent keyed on `req.Name` (a deterministic name derived
from the Proxy's UID), so a repeat call after a crash finds the
existing instance instead of duplicating it.
- `Get` returns `provider.ErrNotFound` as a *normal* outcome — the
reconciler branches on it for replacement and adoption, so don't
treat it as exceptional.
- `Delete` is idempotent: deleting an already-gone instance is not an
error.
- `ListByTag` returns every instance the operator ever tagged, for
orphan GC.
2. **Tag every created resource** with `LabelManaged=true` and
`LabelUID=<Proxy UID>`, and report `CreatedAt` — orphan GC relies on
all three to find owned resources and skip in-flight creates.
3. **Classify every returned error** with `provider.Wrap` into the
four-sentinel taxonomy in
[internal/provider/errors.go](internal/provider/errors.go)
(`ErrNotFound` / `ErrQuotaExceeded` / `ErrTransient` / `ErrPermanent`)
— the reconciler decides retry, slow backoff, or latching `Failed`
purely from that classification, never from provider-specific types.
4. **Wire it up**: add a type-specific config block in
[internal/provider/config.go](internal/provider/config.go), and
register the constructor in the builtins map in
[cmd/main.go](cmd/main.go) (`"<type>": <pkg>.New`). The registry
([internal/provider/registry](internal/provider/registry/registry.go))
handles named instances, and the metrics wrapper is applied
automatically.
Test against a fake API seam rather than the real cloud — see the
`instancesAPI` seam in
[internal/provider/gcp/gcp.go](internal/provider/gcp/gcp.go) for the
pattern.
## Tracing
OpenTelemetry tracing is built in but **off by default** — the operator
only exports spans when pointed at an OTLP receiver:
```bash
# In config/manager/manager.yaml (commented-out block is already there):
# OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger-collector.observability:4318
# Local dev: print spans to stdout instead
OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml
```
Every reconcile, discovery API request, and GC sweep becomes a trace, with
child spans for provider and Kubernetes API calls; log lines inside a traced
operation carry matching `traceID`/`spanID` fields (point Grafana/Loki
derived fields at `traceID`). Sampling and endpoints follow the standard
`OTEL_*` env vars; `--trace-health-probes` additionally traces each health
probe (high volume). Details in `docs/architecture.md` §10.
## Caveats — read these two ## Caveats — read these two
**Changing a proxy changes its IP.** Proxies are immutable cattle: editing **Changing a proxy changes its IP.** Proxies are immutable cattle: editing

View File

@@ -42,6 +42,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/healthz"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/log/zap"
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters" "sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -58,6 +59,7 @@ import (
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version"
// +kubebuilder:scaffold:imports // +kubebuilder:scaffold:imports
) )
@@ -93,6 +95,7 @@ func main() {
var leaseCooldown, maxLeaseTTL time.Duration var leaseCooldown, maxLeaseTTL time.Duration
var showVersion bool var showVersion bool
var gcpWireFullPayloads bool var gcpWireFullPayloads bool
var traceHealthProbes bool
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -133,6 +136,9 @@ func main() {
"Print the commit the binary was built from and exit.") "Print the commit the binary was built from and exit.")
flag.BoolVar(&gcpWireFullPayloads, "gcp-wire-log-full-payloads", false, flag.BoolVar(&gcpWireFullPayloads, "gcp-wire-log-full-payloads", false,
"Log GCP V(5) wire payloads verbatim instead of eliding fields larger than 1KiB.") "Log GCP V(5) wire payloads verbatim instead of eliding fields larger than 1KiB.")
flag.BoolVar(&traceHealthProbes, "trace-health-probes", false,
"Emit one trace span per health probe. Off by default: probes run about "+
"once per second per proxy and would dominate trace volume.")
opts := zap.Options{ opts := zap.Options{
Development: true, Development: true,
@@ -148,6 +154,17 @@ func main() {
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
setupLog.Info("Starting egress-proxies-operator", setupLog.Info("Starting egress-proxies-operator",
"commit", version.Resolve(), "goVersion", goruntime.Version()) "commit", version.Resolve(), "goVersion", goruntime.Version())
// Off (no-op spans, unchanged logs) unless OTEL_* env opts in; see
// internal/tracing. Shutdown is called explicitly after mgr.Start
// returns — the os.Exit paths below skip defers.
tracingShutdown, err := tracing.Setup(logf.IntoContext(context.Background(), setupLog),
"egress-proxies-operator", version.Resolve())
if err != nil {
setupLog.Error(err, "Failed to set up tracing")
os.Exit(1)
}
ctx := ctrl.SetupSignalHandler() ctx := ctrl.SetupSignalHandler()
// Providers load first and fail fast: a manager that comes up without // Providers load first and fail fast: a manager that comes up without
@@ -162,7 +179,11 @@ func main() {
os.Exit(1) os.Exit(1)
} }
providers, err := registry.Build(ctx, cfg, map[string]registry.Constructor{ providers, err := registry.Build(ctx, cfg, map[string]registry.Constructor{
"kubernetes": kubernetes.New, "kubernetes": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
// The kubernetes provider builds its own uncached client;
// wrap its transport so its API calls join the caller's trace.
return kubernetes.NewWithTransportWrapper(ctx, pc, tracing.RestConfigWrapper())
},
"gcp": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) { "gcp": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
return gcp.NewWithWireOptions(ctx, pc, gcp.WireLogOptions{FullPayloads: gcpWireFullPayloads}) return gcp.NewWithWireOptions(ctx, pc, gcp.WireLogOptions{FullPayloads: gcpWireFullPayloads})
}, },
@@ -173,7 +194,8 @@ func main() {
} }
m := metrics.New() m := metrics.New()
for name, p := range providers { for name, p := range providers {
providers[name] = provider.WithMetrics(name, p, m) // Tracing outermost: the span covers the metrics recording too.
providers[name] = provider.WithTracing(name, provider.WithMetrics(name, p, m))
} }
// if the enable-http2 flag is false (the default), http/2 should be disabled // if the enable-http2 flag is false (the default), http/2 should be disabled
@@ -236,7 +258,9 @@ func main() {
cacheOpts.DefaultNamespaces = map[string]cache.Config{proxyNamespace: {}} cacheOpts.DefaultNamespaces = map[string]cache.Config{proxyNamespace: {}}
} }
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ restCfg := ctrl.GetConfigOrDie()
restCfg.Wrap(tracing.RestConfigWrapper())
mgr, err := ctrl.NewManager(restCfg, ctrl.Options{
Scheme: scheme, Scheme: scheme,
Metrics: metricsServerOptions, Metrics: metricsServerOptions,
HealthProbeBindAddress: probeAddr, HealthProbeBindAddress: probeAddr,
@@ -258,6 +282,7 @@ func main() {
engine := health.NewEngine(mgr.GetClient()) engine := health.NewEngine(mgr.GetClient())
engine.Workers = healthWorkers engine.Workers = healthWorkers
engine.Metrics = m engine.Metrics = m
engine.TraceProbes = traceHealthProbes
if err := mgr.Add(engine); err != nil { if err := mgr.Add(engine); err != nil {
setupLog.Error(err, "Failed to add health engine") setupLog.Error(err, "Failed to add health engine")
os.Exit(1) os.Exit(1)
@@ -323,8 +348,19 @@ func main() {
} }
setupLog.Info("Starting manager") setupLog.Info("Starting manager")
if err := mgr.Start(ctx); err != nil { startErr := mgr.Start(ctx)
setupLog.Error(err, "Failed to run manager")
// Flush pending spans on the way out, error path included. Fresh
// context: the signal ctx is already cancelled by the time Start
// returns.
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
if err := tracingShutdown(shutdownCtx); err != nil {
setupLog.Error(err, "Failed to flush traces on shutdown")
}
cancel()
if startErr != nil {
setupLog.Error(startErr, "Failed to run manager")
os.Exit(1) os.Exit(1)
} }
} }

View File

@@ -64,6 +64,8 @@ spec:
- --leader-elect - --leader-elect
- --health-probe-bind-address=:8081 - --health-probe-bind-address=:8081
- --providers-config=/etc/proxy-operator/providers.yaml - --providers-config=/etc/proxy-operator/providers.yaml
# Add --trace-health-probes to emit one span per health probe
# (high volume: ~1/s per proxy).
env: env:
# Bearer token for the discovery API. Optional: without the # Bearer token for the discovery API. Optional: without the
# Secret the API serves unauthenticated (with a loud warning). # Secret the API serves unauthenticated (with a loud warning).
@@ -76,6 +78,33 @@ spec:
name: discovery-token name: discovery-token
key: token key: token
optional: true optional: true
# --- OpenTelemetry tracing ---
# Tracing is OFF until OTEL_EXPORTER_OTLP_ENDPOINT (or
# OTEL_TRACES_EXPORTER) is set; without it the operator runs
# exactly as before. The downward-API vars must stay listed
# before OTEL_RESOURCE_ATTRIBUTES: $(VAR) expansion only sees
# earlier-listed vars.
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: OTEL_SERVICE_NAME
value: egress-proxies-operator
- name: OTEL_RESOURCE_ATTRIBUTES
value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)
# Point at an OTLP receiver (Tempo, Jaeger, otel-collector) to
# enable tracing; http/protobuf (4318) is the default protocol,
# set OTEL_EXPORTER_OTLP_PROTOCOL=grpc for 4317.
# - name: OTEL_EXPORTER_OTLP_ENDPOINT
# value: http://jaeger-collector.observability:4318
# - name: OTEL_TRACES_SAMPLER
# value: parentbased_traceidratio
# - name: OTEL_TRACES_SAMPLER_ARG
# value: "0.1"
image: controller:latest image: controller:latest
name: manager name: manager
ports: ports:

323
docs/api.md Normal file
View File

@@ -0,0 +1,323 @@
# Discovery API reference
The operator serves an HTTP API (the *discovery API*) that crawler clients
use to find and lease egress proxies: list healthy proxies filtered by
attributes, acquire a TTL-based lease on one, release it early, and report
how a target site treated the proxy. It is implemented in
[`internal/discovery`](../internal/discovery/) with lease state in
[`internal/lease`](../internal/lease/); the only Kubernetes interaction is
reading `Proxy` resources from the manager's cache.
## Base URL
The API listens on `:8090` (`--discovery-addr`) inside the manager pod and
is exposed by a Service
([config/default/discovery_service.yaml](../config/default/discovery_service.yaml)).
In-cluster:
```text
http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090
```
From a workstation, port-forward:
```sh
kubectl -n egress-proxies-operator-system port-forward \
svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090 &
```
Set `BASE_URL` to wherever you reach the API; all examples below use it:
```sh
export BASE_URL=localhost:8090 # via the port-forward above
# or, from inside the cluster:
# export BASE_URL=http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090
```
## Authentication
A single static bearer token, read from the `DISCOVERY_TOKEN` environment
variable at startup. The shipped Deployment populates it from the
`discovery-token` Secret (key `token`), which is **optional** — if the
Secret is absent or the token is empty, the API serves **unauthenticated**
(the manager logs a loud warning at startup). Create the Secret:
```sh
kubectl -n egress-proxies-operator-system create secret generic discovery-token \
--from-literal=token="$(openssl rand -hex 24)"
```
Send the token on every request:
```sh
export TOKEN=<the token>
curl -s -H "Authorization: Bearer $TOKEN" "$BASE_URL/v1/proxies" | jq
```
A missing or wrong token gets `401 {"error":"unauthorized",...}`.
`GET /healthz` is always exempt.
The curl examples below omit the `-H "Authorization: Bearer $TOKEN"` flag
for brevity — add it to every call when auth is enabled.
## Conventions
- Requests and responses are JSON. Errors share one envelope:
```json
{"error": "<machine_code>", "message": "<human-readable text>"}
```
- Request bodies are capped at **64 KiB** (larger bodies fail the JSON
decode with `400 invalid_body`).
- Proxies with a deletion timestamp (being finalized) are excluded from
every response and never offered for lease.
## Configuration
| Setting | Default | Meaning |
|---|---|---|
| `--discovery-addr` | `:8090` | Listen address of the API |
| `--max-lease-ttl` | `1h` | Maximum `ttlSeconds` a client may request |
| `--lease-cooldown` | `15m` | Cooldown window applied on `rate_limited`/`banned` reports |
| `DISCOVERY_TOKEN` (env) | empty | Bearer token; empty disables auth |
The shipped Deployment passes none of these flags, so the defaults apply.
## Endpoints
### `GET /healthz`
Liveness check. Unauthenticated, always `200` with body `ok`.
```sh
curl -s "$BASE_URL/healthz"
```
### `GET /v1/proxies` — list proxies
Query parameters (all optional):
| Parameter | Values | Effect |
|---|---|---|
| `healthy` | `true` \| `false` | Keep only proxies whose `Healthy` condition matches. Any other value → `400 invalid_query`. |
| `attr.<key>` | any string | Exact match on `spec.attributes[<key>]`. Repeatable; **all** given pairs must match. |
List everything:
```sh
curl -s "$BASE_URL/v1/proxies" | jq
```
List healthy proxies in a given geo:
```sh
curl -s "$BASE_URL/v1/proxies?healthy=true&attr.geo=eu" | jq
```
Response — `200`, proxies sorted by `id`, an empty match is `200` with
`"count": 0` (never `404`):
```json
{
"proxies": [
{
"id": "default/proxy-kubernetes-sample",
"ip": "10.244.1.7",
"port": 3128,
"attributes": {"geo": "local"},
"phase": "Ready",
"healthy": true,
"latencyMillis": 42,
"activeLeases": 1,
"maxLeases": 5
}
],
"count": 1
}
```
Proxy object fields (the same shape appears inside lease responses):
| Field | Meaning |
|---|---|
| `id` | `namespace/name` of the `Proxy` resource; used as the stable key everywhere |
| `ip` | Effective host — `spec.endpoint.host` for `External` proxies, `status.ip` for `Managed` (empty until the backing VM/pod is up) |
| `port` | Effective port (default `3128`) |
| `attributes` | `spec.attributes` — free-form selection labels (`geo`, `asn`, `purpose`, …); omitted when empty |
| `phase` | `Pending` \| `Provisioning` \| `Ready` \| `Unhealthy` \| `Deleting` \| `Failed` |
| `healthy` | `true` iff the `Healthy` condition is `True` (the through-the-proxy health probe passes) |
| `latencyMillis` | Latency of the last status-affecting health probe |
| `activeLeases` | Currently active leases on this proxy |
| `maxLeases` | Lease capacity (default `5`; an explicit `0` means unleasable) |
### `POST /v1/leases` — acquire a lease
Picks a healthy proxy with free capacity matching the selector and grants
an exclusive-slot, TTL-based lease on it.
Request body (every field optional; `{}` is valid):
```json
{
"selector": {"geo": "eu"},
"ttlSeconds": 300,
"target": "example.com"
}
```
| Field | Default | Meaning |
|---|---|---|
| `selector` | none | Attribute equality filter, same semantics as `attr.<key>` above |
| `ttlSeconds` | `300` (5 min) | Lease lifetime; must be ≤ `--max-lease-ttl` (default 1 h), else `400 invalid_ttl` |
| `target` | none | The site you intend to crawl; enables per-target cooldowns (see below) |
```sh
curl -s -XPOST "$BASE_URL/v1/leases" \
-d '{"selector":{"geo":"eu"},"ttlSeconds":300,"target":"example.com"}' | jq
```
Success — `201`:
```json
{
"leaseID": "P3X6HHQTPCM5UTGVGE3B5UPS3A",
"proxy": {
"id": "default/proxy-eu-1",
"ip": "34.88.10.20",
"port": 3128,
"attributes": {"geo": "eu"},
"phase": "Ready",
"healthy": true,
"latencyMillis": 42,
"activeLeases": 1,
"maxLeases": 5
},
"expiresAt": "2026-08-11T22:05:00Z",
"ttlSeconds": 300
}
```
Use `proxy.ip` and `proxy.port` as an HTTP proxy for the lease's lifetime:
```sh
curl -x http://34.88.10.20:3128 https://example.com
```
No match — `409` with diagnostic counts explaining why nothing qualified:
```json
{
"error": "no_match",
"message": "no healthy proxy with free capacity matched the selector",
"considered": 3,
"atCapacity": 1,
"inCooldown": 1,
"unhealthy": 1
}
```
| Count | Meaning |
|---|---|
| `considered` | Proxies that matched the selector (before health/capacity checks) |
| `atCapacity` | Skipped because `activeLeases >= maxLeases` |
| `inCooldown` | Skipped because of an active cooldown for this target (or a global one) |
| `unhealthy` | Skipped because the `Healthy` condition is not `True` |
Leases expire on their own — releasing is only needed to free the slot
early. There is no renew/extend endpoint; acquire a new lease instead.
### `DELETE /v1/leases/{id}` — release early
Frees the lease's capacity slot immediately. Idempotent: always `204`,
including for unknown or already-expired lease IDs.
```sh
curl -si -XDELETE "$BASE_URL/v1/leases/P3X6HHQTPCM5UTGVGE3B5UPS3A"
```
### `POST /v1/leases/{id}/report` — report an outcome
Tell the operator how the target site treated the proxy. This is the
feedback signal that drives cooldowns.
Request body:
```json
{"result": "rate_limited", "target": "example.com"}
```
| Field | Values | Meaning |
|---|---|---|
| `result` | `ok` \| `rate_limited` \| `banned` | Anything else → `400 invalid_result` |
| `target` | optional | Which site produced the result; falls back to the lease's `target`, then to global |
```sh
curl -si -XPOST "$BASE_URL/v1/leases/P3X6HHQTPCM5UTGVGE3B5UPS3A/report" \
-d '{"result":"rate_limited","target":"example.com"}'
```
Responses: `204` on success, `404 unknown_lease` if the lease ID was never
issued or has aged out.
Semantics:
- `ok` is a pure acknowledgement — nothing is recorded.
- `rate_limited` and `banned` currently behave **identically**: both put
the proxy in one cooldown window (default 15 min, `--lease-cooldown`)
for the resolved target.
- An expired lease remains reportable for one cooldown window past its
TTL, so a late "we got rate-limited" still lands.
## Proxy selection and cooldowns
How `POST /v1/leases` picks among eligible proxies (healthy, matching the
selector, not being deleted, not at capacity, not in cooldown), in order:
1. fewest `activeLeases` (least-loaded),
2. lowest `latencyMillis`,
3. lexicographic `id` (deterministic tie-break).
Cooldowns are keyed by **(proxy, target)**:
- A report **with a target** blocks that proxy only for lease requests
naming the **same target**. Other targets — and requests with no
target — still get the proxy.
- A report **without a target**, on a lease that also had no target,
creates a **global** cooldown: the proxy is blocked for *all* lease
requests until the window passes. Always pass `target` on leases and
reports unless you really mean "this proxy is bad for everyone".
## End-to-end example
```sh
# 1. Acquire a lease for crawling example.com through an EU proxy
LEASE=$(curl -s -XPOST "$BASE_URL/v1/leases" \
-H "Authorization: Bearer $TOKEN" \
-d '{"selector":{"geo":"eu"},"ttlSeconds":600,"target":"example.com"}')
LEASE_ID=$(echo "$LEASE" | jq -r .leaseID)
PROXY=$(echo "$LEASE" | jq -r '"\(.proxy.ip):\(.proxy.port)"')
# 2. Crawl through the leased proxy
curl -x "http://$PROXY" https://example.com/some/page
# 3. Got a 429? Report it — example.com-bound leases will avoid this
# proxy for the next 15 minutes
curl -s -XPOST "$BASE_URL/v1/leases/$LEASE_ID/report" \
-H "Authorization: Bearer $TOKEN" \
-d '{"result":"rate_limited","target":"example.com"}'
# 4. Done early? Release the slot (otherwise the TTL frees it)
curl -s -XDELETE "$BASE_URL/v1/leases/$LEASE_ID" \
-H "Authorization: Bearer $TOKEN"
```
## Caveats
- **Lease and cooldown state is in-memory and per-process.** An operator
restart drops all active leases and cooldowns. Clients must tolerate a
granted lease disappearing (a subsequent report returns `404`).
- **Run a single replica.** The API is served by every manager replica but
is not leader-elected, and lease state is not shared between replicas;
the shipped Deployment pins `replicas: 1`.

View File

@@ -213,6 +213,8 @@ Kubernetes interaction is reading Proxies from the manager's cache. The
server is a non-leader-elected Runnable (all replicas would serve, but the server is a non-leader-elected Runnable (all replicas would serve, but the
deployment ships `replicas: 1` because lease state is per-process — an deployment ships `replicas: 1` because lease state is per-process — an
operator restart drops all leases and cooldowns, a documented caveat). operator restart drops all leases and cooldowns, a documented caveat).
Client-facing reference with request/response schemas and curl examples:
[api.md](api.md).
```text ```text
crawler client crawler client
@@ -298,6 +300,45 @@ Each consuming package defines its own small recorder interface
`metrics.Metrics` satisfies all of them structurally, so no package other `metrics.Metrics` satisfies all of them structurally, so no package other
than `cmd/main.go` imports the metrics package. than `cmd/main.go` imports the metrics package.
### 10. Tracing (`internal/tracing/`)
OpenTelemetry tracing, integrated with — not replacing — the logr/zap
logging. Everything hangs off standard `OTEL_*` env vars: with no
`OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` set, no SDK is
installed, spans are no-ops, and logs are byte-for-byte what they were.
`OTEL_SDK_DISABLED=true` and `OTEL_TRACES_EXPORTER=none` force it off;
`console` prints spans to stdout for local dev; sampling follows
`OTEL_TRACES_SAMPLER(_ARG)`.
Span topology — each unit of work is a new trace (watch events carry no
incoming trace context):
- `Reconcile Proxy` (root, via a `reconcile.Reconciler` decorator) →
`reconcile.managed` / `reconcile.replaceInstance` / `reconcile.delete`,
plus `status.patch` from the deferred flush. `reconcileExternal` does no
I/O and is unspanned.
- `provider.create|get|delete|list` (client spans, `provider.WithTracing`
decorator wired outermost around `WithMetrics`); the GCP SDK's own
otelhttp transport contributes HTTP child spans automatically.
- Kubernetes API calls become client child spans via a wrapped
`rest.Config` transport — gated on an existing parent span, so informer
list/watch long-polls and leader-election renewals never create root
spans. The kube-apiserver ignores incoming `traceparent` by design;
these spans are leaves.
- Discovery API: one server span per request (named from the route
pattern), incoming W3C `traceparent` honored so clients' traces continue
into the operator; `/healthz` excluded.
- `gc.sweep` per GC pass; `health.probe` per probe **only** with
`--trace-health-probes` (default off — probes run ~1/s per proxy).
Log correlation: logr sinks never see a context, so trace IDs ride on the
logger — `tracing.Start` re-derives the ctx logger from a captured base
with `traceID`/`spanID` values (lowerCamel, matching `reconcileID`;
Grafana/Loki derived-field regexes must match `traceID`, not `trace_id`).
Re-deriving from the base rather than layering keeps zap from emitting
duplicate keys on nested spans. GCP V(5) wire logs get the same keys from
the slog handler's ctx.
## Decisions ## Decisions
Judgment calls the spec left open, and deliberate deviations — recorded so Judgment calls the spec left open, and deliberate deviations — recorded so
@@ -402,3 +443,24 @@ they read as choices, not accidents. Chronological by build step.
logger to everything running under the manager — fighting that would logger to everything running under the manager — fighting that would
mean two logging systems in one process. Noted as a deviation rather mean two logging systems in one process. Noted as a deviation rather
than silently ignored. than silently ignored.
- **Tracing is env-gated, not flag-gated:** it activates only when
`OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_TRACES_EXPORTER` is set (user
decision). No collector configured means no SDK installed, no-op spans,
unchanged logs — the safe default for every existing deployment. The
one flag is `--trace-health-probes`, off by default, because probes at
~1/s per proxy would dominate trace volume.
- **Trace keys are `traceID`/`spanID`**, lowerCamel like `reconcileID` and
`providerID`, deliberately not the `trace_id` many Grafana derived-field
examples assume — configure the derived-field regex accordingly. logr
sinks can't read ctx, so the IDs ride on the ctx logger, re-derived from
a captured base per span so zap never emits duplicate keys.
- **Probe→reconcile trace links are not attempted:** the health engine's
`GenericEvent` carries only namespace/name (no ctx), and the workqueue
coalesces events, so any link would be a guess. A health-triggered
reconcile starts a fresh trace; the probe that caused it is findable via
its own (opt-in) span and shared proxy attributes.
- **No `traceparent` toward probe targets:** probe transports stay
uninstrumented so trace headers can never leak through a proxy to
external sites. The kube-apiserver ignores incoming `traceparent` by
design (public endpoint), so k8s client spans are leaves — in-process
traces, not cross-process ones.

94
docs/demo/create-gcp-proxies.sh Executable file
View File

@@ -0,0 +1,94 @@
#!/bin/bash
# Demo: create N Managed proxies backed by the gcp provider, named
# proxy-gcp-demo-1 .. proxy-gcp-demo-N, each in a randomly picked EU zone
# so the fleet gets egress IPs from different locations.
#
# Usage:
# ./create-gcp-proxies.sh <count>
#
# Optional environment:
# NAMESPACE namespace to create the proxies in (default: current context)
# GCP_PROVIDER provider NAME from providers.yaml (default: gcp-eu)
# ZONES space-separated zone list to pick from (default: EU zones below)
set -euo pipefail
if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
echo "usage: $(basename "$0") <count> (positive integer)" >&2
exit 1
fi
count="$1"
provider="${GCP_PROVIDER:-gcp-eu}"
# GCP zones in the EU where e2-micro is generally available. Override with
# ZONES="zone1 zone2 ..." if your project has quota only in some of them.
default_zones=(
europe-west1-b europe-west1-c europe-west1-d # Belgium
europe-west2-a europe-west2-b europe-west2-c # London
europe-west3-a europe-west3-b europe-west3-c # Frankfurt
europe-west4-a europe-west4-b europe-west4-c # Netherlands
europe-west6-a europe-west6-b europe-west6-c # Zurich
europe-west8-a europe-west8-b europe-west8-c # Milan
europe-west9-a europe-west9-b europe-west9-c # Paris
europe-central2-a europe-central2-b europe-central2-c # Warsaw
europe-north1-a europe-north1-b europe-north1-c # Finland
europe-southwest1-a europe-southwest1-b europe-southwest1-c # Madrid
)
if [ -n "${ZONES:-}" ]; then
read -r -a zones <<< "${ZONES}"
else
zones=("${default_zones[@]}")
fi
if ! command -v kubectl &> /dev/null; then
echo "ERROR: kubectl is required but not installed" >&2
exit 1
fi
ns_args=()
if [ -n "${NAMESPACE:-}" ]; then
ns_args=(-n "${NAMESPACE}")
fi
for i in $(seq 1 "${count}"); do
zone="${zones[RANDOM % ${#zones[@]}]}"
echo "Creating proxy-gcp-demo-${i} in ${zone} ..."
kubectl apply "${ns_args[@]}" -f - << EOF
apiVersion: crawl.example.com/v1alpha1
kind: Proxy
metadata:
name: proxy-gcp-demo-${i}
spec:
mode: Managed
provider: ${provider} # must match a provider NAME in providers.yaml
placement:
zone: ${zone}
machineType: e2-micro
# debian-cloud images have no cloud-init, so spec.cloudInit (passed as
# user-data metadata) would be silently ignored there. Ubuntu images do.
image: projects/ubuntu-os-cloud/global/images/family/ubuntu-2404-lts-amd64
port: 3128
cloudInit:
inline: |
#cloud-config
package_update: true
packages:
- squid
write_files:
- path: /etc/squid/conf.d/proxy-operator.conf
content: |
http_access allow all
via off
forwarded_for off
runcmd:
- systemctl restart squid
attributes:
geo: eu
zone: ${zone}
purpose: crawl
EOF
done
echo ""
echo "Created ${count} proxies. VMs take a few minutes to provision and pass"
echo "the health check. Watch them come up with:"
echo " kubectl get px ${ns_args[*]:-} -w"

View File

@@ -0,0 +1,48 @@
#!/bin/bash
# Demo: create N Managed proxies backed by the kubernetes-pod provider,
# named proxy-kubernetes-demo-1 .. proxy-kubernetes-demo-N. Pods share the
# cluster's egress IP — this exercises the full lifecycle, not distinct
# egress paths (use create-gcp-proxies.sh for that).
#
# Usage:
# ./create-kubernetes-proxies.sh <count>
#
# Optional environment:
# NAMESPACE namespace to create the proxies in (default: current context)
set -euo pipefail
if [ $# -ne 1 ] || ! [[ "$1" =~ ^[1-9][0-9]*$ ]]; then
echo "usage: $(basename "$0") <count> (positive integer)" >&2
exit 1
fi
count="$1"
if ! command -v kubectl &> /dev/null; then
echo "ERROR: kubectl is required but not installed" >&2
exit 1
fi
ns_args=()
if [ -n "${NAMESPACE:-}" ]; then
ns_args=(-n "${NAMESPACE}")
fi
for i in $(seq 1 "${count}"); do
echo "Creating proxy-kubernetes-demo-${i} ..."
kubectl apply "${ns_args[@]}" -f - << EOF
apiVersion: crawl.example.com/v1alpha1
kind: Proxy
metadata:
name: proxy-kubernetes-demo-${i}
spec:
mode: Managed
provider: kubernetes
attributes:
geo: local
purpose: crawl
EOF
done
echo ""
echo "Created ${count} proxies. Watch them come up with:"
echo " kubectl get px ${ns_args[*]:-} -w"

67
docs/demo/run-demo.sh Executable file
View File

@@ -0,0 +1,67 @@
#!/bin/bash
# Demo driver: opens a tmux session with a 2x2 pane grid:
#
# top-left: show-egress-ips-table.sh in a 10s loop, run inside the
# netshoot pod against the in-cluster discovery Service
# top-right: watch -n3 kubectl get px
# bottom-left: create-kubernetes-proxies.sh <count>
# bottom-right: create-gcp-proxies.sh <count>
#
# Usage:
# ./run-demo.sh
#
# Optional environment:
# COUNT proxies each create script makes (default: 4)
# SESSION tmux session name (default: proxy-demo; an existing
# session with this name is killed and recreated)
# NETSHOOT_POD pod to exec into for the egress-IP loop (default: netshoot)
# DEMO_DIR where the demo scripts live (default: this script's dir)
set -euo pipefail
COUNT="${COUNT:-4}"
SESSION="${SESSION:-proxy-demo}"
NETSHOOT_POD="${NETSHOOT_POD:-netshoot}"
DEMO_DIR="${DEMO_DIR:-$(cd "$(dirname "$0")" && pwd)}"
BASE_URL="http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090"
for tool in tmux kubectl; do
if ! command -v "${tool}" &> /dev/null; then
echo "ERROR: ${tool} is required but not installed" >&2
exit 1
fi
done
if ! kubectl get pod "${NETSHOOT_POD}" &> /dev/null; then
echo "ERROR: pod ${NETSHOOT_POD} not found — start one with:" >&2
echo " kubectl run netshoot --image=nicolaka/netshoot -- sleep infinity" >&2
exit 1
fi
echo "Copying show-egress-ips-table.sh into pod ${NETSHOOT_POD} ..."
kubectl cp "${DEMO_DIR}/show-egress-ips-table.sh" "${NETSHOOT_POD}:/tmp/show-egress-ips-table.sh"
if tmux has-session -t "${SESSION}" 2> /dev/null; then
echo "Killing existing tmux session ${SESSION}"
tmux kill-session -t "${SESSION}"
fi
# 2x2 grid: after these splits pane indexes are 0 top-left, 1 top-right,
# 2 bottom-left, 3 bottom-right; tiled layout evens them into quarters.
tmux new-session -d -s "${SESSION}"
tmux split-window -h -t "${SESSION}:0"
tmux split-window -v -t "${SESSION}:0.0"
tmux split-window -v -t "${SESSION}:0.1"
tmux select-layout -t "${SESSION}:0" tiled
loop_cmd="BASE_URL=${BASE_URL}; while true; do bash /tmp/show-egress-ips-table.sh \"\$BASE_URL\"; echo; sleep 10; done"
tmux send-keys -t "${SESSION}:0.0" "kubectl exec -it ${NETSHOOT_POD} -- bash -c '${loop_cmd}'" C-m
tmux send-keys -t "${SESSION}:0.1" "watch -n3 kubectl get px" C-m
tmux send-keys -t "${SESSION}:0.2" "bash ${DEMO_DIR}/create-kubernetes-proxies.sh ${COUNT}" C-m
tmux send-keys -t "${SESSION}:0.3" "bash ${DEMO_DIR}/create-gcp-proxies.sh ${COUNT}" C-m
tmux select-pane -t "${SESSION}:0.2"
if [ -n "${TMUX:-}" ]; then
tmux switch-client -t "${SESSION}"
else
tmux attach-session -t "${SESSION}"
fi

View File

@@ -0,0 +1,75 @@
#!/bin/bash
# Demo: condensed-table variant of show-egress-ips.sh, made to fit a small
# tmux pane. One line per proxy: which proxy the request goes through, its
# endpoint, location (zone/geo attribute), and the egress IP the IP-echo
# site saw — or unhealthy/FAILED.
#
# Usage:
# ./show-egress-ips-table.sh <BASE_URL> e.g. ./show-egress-ips-table.sh localhost:8090
#
# Optional environment:
# TOKEN bearer token for the discovery API (see docs/api.md)
# IP_ECHO_URL site that returns the caller's IP as JSON with an "ip" field
# (default: https://api.ipify.org?format=json)
set -euo pipefail
if [ $# -ne 1 ]; then
echo "usage: $(basename "$0") <BASE_URL> (e.g. localhost:8090)" >&2
exit 1
fi
BASE_URL="$1"
IP_ECHO_URL="${IP_ECHO_URL:-https://api.ipify.org?format=json}"
for tool in curl jq; do
if ! command -v "${tool}" &> /dev/null; then
echo "ERROR: ${tool} is required but not installed" >&2
exit 1
fi
done
auth_args=()
if [ -n "${TOKEN:-}" ]; then
auth_args=(-H "Authorization: Bearer ${TOKEN}")
fi
if ! proxies_json=$(curl -sS --fail "${auth_args[@]}" "${BASE_URL}/v1/proxies") \
|| ! echo "${proxies_json}" | jq -e . > /dev/null 2>&1; then
echo "ERROR: could not fetch proxy list from ${BASE_URL}" >&2
exit 1
fi
total=$(echo "${proxies_json}" | jq -r '.count')
fmt="%-31s %-21s %-21s %s\n"
echo "${total} proxies @ $(date +%H:%M:%S)"
# shellcheck disable=SC2059
printf "${fmt}" "PROXY" "ENDPOINT" "LOCATION" "EGRESS-IP"
probed=0
skipped=0
failed=0
while IFS= read -r proxy; do
id=$(echo "${proxy}" | jq -r '.id')
endpoint=$(echo "${proxy}" | jq -r '"\(.ip):\(.port)"')
location=$(echo "${proxy}" | jq -r '.attributes.zone // .attributes.geo // "-"')
healthy=$(echo "${proxy}" | jq -r '.healthy')
if [ "${healthy}" != "true" ]; then
# shellcheck disable=SC2059
printf "${fmt}" "${id}" "${endpoint}" "${location}" "(unhealthy)"
skipped=$((skipped + 1))
continue
fi
if response=$(curl -sS --max-time 10 -x "http://${endpoint}" "${IP_ECHO_URL}" 2> /dev/null); then
egress=$(echo "${response}" | jq -r '.ip // "?"' 2> /dev/null || echo "?")
probed=$((probed + 1))
else
egress="FAILED"
failed=$((failed + 1))
fi
# shellcheck disable=SC2059
printf "${fmt}" "${id}" "${endpoint}" "${location}" "${egress}"
done < <(echo "${proxies_json}" | jq -c '.proxies[]')
echo "-- ${probed} probed, ${skipped} unhealthy, ${failed} failed --"

74
docs/demo/show-egress-ips.sh Executable file
View File

@@ -0,0 +1,74 @@
#!/bin/bash
# Demo: list proxies from the discovery API and show the egress IP each
# healthy one provides, by calling an IP-echo site through it.
#
# Usage:
# ./show-egress-ips.sh <BASE_URL> e.g. ./show-egress-ips.sh localhost:8090
#
# Optional environment:
# TOKEN bearer token for the discovery API (see docs/api.md)
# IP_ECHO_URL site that returns the caller's IP as JSON
# (default: https://api.ipify.org?format=json)
set -euo pipefail
if [ $# -ne 1 ]; then
echo "usage: $(basename "$0") <BASE_URL> (e.g. localhost:8090)" >&2
exit 1
fi
BASE_URL="$1"
IP_ECHO_URL="${IP_ECHO_URL:-https://api.ipify.org?format=json}"
for tool in curl jq; do
if ! command -v "${tool}" &> /dev/null; then
echo "ERROR: ${tool} is required but not installed" >&2
exit 1
fi
done
auth_args=()
if [ -n "${TOKEN:-}" ]; then
auth_args=(-H "Authorization: Bearer ${TOKEN}")
fi
echo "Fetching proxies from ${BASE_URL}/v1/proxies ..."
if ! proxies_json=$(curl -sS --fail "${auth_args[@]}" "${BASE_URL}/v1/proxies"); then
echo "ERROR: could not fetch proxy list from ${BASE_URL}" >&2
exit 1
fi
if ! echo "${proxies_json}" | jq -e . > /dev/null; then
echo "ERROR: response from ${BASE_URL}/v1/proxies is not valid JSON" >&2
exit 1
fi
total=$(echo "${proxies_json}" | jq -r '.count')
echo "Found ${total} proxies"
echo ""
probed=0
skipped=0
failed=0
while IFS= read -r proxy; do
id=$(echo "${proxy}" | jq -r '.id')
ip=$(echo "${proxy}" | jq -r '.ip')
port=$(echo "${proxy}" | jq -r '.port')
healthy=$(echo "${proxy}" | jq -r '.healthy')
if [ "${healthy}" != "true" ]; then
echo "--- skipping ${id} (unhealthy) ---"
echo ""
skipped=$((skipped + 1))
continue
fi
echo "=== via ${id} — http://${ip}:${port} ==="
if response=$(curl -sS --max-time 10 -x "http://${ip}:${port}" "${IP_ECHO_URL}"); then
echo "${response}" | jq . 2> /dev/null || echo "${response}"
probed=$((probed + 1))
else
echo "WARNING: request through ${id} failed" >&2
failed=$((failed + 1))
fi
echo ""
done < <(echo "${proxies_json}" | jq -c '.proxies[]')
echo "Done: ${total} proxies — ${probed} probed, ${skipped} skipped (unhealthy), ${failed} failed"

View File

@@ -0,0 +1,38 @@
# Execution: Discovery API documentation
Plan: [2026-08-11-2152-discovery-api-docs.md](../plans/2026-08-11-2152-discovery-api-docs.md)
- [x] Step 1 — Write `docs/api.md` full API reference
- [x] Step 2 — Add pointers in README and architecture.md
## Step 1 — docs/api.md
Wrote the full reference: base URL (in-cluster FQDN + port-forward), bearer
auth, error envelope, configuration table, all five routes with schemas,
status codes and curl examples, the selection/cooldown semantics section,
an end-to-end curl walkthrough, and the in-memory/single-replica caveats.
All facts were taken from the code, not from memory of prior docs.
Worth noting: the doc explicitly calls out two things no earlier doc
stated for clients — that a report **without** a target on a targetless
lease creates a *global* cooldown (blocking the proxy for everyone), and
that `rate_limited` and `banned` currently behave identically. Both came
straight from `internal/lease/store.go` and are easy to trip over.
## Step 2 — Pointers + verification
Added one-line links to the new doc in README's quickstart (above the curl
block) and in `docs/architecture.md` §7. Verified the documented behavior
against the tree rather than trusting the write-up:
```sh
go build ./... && go test -short ./internal/discovery/ ./internal/lease/
```
Both pass; a grep of `server_test.go` confirmed every documented status
code and error code (`invalid_ttl`, `invalid_query`, `invalid_result`,
`no_match`, `unknown_lease`, 201/204/401/404/409) is asserted by tests.
Worth noting: CHANGELOG entry deliberately deferred until the user
confirms the docs read well, per the CHANGELOG convention's
"once the user confirms it works" clause.

View File

@@ -0,0 +1,55 @@
# Execution: Demo scripts
Plan: [2026-08-11-2220-demo-scripts.md](../plans/2026-08-11-2220-demo-scripts.md)
- [x] Step 0 — Branch `feat/demo-scripts` + plan commit
- [x] Step 1 — `docs/demo/show-egress-ips.sh`
- [x] Extra (added iteratively, not in the original plan) — proxy-creation scripts
## Step 0 + Step 1
Branched off `main`, committed the plan alone, then wrote
`docs/demo/show-egress-ips.sh`: takes `BASE_URL` as its argument, lists
`/v1/proxies` (bearer auth via optional `TOKEN` env), probes each healthy
proxy with `curl -x http://ip:port` against an IP-echo site
(`IP_ECHO_URL`, default ipify JSON), banners which proxy each request goes
through, skips unhealthy ones, and ends with a probed/skipped/failed
summary. Per user request the script was left uncommitted for iteration
and no verification beyond `bash -n` was run.
## Extra — create-kubernetes-proxies.sh, create-gcp-proxies.sh
Added on the same branch before the first commit:
- `create-kubernetes-proxies.sh <count>` — creates
`proxy-kubernetes-demo-1..N` with the kubernetes provider, spec taken
from `config/samples/proxy_kubernetes.yaml`, applied via
`kubectl apply -f -` heredocs. Optional `NAMESPACE` env.
- `create-gcp-proxies.sh <count>` — creates `proxy-gcp-demo-1..N` from the
user-supplied gcp-eu manifest (e2-micro, Ubuntu 24.04, Squid
cloud-init), each with a zone picked randomly from a hardcoded list of
27 EU zones so the fleet gets egress IPs from different locations.
Env overrides: `ZONES`, `GCP_PROVIDER` (default `gcp-eu`), `NAMESPACE`.
## Extra — run-demo.sh, show-egress-ips-table.sh
Second round of iterative additions:
- `run-demo.sh` — tmux demo driver: 2x2 tiled grid with the egress-IP
table looping every 10s inside a netshoot pod (script `kubectl cp`'d
into the pod, `BASE_URL` set to the in-cluster Service FQDN),
`watch -n3 kubectl get px`, and both create scripts auto-running with
`COUNT` proxies each (default 4). Env knobs: `COUNT`, `SESSION`,
`NETSHOOT_POD`, `DEMO_DIR` (defaults to the script's own dir).
- `show-egress-ips-table.sh` — condensed one-line-per-proxy variant of
`show-egress-ips.sh` sized for a tmux pane: PROXY / ENDPOINT /
LOCATION (zone→geo attribute fallback) / EGRESS-IP columns, with
`(unhealthy)` and `FAILED` inline instead of verbose output. The
verbose script stays for standalone use; the demo driver uses the
table variant.
Worth noting: beyond the user's sample manifest, the gcp script also
writes the picked zone into `attributes.zone`, so the discovery API
exposes each proxy's location and leases can select on it. The zone list
is static — if a project lacks quota in some region, `ZONES` narrows the
pool; nothing validates zones against the live project.

View File

@@ -0,0 +1,183 @@
# Execution log: OpenTelemetry tracing integrated with logr/zap logging
Plan: `docs/plans/2026-08-24-1025-otel-tracing.md`
- [x] Step 1 — Dependencies
- [x] Step 2 — New package `internal/tracing`
- [x] Step 3 — `provider.WithTracing` decorator
- [x] Step 4 — `cmd/main.go` wiring
- [x] Step 5 — Reconciler spans
- [x] Step 6 — Discovery server
- [x] Step 7 — GC + health
- [x] Step 8 — GCP wire-log enrichment
- [x] Step 9 — Manifests + docs
## Step 1 — Dependencies
Aligned the pre-existing indirect skew (otel core v1.44.0 vs otlptrace exporters
v1.40.0) and added the new direct deps in one shot:
```bash
go get go.opentelemetry.io/otel@v1.45.0 \
go.opentelemetry.io/otel/sdk@v1.45.0 \
go.opentelemetry.io/otel/trace@v1.45.0 \
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0 \
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp@v1.45.0 \
go.opentelemetry.io/otel/exporters/stdout/stdouttrace@v1.45.0 \
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0
go mod tidy
```
MVS side-effects worth recording: `logr v1.4.3→v1.4.4`, `httpsnoop
v1.0.4→v1.1.0`, `grpc-gateway/v2 v2.27.7→v2.29.0`, `proto/otlp v1.9.0→v1.11.0`,
plus a `genproto/googleapis/api` pseudo-version bump. Full `make test` passed
against the bumped graph (`google.golang.org/api v0.292.0` and k8s v0.36
tolerate otelhttp v0.70.0).
Worth noting: the first `go mod tidy` ran *before* any first-party code
imported `otlptracehttp`/`stdouttrace`, so it silently dropped those two
modules again; the Step 2 tidy re-added them. The plan's semconv question
resolved to `semconv/v1.43.0` — that's what `sdk@v1.45.0/resource/builtin.go`
imports, so first-party code uses the same version to avoid
`ErrSchemaURLConflict` in the common path.
## Step 2 — `internal/tracing` package
Landed as planned: `tracing.go` (env-gated `Setup`, hand-rolled exporter
selection), `logger.go` (`Start`/`StartSpan`/`ContextWithLogger` with the
base-logger ctx key that prevents duplicate `traceID` zap fields on nested
spans), `reconciler.go`, `transport.go`, `http.go`, `options.go`; tests for
all of it (first use of `sdk/trace/tracetest` in the repo).
Two deviations from the plan's letter:
- **k8s transport gating is a custom RoundTripper, not `otelhttp.WithFilter`**
(`parentGatedTransport`): requests without a parent span bypass the otel
transport entirely, so the no-root-spans-from-informers guarantee doesn't
depend on otelhttp filter semantics for transports.
- **`HTTPMiddleware` must copy the route pattern back.** The logger-injecting
inner handler wraps the request via `WithContext` (a shallow copy), so the
mux records the matched pattern on the copy while otelhttp's post-routing
span rename reads the original. Found by the middleware test (span named
`"GET"` instead of `"GET /v1/things/{id}"`); fixed with `r.Pattern =
r2.Pattern` after `next.ServeHTTP`.
Also: both the transport and the middleware pass explicit W3C propagators
instead of relying on the global, so behavior is deterministic under tests
and when tracing is disabled. `Setup` tests reset the global provider to a
fresh noop per case — restoring otel's own default delegate triggers a
"Setting tracer provider to its current value" warning from the SDK.
## Step 3 — `provider.WithTracing`
`internal/provider/tracing.go` mirrors `metrics.go` exactly (same wrap shape,
same `resultLabel` classification reused as the `provider.result` span
attribute); tests reuse `staticProvider` from `metrics_test.go`. `ErrNotFound`
maps to span status Ok as planned — the reconciler polls `Get` to NotFound
during replacement/deletion, so it's an answer, not a failure.
One small API addition to `internal/tracing` for this: exported
`tracing.Tracer(opts...)`, since the decorator lives in package `provider`
and couldn't reach the unexported option resolver. The `cmd/main.go` wiring
(`WithTracing` outermost around `WithMetrics`) lands with Step 4's commit —
same file, one commit.
## Step 4 — `cmd/main.go` wiring
As planned: `tracing.Setup` right after `ctrl.SetLogger`; manager rest.Config
wrapped via `restCfg := ctrl.GetConfigOrDie(); restCfg.Wrap(...)` (the
previous inline call left nowhere to wrap); providers decorated tracing-
outermost; explicit trace flush after `mgr.Start` returns on both the error
and clean paths, with a fresh 10s context since the signal ctx is already
cancelled by then.
The kubernetes provider grew `NewWithTransportWrapper(ctx, cfg, wrap)` (its
client is built from its own `ctrl.GetConfig()`, invisible to the manager's
wrapped config); `New` now delegates with a nil wrapper, so `newWithClient`
tests stayed untouched, and main registers a closure — the same pattern the
gcp constructor already used for wire-log options.
Deviation: the plan put `--trace-health-probes` here, but the flag needs the
`health.Engine.TraceProbes` field that Step 7 introduces — moved there to
keep every commit compiling.
## Step 5 — Reconciler spans
`SetupWithManager` completes with `tracing.NewReconciler("Proxy", r)`;
sub-spans `reconcile.managed` / `reconcile.replaceInstance` /
`reconcile.delete` open at the top of each state machine, and `status.patch`
opens inside the deferred flush closure so it stays within the root span
while its error still folds into the recorded result. `reconcileExternal`
left unspanned as planned (no I/O).
Two small judgment calls: the sub-spans carry no extra attributes — the
provider decorator already records `provider.id`, and the root span carries
the object identity, so duplicating them was noise; and `status.patch` is
emitted every reconcile even when nothing changed (the no-op compare is the
span's content — a real PATCH shows up as its k8s HTTP child). Tests that
call `r.Reconcile` directly bypass the wrapper; with no global tracer set
they see no-op spans, so the existing fake-client and envtest suites run
unchanged.
## Step 6 — Discovery server
Middleware chain is now recover → tracing (server span + request logger) →
request-log → body cap → auth; the request-log middleware and the three
handler error sites log via `logf.FromContext(r.Context())` — which is
`s.log` enriched with the request's traceID/spanID by the tracing
middleware, not a different logger (logr sinks can't read ctx at log time,
so per-request values must ride on the logger instance in the ctx; user
asked, answered in-session).
Deviation from the plan's letter: `recoverMiddleware` keeps `s.log`. It sits
*outside* the tracing layer, so its request ctx never has the enriched
logger — switching it to `FromContext` would silently drop the `"discovery"`
name and gain nothing.
## Step 7 — GC + health
`gc.sweep` runs in a root span with `gc.providers` / `gc.deleted` counters
(a failed orphan delete no longer counts as deleted — the loop grew an
explicit `continue`). The health engine got the deferred `--trace-health-probes`
flag and `TraceProbes` field; the probe call moved into `runProbe`, which
wraps it in a `health.probe` client span only when enabled, ending before
`record` (which stays ctx-free by design). Probe transports remain
uninstrumented so no traceparent can leak through a proxy to external
targets. Probe→reconcile span links stay out of scope (GenericEvent carries
no ctx; the workqueue coalesces events) — recorded in the architecture
Decisions in Step 9.
## Step 8 — GCP wire-log enrichment
`wireFilterHandler.Handle` clones the record and appends `traceID`/`spanID`
when the ctx carries a valid span — same keys as the logr enrichment, added
before the elision pass (both values are well under the 1KiB threshold).
The new `wirelog_test.go` covers the enrichment, that the JWT/token security
filter still drops non-wire Debug records even with a span present, and that
spanless records stay unchanged.
## Step 9 — Manifests, docs, lint
`config/manager/manager.yaml` gained the OTel env block (downward-API
`POD_NAME`/`POD_NAMESPACE` deliberately listed *before*
`OTEL_RESOURCE_ATTRIBUTES``$(VAR)` expansion only sees earlier vars),
with the OTLP endpoint and sampler left as commented examples so deployments
stay tracing-off by default. `docs/architecture.md` got "### 10. Tracing"
plus five Decisions entries; README got a Tracing section.
`make lint` surfaced 16 new-on-branch issues (checked with
`golangci-lint run --new-from-rev main`; the other ~43 pre-date this work):
- one gofmt slip, 13 goconst repeats — fixed with `env*`/`exporter*`
constants in `internal/tracing/tracing.go` shared by the tests;
- two logcheck hits ("function takes both a context and a logger"):
`Setup` now reads its logger from ctx (`logf.FromContext`, caller seeds
via `logf.IntoContext` — the linter's sanctioned pattern), and
`ContextWithLogger` carries a justified `//nolint:logcheck`, since being
the IntoContext-analogue is its purpose.
Worth noting: pre-existing lint debt (50 goconst etc. on `main`) was left
untouched — `make lint` still fails overall; only branch-introduced issues
were cleaned. The lint fixes changed `Setup`'s signature after Step 2/4 had
landed, so those commits show the two-arg version; this commit is where it
settles.

View File

@@ -0,0 +1,58 @@
# Plan: Discovery API documentation (docs/api.md)
**Created:** 2026-08-11 21:52
## Context
The operator serves an HTTP discovery/lease API on `:8090` ([internal/discovery/](internal/discovery/)) that crawler clients use to list proxies, acquire TTL leases, release them, and report rate-limiting. There is no dedicated API reference today: README has four quickstart curls (no auth header, no schemas), and `docs/architecture.md` §7 has an ASCII route map. The user wants full documentation with curl examples for every feature.
**Decisions made with user:** doc lives in a new `docs/api.md`; commit straight to `main` (no MR).
## Deliverable
### 1. New file `docs/api.md` — full API reference
Content (all facts verified against code during planning):
- **Overview & base URL** — what the API is; in-cluster FQDN `http://egress-proxies-operator-controller-manager-discovery-service.egress-proxies-operator-system.svc.cluster.local:8090` (Service: `config/default/discovery_service.yaml`); local access via `kubectl port-forward svc/egress-proxies-operator-controller-manager-discovery-service 8090:8090`.
- **Authentication** — static bearer token from `DISCOVERY_TOKEN` env var (populated from the optional `discovery-token` Secret, key `token`; `config/manager/manager.yaml`). Empty token ⇒ auth disabled with startup warning. Curl: `-H "Authorization: Bearer $TOKEN"` on every example. `/healthz` always exempt.
- **Conventions** — JSON everywhere; error envelope `{"error":"<code>","message":"<text>"}`; request bodies capped at 64 KiB; proxies with a deletion timestamp are excluded from all responses.
- **Configuration table** — `--discovery-addr` (default `:8090`), `--max-lease-ttl` (default 1h), `--lease-cooldown` (default 15m), `DISCOVERY_TOKEN`. Note the shipped Deployment passes none of these flags, so defaults apply.
- **Endpoints**, each with request/response schema, status codes, and a copy-pasteable curl example:
- `GET /healthz` — liveness, unauthenticated.
- `GET /v1/proxies` — filters `healthy=true|false` (else 400 `invalid_query`) and repeatable `attr.<key>=<value>` (verbatim equality on `spec.attributes`, all pairs must match). Response `{"proxies":[proxyView...],"count":N}` sorted by id. Full `proxyView` field table: `id` (ns/name), `ip`, `port`, `attributes`, `phase` (Pending/Provisioning/Ready/Unhealthy/Deleting/Failed), `healthy` (condition `Healthy` == True), `latencyMillis`, `activeLeases`, `maxLeases` (default 5; explicit 0 = unleasable).
- `POST /v1/leases` — body `{selector, ttlSeconds, target}` all optional; TTL defaults 5m, capped at max-lease-ttl (else 400 `invalid_ttl`). 201 `{leaseID, proxy, expiresAt, ttlSeconds}`; 409 `no_match` with `considered/atCapacity/inCooldown/unhealthy` counts (documented meanings).
- `DELETE /v1/leases/{id}` — early release; always 204, idempotent.
- `POST /v1/leases/{id}/report` — body `{result: ok|rate_limited|banned, target}`; 204, 404 `unknown_lease`, 400 `invalid_result`. `ok` is a pure ack; `rate_limited` and `banned` behave identically today (both start one cooldown window).
- **Selection & cooldown semantics** (short section — this is the non-obvious part clients need):
- Selection order: fewest active leases → lowest latency → lexicographic id; deterministic; only healthy, non-deleting proxies with free capacity are candidates.
- Cooldown: 15m default (`--lease-cooldown`), keyed `{proxy, target}`. Report with a target blocks only leases requesting that target; report without a target (and lease without one) creates a **global** cooldown blocking all acquisitions of that proxy — call this footgun out explicitly.
- Expired leases stay reportable for one cooldown window past TTL.
- **End-to-end workflow example** — numbered curl walkthrough: acquire → use `proxy.ip:port` as HTTP proxy (`curl -x`) → report `rate_limited` on 429 → release. Using a `jq`-extracted `leaseID`.
- **Caveats** — lease/cooldown state is in-memory and per-process: single replica only, operator restart drops all leases and cooldowns.
### 2. Pointers to the new doc (small edits)
- `README.md`: one-line link near the quickstart curl section ("full reference: docs/api.md").
- `docs/architecture.md` §7: one-line link to `docs/api.md` as the detailed reference.
### 3. Housekeeping per CLAUDE.md
- First action post-approval: copy this plan to `docs/plans/<timestamp>-discovery-api-docs.md` (timestamp from `date "+%Y-%m-%d-%H%M"`), commit it alone.
- Then write the docs, commit to `main` with `Co-Authored-By: Claude <noreply@anthropic.com>` trailer, push.
- Append execution summary + status checklist to `docs/plans-executions/<same-timestamp>-discovery-api-docs.md` in the docs commit.
- Add `CHANGELOG.md` entry (timestamp via `date "+%Y-%m-%d %H:%M %Z"`) once the user confirms.
## Key source files (facts source of truth)
- [internal/discovery/handlers.go](internal/discovery/handlers.go), [internal/discovery/server.go](internal/discovery/server.go) — routes, schemas, status codes, auth, limits.
- [internal/lease/store.go](internal/lease/store.go) — selection order, cooldown/retention, stats.
- [api/v1alpha1/proxy_types.go](api/v1alpha1/proxy_types.go), [api/v1alpha1/helpers.go](api/v1alpha1/helpers.go) — defaults (port 3128, maxLeases 5), phases, conditions.
- [cmd/main.go](cmd/main.go) — flags/env defaults.
- [config/default/discovery_service.yaml](config/default/discovery_service.yaml), [config/manager/manager.yaml](config/manager/manager.yaml) — service DNS, token secret.
## Verification
- Cross-check every documented status code / field name against `internal/discovery/server_test.go` expectations.
- Sanity-run `go build ./... && go test -short ./internal/discovery/ ./internal/lease/` (no code changes expected — confirms docs match current behavior, not a stale tree).
- Optionally lint the curl JSON bodies by piping each through `jq .` locally.

View File

@@ -0,0 +1,33 @@
# Plan: Demo script — egress IP check through each healthy proxy
**Created:** 2026-08-11 22:20
## Context
First of a planned series of demo scripts under `docs/demo/`. This one showcases the discovery API end to end without leases: list proxies from `$BASE_URL/v1/proxies`, and for each **healthy** one, call an IP-echo site through it (`curl -x`) to show the egress IP that proxy provides — clearly labeling which proxy each request goes through. The user explicitly asked to start by switching to a new branch. The script will be iterated on: **write it but do not commit it** — the user wants to add things to it before anything is committed.
## Steps
### Step 0 — Branch
Create `feat/demo-scripts` off `main`. Commit only the plan copy (`docs/plans/<timestamp>-demo-scripts.md`, timestamp via `date "+%Y-%m-%d-%H%M"`) per CLAUDE.md — nothing else gets committed this round.
### Step 1 — `docs/demo/show-egress-ips.sh` (new file, executable, left uncommitted)
Style: match [.devcontainer/post-install.sh](.devcontainer/post-install.sh) — `#!/bin/bash`, `set -euo pipefail`, `ERROR:`/`WARNING:` messages, `${VAR}` braces.
Behavior:
1. **Args/env:** `BASE_URL` is `$1` (required; missing → usage text + exit 1, e.g. `usage: show-egress-ips.sh <BASE_URL> (e.g. localhost:8090)`). Optional env: `TOKEN` (bearer token, same name docs/api.md uses; when set, send `Authorization: Bearer $TOKEN`), `IP_ECHO_URL` (default `https://api.ipify.org?format=json` — returns `{"ip":"..."}`).
2. **Dependency check:** `command -v curl`, `command -v jq``ERROR` + exit 1 if missing.
3. **Fetch** `"$BASE_URL/v1/proxies"` once (no server-side `healthy` filter — fetch all so unhealthy ones can be shown as skipped, which makes the demo more informative). Fail with a clear error if curl or JSON parsing fails.
4. **Iterate** proxies with `jq -c '.proxies[]'`; for each, extract `id`, `ip`, `port`, `healthy`:
- unhealthy → print `--- skipping <id> (unhealthy) ---`
- healthy → print a clear banner naming the proxy before the request, e.g. `=== via <id> — http://<ip>:<port> ===`, then `curl -sS --max-time 10 -x "http://${ip}:${port}" "$IP_ECHO_URL"`; print the JSON response. A failed probe prints `WARNING: request through <id> failed` and continues (guard so `set -e` doesn't kill the loop).
5. Finish with a one-line summary: N proxies, M probed, K skipped/failed.
Reference for API shapes: [docs/api.md](docs/api.md) (`proxies[].id/ip/port/healthy`; `curl -x http://ip:port` usage is already documented there and in README).
### Deliberately deferred (user will iterate on the script first)
- No commit of the script, no push beyond the plan commit, no MR, no execution summary, no CHANGELOG — all wait until the user says the script (or script set) is ready.

View File

@@ -0,0 +1,107 @@
# Plan: OpenTelemetry tracing integrated with the existing logr/zap logging
**Created:** 2026-08-24 10:25
## Context
The operator has structured logging (controller-runtime zap → logr, `logf.FromContext(ctx)` everywhere) and Prometheus metrics, but no tracing. A single reconcile fans out into k8s API calls, provider calls (GCP VMs / squid Pods), and status patches; the discovery API serves lease requests; GC and health engines run on tickers. Today correlating one operation across those hops means grepping for `reconcileID` or `providerID`. Goal: proper OTel traces — a new trace per unit of work (reconcile, HTTP request, GC sweep, optionally health probe), child spans for k8s/provider/HTTP calls — **and every log line inside a traced operation enriched with the trace context**, without disturbing the existing logr/zap system.
User decisions (confirmed):
- **Enablement:** auto from standard `OTEL_*` env vars — no collector configured ⇒ tracing fully off, zero new flags for the common case.
- **Health probes:** not traced by default; new `--trace-health-probes` flag enables one root span per probe.
- **Backend:** none exists yet — verification uses the console exporter and a throwaway Jaeger all-in-one in kind; manifests ship commented-out OTLP env examples.
## Key design facts (verified, non-obvious)
- **logr can't read ctx:** `logr.LogSink` never sees `context.Context`, so trace IDs cannot be added inside the zap sink. They must be attached as `WithValues` at span-creation points via `logf.IntoContext`. zap does **not** dedupe repeated keys, so enrichment must happen exactly once per ctx chain (see `tracing.Start` below).
- Log keys: **`traceID` / `spanID`** (lowerCamel matches `reconcileID`/`providerID` house style; note in docs that Grafana/Loki derived-field regexes must match `traceID`, not the `trace_id` default).
- **GCP calls trace themselves:** `google.golang.org/api/transport/http` already wraps its transport in `otelhttp.NewTransport` — once a global TracerProvider is set, every Compute call emits an HTTP client span. Do not add HTTP-level spans in the gcp provider; the `provider.*` decorator span becomes their parent.
- **kube-apiserver ignores incoming `traceparent`** by design; our k8s-API client spans are leaves. Don't promise cross-process traces into the apiserver.
- The Go SDK does **not** implement `OTEL_SDK_DISABLED` — handle it ourselves.
- `contrib/exporters/autoexport` drags in metric/log/prometheus exporters (binary + deps on a 128Mi pod) — hand-roll trace-exporter selection (~40 lines) instead.
- Span events are deprecated (OTel spec, Mar 2026): use `span.SetStatus` + `error.type`-style attributes, not `RecordError`/`AddEvent`.
- `rest.Config.Wrap` puts our transport innermost (after auth), so spans see the final request — correct; but informer list/watch + leader election run outside any span ⇒ the transport must **filter out requests with no parent span** or every watch becomes a long-lived root span.
- Manager runnables receive no logger in ctx (only HTTP servers do) — `logf.FromContext` falls back to the global logger there, which is fine; the discovery server must keep seeding from `s.log`.
## Steps
### Step 1 — Dependencies
`go get` (aligning the existing indirect skew: otel core v1.44 / otlptrace v1.40):
`go.opentelemetry.io/otel@v1.45.0`, `otel/sdk@v1.45.0`, `otel/trace@v1.45.0`,
`otel/exporters/otlp/otlptrace/otlptracegrpc@v1.45.0`, `.../otlptracehttp@v1.45.0`,
`otel/exporters/stdout/stdouttrace@v1.45.0`,
`go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.70.0`, plus the
`semconv` version matching the SDK's `resource.Default()` (treat `ErrSchemaURLConflict` from `resource.New` as non-fatal if versions drift).
`go mod tidy`; full build + `make test` to confirm `google.golang.org/api v0.292.0` / k8s v0.36 tolerate the bump (MVS picks the higher otelhttp).
### Step 2 — New package `internal/tracing`
- **`tracing.go``Setup(ctx, log logr.Logger, service, version string) (shutdown func(context.Context) error, err error)`**
- Disabled (returns no-op shutdown, installs nothing) when `OTEL_SDK_DISABLED=true`, or when neither `OTEL_TRACES_EXPORTER` nor `OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is set, or `OTEL_TRACES_EXPORTER=none`. One startup Info line either way stating the effective state/exporter.
- Exporter selection (hand-rolled, no autoexport): `OTEL_TRACES_EXPORTER``otlp` (default) | `console`; for otlp, `OTEL_EXPORTER_OTLP_(TRACES_)PROTOCOL``http/protobuf` (default) | `grpc``otlptracehttp` / `otlptracegrpc` (endpoint/TLS/headers via their native env handling).
- `sdktrace.NewTracerProvider` with batch processor (`WithMaxQueueSize(512)` — bounded on the 128Mi pod; sampler stays the SDK default so `OTEL_TRACES_SAMPLER(_ARG)` works), resource = `resource.New(WithAttributes(service.name, service.version=commit), WithFromEnv())` with env last so it wins.
- Globals: `otel.SetTracerProvider`, `otel.SetTextMapPropagator(TraceContext+Baggage)`, `otel.SetLogger(log.WithName("otel"))`, `otel.SetErrorHandler` → logr at `V(1)` (an unreachable collector errors every few seconds; Error level would spam).
- **`logger.go` — the one logger-enrichment mechanism** (used by everything; prevents duplicate `traceID` keys):
- private ctx key holding the *base* (pre-enrichment) logr.Logger.
- `Start(ctx, spanName, opts...) (context.Context, trace.Span)`: starts a span from the global tracer; if the resulting span context is valid, capture base = stored base or `logf.FromContext(ctx)` on first call, then `logf.IntoContext(ctx, base.WithValues("traceID", ..., "spanID", ...))`. Nested `Start` re-derives from base — never stacks. Documented trade-off: `WithValues` pushed via `logf.IntoContext` *between* two `Start` calls is dropped (nothing first-party does that; controller-runtime's `reconcileID` logger is the base and survives).
- `ContextWithLogger(ctx, base)` seeds the base explicitly (discovery middleware).
- **`reconciler.go``NewReconciler(kind string, inner reconcile.Reconciler, opts...) reconcile.Reconciler`**: root span `Reconcile <kind>` (SpanKind Internal) via `Start`; attrs `k8s.namespace.name`, `k8s.object.name`, `reconcile.id` (`controller.ReconcileIDFromContext`); on return records `reconcile.requeue_after`, sets Error status from err. `WithTracerProvider` option for tests.
- **`transport.go``RestConfigWrapper() transport.WrapperFunc`**: `otelhttp.NewTransport` with `WithFilter` requiring `trace.SpanContextFromContext(r.Context()).IsValid()` — k8s API spans only under an existing trace (excludes informers, leader election, metrics authn).
- **`http.go``HTTPMiddleware(operation string, base logr.Logger) func(http.Handler) http.Handler`**: `otelhttp.NewHandler` (filter `/healthz`; otelhttp renames the span from `r.Pattern` after routing — no custom naming needed) + inner layer calling `ContextWithLogger`-then-enrich so handler logs carry `traceID`. Accepts incoming `traceparent` (clients continue their traces).
Tests (all with `sdktrace` + `tracetest.NewSpanRecorder` via the `WithTracerProvider` options, parallel; `Setup` env tests use `t.Setenv`, not parallel):
- `Start`: nested spans → child span recorded; **funcr-captured log line contains `traceID` exactly once** (pattern: `internal/provider/gcp/gcp_test.go:277`); disabled tracer → no `traceID` in logs.
- `Setup`: table over env combinations (unset→disabled, `none`→disabled, endpoint set→otlp, `console`, `OTEL_SDK_DISABLED`).
- Reconciler decorator: fake inner returns result/err → span name/attrs/status.
- Transport filter: request without parent span produces no span.
- Middleware: `httptest` request → server span named from route pattern; `/healthz` unspanned; handler logger carries `traceID`.
### Step 3 — `provider.WithTracing` decorator
`internal/provider/tracing.go`, mirroring `metrics.go`: `WithTracing(name string, p Provider, opts...) Provider` — one Client span per call (`provider.create|get|delete|list`), attrs `provider.name`, `provider.id` (where present), result class attr from `resultLabel(err)`; status Error only for quota/permanent/transient — `ErrNotFound` stays Ok (expected during deletion polling). Returns the inner error **unmodified**. Uses `tracing.Start` so provider-internal logs (gcp `p.logger(ctx)`) inherit `traceID`. Tests in `internal/provider/tracing_test.go` reusing `staticProvider` from `metrics_test.go:19`.
Wiring in `cmd/main.go:174-177` — tracing outermost: `provider.WithTracing(name, provider.WithMetrics(name, p, m))`.
### Step 4 — `cmd/main.go` wiring
- `tracing.Setup(...)` immediately after `ctrl.SetLogger` (cmd/main.go:148); on error: log + `os.Exit(1)`.
- Explicit `shutdown(context.WithTimeout(context.Background(), 10*time.Second))` (fresh ctx — signal ctx is already cancelled) after `mgr.Start` returns, on **both** the error path before `os.Exit(1)` and the clean path. Earlier `os.Exit` sites have nothing to flush.
- `cfg := ctrl.GetConfigOrDie(); cfg.Wrap(tracing.RestConfigWrapper())` before `ctrl.NewManager` (cmd/main.go:239).
- kubernetes provider: add `kubernetes.NewWithTransportWrapper(ctx, cfg, wrap transport.WrapperFunc)` beside `New` (internal/provider/kubernetes/kubernetes.go:42) and register a closure in main's constructor map (same pattern as gcp at cmd/main.go:166) passing `tracing.RestConfigWrapper()` — keeps `internal/tracing` out of provider packages and `newWithClient` tests untouched.
- New flag `--trace-health-probes` (default false) → `engine.TraceProbes`.
### Step 5 — Reconciler spans
- `SetupWithManager` (proxy_controller.go:412): `b.Complete(tracing.NewReconciler("Proxy", r))`.
- Sub-spans via `tracing.Start`: `reconcile.managed` (:116), `reconcile.replaceInstance` (:226), `reconcile.delete` (:254), and `status.patch` opened **inside the deferred flush closure** (:98-104, runs within the root span; its error already folds into `err`). `reconcileExternal` stays unspanned — no I/O, pure noise.
- Useful attrs where cheap: phase transitions, `providerID`.
### Step 6 — Discovery server
- `handler()` (internal/discovery/server.go:151-156): chain becomes recover → `tracing.HTTPMiddleware("discovery", s.log)` → log → maxBytes → auth (otelhttp must stay outside `maxBytesMiddleware`).
- Handlers and `logMiddleware`/`recoverMiddleware` switch `s.log``logf.FromContext(r.Context())` (handlers.go:85, :141, :234; server.go:163, :191) so request logs carry `traceID`. `s.log` remains for startup lines.
- k8s `List` calls in handlers already use `r.Context()` → child spans appear via the wrapped rest transport.
### Step 7 — GC + health
- `gc.Sweeper.sweep` (internal/gc/gc.go:83): wrap in root span `gc.sweep` via `tracing.Start` (before the `.WithName("orphan-gc")` derivation so it inherits), attrs: providers swept, orphans deleted. Provider/k8s child spans come free from Steps 3/4.
- `health.Engine`: new field `TraceProbes bool`. In the worker loop (engine.go:158) when enabled: root span `health.probe` (SpanKind Client) around `probeFn`, attrs proxy key/ok/latency/error class, ended before `record` (which stays ctx-free). No transport wrap and no propagation — never inject `traceparent` toward external targets through the proxies. Probe→reconcile span links are **out of scope**: `GenericEvent` (engine.go:330) carries no ctx and the workqueue coalesces events; documented in architecture Decisions.
- Lease store sweep: not traced (in-memory only).
### Step 8 — GCP wire-log enrichment
`wireFilterHandler.Handle` (internal/provider/gcp/wirelog.go:46) receives the real request ctx from the SDK's slog calls: append `traceID`/`spanID` attrs when `trace.SpanContextFromContext(ctx)` is valid — V(5) wire logs become correlatable with the `provider.*` span. Add `wirelog_test.go` case.
### Step 9 — Manifests + docs
- `config/manager/manager.yaml` env (after `DISCOVERY_TOKEN`): `POD_NAME`/`POD_NAMESPACE` via downward API **first**, then `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE)` ($(VAR) expansion only sees earlier vars); commented-out `OTEL_EXPORTER_OTLP_ENDPOINT` + `OTEL_TRACES_SAMPLER` examples with a "tracing is off until an endpoint is set" comment. Note `--trace-health-probes` beside the args list.
- `docs/architecture.md`: new "### 10. Tracing" section after Metrics (:281) — span topology, enablement, `traceID` log-key choice; Decisions entries (:303): env-gated enablement, probes off by default, no probe→reconcile links (why), `traceID` naming vs Grafana defaults, no traceparent into apiserver.
- README ops note; CHANGELOG entry (dated via `date`) once the user confirms it works.
- Run `make lint` (CI has no lint step; `lll`/`revive` will hit new files).
## Verification
1. `make test` (unit, includes all new `tracetest` assertions) and `make lint`.
2. Console smoke: `OTEL_TRACES_EXPORTER=console go run ./cmd --providers-config hack/providers-dev.yaml ...` against kind — create a Proxy, confirm (a) `Reconcile Proxy` span JSON on stdout with nested `provider.create` + k8s PATCH spans, (b) the reconcile log lines carry the same `traceID` as the span, (c) no spans from informer watches or leader election.
3. End-to-end in kind: deploy Jaeger all-in-one (`jaegertracing/all-in-one`, OTLP 4318), set `OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318` on the manager Deployment; exercise a reconcile + a `POST /v1/leases` (with and without a client `traceparent`) + wait one GC interval; in the Jaeger UI: reconcile trace shows provider+k8s children; lease trace shows route-named server span with k8s child; grep manager logs for a `traceID` and resolve it in Jaeger.
4. Negative: run with no `OTEL_*` env — startup line says tracing disabled, no export-error spam, logs unchanged (no `traceID` keys).

26
go.mod
View File

@@ -4,10 +4,17 @@ go 1.26.0
require ( require (
cloud.google.com/go/compute v1.65.0 cloud.google.com/go/compute v1.65.0
github.com/go-logr/logr v1.4.3 github.com/go-logr/logr v1.4.4
github.com/onsi/ginkgo/v2 v2.27.4 github.com/onsi/ginkgo/v2 v2.27.4
github.com/onsi/gomega v1.39.0 github.com/onsi/gomega v1.39.0
github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_golang v1.23.2
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
go.opentelemetry.io/otel v1.45.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0
go.opentelemetry.io/otel/sdk v1.45.0
go.opentelemetry.io/otel/trace v1.45.0
google.golang.org/api v0.292.0 google.golang.org/api v0.292.0
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
k8s.io/api v0.36.0 k8s.io/api v0.36.0
@@ -31,7 +38,7 @@ require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect
github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect
@@ -48,7 +55,7 @@ require (
github.com/google/uuid v1.6.0 // indirect github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect
github.com/googleapis/gax-go/v2 v2.23.0 // indirect github.com/googleapis/gax-go/v2 v2.23.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect github.com/json-iterator/go v1.1.12 // indirect
@@ -66,14 +73,9 @@ require (
github.com/stoewer/go-strcase v1.3.0 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect
github.com/x448/float16 v0.8.4 // indirect github.com/x448/float16 v0.8.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.45.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
go.uber.org/multierr v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect go.uber.org/zap v1.27.1 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect
@@ -91,7 +93,7 @@ require (
golang.org/x/tools v0.47.0 // indirect golang.org/x/tools v0.47.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
google.golang.org/grpc v1.83.0 // indirect google.golang.org/grpc v1.83.0 // indirect
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect

56
go.sum
View File

@@ -34,8 +34,8 @@ github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8
github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
@@ -47,8 +47,8 @@ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6O
github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
@@ -86,8 +86,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.19 h1:mMOE7DN2+p76/EdIrm
github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k=
github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE=
github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -170,24 +170,28 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -228,8 +232,8 @@ google.golang.org/api v0.292.0 h1:Ewiwo/GTtiaPZSNAZQUcWLh8AYDEoPmIXyJfeoTSMHU=
google.golang.org/api v0.292.0/go.mod h1:07kjmMnFGm2RQuCza2EZM/5N68G/fVvFb1xKjWqoFA0= google.golang.org/api v0.292.0/go.mod h1:07kjmMnFGm2RQuCza2EZM/5N68G/fVvFb1xKjWqoFA0=
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=

View File

@@ -41,6 +41,7 @@ import (
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1" crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/health"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
) )
// HealthSnapshotter provides the current probe verdict for a proxy. The // HealthSnapshotter provides the current probe verdict for a proxy. The
@@ -96,9 +97,14 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
} }
base := p.DeepCopy() base := p.DeepCopy()
defer func() { defer func() {
// Runs inside the root reconcile span (this defer fires before the
// tracing.NewReconciler wrapper sees the return), and its error is
// folded into err, which that wrapper records.
pctx, span := tracing.Start(ctx, "status.patch")
defer span.End()
// NotFound is expected when this reconcile just removed the last // NotFound is expected when this reconcile just removed the last
// finalizer and the object is already gone. // finalizer and the object is already gone.
if perr := r.patchStatusIfChanged(ctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) { if perr := r.patchStatusIfChanged(pctx, base, &p); perr != nil && !apierrors.IsNotFound(perr) {
err = errors.Join(err, perr) err = errors.Join(err, perr)
} }
}() }()
@@ -114,6 +120,8 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
} }
func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) { func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
ctx, span := tracing.Start(ctx, "reconcile.managed")
defer span.End()
log := logf.FromContext(ctx) log := logf.FromContext(ctx)
if controllerutil.AddFinalizer(p, crawlv1alpha1.FinalizerName) { if controllerutil.AddFinalizer(p, crawlv1alpha1.FinalizerName) {
@@ -224,6 +232,8 @@ func (r *ProxyReconciler) reconcileManaged(ctx context.Context, p *crawlv1alpha1
// exists" — hence: delete, poll to NotFound, only then advance the hash and // exists" — hence: delete, poll to NotFound, only then advance the hash and
// let the create branch run. // let the create branch run.
func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.Proxy, prov provider.Provider, hash string) (ctrl.Result, error) { func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.Proxy, prov provider.Provider, hash string) (ctrl.Result, error) {
ctx, span := tracing.Start(ctx, "reconcile.replaceInstance")
defer span.End()
_, err := prov.Get(ctx, p.Status.ProviderID) _, err := prov.Get(ctx, p.Status.ProviderID)
if provider.Class(err) == provider.ErrNotFound { if provider.Class(err) == provider.ErrNotFound {
// Old instance is gone. The Update inside setSpecHash refreshes p // Old instance is gone. The Update inside setSpecHash refreshes p
@@ -252,6 +262,8 @@ func (r *ProxyReconciler) replaceInstance(ctx context.Context, p *crawlv1alpha1.
} }
func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) { func (r *ProxyReconciler) reconcileDelete(ctx context.Context, p *crawlv1alpha1.Proxy) (ctrl.Result, error) {
ctx, span := tracing.Start(ctx, "reconcile.delete")
defer span.End()
if !controllerutil.ContainsFinalizer(p, crawlv1alpha1.FinalizerName) { if !controllerutil.ContainsFinalizer(p, crawlv1alpha1.FinalizerName) {
return ctrl.Result{}, nil return ctrl.Result{}, nil
} }
@@ -409,7 +421,10 @@ func (r *ProxyReconciler) SetupWithManager(mgr ctrl.Manager) error {
if r.HealthEvents != nil { if r.HealthEvents != nil {
b = b.WatchesRawSource(source.Channel(r.HealthEvents, &handler.EnqueueRequestForObject{})) b = b.WatchesRawSource(source.Channel(r.HealthEvents, &handler.EnqueueRequestForObject{}))
} }
return b.Complete(r) // Root span per reconcile; sub-reconcilers and the status flush hang
// their spans off it. Tests calling r.Reconcile directly bypass the
// wrapper and see no-op spans — the global tracer is never set there.
return b.Complete(tracing.NewReconciler("Proxy", r))
} }
func (r *ProxyReconciler) applyDefaults() { func (r *ProxyReconciler) applyDefaults() {

View File

@@ -11,6 +11,7 @@ import (
apimeta "k8s.io/apimachinery/pkg/api/meta" apimeta "k8s.io/apimachinery/pkg/api/meta"
"sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client"
logf "sigs.k8s.io/controller-runtime/pkg/log"
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1" crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
@@ -82,7 +83,10 @@ func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
var list crawlv1alpha1.ProxyList var list crawlv1alpha1.ProxyList
if err := s.Reader.List(r.Context(), &list); err != nil { if err := s.Reader.List(r.Context(), &list); err != nil {
s.log.Error(err, "listing proxies") // The ctx logger is s.log enriched with this request's
// traceID/spanID by the tracing middleware (logr sinks can't read
// ctx at log time, so per-request values ride on the logger).
logf.FromContext(r.Context()).Error(err, "listing proxies")
writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed") writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
return return
} }
@@ -138,7 +142,7 @@ func (s *Server) handleAcquireLease(w http.ResponseWriter, r *http.Request) {
var list crawlv1alpha1.ProxyList var list crawlv1alpha1.ProxyList
if err := s.Reader.List(r.Context(), &list); err != nil { if err := s.Reader.List(r.Context(), &list); err != nil {
s.log.Error(err, "listing proxies for lease") logf.FromContext(r.Context()).Error(err, "listing proxies for lease")
writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed") writeError(w, http.StatusInternalServerError, "internal", "listing proxies failed")
return return
} }
@@ -231,7 +235,7 @@ func (s *Server) handleReportLease(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusNotFound, "unknown_lease", "no such lease") writeError(w, http.StatusNotFound, "unknown_lease", "no such lease")
return return
} }
s.log.Error(err, "reporting lease", "leaseID", r.PathValue("id")) logf.FromContext(r.Context()).Error(err, "reporting lease", "leaseID", r.PathValue("id"))
writeError(w, http.StatusInternalServerError, "internal", "report failed") writeError(w, http.StatusInternalServerError, "internal", "report failed")
return return
} }

View File

@@ -20,6 +20,7 @@ import (
logf "sigs.k8s.io/controller-runtime/pkg/log" logf "sigs.k8s.io/controller-runtime/pkg/log"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/lease"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
) )
// LeaseStore is what the handlers need from a lease backend. Defined here, // LeaseStore is what the handlers need from a lease backend. Defined here,
@@ -136,7 +137,9 @@ func (s *Server) Start(ctx context.Context) error {
} }
// handler assembles the mux and the middleware chain, outermost first: // handler assembles the mux and the middleware chain, outermost first:
// recover → request-log → body-size cap → bearer auth. // recover → tracing (server span + request logger) → request-log →
// body-size cap → bearer auth. Everything inside the tracing layer logs via
// logf.FromContext(r.Context()) and so carries traceID/spanID.
func (s *Server) handler() http.Handler { func (s *Server) handler() http.Handler {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
@@ -152,6 +155,7 @@ func (s *Server) handler() http.Handler {
h = s.authMiddleware(h) h = s.authMiddleware(h)
h = maxBytesMiddleware(h) h = maxBytesMiddleware(h)
h = s.logMiddleware(h) h = s.logMiddleware(h)
h = tracing.HTTPMiddleware("discovery", s.log)(h)
h = s.recoverMiddleware(h) h = s.recoverMiddleware(h)
return h return h
} }
@@ -188,7 +192,7 @@ func (s *Server) logMiddleware(next http.Handler) http.Handler {
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK} rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
start := time.Now() start := time.Now()
next.ServeHTTP(rec, r) next.ServeHTTP(rec, r)
s.log.Info("request", logf.FromContext(r.Context()).Info("request",
"method", r.Method, "path", r.URL.Path, "method", r.Method, "path", r.URL.Path,
"status", rec.status, "duration", time.Since(start).String()) "status", rec.status, "duration", time.Since(start).String())
}) })

View File

@@ -12,8 +12,11 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client"
logf "sigs.k8s.io/controller-runtime/pkg/log" logf "sigs.k8s.io/controller-runtime/pkg/log"
"go.opentelemetry.io/otel/attribute"
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1" crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider" "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
) )
// Sweeper is the manager Runnable running the sweep loop. // Sweeper is the manager Runnable running the sweep loop.
@@ -81,6 +84,8 @@ func (s *Sweeper) Start(ctx context.Context) error {
// that deletion, and GC racing it would double-delete. A UID becomes // that deletion, and GC racing it would double-delete. A UID becomes
// orphan-eligible only once the object is fully gone. // orphan-eligible only once the object is fully gone.
func (s *Sweeper) sweep(ctx context.Context) { func (s *Sweeper) sweep(ctx context.Context) {
ctx, span := tracing.Start(ctx, "gc.sweep")
defer span.End()
log := logf.FromContext(ctx).WithName("orphan-gc") log := logf.FromContext(ctx).WithName("orphan-gc")
var list crawlv1alpha1.ProxyList var list crawlv1alpha1.ProxyList
@@ -95,6 +100,7 @@ func (s *Sweeper) sweep(ctx context.Context) {
live[string(list.Items[i].UID)] = true live[string(list.Items[i].UID)] = true
} }
deleted := 0
for name, prov := range s.Providers { for name, prov := range s.Providers {
instances, err := prov.ListByTag(ctx) instances, err := prov.ListByTag(ctx)
if err != nil { if err != nil {
@@ -119,7 +125,13 @@ func (s *Sweeper) sweep(ctx context.Context) {
if err := prov.Delete(ctx, inst.ID); err != nil { if err := prov.Delete(ctx, inst.ID); err != nil {
log.Error(err, "deleting orphaned instance", log.Error(err, "deleting orphaned instance",
"provider", name, "providerID", inst.ID, "uid", inst.UID) "provider", name, "providerID", inst.ID, "uid", inst.UID)
continue
} }
deleted++
} }
} }
span.SetAttributes(
attribute.Int("gc.providers", len(s.Providers)),
attribute.Int("gc.deleted", deleted),
)
} }

View File

@@ -10,6 +10,9 @@ import (
"sync" "sync"
"time" "time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
apimeta "k8s.io/apimachinery/pkg/api/meta" apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
@@ -18,6 +21,7 @@ import (
logf "sigs.k8s.io/controller-runtime/pkg/log" logf "sigs.k8s.io/controller-runtime/pkg/log"
crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1" crawlv1alpha1 "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/api/v1alpha1"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
) )
// Snapshot is the engine's current verdict for one proxy, read by the // Snapshot is the engine's current verdict for one proxy, read by the
@@ -99,6 +103,10 @@ type Engine struct {
// transition-only by design, so metrics are where high-frequency // transition-only by design, so metrics are where high-frequency
// signal (true probe recency, every latency sample) lives. // signal (true probe recency, every latency sample) lives.
Metrics ProbeMetrics Metrics ProbeMetrics
// TraceProbes emits one root span per probe (--trace-health-probes).
// Off by default: probes run about once per second per proxy and would
// dominate trace volume.
TraceProbes bool
probeFn func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult probeFn func(context.Context, *url.URL, crawlv1alpha1.HealthCheckSpec, *tls.Config) probeResult
@@ -155,8 +163,7 @@ func (e *Engine) Start(ctx context.Context) error {
case <-ctx.Done(): case <-ctx.Done():
return return
case job := <-jobs: case job := <-jobs:
res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig) e.record(job, e.runProbe(ctx, job), time.Now())
e.record(job, res, time.Now())
} }
} }
}) })
@@ -264,6 +271,29 @@ func newState(p *crawlv1alpha1.Proxy, now time.Time, interval time.Duration) *st
return st return st
} }
// runProbe executes one probe, inside its own root span when TraceProbes is
// on. The span ends before record folds the result (record stays ctx-free),
// and the probe transport is deliberately uninstrumented — no traceparent
// must ever leak through a proxy toward external targets.
func (e *Engine) runProbe(ctx context.Context, job probeJob) probeResult {
if !e.TraceProbes {
return e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
}
ctx, span := tracing.Start(ctx, "health.probe",
trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("proxy", job.key.String())))
defer span.End()
res := e.probeFn(ctx, job.proxyURL, job.hc, e.ProbeTLSConfig)
span.SetAttributes(
attribute.Bool("probe.ok", res.ok),
attribute.Int64("probe.latency_ms", res.latency.Milliseconds()),
)
if !res.ok {
span.SetStatus(codes.Error, res.err.Error())
}
return res
}
// record folds one probe result into the proxy's threshold state and emits // record folds one probe result into the proxy's threshold state and emits
// an event when the result is status-affecting: a first-ever verdict, a // an event when the result is status-affecting: a first-ever verdict, a
// threshold-crossing flip, or a material latency change (beyond // threshold-crossing flip, or a material latency change (beyond

View File

@@ -6,6 +6,7 @@ import (
"log/slog" "log/slog"
"github.com/go-logr/logr" "github.com/go-logr/logr"
"go.opentelemetry.io/otel/trace"
) )
// wireLogMaxFieldBytes is the elision threshold for string fields in wire // wireLogMaxFieldBytes is the elision threshold for string fields in wire
@@ -47,6 +48,15 @@ func (h *wireFilterHandler) Handle(ctx context.Context, rec slog.Record) error {
if rec.Level <= slog.LevelDebug && rec.Message != "api request" && rec.Message != "api response" { if rec.Level <= slog.LevelDebug && rec.Message != "api request" && rec.Message != "api response" {
return nil return nil
} }
// The SDK logs with the request ctx, so wire records can carry the
// surrounding provider span — the one place slog's ctx-aware handlers
// beat logr, and the same keys the logr enrichment uses.
if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
rec = rec.Clone()
rec.AddAttrs(
slog.String("traceID", sc.TraceID().String()),
slog.String("spanID", sc.SpanID().String()))
}
if h.fullPayloads { if h.fullPayloads {
return h.inner.Handle(ctx, rec) return h.inner.Handle(ctx, rec)
} }

View File

@@ -0,0 +1,50 @@
package gcp
import (
"context"
"strings"
"testing"
"github.com/go-logr/logr/funcr"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)
func TestWireFilterHandler_addsTraceContext(t *testing.T) {
t.Parallel()
var lines []string
base := funcr.New(func(prefix, args string) {
lines = append(lines, prefix+" "+args)
}, funcr.Options{Verbosity: 5})
log := wireLogger(base, WireLogOptions{})
tp := sdktrace.NewTracerProvider()
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
ctx, span := tp.Tracer("test").Start(context.Background(), "provider.create")
defer span.End()
log.DebugContext(ctx, "api request", "url", "https://compute.googleapis.com/x")
if len(lines) != 1 {
t.Fatalf("got %d lines, want 1: %v", len(lines), lines)
}
traceID := span.SpanContext().TraceID().String()
if !strings.Contains(lines[0], "traceID") || !strings.Contains(lines[0], traceID) {
t.Errorf("wire record missing trace context %s: %s", traceID, lines[0])
}
// The security filter must still win: non-wire Debug records are
// dropped even when a span is present.
log.DebugContext(ctx, "token exchange", "assertion", "secret-jwt")
if len(lines) != 1 {
t.Fatalf("filtered record leaked: %v", lines[1:])
}
// No span in ctx: record passes through without trace keys.
log.DebugContext(context.Background(), "api response", "status", 200)
if len(lines) != 2 {
t.Fatalf("got %d lines, want 2", len(lines))
}
if strings.Contains(lines[1], "traceID") {
t.Errorf("spanless record must not carry traceID: %s", lines[1])
}
}

View File

@@ -14,6 +14,7 @@ import (
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/cache"
"k8s.io/client-go/transport"
ctrl "sigs.k8s.io/controller-runtime" ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client"
@@ -39,11 +40,22 @@ type Provider struct {
// New builds a kubernetes Provider from its config block. Satisfies // New builds a kubernetes Provider from its config block. Satisfies
// registry.Constructor. // registry.Constructor.
func New(_ context.Context, cfg provider.ProviderConfig) (provider.Provider, error) { func New(ctx context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
return NewWithTransportWrapper(ctx, cfg, nil)
}
// NewWithTransportWrapper is New with an optional transport wrapper applied
// to the provider's own rest.Config (this client is built independently of
// the manager's, so the composition root must wrap it separately for
// tracing). A nil wrapper means a plain client.
func NewWithTransportWrapper(_ context.Context, cfg provider.ProviderConfig, wrap transport.WrapperFunc) (provider.Provider, error) {
restCfg, err := ctrl.GetConfig() restCfg, err := ctrl.GetConfig()
if err != nil { if err != nil {
return nil, fmt.Errorf("kubernetes provider %q: loading kubeconfig: %w", cfg.Name, err) return nil, fmt.Errorf("kubernetes provider %q: loading kubeconfig: %w", cfg.Name, err)
} }
if wrap != nil {
restCfg.Wrap(wrap)
}
scheme := runtime.NewScheme() scheme := runtime.NewScheme()
if err := corev1.AddToScheme(scheme); err != nil { if err := corev1.AddToScheme(scheme); err != nil {
return nil, fmt.Errorf("kubernetes provider %q: %w", cfg.Name, err) return nil, fmt.Errorf("kubernetes provider %q: %w", cfg.Name, err)

View File

@@ -50,17 +50,25 @@ func (i *instrumented) record(op string, err error) {
i.rec.ProviderRequest(i.name, op, resultLabel(err)) i.rec.ProviderRequest(i.name, op, resultLabel(err))
} }
const (
resultOK = "ok"
resultNotFound = "not_found"
resultQuotaExceeded = "quota_exceeded"
resultPermanent = "permanent"
resultTransient = "transient"
)
func resultLabel(err error) string { func resultLabel(err error) string {
switch Class(err) { switch Class(err) {
case nil: case nil:
return "ok" return resultOK
case ErrNotFound: case ErrNotFound:
return "not_found" return resultNotFound
case ErrQuotaExceeded: case ErrQuotaExceeded:
return "quota_exceeded" return resultQuotaExceeded
case ErrPermanent: case ErrPermanent:
return "permanent" return resultPermanent
default: default:
return "transient" return resultTransient
} }
} }

View File

@@ -0,0 +1,80 @@
package provider
import (
"context"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
)
// WithTracing wraps a Provider so every call runs in a client span carrying
// the classified result, mirroring WithMetrics. It goes through
// tracing.StartSpan, so provider-internal logs (e.g. the gcp provider's
// context logger) inherit traceID/spanID. Errors pass through unmodified.
func WithTracing(name string, p Provider, opts ...tracing.Option) Provider {
return &traced{name: name, inner: p, tracer: tracing.Tracer(opts...)}
}
type traced struct {
name string
inner Provider
tracer trace.Tracer
}
func (t *traced) Create(ctx context.Context, req CreateRequest) (string, error) {
ctx, span := t.start(ctx, "provider.create")
id, err := t.inner.Create(ctx, req)
if id != "" {
span.SetAttributes(attribute.String("provider.id", id))
}
t.end(span, err)
return id, err
}
func (t *traced) Get(ctx context.Context, providerID string) (*Instance, error) {
ctx, span := t.start(ctx, "provider.get")
span.SetAttributes(attribute.String("provider.id", providerID))
inst, err := t.inner.Get(ctx, providerID)
t.end(span, err)
return inst, err
}
func (t *traced) Delete(ctx context.Context, providerID string) error {
ctx, span := t.start(ctx, "provider.delete")
span.SetAttributes(attribute.String("provider.id", providerID))
err := t.inner.Delete(ctx, providerID)
t.end(span, err)
return err
}
func (t *traced) ListByTag(ctx context.Context) ([]Instance, error) {
ctx, span := t.start(ctx, "provider.list")
instances, err := t.inner.ListByTag(ctx)
span.SetAttributes(attribute.Int("provider.instances", len(instances)))
t.end(span, err)
return instances, err
}
func (t *traced) start(ctx context.Context, op string) (context.Context, trace.Span) {
return tracing.StartSpan(ctx, t.tracer, op,
trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("provider.name", t.name)))
}
// end records the taxonomy class and closes the span. ErrNotFound stays Ok:
// the reconciler polls Get to NotFound during replacement/deletion, so it is
// an expected answer, not a failure — same reasoning as resultLabel's
// distinct "not_found" bucket.
func (t *traced) end(span trace.Span, err error) {
span.SetAttributes(attribute.String("provider.result", resultLabel(err)))
switch Class(err) {
case nil, ErrNotFound:
span.SetStatus(codes.Ok, "")
default:
span.SetStatus(codes.Error, err.Error())
}
span.End()
}

View File

@@ -0,0 +1,118 @@
package provider
import (
"context"
"errors"
"slices"
"testing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"go.opentelemetry.io/otel/trace"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
)
func TestWithTracing_spanPerCall(t *testing.T) {
t.Parallel()
tests := []struct {
name string
inner *staticProvider
call func(p Provider) error
wantSpan string
wantStatus codes.Code
wantResult string
wantAttr attribute.KeyValue
}{
{
name: "successful create",
inner: &staticProvider{},
call: func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
wantSpan: "provider.create",
wantStatus: codes.Ok,
wantResult: resultOK,
wantAttr: attribute.String("provider.id", "id-1"),
},
{
name: "get NotFound is not a span error",
inner: &staticProvider{getErr: Wrap(ErrNotFound, "get", "x", "id-1", nil)},
call: func(p Provider) error { _, err := p.Get(context.Background(), "id-1"); return err },
wantSpan: "provider.get",
wantStatus: codes.Ok,
wantResult: resultNotFound,
wantAttr: attribute.String("provider.id", "id-1"),
},
{
name: "delete transient error",
inner: &staticProvider{deleteErr: Wrap(ErrTransient, "delete", "x", "id-1", errors.New("503"))},
call: func(p Provider) error { return p.Delete(context.Background(), "id-1") },
wantSpan: "provider.delete",
wantStatus: codes.Error,
wantResult: resultTransient,
wantAttr: attribute.String("provider.id", "id-1"),
},
{
name: "quota exceeded create",
inner: &staticProvider{createErr: Wrap(ErrQuotaExceeded, "create", "x", "", nil)},
call: func(p Provider) error { _, err := p.Create(context.Background(), CreateRequest{}); return err },
wantSpan: "provider.create",
wantStatus: codes.Error,
wantResult: resultQuotaExceeded,
wantAttr: attribute.String("provider.name", "x"),
},
{
name: "list records instance count",
inner: &staticProvider{},
call: func(p Provider) error { _, err := p.ListByTag(context.Background()); return err },
wantSpan: "provider.list",
wantStatus: codes.Ok,
wantResult: resultOK,
wantAttr: attribute.Int("provider.instances", 0),
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
sr := tracetest.NewSpanRecorder()
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
p := WithTracing("x", tc.inner, tracing.WithTracerProvider(tp))
err := tc.call(p)
wantErr := errors.Join(tc.inner.createErr, tc.inner.getErr, tc.inner.deleteErr, tc.inner.listErr)
if (wantErr == nil) != (err == nil) {
t.Fatalf("decorator changed the error: got %v", err)
}
ended := sr.Ended()
if len(ended) != 1 {
t.Fatalf("got %d spans, want 1", len(ended))
}
span := ended[0]
if span.Name() != tc.wantSpan {
t.Errorf("span name = %q, want %q", span.Name(), tc.wantSpan)
}
if span.SpanKind() != trace.SpanKindClient {
t.Errorf("span kind = %v, want client", span.SpanKind())
}
if span.Status().Code != tc.wantStatus {
t.Errorf("status = %v, want %v", span.Status().Code, tc.wantStatus)
}
attrs := span.Attributes()
hasAttr := func(want attribute.KeyValue) bool {
return slices.Contains(attrs, want)
}
if !hasAttr(attribute.String("provider.result", tc.wantResult)) {
t.Errorf("provider.result %q missing in %v", tc.wantResult, attrs)
}
if !hasAttr(tc.wantAttr) {
t.Errorf("attribute %v missing in %v", tc.wantAttr, attrs)
}
})
}
}

41
internal/tracing/http.go Normal file
View File

@@ -0,0 +1,41 @@
package tracing
import (
"net/http"
"github.com/go-logr/logr"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel/propagation"
)
// HTTPMiddleware returns middleware that opens a server span per request
// (named from the mux route pattern once routing has happened) and hands
// handlers a request context whose logf.FromContext logger is base enriched
// with the trace context. Incoming W3C traceparent headers are honored, so
// clients see their own trace continue into the operator. /healthz is never
// traced. With tracing disabled the middleware degrades to injecting base
// unenriched — handlers can rely on logf.FromContext either way.
func HTTPMiddleware(operation string, base logr.Logger, opts ...Option) func(http.Handler) http.Handler {
o := newOptions(opts)
return func(next http.Handler) http.Handler {
inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r2 := r.WithContext(ContextWithLogger(r.Context(), base))
next.ServeHTTP(w, r2)
// WithContext copies the request, so the mux recorded the matched
// route on r2; surface it to otelhttp, which renames the span
// from r.Pattern after the handler returns.
r.Pattern = r2.Pattern
})
otelOpts := []otelhttp.Option{
otelhttp.WithFilter(func(r *http.Request) bool { return r.URL.Path != "/healthz" }),
// Explicit propagators: deterministic regardless of whether the
// global propagator has been installed (tests, disabled tracing).
otelhttp.WithPropagators(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{}, propagation.Baggage{})),
}
if o.tp != nil {
otelOpts = append(otelOpts, otelhttp.WithTracerProvider(o.tp))
}
return otelhttp.NewHandler(inner, operation, otelOpts...)
}
}

View File

@@ -0,0 +1,82 @@
package tracing
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
logf "sigs.k8s.io/controller-runtime/pkg/log"
)
const sampleTraceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
func TestHTTPMiddleware(t *testing.T) {
t.Parallel()
newServer := func(lines *[]string, tp *sdktrace.TracerProvider) http.Handler {
mux := http.NewServeMux()
handler := func(w http.ResponseWriter, r *http.Request) {
logf.FromContext(r.Context()).Info("handling")
w.WriteHeader(http.StatusOK)
}
mux.HandleFunc("GET /healthz", handler)
mux.HandleFunc("GET /v1/things/{id}", handler)
return HTTPMiddleware("discovery", captureLogger(lines), WithTracerProvider(tp))(mux)
}
t.Run("route span, enriched handler logs, traceparent continuation", func(t *testing.T) {
t.Parallel()
sr := tracetest.NewSpanRecorder()
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
var lines []string
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/v1/things/42", nil)
req.Header.Set("traceparent", sampleTraceparent)
newServer(&lines, tp).ServeHTTP(rec, req)
ended := sr.Ended()
if len(ended) != 1 {
t.Fatalf("got %d spans, want 1", len(ended))
}
if got := ended[0].Name(); got != "GET /v1/things/{id}" {
t.Errorf("span name = %q, want route pattern", got)
}
wantTrace := "4bf92f3577b34da6a3ce929d0e0e4736"
if got := ended[0].SpanContext().TraceID().String(); got != wantTrace {
t.Errorf("span traceID = %s, want continuation of client trace %s", got, wantTrace)
}
if len(lines) != 1 {
t.Fatalf("got %d log lines, want 1: %v", len(lines), lines)
}
if n := strings.Count(lines[0], `"traceID"`); n != 1 {
t.Errorf("traceID appears %d times, want 1: %s", n, lines[0])
}
if !strings.Contains(lines[0], wantTrace) {
t.Errorf("handler log missing traceID %s: %s", wantTrace, lines[0])
}
})
t.Run("healthz is not traced but still gets a logger", func(t *testing.T) {
t.Parallel()
sr := tracetest.NewSpanRecorder()
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
var lines []string
rec := httptest.NewRecorder()
newServer(&lines, tp).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
if len(sr.Ended()) != 0 {
t.Fatalf("healthz produced %d spans, want 0", len(sr.Ended()))
}
if len(lines) != 1 || strings.Contains(lines[0], "traceID") {
t.Fatalf("want one unenriched log line, got %v", lines)
}
})
}

View File

@@ -0,0 +1,65 @@
package tracing
import (
"context"
"github.com/go-logr/logr"
"go.opentelemetry.io/otel/trace"
logf "sigs.k8s.io/controller-runtime/pkg/log"
)
// baseLoggerKey stores the pre-enrichment logger. logr sinks never see a
// context, so trace IDs must ride on the logger as WithValues — and zap does
// not dedupe repeated keys, so each nested span must re-derive from the same
// base instead of stacking traceID/spanID onto an already-enriched logger.
type baseLoggerKey struct{}
// Start begins a span from the global tracer and returns a context whose
// logf.FromContext logger carries the span's traceID/spanID. With tracing
// disabled the span is a no-op with an invalid span context and the logger
// is left untouched.
//
// Trade-off, documented: values pushed via logf.IntoContext *between* two
// Start calls are dropped by the inner Start's re-derivation. Nothing
// first-party does that; controller-runtime's per-reconcile logger
// (reconcileID etc.) is captured as the base and survives.
func Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
return StartSpan(ctx, newOptions(nil).tracer(), name, opts...)
}
// StartSpan is Start with an explicit tracer, for decorators that carry
// their own (test-injected) TracerProvider.
func StartSpan(ctx context.Context, tracer trace.Tracer, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
ctx, span := tracer.Start(ctx, name, opts...)
sc := span.SpanContext()
if !sc.IsValid() {
return ctx, span
}
base, ok := ctx.Value(baseLoggerKey{}).(logr.Logger)
if !ok {
base = logf.FromContext(ctx)
ctx = context.WithValue(ctx, baseLoggerKey{}, base)
}
return logf.IntoContext(ctx, withSpanValues(base, sc)), span
}
// ContextWithLogger seeds ctx with base as both the current logger and the
// base for span enrichment. If ctx already carries a valid span (the HTTP
// middleware calls this inside the otelhttp handler), the logger is
// enriched immediately.
//
//nolint:logcheck // IntoContext-analogue: taking ctx and the logger to seed it with is the point.
func ContextWithLogger(ctx context.Context, base logr.Logger) context.Context {
ctx = context.WithValue(ctx, baseLoggerKey{}, base)
if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
return logf.IntoContext(ctx, withSpanValues(base, sc))
}
return logf.IntoContext(ctx, base)
}
// withSpanValues uses lowerCamel keys to match the house style
// (reconcileID, providerID); Grafana/Loki derived fields must match
// "traceID", not the trace_id default.
func withSpanValues(base logr.Logger, sc trace.SpanContext) logr.Logger {
return base.WithValues("traceID", sc.TraceID().String(), "spanID", sc.SpanID().String())
}

View File

@@ -0,0 +1,136 @@
package tracing
import (
"context"
"strings"
"testing"
"github.com/go-logr/logr"
"github.com/go-logr/logr/funcr"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/noop"
logf "sigs.k8s.io/controller-runtime/pkg/log"
)
// captureLogger records every emitted line so tests can assert on the
// rendered key/value output — the only place duplicate zap-style keys
// would show up.
func captureLogger(lines *[]string) logr.Logger {
return funcr.New(func(prefix, args string) {
*lines = append(*lines, prefix+" "+args)
}, funcr.Options{})
}
func recordingTracer(t *testing.T) (trace.Tracer, *tracetest.SpanRecorder) {
t.Helper()
sr := tracetest.NewSpanRecorder()
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
return tp.Tracer("test"), sr
}
func TestStartSpan_enrichesLoggerOncePerNesting(t *testing.T) {
t.Parallel()
tracer, sr := recordingTracer(t)
var lines []string
ctx := logf.IntoContext(context.Background(), captureLogger(&lines))
ctx1, span1 := StartSpan(ctx, tracer, "outer")
logf.FromContext(ctx1).Info("outer work")
ctx2, span2 := StartSpan(ctx1, tracer, "inner")
logf.FromContext(ctx2).Info("inner work")
span2.End()
span1.End()
if len(lines) != 2 {
t.Fatalf("got %d log lines, want 2: %v", len(lines), lines)
}
traceID := span1.SpanContext().TraceID().String()
for i, want := range []string{span1.SpanContext().SpanID().String(), span2.SpanContext().SpanID().String()} {
if n := strings.Count(lines[i], `"traceID"`); n != 1 {
t.Errorf("line %d: traceID appears %d times, want exactly 1: %s", i, n, lines[i])
}
if n := strings.Count(lines[i], `"spanID"`); n != 1 {
t.Errorf("line %d: spanID appears %d times, want exactly 1: %s", i, n, lines[i])
}
if !strings.Contains(lines[i], traceID) {
t.Errorf("line %d: missing traceID %s: %s", i, traceID, lines[i])
}
if !strings.Contains(lines[i], want) {
t.Errorf("line %d: missing spanID %s: %s", i, want, lines[i])
}
}
ended := sr.Ended()
if len(ended) != 2 {
t.Fatalf("got %d spans, want 2", len(ended))
}
// Ended in LIFO order: inner first.
if got := ended[0].Parent().SpanID(); got != span1.SpanContext().SpanID() {
t.Errorf("inner span parent = %s, want %s", got, span1.SpanContext().SpanID())
}
}
func TestStartSpan_noopTracerLeavesLoggerUntouched(t *testing.T) {
t.Parallel()
var lines []string
ctx := logf.IntoContext(context.Background(), captureLogger(&lines))
ctx, span := StartSpan(ctx, noop.NewTracerProvider().Tracer("test"), "op")
defer span.End()
logf.FromContext(ctx).Info("work")
if len(lines) != 1 {
t.Fatalf("got %d log lines, want 1", len(lines))
}
if strings.Contains(lines[0], "traceID") {
t.Errorf("disabled tracing must not add traceID: %s", lines[0])
}
}
func TestContextWithLogger(t *testing.T) {
t.Parallel()
tracer, _ := recordingTracer(t)
t.Run("no span injects base as-is", func(t *testing.T) {
t.Parallel()
var lines []string
ctx := ContextWithLogger(context.Background(), captureLogger(&lines))
logf.FromContext(ctx).Info("plain")
if len(lines) != 1 || strings.Contains(lines[0], "traceID") {
t.Fatalf("want one line without traceID, got %v", lines)
}
})
t.Run("existing span enriches immediately and nested Start does not stack", func(t *testing.T) {
t.Parallel()
var lines []string
ctx, outer := tracer.Start(context.Background(), "server")
defer outer.End()
ctx = ContextWithLogger(ctx, captureLogger(&lines))
logf.FromContext(ctx).Info("handler")
ctx, inner := StartSpan(ctx, tracer, "child")
defer inner.End()
logf.FromContext(ctx).Info("nested")
if len(lines) != 2 {
t.Fatalf("got %d lines, want 2: %v", len(lines), lines)
}
for i, line := range lines {
if n := strings.Count(line, `"traceID"`); n != 1 {
t.Errorf("line %d: traceID appears %d times, want 1: %s", i, n, line)
}
}
if !strings.Contains(lines[1], inner.SpanContext().SpanID().String()) {
t.Errorf("nested line should carry the child spanID: %s", lines[1])
}
})
}

View File

@@ -0,0 +1,48 @@
package tracing
import (
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
)
// tracerName is the instrumentation scope for every span this module emits.
const tracerName = "gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator"
type options struct {
tp trace.TracerProvider
}
// Option configures the tracing decorators. The zero configuration uses the
// global TracerProvider installed by Setup; tests inject their own recorder
// via WithTracerProvider so they can run in parallel without touching
// process-global state.
type Option func(*options)
// WithTracerProvider overrides the global TracerProvider.
func WithTracerProvider(tp trace.TracerProvider) Option {
return func(o *options) { o.tp = tp }
}
func newOptions(opts []Option) options {
var o options
for _, opt := range opts {
opt(&o)
}
return o
}
// Tracer resolves a tracer from the given options, for decorators outside
// this package (provider.WithTracing).
func Tracer(opts ...Option) trace.Tracer {
return newOptions(opts).tracer()
}
// tracer resolves the configured tracer. The global path goes through
// otel.Tracer, which delegates to whatever provider Setup installs later —
// construction order between decorators and Setup does not matter.
func (o options) tracer() trace.Tracer {
if o.tp != nil {
return o.tp.Tracer(tracerName)
}
return otel.Tracer(tracerName)
}

View File

@@ -0,0 +1,52 @@
package tracing
import (
"context"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
// NewReconciler wraps inner so every reconcile runs in a root span named
// "Reconcile <kind>" and every log line under it carries the trace context.
// Reconciles are triggered by watch events with no incoming trace to
// continue, so each one starts a new trace.
func NewReconciler(kind string, inner reconcile.Reconciler, opts ...Option) reconcile.Reconciler {
return &tracedReconciler{
spanName: "Reconcile " + kind,
inner: inner,
tracer: newOptions(opts).tracer(),
}
}
type tracedReconciler struct {
spanName string
inner reconcile.Reconciler
tracer trace.Tracer
}
func (t *tracedReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
ctx, span := StartSpan(ctx, t.tracer, t.spanName,
trace.WithAttributes(
attribute.String("k8s.namespace.name", req.Namespace),
attribute.String("k8s.object.name", req.Name),
))
defer span.End()
if id := controller.ReconcileIDFromContext(ctx); id != "" {
span.SetAttributes(attribute.String("reconcile.id", string(id)))
}
res, err := t.inner.Reconcile(ctx, req)
if err != nil {
span.SetStatus(codes.Error, err.Error())
return res, err
}
span.SetStatus(codes.Ok, "")
if res.RequeueAfter > 0 {
span.SetAttributes(attribute.Int64("reconcile.requeue_after_ms", res.RequeueAfter.Milliseconds()))
}
return res, nil
}

View File

@@ -0,0 +1,86 @@
package tracing
import (
"context"
"errors"
"testing"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
type fakeReconciler struct {
res reconcile.Result
err error
}
func (f *fakeReconciler) Reconcile(context.Context, reconcile.Request) (reconcile.Result, error) {
return f.res, f.err
}
func TestNewReconciler_spanPerReconcile(t *testing.T) {
t.Parallel()
req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "p1"}}
tests := []struct {
name string
inner *fakeReconciler
wantStatus codes.Code
wantAttr attribute.KeyValue
}{
{
name: "success records requeue",
inner: &fakeReconciler{res: reconcile.Result{RequeueAfter: 10 * time.Second}},
wantStatus: codes.Ok,
wantAttr: attribute.Int64("reconcile.requeue_after_ms", 10_000),
},
{
name: "error sets error status",
inner: &fakeReconciler{err: errors.New("boom")},
wantStatus: codes.Error,
wantAttr: attribute.String("k8s.object.name", "p1"),
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
sr := tracetest.NewSpanRecorder()
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
r := NewReconciler("Proxy", tc.inner, WithTracerProvider(tp))
res, err := r.Reconcile(context.Background(), req)
if res != tc.inner.res || !errors.Is(err, tc.inner.err) {
t.Fatalf("decorator changed the result: (%v, %v)", res, err)
}
ended := sr.Ended()
if len(ended) != 1 {
t.Fatalf("got %d spans, want 1", len(ended))
}
span := ended[0]
if span.Name() != "Reconcile Proxy" {
t.Errorf("span name = %q, want %q", span.Name(), "Reconcile Proxy")
}
if span.Status().Code != tc.wantStatus {
t.Errorf("status = %v, want %v", span.Status().Code, tc.wantStatus)
}
found := false
for _, a := range span.Attributes() {
if a == tc.wantAttr {
found = true
}
}
if !found {
t.Errorf("attribute %v missing in %v", tc.wantAttr, span.Attributes())
}
})
}
}

157
internal/tracing/tracing.go Normal file
View File

@@ -0,0 +1,157 @@
// Package tracing wires OpenTelemetry tracing into the operator: SDK setup
// gated on standard OTEL_* environment variables, span helpers that keep the
// logr/zap logging enriched with trace context, and decorators for the
// reconciler, providers, HTTP server, and Kubernetes API transport.
package tracing
import (
"context"
"fmt"
"os"
"strings"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
logf "sigs.k8s.io/controller-runtime/pkg/log"
)
const (
envSDKDisabled = "OTEL_SDK_DISABLED"
envTracesExporter = "OTEL_TRACES_EXPORTER"
envOTLPEndpoint = "OTEL_EXPORTER_OTLP_ENDPOINT"
envOTLPTracesEndpoint = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
envOTLPProtocol = "OTEL_EXPORTER_OTLP_PROTOCOL"
envOTLPTracesProtocol = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"
exporterOTLP = "otlp"
exporterConsole = "console"
exporterNone = "none"
)
// maxQueueSize bounds the batch processor's span buffer; the default 2048
// is oversized for this process (3 concurrent reconciles, tickers) and the
// manager pod runs with a 128Mi memory limit.
const maxQueueSize = 512
// Setup initializes the global tracing pipeline from OTEL_* environment
// variables and returns a shutdown func that flushes pending spans.
//
// Tracing stays fully off — no exporter, no global provider, spans are
// no-ops, logs carry no traceID — unless the environment opts in by setting
// OTEL_TRACES_EXPORTER or an OTLP endpoint. OTEL_SDK_DISABLED=true and
// OTEL_TRACES_EXPORTER=none force it off (the Go SDK does not implement
// OTEL_SDK_DISABLED itself). Sampling, endpoints, TLS, and headers follow
// the standard SDK/exporter env vars (OTEL_TRACES_SAMPLER, OTEL_EXPORTER_OTLP_*).
//
// Setup logs via the logger in ctx (logf.FromContext) rather than a
// parameter — the caller seeds it with logf.IntoContext.
func Setup(ctx context.Context, service, version string) (func(context.Context) error, error) {
log := logf.FromContext(ctx)
noop := func(context.Context) error { return nil }
exporterEnv := strings.ToLower(strings.TrimSpace(os.Getenv(envTracesExporter)))
endpointSet := os.Getenv(envOTLPEndpoint) != "" ||
os.Getenv(envOTLPTracesEndpoint) != ""
sdkDisabled := strings.EqualFold(strings.TrimSpace(os.Getenv(envSDKDisabled)), "true")
if sdkDisabled || exporterEnv == exporterNone || (exporterEnv == "" && !endpointSet) {
log.Info("tracing disabled",
"reason", disabledReason(sdkDisabled, exporterEnv))
return noop, nil
}
exp, expName, err := newExporter(ctx, exporterEnv)
if err != nil {
return noop, fmt.Errorf("tracing setup: %w", err)
}
res, err := buildResource(ctx, service, version)
if err != nil {
if res == nil {
return noop, fmt.Errorf("tracing setup: building resource: %w", err)
}
// Schema-URL conflicts between semconv versions still yield a usable
// merged resource; keep it rather than losing tracing over metadata.
log.V(1).Info("tracing resource merge conflict; continuing", "err", err.Error())
}
tp := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exp, sdktrace.WithMaxQueueSize(maxQueueSize)),
sdktrace.WithResource(res),
)
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{}, propagation.Baggage{}))
otelLog := log.WithName("otel")
otel.SetLogger(otelLog)
// V(1), not Error: an unreachable collector fails every export cycle and
// would otherwise spam the error log every few seconds.
otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
otelLog.V(1).Info("otel error", "err", err.Error())
}))
log.Info("tracing enabled", "exporter", expName, "service", service)
return tp.Shutdown, nil
}
func disabledReason(sdkDisabled bool, exporterEnv string) string {
switch {
case sdkDisabled:
return "OTEL_SDK_DISABLED=true"
case exporterEnv == exporterNone:
return "OTEL_TRACES_EXPORTER=none"
default:
return "no OTEL_TRACES_EXPORTER or OTLP endpoint configured"
}
}
// newExporter hand-rolls the OTEL_TRACES_EXPORTER / OTEL_EXPORTER_OTLP_*
// protocol selection instead of pulling in contrib's autoexport, which drags
// metric/log/prometheus exporters into a trace-only binary.
func newExporter(ctx context.Context, kind string) (sdktrace.SpanExporter, string, error) {
switch kind {
case "", exporterOTLP:
proto := strings.ToLower(strings.TrimSpace(os.Getenv(envOTLPTracesProtocol)))
if proto == "" {
proto = strings.ToLower(strings.TrimSpace(os.Getenv(envOTLPProtocol)))
}
switch proto {
case "", "http/protobuf":
exp, err := otlptracehttp.New(ctx)
return exp, "otlp/http", err
case "grpc":
exp, err := otlptracegrpc.New(ctx)
return exp, "otlp/grpc", err
default:
return nil, "", fmt.Errorf("unsupported OTLP protocol %q (supported: http/protobuf, grpc)", proto)
}
case exporterConsole:
exp, err := stdouttrace.New()
return exp, exporterConsole, err
default:
return nil, "", fmt.Errorf("unsupported OTEL_TRACES_EXPORTER %q (supported: otlp, console, none)", kind)
}
}
// buildResource layers defaults < service identity < environment, so
// OTEL_SERVICE_NAME / OTEL_RESOURCE_ATTRIBUTES always win.
func buildResource(ctx context.Context, service, version string) (*resource.Resource, error) {
base, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL,
semconv.ServiceName(service),
semconv.ServiceVersion(version),
))
if err != nil {
return base, err
}
env, err := resource.New(ctx, resource.WithFromEnv())
if err != nil {
return base, err
}
return resource.Merge(base, env)
}

View File

@@ -0,0 +1,130 @@
package tracing
import (
"context"
"testing"
"time"
"go.opentelemetry.io/otel"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace/noop"
)
const testOTLPEndpoint = "http://localhost:4318"
// clearOTelEnv pins every env var Setup reads, so developer shells with
// OTEL_* set can't change test outcomes. Not parallel-safe by design
// (t.Setenv forbids t.Parallel).
func clearOTelEnv(t *testing.T) {
t.Helper()
for _, k := range []string{
envSDKDisabled,
envTracesExporter,
envOTLPEndpoint,
envOTLPTracesEndpoint,
envOTLPProtocol,
envOTLPTracesProtocol,
} {
t.Setenv(k, "")
}
}
func TestSetup_envGating(t *testing.T) {
tests := []struct {
name string
env map[string]string
wantEnabled bool
wantErr bool
}{
{name: "no env means disabled", env: nil, wantEnabled: false},
{
name: "endpoint enables otlp",
env: map[string]string{envOTLPEndpoint: testOTLPEndpoint},
wantEnabled: true,
},
{
name: "traces endpoint enables otlp",
env: map[string]string{envOTLPTracesEndpoint: testOTLPEndpoint},
wantEnabled: true,
},
{
name: "explicit console exporter",
env: map[string]string{envTracesExporter: exporterConsole},
wantEnabled: true,
},
{
name: "grpc protocol",
env: map[string]string{envTracesExporter: exporterOTLP, envOTLPProtocol: "grpc"},
wantEnabled: true,
},
{
name: "exporter none wins over endpoint",
env: map[string]string{
envTracesExporter: exporterNone,
envOTLPEndpoint: testOTLPEndpoint,
},
wantEnabled: false,
},
{
name: "OTEL_SDK_DISABLED wins over everything",
env: map[string]string{
envSDKDisabled: "true",
envOTLPEndpoint: testOTLPEndpoint,
},
wantEnabled: false,
},
{
name: "unsupported exporter errors",
env: map[string]string{envTracesExporter: "jaeger"},
wantErr: true,
},
{
name: "unsupported protocol errors",
env: map[string]string{
envTracesExporter: exporterOTLP,
envOTLPProtocol: "http/json",
},
wantErr: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
clearOTelEnv(t)
for k, v := range tc.env {
t.Setenv(k, v)
}
// Fresh noop global per case: disabled paths must leave it
// untouched, and resetting avoids leaking one case's SDK
// provider into the next (or warning-prone restores of the
// process-default delegate).
before := noop.NewTracerProvider()
otel.SetTracerProvider(before)
shutdown, err := Setup(context.Background(), "test-svc", "abc123")
if tc.wantErr {
if err == nil {
t.Fatal("want error, got nil")
}
return
}
if err != nil {
t.Fatalf("Setup: %v", err)
}
_, isSDK := otel.GetTracerProvider().(*sdktrace.TracerProvider)
if isSDK != tc.wantEnabled {
t.Errorf("global provider is SDK = %v, want %v", isSDK, tc.wantEnabled)
}
if tc.wantEnabled && otel.GetTracerProvider() == before {
t.Error("enabled Setup must install a new global provider")
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := shutdown(ctx); err != nil {
t.Errorf("shutdown: %v", err)
}
})
}
}

View File

@@ -0,0 +1,51 @@
package tracing
import (
"net/http"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
"k8s.io/client-go/transport"
)
// RestConfigWrapper returns a rest.Config.Wrap-compatible wrapper that adds
// a client span to every Kubernetes API request already running inside a
// trace. Requests with no parent span pass through untouched: informer
// list/watch long-polls, leader-election renewals, and metrics authn would
// otherwise each become meaningless (and in the watch case, minutes-long)
// root spans. client-go applies Wrap innermost, so the span sees the final
// authenticated request.
func RestConfigWrapper(opts ...Option) transport.WrapperFunc {
o := newOptions(opts)
return func(rt http.RoundTripper) http.RoundTripper {
otelOpts := []otelhttp.Option{
// Explicit propagators: deterministic regardless of global
// state. The apiserver ignores incoming traceparent (by
// design), so injection is harmless there.
otelhttp.WithPropagators(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{}, propagation.Baggage{})),
}
if o.tp != nil {
otelOpts = append(otelOpts, otelhttp.WithTracerProvider(o.tp))
}
return &parentGatedTransport{
traced: otelhttp.NewTransport(rt, otelOpts...),
plain: rt,
}
}
}
// parentGatedTransport enforces the parent-span requirement itself rather
// than relying on otelhttp filter semantics for transports.
type parentGatedTransport struct {
traced http.RoundTripper
plain http.RoundTripper
}
func (t *parentGatedTransport) RoundTrip(r *http.Request) (*http.Response, error) {
if trace.SpanContextFromContext(r.Context()).IsValid() {
return t.traced.RoundTrip(r)
}
return t.plain.RoundTrip(r)
}

View File

@@ -0,0 +1,73 @@
package tracing
import (
"context"
"io"
"net/http"
"net/http/httptest"
"testing"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"go.opentelemetry.io/otel/trace"
)
func TestRestConfigWrapper_parentGated(t *testing.T) {
t.Parallel()
var gotTraceparent string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotTraceparent = r.Header.Get("traceparent")
w.WriteHeader(http.StatusOK)
}))
t.Cleanup(srv.Close)
sr := tracetest.NewSpanRecorder()
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
rt := RestConfigWrapper(WithTracerProvider(tp))(http.DefaultTransport)
do := func(ctx context.Context) {
t.Helper()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL, nil)
if err != nil {
t.Fatal(err)
}
resp, err := rt.RoundTrip(req)
if err != nil {
t.Fatal(err)
}
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}
// No parent span: informer watches, leader election. Must not trace.
do(context.Background())
if len(sr.Ended()) != 0 {
t.Fatalf("request without parent span produced %d spans, want 0", len(sr.Ended()))
}
if gotTraceparent != "" {
t.Fatalf("request without parent span injected traceparent %q", gotTraceparent)
}
// Under a parent span: one client child span, traceparent injected.
ctx, parent := tp.Tracer("test").Start(context.Background(), "reconcile")
do(ctx)
parent.End()
var client sdktrace.ReadOnlySpan
for _, s := range sr.Ended() {
if s.SpanKind() == trace.SpanKindClient {
client = s
}
}
if client == nil {
t.Fatalf("no client span recorded, got %d spans", len(sr.Ended()))
}
if got := client.Parent().SpanID(); got != parent.SpanContext().SpanID() {
t.Errorf("client span parent = %s, want %s", got, parent.SpanContext().SpanID())
}
if gotTraceparent == "" {
t.Error("traceparent header not injected under a parent span")
}
}