OpenTelemetry tracing integrated with logr/zap logging #4

Open
kacerr wants to merge 13 commits from feat/otel-tracing into main
3 changed files with 68 additions and 7 deletions
Showing only changes of commit 8e4f3b9b13 - Show all commits

View File

@@ -58,6 +58,7 @@ import (
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/gcp"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/kubernetes"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/registry"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/version"
// +kubebuilder:scaffold:imports
)
@@ -148,6 +149,17 @@ func main() {
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
setupLog.Info("Starting egress-proxies-operator",
"commit", version.Resolve(), "goVersion", goruntime.Version())
// Off (no-op spans, unchanged logs) unless OTEL_* env opts in; see
// internal/tracing. Shutdown is called explicitly after mgr.Start
// returns — the os.Exit paths below skip defers.
tracingShutdown, err := tracing.Setup(context.Background(), setupLog,
"egress-proxies-operator", version.Resolve())
if err != nil {
setupLog.Error(err, "Failed to set up tracing")
os.Exit(1)
}
ctx := ctrl.SetupSignalHandler()
// Providers load first and fail fast: a manager that comes up without
@@ -162,7 +174,11 @@ func main() {
os.Exit(1)
}
providers, err := registry.Build(ctx, cfg, map[string]registry.Constructor{
"kubernetes": kubernetes.New,
"kubernetes": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
// The kubernetes provider builds its own uncached client;
// wrap its transport so its API calls join the caller's trace.
return kubernetes.NewWithTransportWrapper(ctx, pc, tracing.RestConfigWrapper())
},
"gcp": func(ctx context.Context, pc provider.ProviderConfig) (provider.Provider, error) {
return gcp.NewWithWireOptions(ctx, pc, gcp.WireLogOptions{FullPayloads: gcpWireFullPayloads})
},
@@ -173,7 +189,8 @@ func main() {
}
m := metrics.New()
for name, p := range providers {
providers[name] = provider.WithMetrics(name, p, m)
// Tracing outermost: the span covers the metrics recording too.
providers[name] = provider.WithTracing(name, provider.WithMetrics(name, p, m))
}
// if the enable-http2 flag is false (the default), http/2 should be disabled
@@ -236,7 +253,9 @@ func main() {
cacheOpts.DefaultNamespaces = map[string]cache.Config{proxyNamespace: {}}
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
restCfg := ctrl.GetConfigOrDie()
restCfg.Wrap(tracing.RestConfigWrapper())
mgr, err := ctrl.NewManager(restCfg, ctrl.Options{
Scheme: scheme,
Metrics: metricsServerOptions,
HealthProbeBindAddress: probeAddr,
@@ -323,8 +342,19 @@ func main() {
}
setupLog.Info("Starting manager")
if err := mgr.Start(ctx); err != nil {
setupLog.Error(err, "Failed to run manager")
startErr := mgr.Start(ctx)
// Flush pending spans on the way out, error path included. Fresh
// context: the signal ctx is already cancelled by the time Start
// returns.
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
if err := tracingShutdown(shutdownCtx); err != nil {
setupLog.Error(err, "Failed to flush traces on shutdown")
}
cancel()
if startErr != nil {
setupLog.Error(startErr, "Failed to run manager")
os.Exit(1)
}
}

View File

@@ -5,7 +5,7 @@ Plan: `docs/plans/2026-08-24-1025-otel-tracing.md`
- [x] Step 1 — Dependencies
- [x] Step 2 — New package `internal/tracing`
- [x] Step 3 — `provider.WithTracing` decorator
- [ ] Step 4 — `cmd/main.go` wiring
- [x] Step 4 — `cmd/main.go` wiring
- [ ] Step 5 — Reconciler spans
- [ ] Step 6 — Discovery server
- [ ] Step 7 — GC + health
@@ -81,3 +81,22 @@ One small API addition to `internal/tracing` for this: exported
and couldn't reach the unexported option resolver. The `cmd/main.go` wiring
(`WithTracing` outermost around `WithMetrics`) lands with Step 4's commit —
same file, one commit.
## Step 4 — `cmd/main.go` wiring
As planned: `tracing.Setup` right after `ctrl.SetLogger`; manager rest.Config
wrapped via `restCfg := ctrl.GetConfigOrDie(); restCfg.Wrap(...)` (the
previous inline call left nowhere to wrap); providers decorated tracing-
outermost; explicit trace flush after `mgr.Start` returns on both the error
and clean paths, with a fresh 10s context since the signal ctx is already
cancelled by then.
The kubernetes provider grew `NewWithTransportWrapper(ctx, cfg, wrap)` (its
client is built from its own `ctrl.GetConfig()`, invisible to the manager's
wrapped config); `New` now delegates with a nil wrapper, so `newWithClient`
tests stayed untouched, and main registers a closure — the same pattern the
gcp constructor already used for wire-log options.
Deviation: the plan put `--trace-health-probes` here, but the flag needs the
`health.Engine.TraceProbes` field that Step 7 introduces — moved there to
keep every commit compiling.

View File

@@ -14,6 +14,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/transport"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -39,11 +40,22 @@ type Provider struct {
// New builds a kubernetes Provider from its config block. Satisfies
// registry.Constructor.
func New(_ context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
func New(ctx context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
return NewWithTransportWrapper(ctx, cfg, nil)
}
// NewWithTransportWrapper is New with an optional transport wrapper applied
// to the provider's own rest.Config (this client is built independently of
// the manager's, so the composition root must wrap it separately for
// tracing). A nil wrapper means a plain client.
func NewWithTransportWrapper(_ context.Context, cfg provider.ProviderConfig, wrap transport.WrapperFunc) (provider.Provider, error) {
restCfg, err := ctrl.GetConfig()
if err != nil {
return nil, fmt.Errorf("kubernetes provider %q: loading kubeconfig: %w", cfg.Name, err)
}
if wrap != nil {
restCfg.Wrap(wrap)
}
scheme := runtime.NewScheme()
if err := corev1.AddToScheme(scheme); err != nil {
return nil, fmt.Errorf("kubernetes provider %q: %w", cfg.Name, err)