From 095b1cd25199b46d3bc86df6aa1e676aeea925a0 Mon Sep 17 00:00:00 2001 From: Hans van den Bogert Date: Mon, 2 Mar 2026 11:23:39 +0100 Subject: [PATCH] feat: add otlp logging support - Adds a loggerprovider based on otlp logger - In demo directory of oltp: - Added grafana for unified view of both traces and logs - tracing now uses oltp from the collector to the jaeger instance Signed-off-by: Hans van den Bogert --- cmd/podinfo/main.go | 57 ++++++++++++++++++++++++++++++++--- go.mod | 4 +++ go.sum | 12 ++++++++ otel/README.md | 30 +++++++++++------- otel/docker-compose.yaml | 31 +++++++++++-------- otel/grafana-datasources.yaml | 10 ++++++ otel/otel-config.yaml | 13 ++++++-- pkg/signals/shutdown.go | 13 ++++++++ 8 files changed, 140 insertions(+), 30 deletions(-) create mode 100644 otel/grafana-datasources.yaml diff --git a/cmd/podinfo/main.go b/cmd/podinfo/main.go index f8f804a..06b1976 100644 --- a/cmd/podinfo/main.go +++ b/cmd/podinfo/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "os" "path/filepath" @@ -10,6 +11,11 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" + "go.opentelemetry.io/contrib/bridges/otelzap" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" + sdklog "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.7.0" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -53,7 +59,7 @@ func main() { fs.Int("stress-cpu", 0, "number of CPU cores with 100 load") fs.Int("stress-memory", 0, "MB of data to load into memory") fs.String("cache-server", "", "Redis address in the format 'tcp://:'") - fs.String("otel-service-name", "", "service name for reporting to open telemetry address, when not set tracing is disabled") + fs.String("otel-service-name", "", "service name for OpenTelemetry, when not set tracing and log export are disabled") versionFlag := fs.BoolP("version", "v", false, "get version number") @@ -93,8 +99,18 @@ func main() { } } + // initialize OTel log provider if service name is set + var loggerProvider *sdklog.LoggerProvider + if otelServiceName := viper.GetString("otel-service-name"); otelServiceName != "" { + var err error + loggerProvider, err = initLoggerProvider(context.Background(), otelServiceName) + if err != nil { + fmt.Fprintf(os.Stderr, "Error initializing OTel log provider: %s\n", err.Error()) + } + } + // configure logging - logger, _ := initZap(viper.GetString("level")) + logger, _ := initZap(viper.GetString("level"), loggerProvider) defer logger.Sync() stdLog := zap.RedirectStdLog(logger) defer stdLog() @@ -163,10 +179,29 @@ func main() { // graceful shutdown stopCh := signals.SetupSignalHandler() sd, _ := signals.NewShutdown(srvCfg.ServerShutdownTimeout, logger) + sd.SetLoggerProvider(loggerProvider) sd.Graceful(stopCh, httpServer, httpsServer, grpcServer, healthy, ready) } -func initZap(logLevel string) (*zap.Logger, error) { +func initLoggerProvider(ctx context.Context, serviceName string) (*sdklog.LoggerProvider, error) { + exporter, err := otlploggrpc.New(ctx) + if err != nil { + return nil, fmt.Errorf("creating OTLP log exporter: %w", err) + } + + provider := sdklog.NewLoggerProvider( + sdklog.WithProcessor(sdklog.NewBatchProcessor(exporter)), + sdklog.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(serviceName), + semconv.ServiceVersionKey.String(version.VERSION), + )), + ) + + return provider, nil +} + +func initZap(logLevel string, loggerProvider *sdklog.LoggerProvider) (*zap.Logger, error) { level := zap.NewAtomicLevelAt(zapcore.InfoLevel) switch logLevel { case "debug": @@ -210,7 +245,21 @@ func initZap(logLevel string) (*zap.Logger, error) { ErrorOutputPaths: []string{"stderr"}, } - return zapConfig.Build() + logger, err := zapConfig.Build() + if err != nil { + return nil, err + } + + if loggerProvider != nil { + otelCore := otelzap.NewCore("github.com/stefanprodan/podinfo", + otelzap.WithLoggerProvider(loggerProvider), + ) + logger = logger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + return zapcore.NewTee(core, otelCore) + })) + } + + return logger, nil } var stressMemoryPayload []byte diff --git a/go.mod b/go.mod index 5190278..d17e163 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/spf13/viper v1.21.0 github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.6 + go.opentelemetry.io/contrib/bridges/otelzap v0.15.0 go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.65.0 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.65.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 @@ -24,9 +25,11 @@ require ( go.opentelemetry.io/contrib/propagators/jaeger v1.40.0 go.opentelemetry.io/contrib/propagators/ot v1.40.0 go.opentelemetry.io/otel v1.40.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 go.opentelemetry.io/otel/sdk v1.40.0 + go.opentelemetry.io/otel/sdk/log v0.16.0 go.opentelemetry.io/otel/trace v1.40.0 go.uber.org/zap v1.27.1 golang.org/x/net v0.51.0 @@ -66,6 +69,7 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/swaggo/files v1.0.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/log v0.16.0 // indirect go.opentelemetry.io/otel/metric v1.40.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index b2c28b3..7b37b08 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,8 @@ github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/bridges/otelzap v0.15.0 h1:x4qzjKkTl2hXmLl+IviSXvzaTyCJSYvpFZL5SRVLBxs= +go.opentelemetry.io/contrib/bridges/otelzap v0.15.0/go.mod h1:h7dZHJgqkzUiKFXCTJBrPWH0LEZaZXBFzKWstjWBRxw= go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.65.0 h1:LIMn2KWRS0jRDDHYyIEYgKWsMwufA9GXusJiwik0u64= go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.65.0/go.mod h1:JwJa4o3Wq+4Yz2BjlYFGWyx2h0Fw1lnoj5kpsaTI97o= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.65.0 h1:ab5U7DpTjjN8pNgwqlA/s0Csb+N2Raqo9eTSDhfg4Z8= @@ -133,16 +135,26 @@ go.opentelemetry.io/contrib/propagators/ot v1.40.0 h1:Lon8J5SPmWaL1Ko2TIlCNHJ42/ go.opentelemetry.io/contrib/propagators/ot v1.40.0/go.mod h1:dKWtJTlp1Yj+8Cneye5idO46eRPIbi23qVuJYKjNnvY= go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0 h1:ZVg+kCXxd9LtAaQNKBxAvJ5NpMf7LpvEr4MIZqb0TMQ= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0/go.mod h1:hh0tMeZ75CCXrHd9OXRYxTlCAdxcXioWHFIpYw2rZu8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.40.0 h1:MzfofMZN8ulNqobCmCAVbqVL5syHw+eB2qPRkCMA/fQ= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.40.0/go.mod h1:E73G9UFtKRXrxhBsHtG00TB5WxX57lpsQzogDkqBTz8= +go.opentelemetry.io/otel/log v0.16.0 h1:DeuBPqCi6pQwtCK0pO4fvMB5eBq6sNxEnuTs88pjsN4= +go.opentelemetry.io/otel/log v0.16.0/go.mod h1:rWsmqNVTLIA8UnwYVOItjyEZDbKIkMxdQunsIhpUMes= +go.opentelemetry.io/otel/log/logtest v0.16.0 h1:jr1CG3Z6FD9pwUaL/D0s0X4lY2ZVm1jP3JfCtzGxUmE= +go.opentelemetry.io/otel/log/logtest v0.16.0/go.mod h1:qeeZw+cI/rAtCzZ03Kq1ozq6C4z/PCa+K+bb0eJfKNs= go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/log v0.16.0 h1:e/b4bdlQwC5fnGtG3dlXUrNOnP7c8YLVSpSfEBIkTnI= +go.opentelemetry.io/otel/sdk/log v0.16.0/go.mod h1:JKfP3T6ycy7QEuv3Hj8oKDy7KItrEkus8XJE6EoSzw4= +go.opentelemetry.io/otel/sdk/log/logtest v0.16.0 h1:/XVkpZ41rVRTP4DfMgYv1nEtNmf65XPPyAdqV90TMy4= +go.opentelemetry.io/otel/sdk/log/logtest v0.16.0/go.mod h1:iOOPgQr5MY9oac/F5W86mXdeyWZGleIx3uXO98X2R6Y= go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= diff --git a/otel/README.md b/otel/README.md index 8a23f58..7b05ca9 100644 --- a/otel/README.md +++ b/otel/README.md @@ -1,27 +1,34 @@ -# Tracing Demo +# Tracing & Logging Demo The directory contains sample [OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector) -and [Jaeger](https://www.jaegertracing.io) configurations for a tracing demo. +and [Jaeger](https://www.jaegertracing.io) / [Loki](https://grafana.com/oss/loki/) configurations for a tracing and logging demo. ## Configuration -The provided [docker-compose.yaml](docker-compose.yaml) sets up 4 Containers +The provided [docker-compose.yaml](docker-compose.yaml) sets up 6 containers: 1. PodInfo Frontend on port 9898 2. PodInfo Backend on port 9899 3. OpenTelemetry Collector listening on port 4317 for GRPC -4. Jaeger all-in-one listening on multiple ports +4. Jaeger all-in-one with UI on port 16686 +5. Loki on port 3100 +6. Grafana on port 3000 ## How does it work? -The frontend pods are configured to call onto the backend pods. Both the podinfo -pods are configured to send traces over to the collector at port 4317 using GRPC. -The collector forwards all received spans to Jaeger over port 14250 and Jaeger -exposes a UI over port `16686`. +The frontend pod is configured to call the backend pod. Both podinfo pods send traces +and logs to the collector at port 4317 using OTLP gRPC. + +The collector forwards: +- **Traces** to Jaeger via OTLP gRPC on port 4317 +- **Logs** to Loki via OTLP HTTP on port 3100 + +Jaeger exposes its UI on port `16686`. Grafana exposes its UI on port `3000` and is +pre-configured with both Jaeger and Loki as datasources. ## Running it locally -1. Start all the Containers +1. Start all the containers ```shell make run ``` @@ -30,8 +37,9 @@ make run curl -v http://localhost:9898/status/200 curl -X POST -v http://localhost:9898/api/echo ``` -3. Visit `http://localhost:16686/` to see the spans -4. Stop all the containers +3. Visit `http://localhost:16686/` to see traces in Jaeger +4. Visit `http://localhost:3000/` to explore logs in Grafana (Explore → Loki) and traces (Explore → Jaeger) +5. Stop all the containers ```shell make stop ``` diff --git a/otel/docker-compose.yaml b/otel/docker-compose.yaml index ab9b56b..9c9b6ab 100644 --- a/otel/docker-compose.yaml +++ b/otel/docker-compose.yaml @@ -5,31 +5,38 @@ services: build: .. command: ./podinfo --backend-url http://podinfo_backend:9899/status/200 --otel-service-name=podinfo_frontend environment: - - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel:4317 + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel:4317 ports: - "9898:9898" podinfo_backend: build: .. command: ./podinfo --port 9899 --otel-service-name=podinfo_backend environment: - - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel:4317 + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel:4317 ports: - "9899:9899" otel: command: --config otel-config.yaml - image: otel/opentelemetry-collector:0.41.0 + image: otel/opentelemetry-collector-contrib:0.116.1 ports: - "4317:4317" volumes: - ${PWD}/otel-config.yaml:/otel-config.yaml - jaeger: - image: jaegertracing/all-in-one:1.29.0 + loki: + image: grafana/loki:3.0.0 + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + grafana: + image: grafana/grafana:10.4.0 + ports: + - "3000:3000" + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + volumes: + - ${PWD}/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + jaeger: + image: jaegertracing/all-in-one:1.57.0 ports: - - "5775:5775/udp" - - "6831:6831/udp" - - "6832:6832/udp" - - "5778:5778" - "16686:16686" - - "14268:14268" - - "14250:14250" - - "9411:9411" diff --git a/otel/grafana-datasources.yaml b/otel/grafana-datasources.yaml new file mode 100644 index 0000000..534f173 --- /dev/null +++ b/otel/grafana-datasources.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Loki + type: loki + url: http://loki:3100 + isDefault: true + - name: Jaeger + type: jaeger + url: http://jaeger:16686 diff --git a/otel/otel-config.yaml b/otel/otel-config.yaml index f123d4d..05c3abf 100644 --- a/otel/otel-config.yaml +++ b/otel/otel-config.yaml @@ -2,15 +2,18 @@ receivers: otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: processors: exporters: - jaeger: - endpoint: jaeger:14250 + otlp/jaeger: + endpoint: jaeger:4317 tls: insecure: true + otlphttp/loki: + endpoint: http://loki:3100/otlp extensions: health_check: @@ -23,4 +26,8 @@ service: traces: receivers: [otlp] processors: [] - exporters: [jaeger] + exporters: [otlp/jaeger] + logs: + receivers: [otlp] + processors: [] + exporters: [otlphttp/loki] diff --git a/pkg/signals/shutdown.go b/pkg/signals/shutdown.go index 2d8da7b..6b57fc0 100644 --- a/pkg/signals/shutdown.go +++ b/pkg/signals/shutdown.go @@ -8,6 +8,7 @@ import ( "github.com/gomodule/redigo/redis" "github.com/spf13/viper" + sdklog "go.opentelemetry.io/otel/sdk/log" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.uber.org/zap" "google.golang.org/grpc" @@ -17,9 +18,14 @@ type Shutdown struct { logger *zap.Logger pool *redis.Pool tracerProvider *sdktrace.TracerProvider + loggerProvider *sdklog.LoggerProvider serverShutdownTimeout time.Duration } +func (s *Shutdown) SetLoggerProvider(lp *sdklog.LoggerProvider) { + s.loggerProvider = lp +} + func NewShutdown(serverShutdownTimeout time.Duration, logger *zap.Logger) (*Shutdown, error) { srv := &Shutdown{ logger: logger, @@ -62,6 +68,13 @@ func (s *Shutdown) Graceful(stopCh <-chan struct{}, httpServer *http.Server, htt } } + // stop OpenTelemetry logger provider + if s.loggerProvider != nil { + if err := s.loggerProvider.Shutdown(ctx); err != nil { + s.logger.Warn("stopping logger provider", zap.Error(err)) + } + } + // determine if the GRPC was started if grpcServer != nil { s.logger.Info("Shutting down GRPC server", zap.Duration("timeout", s.serverShutdownTimeout))