feat: add otlp logging support

- Adds a loggerprovider based on otlp logger - In demo directory of oltp: - Added grafana for unified view of both traces and logs - tracing now uses oltp from the collector to the jaeger instance Signed-off-by: Hans van den Bogert <hansbogert@gmail.com>
2026-03-31 16:16:53 +00:00 · 2026-03-02 11:23:39 +01:00
parent 0a27dbe40c
commit 095b1cd251
8 changed files with 140 additions and 30 deletions
--- a/cmd/podinfo/main.go
+++ b/cmd/podinfo/main.go
@@ -1,6 +1,7 @@
 package main

 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -10,6 +11,11 @@ import (

 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+	"go.opentelemetry.io/contrib/bridges/otelzap"
+	"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc"
+	sdklog "go.opentelemetry.io/otel/sdk/log"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"

@@ -53,7 +59,7 @@ func main() {
 	fs.Int("stress-cpu", 0, "number of CPU cores with 100 load")
 	fs.Int("stress-memory", 0, "MB of data to load into memory")
 	fs.String("cache-server", "", "Redis address in the format 'tcp://<host>:<port>'")
-	fs.String("otel-service-name", "", "service name for reporting to open telemetry address, when not set tracing is disabled")
+	fs.String("otel-service-name", "", "service name for OpenTelemetry, when not set tracing and log export are disabled")

 	versionFlag := fs.BoolP("version", "v", false, "get version number")

@@ -93,8 +99,18 @@ func main() {
 		}
 	}

+	// initialize OTel log provider if service name is set
+	var loggerProvider *sdklog.LoggerProvider
+	if otelServiceName := viper.GetString("otel-service-name"); otelServiceName != "" {
+		var err error
+		loggerProvider, err = initLoggerProvider(context.Background(), otelServiceName)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error initializing OTel log provider: %s\n", err.Error())
+		}
+	}
+
 	// configure logging
-	logger, _ := initZap(viper.GetString("level"))
+	logger, _ := initZap(viper.GetString("level"), loggerProvider)
 	defer logger.Sync()
 	stdLog := zap.RedirectStdLog(logger)
 	defer stdLog()
@@ -163,10 +179,29 @@ func main() {
 	// graceful shutdown
 	stopCh := signals.SetupSignalHandler()
 	sd, _ := signals.NewShutdown(srvCfg.ServerShutdownTimeout, logger)
+	sd.SetLoggerProvider(loggerProvider)
 	sd.Graceful(stopCh, httpServer, httpsServer, grpcServer, healthy, ready)
 }

-func initZap(logLevel string) (*zap.Logger, error) {
+func initLoggerProvider(ctx context.Context, serviceName string) (*sdklog.LoggerProvider, error) {
+	exporter, err := otlploggrpc.New(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("creating OTLP log exporter: %w", err)
+	}
+
+	provider := sdklog.NewLoggerProvider(
+		sdklog.WithProcessor(sdklog.NewBatchProcessor(exporter)),
+		sdklog.WithResource(resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceNameKey.String(serviceName),
+			semconv.ServiceVersionKey.String(version.VERSION),
+		)),
+	)
+
+	return provider, nil
+}
+
+func initZap(logLevel string, loggerProvider *sdklog.LoggerProvider) (*zap.Logger, error) {
 	level := zap.NewAtomicLevelAt(zapcore.InfoLevel)
 	switch logLevel {
 	case "debug":
@@ -210,7 +245,21 @@ func initZap(logLevel string) (*zap.Logger, error) {
 		ErrorOutputPaths: []string{"stderr"},
 	}

-	return zapConfig.Build()
+	logger, err := zapConfig.Build()
+	if err != nil {
+		return nil, err
+	}
+
+	if loggerProvider != nil {
+		otelCore := otelzap.NewCore("github.com/stefanprodan/podinfo",
+			otelzap.WithLoggerProvider(loggerProvider),
+		)
+		logger = logger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
+			return zapcore.NewTee(core, otelCore)
+		}))
+	}
+
+	return logger, nil
 }

 var stressMemoryPayload []byte
--- a/go.mod
+++ b/go.mod
@@ -16,6 +16,7 @@ require (
 	github.com/spf13/viper v1.21.0
 	github.com/swaggo/http-swagger v1.3.4
 	github.com/swaggo/swag v1.16.6
+	go.opentelemetry.io/contrib/bridges/otelzap v0.15.0
 	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.65.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.65.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0
@@ -24,9 +25,11 @@ require (
 	go.opentelemetry.io/contrib/propagators/jaeger v1.40.0
 	go.opentelemetry.io/contrib/propagators/ot v1.40.0
 	go.opentelemetry.io/otel v1.40.0
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0
 	go.opentelemetry.io/otel/sdk v1.40.0
+	go.opentelemetry.io/otel/sdk/log v0.16.0
 	go.opentelemetry.io/otel/trace v1.40.0
 	go.uber.org/zap v1.27.1
 	golang.org/x/net v0.51.0
@@ -66,6 +69,7 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/swaggo/files v1.0.1 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel/log v0.16.0 // indirect
 	go.opentelemetry.io/otel/metric v1.40.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
--- a/go.sum
+++ b/go.sum
@@ -117,6 +117,8 @@ github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/contrib/bridges/otelzap v0.15.0 h1:x4qzjKkTl2hXmLl+IviSXvzaTyCJSYvpFZL5SRVLBxs=
+go.opentelemetry.io/contrib/bridges/otelzap v0.15.0/go.mod h1:h7dZHJgqkzUiKFXCTJBrPWH0LEZaZXBFzKWstjWBRxw=
 go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.65.0 h1:LIMn2KWRS0jRDDHYyIEYgKWsMwufA9GXusJiwik0u64=
 go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.65.0/go.mod h1:JwJa4o3Wq+4Yz2BjlYFGWyx2h0Fw1lnoj5kpsaTI97o=
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.65.0 h1:ab5U7DpTjjN8pNgwqlA/s0Csb+N2Raqo9eTSDhfg4Z8=
@@ -133,16 +135,26 @@ go.opentelemetry.io/contrib/propagators/ot v1.40.0 h1:Lon8J5SPmWaL1Ko2TIlCNHJ42/
 go.opentelemetry.io/contrib/propagators/ot v1.40.0/go.mod h1:dKWtJTlp1Yj+8Cneye5idO46eRPIbi23qVuJYKjNnvY=
 go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
 go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0 h1:ZVg+kCXxd9LtAaQNKBxAvJ5NpMf7LpvEr4MIZqb0TMQ=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0/go.mod h1:hh0tMeZ75CCXrHd9OXRYxTlCAdxcXioWHFIpYw2rZu8=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.40.0 h1:MzfofMZN8ulNqobCmCAVbqVL5syHw+eB2qPRkCMA/fQ=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.40.0/go.mod h1:E73G9UFtKRXrxhBsHtG00TB5WxX57lpsQzogDkqBTz8=
+go.opentelemetry.io/otel/log v0.16.0 h1:DeuBPqCi6pQwtCK0pO4fvMB5eBq6sNxEnuTs88pjsN4=
+go.opentelemetry.io/otel/log v0.16.0/go.mod h1:rWsmqNVTLIA8UnwYVOItjyEZDbKIkMxdQunsIhpUMes=
+go.opentelemetry.io/otel/log/logtest v0.16.0 h1:jr1CG3Z6FD9pwUaL/D0s0X4lY2ZVm1jP3JfCtzGxUmE=
+go.opentelemetry.io/otel/log/logtest v0.16.0/go.mod h1:qeeZw+cI/rAtCzZ03Kq1ozq6C4z/PCa+K+bb0eJfKNs=
 go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
 go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
 go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
 go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
+go.opentelemetry.io/otel/sdk/log v0.16.0 h1:e/b4bdlQwC5fnGtG3dlXUrNOnP7c8YLVSpSfEBIkTnI=
+go.opentelemetry.io/otel/sdk/log v0.16.0/go.mod h1:JKfP3T6ycy7QEuv3Hj8oKDy7KItrEkus8XJE6EoSzw4=
+go.opentelemetry.io/otel/sdk/log/logtest v0.16.0 h1:/XVkpZ41rVRTP4DfMgYv1nEtNmf65XPPyAdqV90TMy4=
+go.opentelemetry.io/otel/sdk/log/logtest v0.16.0/go.mod h1:iOOPgQr5MY9oac/F5W86mXdeyWZGleIx3uXO98X2R6Y=
 go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
 go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
 go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
--- a/otel/README.md
+++ b/otel/README.md
@@ -1,27 +1,34 @@
-# Tracing Demo
+# Tracing & Logging Demo

 The directory contains sample [OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector)
-and [Jaeger](https://www.jaegertracing.io) configurations for a tracing demo.
+and [Jaeger](https://www.jaegertracing.io) / [Loki](https://grafana.com/oss/loki/) configurations for a tracing and logging demo.

 ## Configuration

-The provided [docker-compose.yaml](docker-compose.yaml) sets up 4 Containers
+The provided [docker-compose.yaml](docker-compose.yaml) sets up 6 containers:

 1. PodInfo Frontend on port 9898
 2. PodInfo Backend on port 9899
 3. OpenTelemetry Collector listening on port 4317 for GRPC
-4. Jaeger all-in-one listening on multiple ports
+4. Jaeger all-in-one with UI on port 16686
+5. Loki on port 3100
+6. Grafana on port 3000

 ## How does it work?

-The frontend pods are configured to call onto the backend pods. Both the podinfo
-pods are configured to send traces over to the collector at port 4317 using GRPC.
-The collector forwards all received spans to Jaeger over port 14250 and Jaeger
-exposes a UI over port `16686`.
+The frontend pod is configured to call the backend pod. Both podinfo pods send traces
+and logs to the collector at port 4317 using OTLP gRPC.
+
+The collector forwards:
+- **Traces** to Jaeger via OTLP gRPC on port 4317
+- **Logs** to Loki via OTLP HTTP on port 3100
+
+Jaeger exposes its UI on port `16686`. Grafana exposes its UI on port `3000` and is
+pre-configured with both Jaeger and Loki as datasources.

 ## Running it locally

-1. Start all the Containers
+1. Start all the containers
 ```shell
 make run
 ```
@@ -30,8 +37,9 @@ make run
 curl -v http://localhost:9898/status/200
 curl -X POST -v http://localhost:9898/api/echo
 ```
-3. Visit `http://localhost:16686/` to see the spans
-4. Stop all the containers
+3. Visit `http://localhost:16686/` to see traces in Jaeger
+4. Visit `http://localhost:3000/` to explore logs in Grafana (Explore → Loki) and traces (Explore → Jaeger)
+5. Stop all the containers
 ```shell
 make stop
 ```
--- a/otel/docker-compose.yaml
+++ b/otel/docker-compose.yaml
@@ -5,31 +5,38 @@ services:
    build: ..
    command: ./podinfo --backend-url http://podinfo_backend:9899/status/200 --otel-service-name=podinfo_frontend
    environment:
-      - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel:4317
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel:4317
    ports:
      - "9898:9898"
  podinfo_backend:
    build: ..
    command: ./podinfo --port 9899 --otel-service-name=podinfo_backend
    environment:
-      - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel:4317
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel:4317
    ports:
      - "9899:9899"
  otel:
    command: --config otel-config.yaml
-    image: otel/opentelemetry-collector:0.41.0
+    image: otel/opentelemetry-collector-contrib:0.116.1
    ports:
      - "4317:4317"
    volumes:
      - ${PWD}/otel-config.yaml:/otel-config.yaml
-  jaeger:
-    image: jaegertracing/all-in-one:1.29.0
+  loki:
+    image: grafana/loki:3.0.0
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+  grafana:
+    image: grafana/grafana:10.4.0
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    volumes:
+      - ${PWD}/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+  jaeger:
+    image: jaegertracing/all-in-one:1.57.0
    ports:
-      - "5775:5775/udp"
-      - "6831:6831/udp"
-      - "6832:6832/udp"
-      - "5778:5778"
      - "16686:16686"
-      - "14268:14268"
-      - "14250:14250"
-      - "9411:9411"
--- a/otel/grafana-datasources.yaml
+++ b/otel/grafana-datasources.yaml
@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    url: http://loki:3100
+    isDefault: true
+  - name: Jaeger
+    type: jaeger
+    url: http://jaeger:16686
--- a/otel/otel-config.yaml
+++ b/otel/otel-config.yaml
@@ -2,15 +2,18 @@ receivers:
  otlp:
    protocols:
      grpc:
+        endpoint: 0.0.0.0:4317
      http:

 processors:

 exporters:
-  jaeger:
-    endpoint: jaeger:14250
+  otlp/jaeger:
+    endpoint: jaeger:4317
    tls:
      insecure: true
+  otlphttp/loki:
+    endpoint: http://loki:3100/otlp

 extensions:
  health_check:
@@ -23,4 +26,8 @@ service:
    traces:
      receivers: [otlp]
      processors: []
-      exporters: [jaeger]
+      exporters: [otlp/jaeger]
+    logs:
+      receivers: [otlp]
+      processors: []
+      exporters: [otlphttp/loki]
--- a/pkg/signals/shutdown.go
+++ b/pkg/signals/shutdown.go
@@ -8,6 +8,7 @@ import (

 	"github.com/gomodule/redigo/redis"
 	"github.com/spf13/viper"
+	sdklog "go.opentelemetry.io/otel/sdk/log"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
@@ -17,9 +18,14 @@ type Shutdown struct {
 	logger                *zap.Logger
 	pool                  *redis.Pool
 	tracerProvider        *sdktrace.TracerProvider
+	loggerProvider        *sdklog.LoggerProvider
 	serverShutdownTimeout time.Duration
 }

+func (s *Shutdown) SetLoggerProvider(lp *sdklog.LoggerProvider) {
+	s.loggerProvider = lp
+}
+
 func NewShutdown(serverShutdownTimeout time.Duration, logger *zap.Logger) (*Shutdown, error) {
 	srv := &Shutdown{
 		logger:                logger,
@@ -62,6 +68,13 @@ func (s *Shutdown) Graceful(stopCh <-chan struct{}, httpServer *http.Server, htt
 		}
 	}

+	// stop OpenTelemetry logger provider
+	if s.loggerProvider != nil {
+		if err := s.loggerProvider.Shutdown(ctx); err != nil {
+			s.logger.Warn("stopping logger provider", zap.Error(err))
+		}
+	}
+
 	// determine if the GRPC was started
 	if grpcServer != nil {
 		s.logger.Info("Shutting down GRPC server", zap.Duration("timeout", s.serverShutdownTimeout))