Olivine:以 Go 建構與 Redis 相容的服務 · 章節目錄

第 18 章

Day 18:OpenTelemetry

OpenTelemetry 是雲端原生生態系的 observability 標準,涵蓋:

  • Tracing:追蹤請求如何處理,以及呼叫了哪些函式。
  • Metrics:收集 CPU、記憶體、平均回應時間等指標。
  • Logging。

注意:對目前的 Olivine 而言,這個主題還有些早。本章提供範例,但不深入解釋所有設計原因。

OpenTelemetry SDK

OpenTelemetry 提供一組 SDK:

$ go get go.opentelemetry.io/otel go.opentelemetry.io/otel/sdk go.opentelemetry.io/otel/sdk/log go.opentelemetry.io/otel/sdk/metric go.opentelemetry.io/contrib/bridges/otelslog go.opentelemetry.io/contrib/exporters/autoexport go.opentelemetry.io/contrib/instrumentation/runtime

在 internal/service/otel.go 初始化:

package service

import (
	"context"
	"errors"
	"io"
	"log"
	"log/slog"
	"os"

	"go.opentelemetry.io/contrib/bridges/otelslog"
	"go.opentelemetry.io/contrib/exporters/autoexport"
	"go.opentelemetry.io/otel"
	otellog "go.opentelemetry.io/otel/log"
	"go.opentelemetry.io/otel/log/global"
	"go.opentelemetry.io/otel/propagation"
	sdklog "go.opentelemetry.io/otel/sdk/log"
	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	"go.opentelemetry.io/otel/sdk/resource"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.41.0"
)

const (
	otelEnabledEnv = "OTEL_ENABLED"
	serviceName    = "olivine"
)

func OTelEnabled() bool {
	return os.Getenv(otelEnabledEnv) == "1"
}

// SetupOTelSDK bootstraps the OpenTelemetry pipeline.
// If it does not return an error, make sure to call shutdown for proper cleanup.
func SetupOTelSDK(ctx context.Context) (func(context.Context) error, error) {
	var shutdownFuncs []func(context.Context) error
	var err error

	// shutdown calls cleanup functions registered via shutdownFuncs.
	// The errors from the calls are joined.
	// Each registered cleanup will be invoked once.
	shutdown := func(ctx context.Context) error {
		var err error
		for _, fn := range shutdownFuncs {
			err = errors.Join(err, fn(ctx))
		}
		shutdownFuncs = nil
		return err
	}

	// handleErr calls shutdown for cleanup and makes sure that all errors are returned.
	handleErr := func(inErr error) {
		err = errors.Join(inErr, shutdown(ctx))
	}

	// Set up propagator.
	prop := newPropagator()
	otel.SetTextMapPropagator(prop)

	res, err := newResource()
	if err != nil {
		handleErr(err)
		return shutdown, err
	}

	// Set up trace provider.
	tracerProvider, err := newTracerProvider(ctx, res)
	if err != nil {
		handleErr(err)
		return shutdown, err
	}
	shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown)
	otel.SetTracerProvider(tracerProvider)

	// Set up meter provider.
	meterProvider, err := newMeterProvider(ctx, res)
	if err != nil {
		handleErr(err)
		return shutdown, err
	}
	shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown)
	otel.SetMeterProvider(meterProvider)

	// Set up logger provider.
	loggerProvider, err := newLoggerProvider(ctx, res)
	if err != nil {
		handleErr(err)
		return shutdown, err
	}
	shutdownFuncs = append(shutdownFuncs, loggerProvider.Shutdown)
	global.SetLoggerProvider(loggerProvider)

	// Bridge the standard slog logger to OpenTelemetry while preserving its
	// existing console output.
	defaultLogger := slog.Default()
	defaultLogWriter := log.Writer()
	defaultLogFlags := log.Flags()
	defaultLogPrefix := log.Prefix()
	slog.SetDefault(newSlogLogger(os.Stderr, loggerProvider))
	shutdownFuncs = append(shutdownFuncs, func(context.Context) error {
		slog.SetDefault(defaultLogger)
		log.SetOutput(defaultLogWriter)
		log.SetFlags(defaultLogFlags)
		log.SetPrefix(defaultLogPrefix)
		return nil
	})

	return shutdown, err
}

func newSlogLogger(consoleWriter io.Writer, loggerProvider otellog.LoggerProvider) *slog.Logger {
	consoleHandler := slog.NewTextHandler(consoleWriter, nil)
	otelHandler := otelslog.NewHandler(
		"olivine",
		otelslog.WithLoggerProvider(loggerProvider),
	)
	return slog.New(slog.NewMultiHandler(consoleHandler, otelHandler))
}

func newPropagator() propagation.TextMapPropagator {
	return propagation.NewCompositeTextMapPropagator(
		propagation.TraceContext{},
		propagation.Baggage{},
	)
}

func newResource() (*resource.Resource, error) {
	return resource.Merge(
		resource.Default(),
		resource.NewSchemaless(semconv.ServiceName(serviceName)),
	)
}

func newTracerProvider(ctx context.Context, res *resource.Resource) (*sdktrace.TracerProvider, error) {
	spanExporter, err := autoexport.NewSpanExporter(ctx)
	if err != nil {
		return nil, err
	}
	tracerProvider := sdktrace.NewTracerProvider(
		sdktrace.WithBatcher(spanExporter),
		sdktrace.WithResource(res),
	)
	return tracerProvider, nil
}

func newMeterProvider(ctx context.Context, res *resource.Resource) (*sdkmetric.MeterProvider, error) {
	metricReader, err := autoexport.NewMetricReader(ctx)
	if err != nil {
		return nil, err
	}

	meterProvider := sdkmetric.NewMeterProvider(
		sdkmetric.WithReader(metricReader),
		sdkmetric.WithResource(res),
	)
	return meterProvider, nil
}

func newLoggerProvider(ctx context.Context, res *resource.Resource) (*sdklog.LoggerProvider, error) {
	logExporter, err := autoexport.NewLogExporter(ctx)
	if err != nil {
		return nil, err
	}

	loggerProvider := sdklog.NewLoggerProvider(
		sdklog.WithProcessor(sdklog.NewBatchProcessor(logExporter)),
		sdklog.WithResource(res),
	)
	return loggerProvider, nil
}

這段程式以 OpenTelemetry Getting Started 為基礎。

在 cmd/olivine/app.go 啟用 SDK:

// file: cmd/olivine/app.go
func (app *App) Run() error {
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

+	if service.OTelEnabled() {
+		otelShutdown, err := service.SetupOTelSDK(ctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := otelShutdown(context.Background()); err != nil {
+				slog.Error("failed to shutdown otel", slog.Any("error", err))
+			}
+		}()
+	}

	slog.Info("restoring data from disk")
	// ...
}

Metrics

go.opentelemetry.io/contrib/instrumentation/runtime 提供 Go runtime 的 metrics instrumentation:

if service.OTelEnabled() {
	otelShutdown, err := service.SetupOTelSDK(ctx)
	if err != nil {
		return err
	}
	defer func() {
		if err := otelShutdown(context.Background()); err != nil {
			slog.Error("failed to shutdown otel", slog.Any("error", err))
		}
	}()
+	if err := runtime.Start(runtime.WithMinimumReadMemStatsInterval(time.Second)); err != nil {
+		return err
+	}
}

試著啟動 Olivine:

$ OTEL_ENABLED=1 OTEL_METRICS_EXPORTER=console go run ./cmd/olivine

注意:OTEL_METRICS_EXPORTER 由 go.opentelemetry.io/contrib/exporters/autoexport 處理,可參考它的 godoc。

也能自行設計 instrumentation,例如 storage 的 key 數量、記憶體用量等。

Tracing

Client-server 應用程式需要追蹤請求流程。使用 OpenTelemetry tracer 建立 span:

// file: internal/server/handler.go
func (h *simpleHandler) ServeRESP(ctx context.Context, rd *resp.Reader) (ret resp.Value, err error) {
+	ctx, span := otel.Tracer("olivine").Start(ctx, "server.(*simpleHandler).ServeRESP")
+	defer span.End()

	command, err := resp.ReadCommand(rd)
// file: internal/service/cmd/ping.go
func (c *Ping) Exec(ctx context.Context, cmd *resp.Command) (ret resp.Value, err error) {
+	_, span := otel.Tracer("olivine").Start(ctx, "cmd.(*Ping).Exec")
+	defer span.End()

	if err := c.parse(cmd); err != nil {
		return nil, err
	}

錯誤處理

只回傳 err != nil 不會自動讓 span 記錄為失敗。可以主動記錄錯誤與設定狀態:

if err != nil {
		span.RecordError(err)
		span.SetStatus(codes.Error, err.Error())
}

在 internal/util/otel.go 建立 EndSpan():

func EndSpan(span trace.Span, err error) {
	if err != nil {
		span.RecordError(err)
		span.SetStatus(codes.Error, err.Error())
	}
	span.End()
}

以 defer func() { util.EndSpan(span, err) }() 取代 defer span.End()。

Logging

因為已初始化 slog logger bridge,不需要額外修改每個 logging 呼叫。

Jaeger、Prometheus 與 Loki

只將 telemetry 印到 console 不便分析,輸出太多時也容易淹沒資訊。可以使用:

以下列環境變數啟動 Olivine:

$ OTEL_ENABLED=1 OTEL_SERVICE_NAME=olivine OTEL_TRACES_EXPORTER=otlp OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4318/v1/traces OTEL_METRICS_EXPORTER=otlp OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=http/protobuf OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:9090/api/v1/otlp/v1/metrics OTEL_LOGS_EXPORTER=otlp OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=http/protobuf OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://localhost:3100/otlp/v1/logs go run ./cmd/olivine

OpenTelemetry Collector

Collector 能統一接收 traces、metrics、logs,讓每個服務不必各自設定所有後端 endpoints。我建議正式部署時使用 Collector。

Grafana

Grafana 提供整合觀測資料的 dashboard 與 alert。

注意:我另外實作了 docker-compose.dev.yml 與 deployments/ 整合服務,但不在本章範圍。

目前狀態

我認為 d18/otel 的整合仍缺少許多正式環境所需的 instrumentation 與 tracing,因此尚不足以合併進 main。目前先以這個分支進行實驗。