feat(OpenTelemetry): Integrate OpenTelemetry into agent

README.md

@@ -89,6 +89,7 @@ BLADE_LISTEN_METRICS=":1234"
 | `BLADE_FAN_SPEED_PERCENT=80`                      | Set static fan speed                     |
 | `BLADE_CRITICAL_TEMPERATURE_THRESHOLD=60`         | Set critical temp threshold (°C)         |
 | `BLADE_HAL_RPM_REPORTING_STANDARD_FAN_UNIT=false` | Disable RPM monitoring for lower CPU use |
+| `OTEL_EXPORTER_OTLP_ENDPOINT`                     | Endpoint for the OTLP exporter           |
 
 ## Exposing the gRPC API for Remote Access
 

cmd/agent/main.go

@@ -14,6 +14,8 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/spechtlabs/go-otel-utils/otelprovider"
+	"github.com/spechtlabs/go-otel-utils/otelzap"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"github.com/uptime-industries/compute-blade-agent/internal/agent"
@@ -55,21 +57,78 @@ func main() {
 		baseLogger = zap.Must(zap.NewProduction())
 	}
 
-	zapLogger := baseLogger.With(zap.String("app", "compute-blade-agent"))
+	zapLogger := baseLogger.With(
+		zap.String("app", "compute-blade-agent"),
+		zap.String("scope", "global"),
+		zap.String("version", Version),
+		zap.String("commit", Commit),
+		zap.String("date", Date),
+	)
 	defer func() {
 		_ = zapLogger.Sync()
 	}()
-	_ = zap.ReplaceGlobals(zapLogger.With(zap.String("scope", "global")))
-	baseCtx := log.IntoContext(context.Background(), zapLogger)
 
+	// Replace zap global
+	undoZapGlobals := zap.ReplaceGlobals(zapLogger)
+
+	// Redirect stdlib log to zap
+	undoStdLogRedirect := zap.RedirectStdLog(zapLogger)
+
+	// Create OpenTelemetry Log and Trace provider
+	logProvider := otelprovider.NewLogger(
+		otelprovider.WithLogAutomaticEnv(),
+	)
+
+	traceProvider := otelprovider.NewTracer(
+		otelprovider.WithTraceAutomaticEnv(),
+	)
+
+	// Create otelLogger
+	otelZapLogger := otelzap.New(zapLogger,
+		otelzap.WithCaller(true),
+		otelzap.WithMinLevel(zap.InfoLevel),
+		otelzap.WithAnnotateLevel(zap.WarnLevel),
+		otelzap.WithErrorStatusLevel(zap.ErrorLevel),
+		otelzap.WithStackTrace(false),
+		otelzap.WithLoggerProvider(logProvider),
+	)
+
+	// Replace global otelZap logger
+	undoOtelZapGlobals := otelzap.ReplaceGlobals(otelZapLogger)
+	defer undoOtelZapGlobals()
+
+	// Cleanup Logging and Tracing
+	defer func() {
+		if err := traceProvider.ForceFlush(context.Background()); err != nil {
+			otelzap.L().Warn("failed to flush traces")
+		}
+
+		if err := logProvider.ForceFlush(context.Background()); err != nil {
+			otelzap.L().Warn("failed to flush logs")
+		}
+
+		if err := traceProvider.Shutdown(context.Background()); err != nil {
+			panic(err)
+		}
+
+		if err := logProvider.Shutdown(context.Background()); err != nil {
+			panic(err)
+		}
+
+		undoStdLogRedirect()
+		undoZapGlobals()
+	}()
+
+	// Setup context
+	baseCtx := log.IntoContext(context.Background(), otelZapLogger)
 	ctx, cancelCtx := context.WithCancelCause(baseCtx)
 	defer cancelCtx(context.Canceled)
 
 	// load configuration
 	var cbAgentConfig agent.ComputeBladeAgentConfig
 	if err := viper.Unmarshal(&cbAgentConfig); err != nil {
 		cancelCtx(err)
-		log.FromContext(ctx).Fatal("Failed to load configuration", zap.Error(err))
+		log.FromContext(ctx).WithError(err).Fatal("Failed to load configuration")
 	}
 
 	// setup stop signal handlers
@@ -101,7 +160,7 @@ func main() {
 	computebladeAgent, err := agent.NewComputeBladeAgent(ctx, cbAgentConfig)
 	if err != nil {
 		cancelCtx(err)
-		log.FromContext(ctx).Fatal("Failed to create agent", zap.Error(err))
+		log.FromContext(ctx).WithError(err).Fatal("Failed to create agent")
 	}
 
 	// Run agent
@@ -143,15 +202,15 @@ func main() {
 		defer shutdownCtxCancel()
 
 		if err := promServer.Shutdown(shutdownCtx); err != nil {
-			log.FromContext(ctx).Error("Failed to shutdown prometheus/pprof server", zap.Error(err))
+			log.FromContext(ctx).WithError(err).Error("Failed to shutdown prometheus/pprof server")
 		}
 	}()
 
 	wg.Wait()
 
 	// Wait for context cancel
 	if err := ctx.Err(); !errors.Is(err, context.Canceled) {
-		log.FromContext(ctx).Fatal("Exiting", zap.Error(err))
+		log.FromContext(ctx).WithError(err).Fatal("Exiting")
 	} else {
 		log.FromContext(ctx).Info("Exiting")
 	}
@@ -172,7 +231,7 @@ func runPrometheusEndpoint(ctx context.Context, cancel context.CancelCauseFunc,
 	go func() {
 		err := server.ListenAndServe()
 		if err != nil && !errors.Is(err, http.ErrServerClosed) {
-			log.FromContext(ctx).Error("Failed to start prometheus/pprof server", zap.Error(err))
+			log.FromContext(ctx).WithError(err).Error("Failed to start prometheus/pprof server")
 			cancel(err)
 		}
 	}()

cmd/bladectl/config/config.go

@@ -6,7 +6,7 @@ import (
 	"path/filepath"
 
 	"github.com/sierrasoftworks/humane-errors-go"
-	"go.uber.org/zap"
+	"github.com/spechtlabs/go-otel-utils/otelzap"
 )
 
 type BladectlConfig struct {
@@ -58,7 +58,7 @@ func NewAuthenticatedBladectlConfig(server string, caPEM []byte, clientCertDER [
 func NewBladectlConfig(server string) *BladectlConfig {
 	hostname, err := os.Hostname()
 	if err != nil {
-		zap.L().Fatal("Failed to extract hostname", zap.Error(err))
+		otelzap.L().WithError(err).Fatal("Failed to extract hostname")
 	}
 
 	return &BladectlConfig{

go.mod

@@ -1,35 +1,42 @@
 module github.com/uptime-industries/compute-blade-agent
 
-go 1.23.0
-
-toolchain go1.24.0
+go 1.24.0
 
 require (
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2
 	github.com/prometheus/client_golang v1.16.0
 	github.com/sierrasoftworks/humane-errors-go v0.0.0-20241125132722-d032d7dd359e
+	github.com/spechtlabs/go-otel-utils/otelprovider v0.0.8
+	github.com/spechtlabs/go-otel-utils/otelzap v0.0.9
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.10.0
 	github.com/warthog618/gpiod v0.8.1
 	go.bug.st/serial v1.6.1
-	go.uber.org/zap v1.24.0
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0
+	go.uber.org/zap v1.27.0
 	golang.org/x/sync v0.12.0
-	google.golang.org/grpc v1.67.1
+	google.golang.org/grpc v1.71.0
 	google.golang.org/protobuf v1.36.6
 	gopkg.in/yaml.v3 v3.0.1
 	tinygo.org/x/drivers v0.26.0
 )
 
 require (
+	github.com/aws/smithy-go v1.22.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/creack/goselect v0.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
@@ -45,7 +52,19 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/otel v1.35.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.11.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.11.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 // indirect
+	go.opentelemetry.io/otel/log v0.11.0 // indirect
+	go.opentelemetry.io/otel/metric v1.35.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.35.0 // indirect
+	go.opentelemetry.io/otel/sdk/log v0.11.0 // indirect
+	go.opentelemetry.io/otel/trace v1.35.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.5.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/net v0.38.0 // indirect
 	golang.org/x/sys v0.31.0 // indirect