feat(OpenTelemetry): Integrate OpenTelemetry into agent

README.md

@@ -89,6 +89,7 @@ BLADE_LISTEN_METRICS=":1234"
 | `BLADE_FAN_SPEED_PERCENT=80`                      | Set static fan speed                     |
 | `BLADE_CRITICAL_TEMPERATURE_THRESHOLD=60`         | Set critical temp threshold (°C)         |
 | `BLADE_HAL_RPM_REPORTING_STANDARD_FAN_UNIT=false` | Disable RPM monitoring for lower CPU use |
+| `OTEL_EXPORTER_OTLP_ENDPOINT`                     | Endpoint for the OTLP exporter           |
 
 ## Exposing the gRPC API for Remote Access
 

cmd/agent/main.go

@@ -17,6 +17,8 @@ import (
 	"github.com/compute-blade-community/compute-blade-agent/internal/api"
 	"github.com/compute-blade-community/compute-blade-agent/pkg/log"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/spechtlabs/go-otel-utils/otelprovider"
+	"github.com/spechtlabs/go-otel-utils/otelzap"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"go.uber.org/zap"
@@ -25,7 +27,6 @@ import (
 var (
 	Version string
 	Commit  string
-	Date    string
 )
 
 var debug = pflag.BoolP("debug", "v", false, "enable verbose logging")
@@ -55,21 +56,76 @@ func main() {
 		baseLogger = zap.Must(zap.NewProduction())
 	}
 
-	zapLogger := baseLogger.With(zap.String("app", "compute-blade-agent"))
+	zapLogger := baseLogger.With(
+		zap.String("app", "compute-blade-agent"),
+		zap.String("version", Version),
+		zap.String("commit", Commit),
+	)
 	defer func() {
 		_ = zapLogger.Sync()
 	}()
-	_ = zap.ReplaceGlobals(zapLogger.With(zap.String("scope", "global")))
-	baseCtx := log.IntoContext(context.Background(), zapLogger)
 
+	// Replace zap global
+	undoZapGlobals := zap.ReplaceGlobals(zapLogger)
+
+	// Redirect stdlib log to zap
+	undoStdLogRedirect := zap.RedirectStdLog(zapLogger)
+
+	// Create OpenTelemetry Log and Trace provider
+	logProvider := otelprovider.NewLogger(
+		otelprovider.WithLogAutomaticEnv(),
+	)
+
+	traceProvider := otelprovider.NewTracer(
+		otelprovider.WithTraceAutomaticEnv(),
+	)
+
+	// Create otelLogger
+	otelZapLogger := otelzap.New(zapLogger,
+		otelzap.WithCaller(true),
+		otelzap.WithMinLevel(zap.InfoLevel),
+		otelzap.WithAnnotateLevel(zap.WarnLevel),
+		otelzap.WithErrorStatusLevel(zap.ErrorLevel),
+		otelzap.WithStackTrace(false),
+		otelzap.WithLoggerProvider(logProvider),
+	)
+
+	// Replace global otelZap logger
+	undoOtelZapGlobals := otelzap.ReplaceGlobals(otelZapLogger)
+	defer undoOtelZapGlobals()
+
+	// Cleanup Logging and Tracing
+	defer func() {
+		if err := traceProvider.ForceFlush(context.Background()); err != nil {
+			otelzap.L().Warn("failed to flush traces")
+		}
+
+		if err := logProvider.ForceFlush(context.Background()); err != nil {
+			otelzap.L().Warn("failed to flush logs")
+		}
+
+		if err := traceProvider.Shutdown(context.Background()); err != nil {
+			panic(err)
+		}
+
+		if err := logProvider.Shutdown(context.Background()); err != nil {
+			panic(err)
+		}
+
+		undoStdLogRedirect()
+		undoZapGlobals()
+	}()
+
+	// Setup context
+	baseCtx := log.IntoContext(context.Background(), otelZapLogger)
 	ctx, cancelCtx := context.WithCancelCause(baseCtx)
 	defer cancelCtx(context.Canceled)
 
 	// load configuration
 	var cbAgentConfig agent.ComputeBladeAgentConfig
 	if err := viper.Unmarshal(&cbAgentConfig); err != nil {
 		cancelCtx(err)
-		log.FromContext(ctx).Fatal("Failed to load configuration", zap.Error(err))
+		log.FromContext(ctx).WithError(err).Fatal("Failed to load configuration")
 	}
 
 	// setup stop signal handlers
@@ -97,11 +153,11 @@ func main() {
 		}
 	}()
 
-	log.FromContext(ctx).Info("Bootstrapping compute-blade-agent", zap.String("version", Version), zap.String("commit", Commit), zap.String("date", Date))
+	log.FromContext(ctx).Info("Bootstrapping compute-blade-agent")
 	computebladeAgent, err := agent.NewComputeBladeAgent(ctx, cbAgentConfig)
 	if err != nil {
 		cancelCtx(err)
-		log.FromContext(ctx).Fatal("Failed to create agent", zap.Error(err))
+		log.FromContext(ctx).WithError(err).Fatal("Failed to create agent")
 	}
 
 	// Run agent
@@ -124,13 +180,17 @@ func main() {
 	// Wait for done
 	<-ctx.Done()
 
+	// Since ctx is now done, we can no longer use it to get `log.FromContext(ctx)`
+	// but we must use otelzap.L() to get a logger
+
+	// Shut down gRPC and Prom Servers async
 	var wg sync.WaitGroup
 
 	// Shut-Down GRPC Server
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		log.FromContext(ctx).Info("Shutting down grpc server")
+		otelzap.L().Info("Shutting down grpc server")
 		grpcServer.GracefulStop()
 	}()
 
@@ -142,18 +202,19 @@ func main() {
 		shutdownCtx, shutdownCtxCancel := context.WithTimeout(context.Background(), 5*time.Second)
 		defer shutdownCtxCancel()
 
+		otelzap.L().Info("Shutting down prometheus/pprof server")
 		if err := promServer.Shutdown(shutdownCtx); err != nil {
-			log.FromContext(ctx).Error("Failed to shutdown prometheus/pprof server", zap.Error(err))
+			otelzap.L().WithError(err).Error("Failed to shutdown prometheus/pprof server")
 		}
 	}()
 
 	wg.Wait()
 
-	// Wait for context cancel
+	// Terminate accordingly
 	if err := ctx.Err(); !errors.Is(err, context.Canceled) {
-		log.FromContext(ctx).Fatal("Exiting", zap.Error(err))
+		otelzap.L().WithError(err).Fatal("Exiting")
 	} else {
-		log.FromContext(ctx).Info("Exiting")
+		otelzap.L().Info("Exiting")
 	}
 }
 
@@ -172,7 +233,7 @@ func runPrometheusEndpoint(ctx context.Context, cancel context.CancelCauseFunc,
 	go func() {
 		err := server.ListenAndServe()
 		if err != nil && !errors.Is(err, http.ErrServerClosed) {
-			log.FromContext(ctx).Error("Failed to start prometheus/pprof server", zap.Error(err))
+			log.FromContext(ctx).WithError(err).Error("Failed to start prometheus/pprof server")
 			cancel(err)
 		}
 	}()

cmd/bladectl/cmd_identify.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"errors"
+
 	bladeapiv1alpha1 "github.com/compute-blade-community/compute-blade-agent/api/bladeapi/v1alpha1"
 	"github.com/sierrasoftworks/humane-errors-go"
 	"github.com/spf13/cobra"

cmd/bladectl/config/config.go

@@ -6,7 +6,7 @@ import (
 	"path/filepath"
 
 	"github.com/sierrasoftworks/humane-errors-go"
-	"go.uber.org/zap"
+	"github.com/spechtlabs/go-otel-utils/otelzap"
 )
 
 type BladectlConfig struct {
@@ -58,7 +58,7 @@ func NewAuthenticatedBladectlConfig(server string, caPEM []byte, clientCertDER [
 func NewBladectlConfig(server string) *BladectlConfig {
 	hostname, err := os.Hostname()
 	if err != nil {
-		zap.L().Fatal("Failed to extract hostname", zap.Error(err))
+		otelzap.L().WithError(err).Fatal("Failed to extract hostname")
 	}
 
 	return &BladectlConfig{

go.mod

@@ -6,12 +6,15 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2
 	github.com/prometheus/client_golang v1.22.0
 	github.com/sierrasoftworks/humane-errors-go v0.0.0-20250507223502-4bb667dc1e16
+	github.com/spechtlabs/go-otel-utils/otelprovider v0.0.10
+	github.com/spechtlabs/go-otel-utils/otelzap v0.0.10
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/pflag v1.0.6
 	github.com/spf13/viper v1.20.1
 	github.com/stretchr/testify v1.10.0
 	github.com/warthog618/gpiod v0.8.1
 	go.bug.st/serial v1.6.4
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/sync v0.15.0
 	google.golang.org/grpc v1.73.0
@@ -21,13 +24,19 @@ require (
 )
 
 require (
+	github.com/aws/smithy-go v1.22.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/creack/goselect v0.1.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
@@ -41,9 +50,23 @@ require (
 	github.com/spf13/cast v1.8.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.11.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.11.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 // indirect
+	go.opentelemetry.io/otel/log v0.11.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk/log v0.11.0 // indirect
+	go.opentelemetry.io/otel/trace v1.36.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.5.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/net v0.40.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
 	golang.org/x/text v0.25.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect
 )