canonical · kayra1 · Oct 8, 2025
@@ -5,6 +5,7 @@ NOTARY_UI_FILES := $(shell find ui/src/ -type f) ui/package.json ui/package-lock
 
 NOTARY_ARTIFACT_NAME := notary
 NOTARY_CONFIG_FILE := config.yaml
+NOTARY_DOCKER_CONFIG_FILE := config-docker.yaml
 NOTARY_TLS_CERT := cert.pem
 NOTARY_TLS_KEY := key.pem
 ROCK_ARTIFACT_NAME := notary.rock
@@ -30,7 +31,7 @@ hotswap:
 	lxc exec notary -- docker cp ./notary notary:/bin/notary
 	lxc exec notary -- docker exec notary pebble restart notary
 
-deploy: $(ARTIFACT_FOLDER)/$(ROCK_ARTIFACT_NAME)
+deploy: $(ARTIFACT_FOLDER)/$(ROCK_ARTIFACT_NAME) $(ARTIFACT_FOLDER)/$(NOTARY_DOCKER_CONFIG_FILE) $(ARTIFACT_FOLDER)/$(NOTARY_TLS_CERT) $(ARTIFACT_FOLDER)/$(NOTARY_TLS_KEY)
 	@# Start notary container if it's not available
 	@if [ "$$(lxc list 2> /dev/null | grep notary > /dev/null; echo $$?)" = 1 ]; then \
 		echo "creating new notary VM instance in LXD"; \
@@ -45,10 +46,30 @@ deploy: $(ARTIFACT_FOLDER)/$(ROCK_ARTIFACT_NAME)
 	    lxc exec notary -- snap install rockcraft --classic ;\
 		\
 		echo "pushing config files"; \
-		lxc file push $(ARTIFACT_FOLDER)/$(ROCK_ARTIFACT_NAME) notary/root/$(ROCK_ARTIFACT_NAME); \
-		lxc file push $(ARTIFACT_FOLDER)/$(NOTARY_CONFIG_FILE) notary/root/$(NOTARY_CONFIG_FILE); \
-		lxc file push $(ARTIFACT_FOLDER)/$(NOTARY_TLS_CERT) notary/root/$(NOTARY_TLS_CERT); \
-		lxc file push $(ARTIFACT_FOLDER)/$(NOTARY_TLS_KEY) notary/root/$(NOTARY_TLS_KEY); \
+		lxc file push -p $(ARTIFACT_FOLDER)/$(ROCK_ARTIFACT_NAME) notary/root/$(ROCK_ARTIFACT_NAME); \
+		lxc file push -p $(ARTIFACT_FOLDER)/$(NOTARY_DOCKER_CONFIG_FILE) notary/root/$(NOTARY_DOCKER_CONFIG_FILE); \
+		lxc file push -p $(ARTIFACT_FOLDER)/$(NOTARY_TLS_CERT) notary/root/$(NOTARY_TLS_CERT); \
+		lxc file push -p $(ARTIFACT_FOLDER)/$(NOTARY_TLS_KEY) notary/root/$(NOTARY_TLS_KEY); \
+	fi
+
+	@# Deploy Jaeger if it hasn't been deployed yet
+	@if [ "$$(lxc exec notary -- docker ps 2> /dev/null | grep jaeger > /dev/null; echo $$?)" = 1 ]; then \
+	    echo "creating and running jaeger in Docker"; \
+		# The forwarded ports are, in order: \
+        # HTTP, /api/v3/*, OTLP-based JSON over HTTP (GUI) \
+        # gRPC, ExportTraceServiceRequest, OTLP Protobuf \
+		# HTTP, /v1/traces, OTLP Protobuf or OTLP JSON \
+		# HTTP, /sampling, sampling.proto_via Protobuf-to-JSON mapping_ \
+		# HTTP, /api/v2/spans, Zipkin v2 JSON or Protobuf \
+		lxc exec notary -- docker run -d --name jaeger \
+			--network host \
+            -p 16686:16686 \
+		    -p 4317:4317   \
+		    -p 4318:4318   \
+		    -p 5778:5778   \
+		    -p 9411:9411   \
+		    jaegertracing/jaeger:2.6.0; \
+		sleep 10; \
 	fi
 
 	@# Remove the old notary if it was still there
@@ -64,7 +85,7 @@ deploy: $(ARTIFACT_FOLDER)/$(ROCK_ARTIFACT_NAME)
 		-v /root:/config \
 		--network host \
 		-p 2111:2111 \
-		notary:latest --args notary start --config /config/config.yaml;
+		notary:latest --args notary start -m --config /config/$(NOTARY_DOCKER_CONFIG_FILE);
 	@echo "You can access notary at $$(lxc info notary | grep enp5s0 -A 15 | grep inet: | grep -oE '([0-9]{1,3}\.){3}[0-9]{1,3}'):2111"
 
 logs:
@@ -83,14 +104,27 @@ $(ARTIFACT_FOLDER)/$(NOTARY_CONFIG_FILE):
 	@echo 'key_path: "artifacts/key.pem"'      >> $@;\
      echo 'cert_path: "artifacts/cert.pem"'    >> $@;\
      echo 'db_path: "artifacts/notary.db"'     >> $@;\
+     echo 'port: 2111'                         >> $@;\
+	 echo 'pebble_notifications: false'        >> $@;\
+	 echo 'encryption_backend:'                >> $@;\
+	 echo '  type: "none"'                     >> $@;\
+
+$(ARTIFACT_FOLDER)/$(NOTARY_DOCKER_CONFIG_FILE):
+	@echo 'key_path: "/config/key.pem"'        >> $@;\
+     echo 'cert_path: "/config/cert.pem"'      >> $@;\
+     echo 'db_path: "/config/notary.db"'       >> $@;\
      echo 'port: 2111'                         >> $@;\
 	 echo 'pebble_notifications: false'        >> $@;\
 	 echo 'logging:'                           >> $@;\
 	 echo '  system:'                          >> $@;\
 	 echo '    level: "debug"'                 >> $@;\
-	 echo '    output: "artifacts/notary.log"' >> $@;\
-	 echo 'encryption_backend:'             >> $@;\
-	 echo '  type: "none"'                   >> $@;\
+	 echo '    output: "/config/notary.log"'   >> $@;\
+	 echo 'encryption_backend:'                >> $@;\
+	 echo '  type: "none"'                     >> $@;\
+	 echo 'tracing:'                           >> $@;\
+	 echo '  service_name: "notary"'           >> $@;\
+	 echo '  endpoint: "127.0.0.1:4317"'       >> $@;\
+	 echo '  sampling_rate: "100%"'            >> $@
 
 $(ARTIFACT_FOLDER)/$(NOTARY_TLS_CERT) $(ARTIFACT_FOLDER)/$(NOTARY_TLS_KEY):
 	openssl req -newkey rsa:2048 -nodes -keyout $(ARTIFACT_FOLDER)/$(NOTARY_TLS_KEY) -x509 -days 1 -out $(ARTIFACT_FOLDER)/$(NOTARY_TLS_CERT) -subj "/CN=example.com"

@@ -50,6 +50,7 @@ https://canonical-notary.readthedocs-hosted.com/en/latest/reference/config_file/
 			ExternalHostname:          appContext.ExternalHostname,
 			EnablePebbleNotifications: appContext.PebbleNotificationsEnabled,
 			Logger:                    appContext.Logger,
+			Tracer: 				   appContext.Tracer,
 			PublicConfig:              appContext.PublicConfig,
 		})
 		if err != nil {

@@ -36,3 +36,6 @@ RBAC
 GCM
 HashiCorp
 Requestor
+Grafana
+OpenTelemetry
+url
@@ -31,6 +31,12 @@ Or If you are using the snap you can modify the config under `/var/snap/notary/c
       - `approle_secret_id` (string): Secret ID for AppRole authentication.
       - `tls_ca_cert` (string): Path to the CA certificate for TLS verification (optional).
       - `tls_skip_verify` (boolean): Whether to skip TLS certificate verification (optional, defaults to `false`). It is strongly discouraged to set this to `true` outside of development environments
+- `tracing` (object): Configuration for tracing.
+  - `service_name` (string): The name that will identify your service in the tracing system
+  - `endpoint` (string): The URL of your OpenTelemetry collector endpoint
+  - `sampling_rate` (string): The percentage of traces to sample. Can be specified as a percentage (50%)
+  or a decimal value between 0.0 and 1.0 (0.0, 0.5, 1.0).
+
 
 ## Examples
 
@@ -47,6 +53,10 @@ logging:
     level: "info"
     output: "stdout"
 encryption_backend: {}
+tracing:
+  service_name: "notary"
+  endpoint: "127.0.0.1:4317"
+  sampling_rate: "100%"
 ```
 
 ### With HSM as an Encryption Backend
@@ -67,4 +77,8 @@ encryption_backend:
       lib_path: "/path/to/yubihsm_pkcs11.so"
       pin: "0001password"
       aes_encryption_key_id: 0x1234
+tracing:
+  service_name: "notary"
+  endpoint: "127.0.0.1:4317"
+  sampling_rate: "100%"
 ```
@@ -9,4 +9,5 @@ api/index.md
 config_file.md
 metrics.md
 roles.md
+tracing.md
 ```
@@ -0,0 +1,96 @@
+# Distributed Tracing in Notary
+
+Notary supports distributed tracing using OpenTelemetry, which allows you to monitor and troubleshoot request flows across the service. Traces can be viewed in any compatible visualization tool, such as Grafana Tempo.
+
+## Configuration
+
+To enable tracing in Notary, add the following configuration to your YAML configuration file:
+
+```yaml
+tracing:
+  service_name: "notary"  # Optional, defaults to "notary"
+  endpoint: "tempo:4317" # Required if enabled, the OpenTelemetry gRPC endpoint
+  sampling_rate: "100%"   # Optional, defaults to 100% (1.0)
+```
+
+### Configuration Options
+
+- **service_name**: The name that will identify your service in the tracing system
+- **endpoint**: The URL of your Tempo (or other OpenTelemetry collector) endpoint
+- **sampling_rate**: The percentage of traces to sample. Can be specified as:
+  - A percentage (e.g., "10%", "50%", "100%")
+  - A decimal value between 0.0 and 1.0 (e.g., "0.1", "0.5", "1.0")
+
+## Viewing Traces
+
+Traces are sent to the configured Tempo URL, where they can be visualized using Grafana or any other compatible tool.
+
+### Example Tempo Configuration with Docker Compose
+
+Here's a minimal example of how to set up Tempo with Docker Compose:
+
+```yaml
+version: '3'
+services:
+  tempo:
+    image: grafana/tempo:latest
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml
+    ports:
+      - "3200:3200"  # Tempo server
+      - "4317:4317"  # OTLP gRPC
+
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+    ports:
+      - "3000:3000"
+    depends_on:
+      - tempo
+```
+
+### trace flow
+
+Notary's tracing implementation tracks HTTP requests through the system, including:
+
+- HTTP method and path
+- Response status codes
+- Error information
+- Duration of requests
+
+## Troubleshooting
+
+If traces are not appearing in your visualization tool:
+
+1. Verify that tracing is enabled in your configuration
+2. Check that the `endpoint` is correct and accessible from your Notary instance
+3. Examine the Notary logs for any errors related to tracing
+4. Ensure that your sampling rate is high enough to capture traces (set to "100%" for testing)
+
+## Performance Considerations
+
+Tracing adds a small overhead to request processing. In production environments, consider:
+
+- Using a lower sampling rate (e.g., "10%") to reduce overhead
+- Monitoring the impact on service response times
+- Adjusting batch settings if handling very high volumes
+
+## Extending Tracing
+
+For developers extending Notary, you can add custom spans to functions by accessing the tracer:
+
+```go
+// Example of adding a custom span
+func myFunction(ctx context.Context) {
+    tracer := otel.Tracer("github.com/canonical/notary")
+    ctx, span := tracer.Start(ctx, "myFunction")
+    defer span.End()
+
+    // Add attributes to the span
+    span.SetAttributes(attribute.String("key", "value"))
+
+    // Your function code here
+}
+```
@@ -6,42 +6,63 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/google/go-cmp v0.7.0
 	github.com/hashicorp/vault-client-go v0.4.3
-	github.com/mattn/go-sqlite3 v1.14.28
-	github.com/pressly/goose/v3 v3.25.0
-	github.com/prometheus/client_golang v1.22.0
-	github.com/spf13/cobra v1.9.1
-	github.com/spf13/pflag v1.0.7
-	github.com/spf13/viper v1.20.1
+	github.com/mattn/go-sqlite3 v1.14.24
+	github.com/pressly/goose/v3 v3.26.0
+	github.com/prometheus/client_golang v1.21.1
+	github.com/spf13/cobra v1.10.1
+	github.com/spf13/pflag v1.0.10
+	github.com/spf13/viper v1.21.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0
+	go.opentelemetry.io/otel v1.37.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0
+	go.opentelemetry.io/otel/sdk v1.29.0
+	go.opentelemetry.io/otel/trace v1.37.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.40.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mfridman/interpolate v0.0.2 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	github.com/sagikazarmark/locafero v0.7.0 // indirect
+	github.com/sagikazarmark/locafero v0.11.0 // indirect
 	github.com/sethvargo/go-retry v0.3.0 // indirect
-	github.com/sourcegraph/conc v0.3.0 // indirect
-	github.com/spf13/afero v1.12.0 // indirect
-	github.com/spf13/cast v1.7.1 // indirect
+	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
+	github.com/spf13/afero v1.15.0 // indirect
+	github.com/spf13/cast v1.10.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/otel/metric v1.37.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.1.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/net v0.42.0 // indirect
 	golang.org/x/sync v0.16.0 // indirect
-	golang.org/x/text v0.27.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
 	golang.org/x/time v0.8.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 // indirect
+	google.golang.org/grpc v1.67.3 // indirect
 )
 
 require (