From 6968f6e6cd0401e2a9653b2c70f8ae5f1716c69c Mon Sep 17 00:00:00 2001 From: Sahitya Damera Date: Tue, 14 Oct 2025 14:41:24 +0100 Subject: [PATCH] Add integration test for OTel integration Signed-off-by: Sahitya Damera --- .github/workflows/integration.yml | 53 +++++++++++++ mountpoint-s3/tests/otel_metrics.sh | 119 ++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 mountpoint-s3/tests/otel_metrics.sh diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9b715c5cc..103f4baa6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -219,6 +219,59 @@ jobs: - name: Run fstab integration tests run: ./mountpoint-s3/tests/fstab/basic_checks.sh + metrics: + name: Metrics Tests (${{ matrix.runner.name }}) + runs-on: ${{ matrix.runner.tags }} + + needs: approval + + strategy: + fail-fast: false + matrix: + runner: + - name: Ubuntu x86 + tags: [ubuntu-22.04] + - name: AL2 arm + tags: [self-hosted, linux, arm64] + + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ vars.ACTIONS_IAM_ROLE }} + aws-region: ${{ vars.S3_REGION }} + - name: Checkout code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + submodules: true + persist-credentials: false + - name: Set up Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "" + - name: Install operating system dependencies + uses: ./.github/actions/install-dependencies + with: + fuseVersion: 2 + - name: Install OpenTelemetry Collector + run: | + OS=$(uname -s | tr '[:upper:]' '[:lower:]') + ARCH=$(uname -m) + if [[ "$ARCH" == "x86_64" ]]; then + ARCH="amd64" + elif [[ "$ARCH" == "aarch64" ]] || [[ "$ARCH" == "arm64" ]]; then + ARCH="arm64" + fi + VERSION=$(curl -s https://api.github.com/repos/open-telemetry/opentelemetry-collector-releases/releases/latest | jq -r '.tag_name' | sed 's/^v//') + curl -L "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${VERSION}/otelcol_${VERSION}_${OS}_${ARCH}.tar.gz" \ + -o /tmp/otelcol.tar.gz + tar -xzf /tmp/otelcol.tar.gz -C /tmp + chmod +x /tmp/otelcol + rm /tmp/otelcol.tar.gz + - name: Run OTLP metrics tests + run: ./mountpoint-s3/tests/otel_metrics.sh + asan: name: Address sanitizer runs-on: [self-hosted, linux, arm64] diff --git a/mountpoint-s3/tests/otel_metrics.sh b/mountpoint-s3/tests/otel_metrics.sh new file mode 100644 index 000000000..6c3ce6e0a --- /dev/null +++ b/mountpoint-s3/tests/otel_metrics.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +# Exit on errors +set -e + +echo "Testing OTLP metrics export using OTel collector..." + +# Build mountpoint with otlp_integration feature +build_out=$(cargo build --bin mount-s3 --release --features otlp_integration --message-format=json-render-diagnostics) +MOUNTPOINT_PATH=$(printf "%s" "$build_out" | jq -js '[.[] | select(.reason == "compiler-artifact") | select(.executable != null)] | last | .executable') +echo "Mountpoint path: $MOUNTPOINT_PATH" + +# Create mount directory +sudo mkdir -p /mnt/metrics_test + +# Clean up previous test files +rm -f /tmp/otlp-metrics.log + +# Create OTLP collector config with file export +cat > /tmp/otel-config.yaml << EOF +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +exporters: + file: + path: /tmp/otlp-metrics.log + logging: + loglevel: info + +service: + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [file, logging] +EOF + +# Verify OTLP collector is available +if [[ ! -f /tmp/otelcol ]]; then + echo "Error: OTLP collector not found at /tmp/otelcol" + exit 1 +fi + +# Start OTLP collector +/tmp/otelcol --config=/tmp/otel-config.yaml > /tmp/collector.log 2>&1 & +COLLECTOR_PID=$! + +# Check if collector started successfully +sleep 1 +if ! kill -0 $COLLECTOR_PID 2>/dev/null; then + echo "OTLP collector failed to start" + exit 1 +fi + +# Run mountpoint with OTLP endpoint configured +"$MOUNTPOINT_PATH" --otlp-endpoint "http://localhost:4318" \ + --otlp-export-interval 2 \ + "${S3_BUCKET_NAME}" /mnt/metrics_test & +MOUNT_PID=$! + +sleep 1 +if ! kill -0 $MOUNT_PID 2>/dev/null; then + echo "Mountpoint failed to start" + exit 1 +fi + +# File operations to export metrics to OTLP +dd if=/dev/zero of=/mnt/metrics_test/metrics_test.txt bs=1M count=64 2>/dev/null || true + +# Random reads to force prefetch reset state +echo "Performing random reads..." +SKIP_OFFSETS=(0 48 32 8) +for skip in "${SKIP_OFFSETS[@]}"; do + dd if=/mnt/metrics_test/metrics_test.txt of=/dev/null bs=1M skip=$skip count=1 iflag=direct 2>/dev/null || true +done + +# Non existent file access to test S3 and FUSE request errors +cat /mnt/metrics_test/nonexistent_file.txt 2>/dev/null || true + +# Wait for periodic metrics to be exported +sleep 10 + +# Unmount the filesystem (mountpoint will exit gracefully) +fusermount -u /mnt/metrics_test 2>/dev/null || fusermount3 -u /mnt/metrics_test 2>/dev/null || true + +# Check for module-level metrics (relaxed validation) +MODULES=("process." "fuse." "s3." "prefetch.") +FOUND_MODULES=() + +for module in "${MODULES[@]}"; do + if grep -q "$module" /tmp/otlp-metrics.log 2>/dev/null; then + FOUND_MODULES+=("$module") + fi +done + +if [[ ${#FOUND_MODULES[@]} -gt 0 ]]; then + echo "✓ OTLP metrics found for modules: ${FOUND_MODULES[*]}" + echo "Available metrics:" + grep -E "process\.|fuse\.|s3\.|prefetch\.|experimental\." /tmp/otlp-metrics.log | head -10 +else + echo "✗ No expected module metrics found" + [[ -f /tmp/otlp-metrics.log ]] && echo "Available metrics:" && cat /tmp/otlp-metrics.log || echo "No metrics found in file" + exit 1 +fi + +# Cleanup collector +kill $COLLECTOR_PID 2>/dev/null || true + +# Cleanup +sudo rmdir /mnt/metrics_test 2>/dev/null || true +rm -f /tmp/otel-config.yaml /tmp/otlp-metrics.log /tmp/collector.log + +echo "OTLP metrics test passed" \ No newline at end of file