diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7683c698a4..17187310ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -386,6 +386,9 @@ jobs: needs: smokeTests if: always() steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Download all smoke tests artifacts uses: actions/download-artifact@v4 with: @@ -405,6 +408,11 @@ jobs: platform=Linux " > environment.properties + - name: Convert metrics JSON to Prometheus format + working-directory: './packages/e2e-tests/tools/' + run: | + ./convert_metrics_to_prometheus.sh "${{ github.workflow }}" "${{ github.run_id }}" ../../../metrics + - name: Publish allure report to S3 uses: andrcuns/allure-publish-action@v2.9.0 if: always() @@ -428,11 +436,68 @@ jobs: with: name: test-artifacts path: | - ./packages/e2e-tests/screenshots - ./packages/e2e-tests/logs - ./packages/e2e-tests/reports + ./screenshots + ./logs + ./reports + ./metrics retention-days: 5 + - name: Create Prometheus config + env: + PUSH_URL: ${{ secrets.GRAFANA_PUSH_URL }} # e.g., https://prometheus-us-central1.grafana.net/api/prom/push + PUSH_USER: ${{ secrets.GRAFANA_USERNAME }} # e.g., 787878 + PUSH_PASS: ${{ secrets.GRAFANA_PASSWORD }} # e.g., eyJrIjoxxxxxxxxxxxxxxyMX0= + run: | + cat >prometheus.yml < http.log 2>&1 + printf '\n*****************\n' + echo 'This is my current directory:' + pwd + printf '\n*****************\n' + echo 'This is ls -al in that directory' + ls -al + printf '\n*****************\n' + + - name: Run Prometheus + run: | + # Enable logs for debugging + #docker run --name prometheus -v $(pwd):/etc/prometheus -p 9090:9090 --link pushgateway:pushgateway prom/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:9090 --log.level=debug + docker run -d --name prometheus -v $(pwd):/etc/prometheus -p 9090:9090 --link pushgateway:pushgateway prom/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:9090 > prom.log 2>&1 + sleep 60 + # Uncomment for troubleshooting. + printf '\n*****************\n' + printf '\n curl e2e_cpu_seconds_total \n' + curl http://localhost:9090/api/v1/query?query=e2e_cpu_seconds_total_v1 + printf '\n*****************\n' + printf '\n cat prom.log \n' + cat prom.log + #printf '\n*****************\n' + #printf '\n send_failures_total \n' + #curl 'http://localhost:9090/api/v1/query?query=prometheus_remote_storage_queue_send_failures_total' + #printf '\n*****************\n' + #printf '\n storage_retries_total \n' + #curl 'http://localhost:9090/api/v1/query?query=prometheus_remote_storage_retries_total' + - run: | if [[ ${{ needs.smokeTests.result }} == "success" ]]; then exit 0 diff --git a/packages/e2e-tests/package.json b/packages/e2e-tests/package.json index 9c092ebdbb..adedd401c4 100755 --- a/packages/e2e-tests/package.json +++ b/packages/e2e-tests/package.json @@ -10,7 +10,7 @@ "allure:generate": "yarn allure generate -c ./reports/allure/results -o ./reports/allure/report", "allure:open": "allure open ./reports/allure/report/", "build": "yarn exec echo 'No build required'", - "cleanup": "rm -rf logs node_modules reports screenshots", + "cleanup": "rm -rf logs node_modules reports screenshots metrics", "lint": "run -T eslint -c .eslintrc.cjs .", "lint:fix": "run -T eslint -c .eslintrc.cjs . --fix", "pack-chrome-extension": "node ./src/utils/packExtension.ts", diff --git a/packages/e2e-tests/src/support/PidMonitor.ts b/packages/e2e-tests/src/support/PidMonitor.ts index d0190330e4..92bb4df47d 100644 --- a/packages/e2e-tests/src/support/PidMonitor.ts +++ b/packages/e2e-tests/src/support/PidMonitor.ts @@ -9,7 +9,7 @@ import { Logger } from './logger'; const execAsync = promisify(exec); interface UsageData { - timestamp: string; + timestamp: number; cpu: number; memory: number; } @@ -81,7 +81,7 @@ class PidMonitor { try { const stats = await pidusage(this.pid); this._data.push({ - timestamp: new Date().toISOString(), + timestamp: Date.now(), cpu: stats.cpu, memory: stats.memory }); diff --git a/packages/e2e-tests/tools/convertChromeExtToSafari.sh b/packages/e2e-tests/tools/convertChromeExtToSafari.sh deleted file mode 100644 index 9067ae3589..0000000000 --- a/packages/e2e-tests/tools/convertChromeExtToSafari.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -EXTENSION_DIR=../../apps/browser-extension-wallet -EXTENSION_BUILD_DIR="$EXTENSION_DIR/dist" -SAFARI_EXTENSION_OUTPUT_RELATIVE_PATH=./wallet-extension-safari-build -SAFARI_BUILT_EXTENSION_DIR="$SAFARI_EXTENSION_OUTPUT_RELATIVE_PATH/extension-build" -SAFARI_DEBUG_BUILD_PATH="$SAFARI_BUILT_EXTENSION_DIR/Build/Products/Release" - -if [ ! -d $EXTENSION_BUILD_DIR ]; then - echo "Could not find web extension build. Please build extension first" - exit -fi - -xcrun safari-web-extension-converter $EXTENSION_BUILD_DIR --no-prompt --copy-resources --no-open --project-location $SAFARI_EXTENSION_OUTPUT_RELATIVE_PATH --bundle-identifier io.lace.Lace - -cd "$SAFARI_EXTENSION_OUTPUT_RELATIVE_PATH/Lace" - -xcodebuild -configuration Release -scheme "Lace (macOS)" -derivedDataPath $SAFARI_BUILT_EXTENSION_DIR diff --git a/packages/e2e-tests/tools/convert_metrics_to_prometheus.sh b/packages/e2e-tests/tools/convert_metrics_to_prometheus.sh new file mode 100755 index 0000000000..1f2274870d --- /dev/null +++ b/packages/e2e-tests/tools/convert_metrics_to_prometheus.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -euo pipefail + +JOB=${1:-} +INSTANCE=${2:-} +METRICS_DIR=${3:-metrics} + +if [[ -z "$JOB" || -z "$INSTANCE" ]]; then + echo "Usage: $0 [metrics_dir]" + echo " job - GitHub workflow name" + echo " instance - GitHub run ID" + echo " metrics_dir - (optional) Directory containing JSON metrics files (default: metrics)" + exit 1 +fi + +OUTPUT_FILE="$METRICS_DIR/prometheus.txt" + +if [[ ! -d "$METRICS_DIR" ]]; then + echo "Metrics directory '$METRICS_DIR' does not exist." + exit 0 +fi + +shopt -s nullglob +FILES=("$METRICS_DIR"/*.json) + +# Filter out the output file from being processed +FILES=("${FILES[@]/$OUTPUT_FILE}") + +if [[ ${#FILES[@]} -eq 0 ]]; then + echo "No JSON files found in '$METRICS_DIR'." + exit 0 +fi + +> "$OUTPUT_FILE" + +for file in "${FILES[@]}"; do + if [[ ! -s "$file" ]]; then + echo "Skipping empty file: $file" + continue + fi + + filename=$(basename "$file") + scenario_id="${filename%%-chrome-usage.json}" + scenario_name=$(jq -r '.scenarioName // empty' "$file" | sed 's/"/\\"/g') + data_length=$(jq '.data | length' "$file") + + echo "# Metrics from $filename" >> "$OUTPUT_FILE" + + if [[ "$data_length" -eq 0 ]]; then + echo "e2e_cpu_seconds_total{scenario_name=\"$scenario_name\",scenario_id=\"$scenario_id\",job=\"$JOB\",instance=\"$INSTANCE\"} 0" >> "$OUTPUT_FILE" + echo "e2e_memory_rss_bytes{scenario_name=\"$scenario_name\",scenario_id=\"$scenario_id\",job=\"$JOB\",instance=\"$INSTANCE\"} 0" >> "$OUTPUT_FILE" + else + jq -c '.data[]' "$file" | while read -r entry; do + timestamp=$(echo "$entry" | jq -r '.timestamp') + cpu=$(echo "$entry" | jq -r '.cpu') + memory=$(echo "$entry" | jq -r '.memory') + + echo "e2e_cpu_seconds_total{scenario_name=\"$scenario_name\",scenario_id=\"$scenario_id\",job=\"$JOB\",instance=\"$INSTANCE\",timestamp=\"$timestamp\"} $cpu" >> "$OUTPUT_FILE" + echo "e2e_memory_rss_bytes{scenario_name=\"$scenario_name\",scenario_id=\"$scenario_id\",job=\"$JOB\",instance=\"$INSTANCE\",timestamp=\"$timestamp\"} $memory" >> "$OUTPUT_FILE" + done + fi +done + +echo "Metrics converted to Prometheus format: $OUTPUT_FILE" diff --git a/packages/e2e-tests/tools/openSafariExtension.sh b/packages/e2e-tests/tools/openSafariExtension.sh deleted file mode 100644 index fda8d696b2..0000000000 --- a/packages/e2e-tests/tools/openSafariExtension.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -SAFARI_EXTENSION_OUTPUT_RELATIVE_PATH=./wallet-extension-safari-build -SAFARI_BUILT_EXTENSION_DIR="$SAFARI_EXTENSION_OUTPUT_RELATIVE_PATH/extension-build" -SAFARI_DEBUG_BUILD_PATH="$SAFARI_BUILT_EXTENSION_DIR/Build/Products/Release" - -cd "$SAFARI_EXTENSION_OUTPUT_RELATIVE_PATH/Lace" -open -n "$SAFARI_DEBUG_BUILD_PATH/Lace.app"