Merge branch 'scalability/minor-week-22' into 'master'

Benchmark queries w/ caching + retry create challenge in delegation experiment.

If this works, can do something similar for other failing calls. 

See merge request dfinity-lab/public/ic!12627

Author: Dfinity-skaestle

gitlab-ci/config/60--benchmark-test--spawn-benchmark-pipeline.yml

@@ -77,7 +77,22 @@ maximum-capacity-system-baseline-query:
       $SHELL_WRAPPER timeout 1h ./testnet/tools/icos_deploy.sh $TESTNET --git-revision "$GIT_REVISION" --no-boundary-nodes
       cd ./scalability
 
-      $SHELL_WRAPPER python3 experiments/run_system_baseline_experiment.py --use_random_payload=True --num_canisters 2 --testnet $TESTNET --wg_subnet 2 --wg_testnet $TESTNET --datapoints 100,3500,4500,5500,7500,8500,12000,17200,40000
+      # First run, no caching
+      # -------------------------------------------------
+
+      $SHELL_WRAPPER python3 experiments/run_system_baseline_experiment.py --use_random_payload=True --num_canisters 2 --testnet $TESTNET --wg_subnet 2 --wg_testnet $TESTNET --datapoints 100,3500,4500,5500,7500,12000,17200,40000
+
+      TIMESTAMP=$(find results/"$GIT_REVISION" -maxdepth 1 -mindepth 1 -type d -printf "%f\n" | sort -nr | head -1)
+      $SHELL_WRAPPER python3 common/generate_report.py --base_dir="results/" --git_revision="$GIT_REVISION" --timestamp="$TIMESTAMP"
+      $SHELL_WRAPPER python3 common/notify_dashboard.py --base_dir="results/" --git_revision="$GIT_REVISION" --timestamp="$TIMESTAMP" --is_max_capacity_run="True" --branch="$CURRENT_BRANCH" --gitlab_job_id="$CI_JOB_ID"
+
+      # Allow the system some time to recover from previous benchmark
+      sleep 60
+
+      # Second run, with caching
+      # -------------------------------------------------
+
+      $SHELL_WRAPPER python3 experiments/run_system_baseline_experiment.py --num_canisters 2 --testnet $TESTNET --wg_subnet 2 --wg_testnet $TESTNET --datapoints 100,3500,4500,5500,7500,12000,17200,40000
 
       TIMESTAMP=$(find results/"$GIT_REVISION" -maxdepth 1 -mindepth 1 -type d -printf "%f\n" | sort -nr | head -1)
       $SHELL_WRAPPER python3 common/generate_report.py --base_dir="results/" --git_revision="$GIT_REVISION" --timestamp="$TIMESTAMP"
@@ -86,8 +101,10 @@ maximum-capacity-system-baseline-query:
       find . -name  'workload-generator*stderr.txt' -print0 | xargs -0 pigz
       cd -
 
+      # Upload results from both run (the entire folder of that GIT revision)
       $SHELL_WRAPPER rclone --config="${CI_PROJECT_DIR}/.rclone.conf"  copyto "scalability/results/$GIT_REVISION" "performance-testing:performance-testing-results/$GIT_REVISION"
 
+
 # BENCHMARK SUITE maximum_capacity_system_baseline with update calls
 maximum-capacity-system-baseline-update:
   extends: .benchmark-test

scalability/common/cdweekly_results.py

@@ -35,7 +35,7 @@
 # Guidance: https://plotly.com/javascript/hover-text-and-formatting/
 HOVERTEMPLATE = """
 <b>Click for full report</b><br><br>
-%{yaxis.title.text}: %{y:,.0f}<br>
+%{yaxis.title.text}: %{y:,.2f}<br>
 %{xaxis.title.text}: %{x}<br>
 <br>
 %{text}

scalability/common/delegation.py

@@ -9,6 +9,7 @@
 from ic.canister import Canister
 from ic.client import Client
 from ic.identity import DelegateIdentity, Identity
+from retry.api import retry_call
 
 logging.basicConfig(level=logging.INFO)
 
@@ -100,7 +101,7 @@ def get_delegation(host_url, ii_canister_id):
 
     identityCanister = Canister(agent=agent, canister_id=ii_canister_id, candid=identity_canister_did)
     logging.info(f"II: Creating challenge .. on {host_url} using canister {ii_canister_id}")
-    challenge = identityCanister.create_challenge()
+    challenge = retry_call(identityCanister.create_challenge)
 
     # Call still failed after reinstalling
     if challenge is None:

scalability/experiments/run_system_baseline_experiment.py

@@ -183,7 +183,7 @@ def run_iterations(self, iterations=None):
 
             # Write summary file in each iteration including experiment specific data.
             self.write_summary_file(
-                "run_system_baseline_experiment",
+                "run_system_baseline_experiment" if FLAGS.use_random_payload else "run_system_baseline_experiment_cached",
                 {
                     "is_update": FLAGS.use_updates,
                     "total_requests": total_requests,